Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .dumplingconf.example
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ last_login = { strategy = "datetime_fuzz" }
wake_time = { strategy = "time_fuzz", min_seconds = -3600, max_seconds = 3600 }

[rules."public.orders"]
# credit card — redact entirely; force as quoted string
credit_card = { strategy = "redact", as_string = true }
# credit card — Luhn-valid synthetic PAN (length 13–19); use domain for stable FKs across dumps
credit_card = { strategy = "payment_card", length = 16, domain = "order_pan" }
# monetary / numeric — random decimal in range with fixed fractional digits
order_total = { strategy = "decimal", min = 0, max = 99999, scale = 2, domain = "order_amount" }
# keep the same anonymized email as users table via shared domain
customer_email = { strategy = "faker", faker = "internet::SafeEmail", domain = "customer_identity" }

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
components: rustfmt, clippy

- name: Cache Cargo build artifacts
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@v2.9.1

- name: Check formatting
run: cargo fmt --all -- --check
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/platform-compat-latest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable

- name: Cache Cargo build artifacts
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@v2.9.1

- name: Build release binary
run: cargo build --release --locked
2 changes: 1 addition & 1 deletion .github/workflows/platform-compat-matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable

- name: Cache Cargo build artifacts
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@v2.9.1

- name: Build release binary
run: cargo build --release --locked
2 changes: 1 addition & 1 deletion .github/workflows/policy-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable

- name: Cache Cargo build artifacts
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@v2.9.1

- name: Build dumpling
run: cargo build --release --locked
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable

- name: Cache Cargo build artifacts
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@v2.9.1

- name: Set up Python
uses: actions/setup-python@v6
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
components: rustfmt, clippy

- name: Cache Cargo build artifacts
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@v2.9.1

- name: Validate formatting
run: cargo fmt --all -- --check
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
uses: dtolnay/rust-toolchain@stable

- name: Cache Cargo build artifacts
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@v2.9.1

- name: Run cargo tests
run: cargo test --all-targets --all-features
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ Follow these steps in order. Do not skip any step.

7. **Tests**: Add `#[test]` functions in `src/transform.rs` (unit-test strategy output values) and in `src/sql.rs` (end-to-end pipeline test). Use `set_random_seed(N)` for reproducibility.

8. **`README.md`**: Add a row to the "Anonymization strategies" table.
8. **`README.md`**: Document the strategy under *Configuration → Anonymization strategies* (per-strategy subsection with accepted options), and mention any new spec fields in `AnonymizerSpec`’s doc comment in `settings.rs`.

**`faker` strategy:** Config only carries string identifiers; Dumpling never evaluates user Rust from config. To ship a new generator, add dispatch in `src/faker_dispatch.rs` and validation in `validate_anonymizer_spec` for the `faker` branch. Upstream reference: [`fake` on docs.rs](https://docs.rs/fake/latest/fake/), [`fake::faker` module index](https://docs.rs/fake/latest/fake/faker/index.html), [source on GitHub](https://github.com/cksac/fake-rs).

Expand Down
130 changes: 95 additions & 35 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ ssn = { strategy = "hash", salt = "${env:DUMPLING_USERS_SSN_SALT}", as_string
age = { strategy = "int_range", min = 18, max = 90 }

[rules."orders"]
credit_card = { strategy = "redact", as_string = true }
credit_card = { strategy = "payment_card", length = 16, domain = "order_pan" }
amount = { strategy = "decimal", min = 0, max = 9999, scale = 2, domain = "order_amount" }

# Optional explicit sensitive columns policy list (for strict coverage)
[sensitive_columns]
Expand Down Expand Up @@ -166,29 +167,99 @@ token = "high"

### Anonymization strategies

| Strategy | Description |
|---|---|
| `null` | Set field to SQL `NULL` |
| `redact` | Replace with `REDACTED` (string) |
| `uuid` | Random UUIDv4-like string |
| `hash` | SHA-256 hex of original value; supports per-column `salt` and global `salt` |
| `email` | Safe email address (same generator as `faker = "internet::SafeEmail"`); supports `locale` |
| `name` | Full name (same as `faker = "name::Name"`); supports `locale` |
| `first_name` | First name (same as `faker = "name::FirstName"`); supports `locale` |
| `last_name` | Last name (same as `faker = "name::LastName"`); supports `locale` |
| `phone` | Locale-aware fake phone number (configurable via `locale`); defaults to English format |
| `faker` | Values from the Rust [`fake`](https://crates.io/crates/fake) crate ([docs.rs](https://docs.rs/fake/latest/fake/), [`faker` modules](https://docs.rs/fake/latest/fake/faker/index.html)), chosen by a **string identifier** only (`faker = "module::Type"`, e.g. `internet::SafeEmail`). Config is **data only**: nothing from TOML is compiled or executed as Rust at runtime. Use `locale` for locale-aware generators; optional `min`/`max`, `length`, `format` as documented. Unsupported targets fail at config load. New generators require a **new Dumpling release** (or your own fork), not config-side code. |
| `int_range` | Random integer in `[min, max]` |
| `string` | Random alphanumeric string (`length = 12` by default) |
| `date_fuzz` | Shifts a date by a random number of days in `[min_days, max_days]` (defaults: `-30..30`) |
| `time_fuzz` | Shifts a time-of-day by a random number of seconds in `[min_seconds, max_seconds]` with 24h wraparound (defaults: `-300..300`) |
| `datetime_fuzz` | Shifts a timestamp/timestamptz by a random number of seconds in `[min_seconds, max_seconds]` (defaults: `-86400..86400`) |

**`faker` reference (upstream `fake` crate):** Dumpling’s `faker = "module::Type"` strings mirror the Rust [`fake`](https://crates.io/crates/fake) crate’s [`faker`](https://docs.rs/fake/latest/fake/faker/index.html) module layout. Use these when picking or extending generators:

- [docs.rs — `fake` crate root](https://docs.rs/fake/latest/fake/) (overview, `Fake` / `Dummy` traits, locales)
- [docs.rs — `fake::faker` module index](https://docs.rs/fake/latest/fake/faker/index.html) (per-domain submodules: `address`, `internet`, `name`, …)
- [GitHub — `cksac/fake-rs`](https://github.com/cksac/fake-rs) (source, README with the CLI’s generator name list)
Each column rule is a TOML inline table: `{ strategy = "<name>", ... }`. **Strategy-specific keys** are documented next to the strategy that accepts them. A few keys apply across many strategies; see [Cross-cutting options](#cross-cutting-options) below.

#### Choosing a strategy (cheaper vs more realistic)

Prefer **lightweight** strategies when nothing downstream requires lifelike values: **`null`**, **`redact`**, **`blank`**, **`empty_array`**, **`empty_object`**, **`string`**, **`int_range`**, and **`decimal`** are cheap to generate (simple constants, random digits/alnum, or bounded numeric shapes). Use **`blank`** for NOT NULL text where you must clear content without SQL NULL; use **`empty_array`** / **`empty_object`** on JSON path rules (or text columns holding JSON) when the document must keep `[]` / `{}` instead of `null` or `""`.

Reach for **richer** strategies when realism matters for restores, demos, or tests that exercise parsers and validators: **`email`**, **`name`**, **`first_name`**, **`last_name`**, **`phone`**, **`faker`**, **`uuid`**, **`hash`**, **`payment_card`**, and the **`date_fuzz` / `time_fuzz` / `datetime_fuzz`** family do more work (formatting, parsing, digest, or upstream generators). If a cheap strategy would break **CHECK constraints**, **NOT NULL**, **foreign-key shape**, or **import tooling** that validates formats, switch to a strategy that emits compatible values—or keep **`domain`** on the heavier strategy so referential consistency is preserved where you need it.

#### `null`

- **Behavior:** emit SQL `NULL` for the cell.
- **Options:** none. (`domain` is rejected.)

#### `redact`

- **Behavior:** replace with the literal `REDACTED`.
- **`as_string`:** if `true`, the replacement is always a single-quoted SQL string; if `false`, it is emitted without quotes (still valid as an identifier-like token in many dumps). When the **original** cell was already a quoted string, Dumpling quotes the output even when `as_string` is omitted—see [Cross-cutting options](#cross-cutting-options).

#### `blank`

- **Behavior:** replace with an **empty string** (`''` in SQL when quoted). If the source cell is SQL **`NULL`**, the cell stays **`NULL`** (same as `null` / `redact` semantics for missing values).
- **Options:** none. (`domain` is rejected.) **`as_string`** is ignored; output is always the empty string literal when non-NULL.

#### `empty_array` / `empty_object`

- **Behavior:** replace with the JSON tokens **`[]`** and **`{}`** as **unquoted** SQL/COPY tokens (so they parse as JSON when the column holds JSON). If the source cell is SQL **`NULL`**, the cell stays **`NULL`**.
- **JSON path rules:** use these on leaves that are JSON **arrays** or **objects** when you need a typed empty container instead of `null` or `""`.
- **Options:** none. (`domain` is rejected.)

#### `uuid`

- **Behavior:** random UUIDv4-like hyphenated hex string.
- **`as_string`:** same meaning as for `redact` / `hash` (force quoted literal vs. unquoted token).

#### `hash`

- **Behavior:** salted digest of the original cell value (SHA-256 in standard profile; HMAC-SHA-256 in `--security-profile hardened`).
- **`salt`:** optional per-column salt; otherwise the top-level `salt` or registry default applies.
- **`as_string`:** if `true`, force a quoted string literal; if `false`, unquoted hex. Quoted **source** cells are still written quoted when `as_string` is omitted.

#### `email`, `name`, `first_name`, `last_name`, `phone`

- **Behavior:** locale-aware fake values (same underlying generators as the matching `faker` targets).
- **`locale`:** optional; one of `en`, `fr_fr`, `de_de`, `it_it`, `pt_br`, `pt_pt`, `ar_sa`, `zh_cn`, `zh_tw`, `ja_jp`, `cy_gb` (default `en`).
- **Output:** always emitted as a quoted string replacement.

#### `int_range`

- **Behavior:** random integer in the inclusive range `[min, max]` (defaults `min = 0`, `max = 1_000_000`).
- **`min` / `max`:** inclusive bounds; `min` must be ≤ `max`.
- **Output:** always unquoted digits (suitable for integer / JSON number columns).

#### `decimal`

- **Behavior:** random decimal with integer part in `[min, max]` and fractional part of **`scale`** digits (defaults `min = 0`, `max = 1_000_000`, `scale = 2`). Use `scale = 0` for a plain integer string in the same range.
- **`min` / `max`:** inclusive integer-part bounds.
- **`scale`:** number of digits after `.` (0–38).
- **`as_string`:** same as `hash` / `redact` for quoting the full literal.

#### `payment_card`

- **Behavior:** random digit string of length **`length`** (default **16**) with a **valid Luhn check digit**, so `--scan-output` PAN detection treats synthetic values like test cards, not arbitrary digit runs.
- **`length`:** total digit count including check digit; must be 13–19 (PAN lengths).
- **Output:** always a quoted string of digits (no separators).

#### `string`

- **Behavior:** random alphanumeric string.
- **`length`:** character count (default 12); must be ≥ 1 when set.

#### `faker`

- **Behavior:** values from the Rust [`fake`](https://crates.io/crates/fake) crate ([`faker` modules](https://docs.rs/fake/latest/fake/faker/index.html)), selected only by the string **`faker = "module::Type"`** (e.g. `internet::SafeEmail`). Config is **data only**—nothing from TOML is compiled as Rust. Unsupported pairs fail at config load; new generators require a **new Dumpling release** (or a fork), not config-side code.
- **`faker`:** required; maps to a built-in allowlist in `src/faker_dispatch.rs`.
- **`locale`:** optional; same set as the built-in PII strategies when the upstream generator is locale-aware.
- **`min` / `max` / `length` / `format`:** only for/faker combinations that upstream supports (e.g. `number::NumberWithFormat` uses **`format`**: `#` = any digit, `^` = 1–9 per [`fake` docs](https://docs.rs/fake/latest/fake/)).

**Upstream reference:** [docs.rs — `fake`](https://docs.rs/fake/latest/fake/), [docs.rs — `fake::faker`](https://docs.rs/fake/latest/fake/faker/index.html), [GitHub — cksac/fake-rs](https://github.com/cksac/fake-rs).

#### `date_fuzz`, `time_fuzz`, `datetime_fuzz`

- **Behavior:** parse the existing value when possible and shift by a random offset; on parse failure the original string is kept.
- **`date_fuzz`:** **`min_days` / `max_days`** (defaults `-30` … `30`).
- **`time_fuzz` / `datetime_fuzz`:** **`min_seconds` / `max_seconds`** (`time_fuzz` defaults `-300` … `300`; `datetime_fuzz` defaults `-86400` … `86400`).
- **`as_string`:** force quoted literal vs. unquoted token for the emitted date/time/timestamp string.

### Cross-cutting options

These keys are valid on **multiple** strategies (unless validation says otherwise):

- **`domain`:** deterministic mapping bucket. The same non-NULL source value maps to the same pseudonym for that strategy inside the domain (across tables/columns). **SQL `NULL` is always preserved**—no fabricated FK targets.
- **`unique_within_domain`:** when `true` (requires `domain`), different source values are assigned distinct pseudonyms within the domain.
- **`as_string`:** when `true`, force the replacement to render as a **single-quoted SQL string literal**. When `false` or omitted, Dumpling still quotes the output if the **original** cell was quoted (`render_cell` uses `force_quoted || original.was_quoted`). Set `as_string = true` when the source may be unquoted (numeric-looking literals, some `COPY` shapes) but you need a string literal in the dump.

### Secret references

Expand Down Expand Up @@ -230,17 +301,6 @@ dumpling --input dump.sql --check --strict-coverage --report coverage.json
dumpling --security-profile hardened --input dump.sql --check
```

### Common column options

- `as_string`: if true, forces the anonymized value to be rendered as a quoted SQL string literal. By default Dumpling preserves the original quoting where possible.
- `domain`: deterministic mapping domain. When set, the same source value always maps to the same pseudonym inside that domain (across tables/columns). **SQL `NULL` inputs are always preserved as `NULL`** — a null FK reference has no source value to map, so no pseudonym is fabricated.
- `unique_within_domain`: when true, different source values are assigned unique pseudonyms within the configured `domain`. NULL values are unaffected and always remain NULL.
- `min_days` / `max_days`: used by `date_fuzz`.
- `min_seconds` / `max_seconds`: used by `time_fuzz` and `datetime_fuzz`.
- `locale`: selects the language/regional format for `email`, `name`, `first_name`, `last_name`, `faker`, and `phone`. Supported values: `en`, `fr_fr`, `de_de`, `it_it`, `pt_br`, `pt_pt`, `ar_sa`, `zh_cn`, `zh_tw`, `ja_jp`, `cy_gb`. Defaults to `en` when not specified.
- `faker`: required when `strategy = "faker"`. A plain string `"module::Type"` (case-insensitive) that maps to a **built-in** generator compiled into Dumpling—not arbitrary Rust or expressions. Names follow [`fake::faker`](https://docs.rs/fake/latest/fake/faker/index.html) (e.g. `internet::SafeEmail` → `faker::internet::SafeEmail` in the crate).
- `format`: used with `faker = "number::NumberWithFormat"`; pattern uses `#` (0–9) and `^` (1–9) per the [`fake` crate docs](https://docs.rs/fake/latest/fake/).

> **Note:** `table_options` are no longer supported; use explicit `rules` and optional `column_cases`.

---
Expand Down
4 changes: 4 additions & 0 deletions docs/src/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ When you use `strategy = "faker"` with `faker = "module::Type"`, those names ali

Dumpling only exposes a **subset** wired in `src/faker_dispatch.rs`; unsupported `module::Type` pairs fail at config load.

## Anonymization strategies

Strategy names and **per-strategy options** (`min`, `scale`, `as_string`, `faker`, …) are documented in the repository **README** under *Configuration → Anonymization strategies* (each strategy lists only the keys it accepts, plus **Choosing a strategy** for when to prefer cheap vs realistic transforms, and **Cross-cutting options** for `domain`, `unique_within_domain`, and `as_string`).

## Baseline config template

```toml
Expand Down
81 changes: 81 additions & 0 deletions src/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,7 @@ mod tests {
salt: None,
min: None,
max: None,
scale: None,
length: Some(4),
min_days: None,
max_days: None,
Expand Down Expand Up @@ -786,6 +787,7 @@ mod tests {
salt: None,
min: Some(0),
max: Some(9),
scale: None,
length: None,
min_days: None,
max_days: None,
Expand Down Expand Up @@ -820,6 +822,7 @@ mod tests {
salt: None,
min: Some(0),
max: Some(0),
scale: None,
length: None,
min_days: None,
max_days: None,
Expand Down Expand Up @@ -848,4 +851,82 @@ mod tests {
);
assert_eq!(v2["b"], false);
}

#[test]
fn rewrite_json_paths_empty_array_and_empty_object_strategies() {
let mut rules: HashMap<String, HashMap<String, AnonymizerSpec>> = HashMap::new();
rules.insert("public.t".to_string(), HashMap::new());
let cfg = ResolvedConfig {
salt: None,
rules,
row_filters: HashMap::new(),
column_cases: HashMap::new(),
sensitive_columns: HashMap::new(),
output_scan: crate::settings::OutputScanConfig::default(),
source_path: None,
};
let registry = AnonymizerRegistry::from_config(&cfg);

let arr_spec = AnonymizerSpec {
strategy: "empty_array".to_string(),
salt: None,
min: None,
max: None,
scale: None,
length: None,
min_days: None,
max_days: None,
min_seconds: None,
max_seconds: None,
domain: None,
unique_within_domain: None,
as_string: None,
locale: None,
faker: None,
format: None,
};
let out = rewrite_json_paths_with_rules(
&registry,
None,
&[(vec!["items".to_string()], arr_spec)],
r#"{"items":[1,2],"meta":{}}"#,
)
.unwrap()
.unwrap();
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
assert!(v["items"].is_array());
assert_eq!(v["items"], serde_json::json!([]));
assert_eq!(v["meta"], serde_json::json!({}));

let obj_spec = AnonymizerSpec {
strategy: "empty_object".to_string(),
salt: None,
min: None,
max: None,
scale: None,
length: None,
min_days: None,
max_days: None,
min_seconds: None,
max_seconds: None,
domain: None,
unique_within_domain: None,
as_string: None,
locale: None,
faker: None,
format: None,
};
let out2 = rewrite_json_paths_with_rules(
&registry,
None,
&[(vec!["meta".to_string()], obj_spec)],
r#"{"items":[],"meta":{"k":1}}"#,
)
.unwrap()
.unwrap();
let v2: serde_json::Value = serde_json::from_str(&out2).unwrap();
assert!(v2["meta"].is_object());
assert_eq!(v2["meta"], serde_json::json!({}));
assert_eq!(v2["items"], serde_json::json!([]));
}
}
Loading
Loading