Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .dumplingconf.example
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@ salt = "${DUMPLING_GLOBAL_SALT}"
# Faker modules: https://docs.rs/fake/latest/fake/faker/index.html
# Upstream repo: https://github.com/cksac/fake-rs
[rules."public.users"]
# email — fake email via Rust `fake` crate; force quoted string output
email = { strategy = "faker", faker = "internet::SafeEmail", domain = "customer_identity", unique_within_domain = true }
# name — locale-aware full name (see `locale`); other generators use `faker = "module::Type"`
full_name = { strategy = "faker", faker = "name::Name" }
first_name = { strategy = "faker", faker = "name::FirstName" }
last_name = { strategy = "faker", faker = "name::LastName" }
# email — safe fake email (built-in); force quoted string output
email = { strategy = "email", domain = "customer_identity", unique_within_domain = true }
# name — locale-aware full name (see `locale`)
full_name = { strategy = "name" }
first_name = { strategy = "first_name" }
last_name = { strategy = "last_name" }
# phone — US-style (xxx) xxx-xxxx
phone = { strategy = "phone" }
# ssn — SHA-256 hex of original; use per-column salt for extra protection
Expand Down
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht

## [Unreleased]

### Added

- **First-class strategies** `email`, `name`, `first_name`, `last_name`, and `phone` in config (same generators as `faker = "internet::SafeEmail"`, `name::Name`, `name::FirstName`, `name::LastName`, and locale-aware phone). Strategy names are normalized to lowercase at load.

### Changed

- **Random-path faker/phone/PII**: one reused `StdRng` on `AnonymizerRegistry` instead of re-seeding per cell.
- **`faker` locale resolution**: `resolved_locale_key` avoids allocating a `String` per faker call when locale is `en` or absent.

## [0.4.3] - 2026-05-03

### Fixed
Expand Down
16 changes: 10 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,8 +121,8 @@ salt = "${DUMPLING_GLOBAL_SALT}"

# Rules are keyed by either "table" or "schema.table"
[rules."public.users"]
email = { strategy = "faker", faker = "internet::SafeEmail", domain = "customer_identity", unique_within_domain = true }
name = { strategy = "faker", faker = "name::Name", locale = "de_de" } # German-locale name
email = { strategy = "email", domain = "customer_identity", unique_within_domain = true }
name = { strategy = "name", locale = "de_de" } # German-locale name
ssn = { strategy = "hash", salt = "${env:DUMPLING_USERS_SSN_SALT}", as_string = true } # SHA-256 of original (salted)
age = { strategy = "int_range", min = 18, max = 90 }

Expand Down Expand Up @@ -162,8 +162,12 @@ token = "high"
| `redact` | Replace with `REDACTED` (string) |
| `uuid` | Random UUIDv4-like string |
| `hash` | SHA-256 hex of original value; supports per-column `salt` and global `salt` |
| `faker` | Values from the Rust [`fake`](https://crates.io/crates/fake) crate ([docs.rs](https://docs.rs/fake/latest/fake/), [`faker` modules](https://docs.rs/fake/latest/fake/faker/index.html)), chosen by a **string identifier** only (`faker = "module::Type"`, e.g. `internet::SafeEmail`). Config is **data only**: nothing from TOML is compiled or executed as Rust at runtime. Use `locale` for locale-aware generators; optional `min`/`max`, `length`, `format` as documented. Unsupported targets fail at config load. New generators require a **new Dumpling release** (or your own fork), not config-side code. |
| `email` | Safe email address (same generator as `faker = "internet::SafeEmail"`); supports `locale` |
| `name` | Full name (same as `faker = "name::Name"`); supports `locale` |
| `first_name` | First name (same as `faker = "name::FirstName"`); supports `locale` |
| `last_name` | Last name (same as `faker = "name::LastName"`); supports `locale` |
| `phone` | Locale-aware fake phone number (configurable via `locale`); defaults to English format |
| `faker` | Values from the Rust [`fake`](https://crates.io/crates/fake) crate ([docs.rs](https://docs.rs/fake/latest/fake/), [`faker` modules](https://docs.rs/fake/latest/fake/faker/index.html)), chosen by a **string identifier** only (`faker = "module::Type"`, e.g. `internet::SafeEmail`). Config is **data only**: nothing from TOML is compiled or executed as Rust at runtime. Use `locale` for locale-aware generators; optional `min`/`max`, `length`, `format` as documented. Unsupported targets fail at config load. New generators require a **new Dumpling release** (or your own fork), not config-side code. |
| `int_range` | Random integer in `[min, max]` |
| `string` | Random alphanumeric string (`length = 12` by default) |
| `date_fuzz` | Shifts a date by a random number of days in `[min_days, max_days]` (defaults: `-30..30`) |
Expand Down Expand Up @@ -223,7 +227,7 @@ dumpling --security-profile hardened --input dump.sql --check
- `unique_within_domain`: when true, different source values are assigned unique pseudonyms within the configured `domain`. NULL values are unaffected and always remain NULL.
- `min_days` / `max_days`: used by `date_fuzz`.
- `min_seconds` / `max_seconds`: used by `time_fuzz` and `datetime_fuzz`.
- `locale`: selects the language/regional format for the `faker` and `phone` strategies. Supported values: `en`, `fr_fr`, `de_de`, `it_it`, `pt_br`, `pt_pt`, `ar_sa`, `zh_cn`, `zh_tw`, `ja_jp`, `cy_gb`. Defaults to `en` when not specified.
- `locale`: selects the language/regional format for `email`, `name`, `first_name`, `last_name`, `faker`, and `phone`. Supported values: `en`, `fr_fr`, `de_de`, `it_it`, `pt_br`, `pt_pt`, `ar_sa`, `zh_cn`, `zh_tw`, `ja_jp`, `cy_gb`. Defaults to `en` when not specified.
- `faker`: required when `strategy = "faker"`. A plain string `"module::Type"` (case-insensitive) that maps to a **built-in** generator compiled into Dumpling—not arbitrary Rust or expressions. Names follow [`fake::faker`](https://docs.rs/fake/latest/fake/faker/index.html) (e.g. `internet::SafeEmail` → `faker::internet::SafeEmail` in the crate).
- `format`: used with `faker = "number::NumberWithFormat"`; pattern uses `#` (0–9) and `^` (1–9) per the [`fake` crate docs](https://docs.rs/fake/latest/fake/).

Expand Down Expand Up @@ -394,7 +398,7 @@ Define default strategies in `rules."<table>"` and add ordered per-column cases
```toml
[rules."public.users"]
email = { strategy = "hash", as_string = true } # default
name = { strategy = "faker", faker = "name::Name" }
name = { strategy = "name" }

[[column_cases."public.users".email]]
when.any = [{ column = "is_admin", op = "eq", value = "true" }]
Expand Down Expand Up @@ -445,7 +449,7 @@ salt = "${DUMPLING_HMAC_KEY}"

[rules."public.users"]
ssn = { strategy = "hash", as_string = true }
email = { strategy = "faker", faker = "internet::SafeEmail", domain = "users" }
email = { strategy = "email", domain = "users" }
```

```bash
Expand Down
72 changes: 66 additions & 6 deletions src/faker_dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,71 @@ pub fn parse_faker_path(faker: &str) -> Option<(&str, &str)> {
Some((module, typ))
}

/// Normalized locale key for `faker`, `phone`, and built-in PII strategies (`email`, `name`, …).
/// Uses ASCII case-insensitive matching without allocating.
pub fn resolved_locale_key(spec: &AnonymizerSpec) -> &'static str {
let s = spec.locale.as_deref().map(str::trim).unwrap_or("");
if s.is_empty() || s.eq_ignore_ascii_case("en") {
return "en";
}
if s.eq_ignore_ascii_case("fr_fr") {
return "fr_fr";
}
if s.eq_ignore_ascii_case("de_de") {
return "de_de";
}
if s.eq_ignore_ascii_case("it_it") {
return "it_it";
}
if s.eq_ignore_ascii_case("pt_br") {
return "pt_br";
}
if s.eq_ignore_ascii_case("pt_pt") {
return "pt_pt";
}
if s.eq_ignore_ascii_case("ar_sa") {
return "ar_sa";
}
if s.eq_ignore_ascii_case("zh_cn") {
return "zh_cn";
}
if s.eq_ignore_ascii_case("zh_tw") {
return "zh_tw";
}
if s.eq_ignore_ascii_case("ja_jp") {
return "ja_jp";
}
if s.eq_ignore_ascii_case("cy_gb") {
return "cy_gb";
}
"en"
}

/// Built-in `strategy = "email"` — same generator as `faker = "internet::SafeEmail"`.
pub fn pii_safe_email(loc: &str, rng: &mut StdRng) -> String {
fl!(loc, rng, SafeEmail)
}

/// Built-in `strategy = "name"` — full name.
pub fn pii_full_name(loc: &str, rng: &mut StdRng) -> String {
fl!(loc, rng, Name)
}

/// Built-in `strategy = "first_name"`.
pub fn pii_first_name(loc: &str, rng: &mut StdRng) -> String {
fl!(loc, rng, FirstName)
}

/// Built-in `strategy = "last_name"`.
pub fn pii_last_name(loc: &str, rng: &mut StdRng) -> String {
fl!(loc, rng, LastName)
}

/// Built-in `strategy = "phone"` — same generator as `faker` phone_number fakers.
pub fn pii_phone_number(loc: &str, rng: &mut StdRng) -> String {
fl!(loc, rng, PhoneNumber)
}

pub fn faker_string_with_rng(spec: &AnonymizerSpec, rng: &mut StdRng) -> Option<String> {
let faker = spec.faker.as_deref()?.trim();
if faker.is_empty() {
Expand All @@ -97,12 +162,7 @@ pub fn faker_string_with_rng(spec: &AnonymizerSpec, rng: &mut StdRng) -> Option<
let (module, typ) = parse_faker_path(faker)?;
let module_lc = module.to_ascii_lowercase();
let typ_lc = typ.to_ascii_lowercase();
let locale = spec
.locale
.as_deref()
.map(|l| l.trim().to_ascii_lowercase())
.unwrap_or_else(|| "en".to_string());
let loc = locale.as_str();
let loc = resolved_locale_key(spec);

let s: String = match (module_lc.as_str(), typ_lc.as_str()) {
("name", "firstname") => fl!(loc, rng, FirstName),
Expand Down
37 changes: 13 additions & 24 deletions src/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub fn should_keep_row(
schema: Option<&str>,
table: &str,
columns: &[String],
cells: &[Option<String>], // unescaped strings; None for NULL
cells: &[Option<&str>], // unescaped strings; None for NULL
) -> bool {
let set = match lookup_row_filters(cfg, schema, table) {
Some(s) => s,
Expand Down Expand Up @@ -43,7 +43,7 @@ pub fn should_keep_row(
true
}

fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option<String>]) -> bool {
fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option<&str>]) -> bool {
let targets = match extract_predicate_targets(pred, columns, cells) {
Some(values) => values,
None => return false, // top-level column missing -> does not match
Expand Down Expand Up @@ -121,13 +121,14 @@ fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option<Strin
fn extract_predicate_targets(
pred: &Predicate,
columns: &[String],
cells: &[Option<String>],
cells: &[Option<&str>],
) -> Option<Vec<Option<String>>> {
if let Some(i) = columns
.iter()
.position(|c| c.eq_ignore_ascii_case(&pred.column))
{
return Some(vec![cells.get(i).cloned().unwrap_or(None)]);
let cell = cells.get(i).copied().flatten();
return Some(vec![cell.map(|s| s.to_string())]);
}

let (base_column, path) = parse_json_column_key(&pred.column);
Expand Down Expand Up @@ -413,7 +414,7 @@ where
Ok(())
}

pub fn when_matches(when: &When, columns: &[String], cells: &[Option<String>]) -> bool {
pub fn when_matches(when: &When, columns: &[String], cells: &[Option<&str>]) -> bool {
// If any is non-empty, require at least one to match
if !when.any.is_empty() {
let mut matched_any = false;
Expand Down Expand Up @@ -603,35 +604,23 @@ mod tests {
Some("public"),
"users",
&cols,
&[
Some("1".to_string()),
Some("alice@myco.com".to_string()),
Some("US".to_string())
]
&[Some("1"), Some("alice@myco.com"), Some("US")]
));
// Case-insensitive keep (iregex)
assert!(should_keep_row(
&cfg,
Some("public"),
"users",
&cols,
&[
Some("2".to_string()),
Some("Carol@MYCO.COM".to_string()),
Some("GB".to_string())
]
&[Some("2"), Some("Carol@MYCO.COM"), Some("GB")]
));
// Delete example.com
assert!(!should_keep_row(
&cfg,
Some("public"),
"users",
&cols,
&[
Some("3".to_string()),
Some("bob@example.com".to_string()),
Some("US".to_string())
]
&[Some("3"), Some("bob@example.com"), Some("US")]
));
}

Expand Down Expand Up @@ -668,14 +657,14 @@ mod tests {
Some("public"),
"events",
&cols,
&[Some(r#"{"profile":{"tier":"gold"}}"#.to_string())]
&[Some(r#"{"profile":{"tier":"gold"}}"#)]
));
assert!(!should_keep_row(
&cfg,
Some("public"),
"events",
&cols,
&[Some(r#"{"profile":{"tier":"silver"}}"#.to_string())]
&[Some(r#"{"profile":{"tier":"silver"}}"#)]
));
}

Expand Down Expand Up @@ -713,15 +702,15 @@ mod tests {
"events",
&cols,
&[Some(
r#"{"items":[{"kind":"secondary"},{"kind":"primary"}]}"#.to_string()
r#"{"items":[{"kind":"secondary"},{"kind":"primary"}]}"#
)]
));
assert!(!should_keep_row(
&cfg,
Some("public"),
"events",
&cols,
&[Some(r#"{"items":[{"kind":"secondary"}]}"#.to_string())]
&[Some(r#"{"items":[{"kind":"secondary"}]}"#)]
));
}

Expand Down
28 changes: 23 additions & 5 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
use std::fs::File;
use std::io::{self, BufRead, BufReader, BufWriter, Write};

/// Larger than default 8 KiB to reduce syscall overhead on big dumps.
const IO_BUF_CAPACITY: usize = 256 * 1024;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

Expand Down Expand Up @@ -343,7 +346,10 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> {
.take()
.ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?;
pg_restore_child = Some(child);
(Box::new(BufReader::new(stdout)), Some(archive_path.clone()))
(
Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)),
Some(archive_path.clone()),
)
} else {
match &cli.input {
Some(path) => {
Expand All @@ -360,13 +366,19 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> {
);
}
let f = File::open(path)?;
(Box::new(BufReader::new(f)), Some(path.clone()))
(
Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)),
Some(path.clone()),
)
}
None => {
if !cli.allow_ext.is_empty() {
eprintln!("dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin");
}
(Box::new(BufReader::new(io::stdin())), None)
(
Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, io::stdin())),
None,
)
}
}
};
Expand All @@ -380,9 +392,15 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> {
.ok_or_else(|| anyhow::anyhow!("--in-place requires an --input path"))?;
let mut tmp = input_path.clone();
tmp.set_extension("sql.dumpling.tmp");
Box::new(BufWriter::new(File::create(&tmp)?))
Box::new(BufWriter::with_capacity(
IO_BUF_CAPACITY,
File::create(&tmp)?,
))
} else if let Some(path) = &cli.output {
Box::new(BufWriter::new(File::create(path)?))
Box::new(BufWriter::with_capacity(
IO_BUF_CAPACITY,
File::create(path)?,
))
} else {
Box::new(BufWriter::new(io::stdout()))
};
Expand Down
Loading
Loading