diff --git a/.dumplingconf.example b/.dumplingconf.example index 4894f90..21ff505 100644 --- a/.dumplingconf.example +++ b/.dumplingconf.example @@ -31,12 +31,12 @@ salt = "${DUMPLING_GLOBAL_SALT}" # Faker modules: https://docs.rs/fake/latest/fake/faker/index.html # Upstream repo: https://github.com/cksac/fake-rs [rules."public.users"] -# email — fake email via Rust `fake` crate; force quoted string output -email = { strategy = "faker", faker = "internet::SafeEmail", domain = "customer_identity", unique_within_domain = true } -# name — locale-aware full name (see `locale`); other generators use `faker = "module::Type"` -full_name = { strategy = "faker", faker = "name::Name" } -first_name = { strategy = "faker", faker = "name::FirstName" } -last_name = { strategy = "faker", faker = "name::LastName" } +# email — safe fake email (built-in); force quoted string output +email = { strategy = "email", domain = "customer_identity", unique_within_domain = true } +# name — locale-aware full name (see `locale`) +full_name = { strategy = "name" } +first_name = { strategy = "first_name" } +last_name = { strategy = "last_name" } # phone — US-style (xxx) xxx-xxxx phone = { strategy = "phone" } # ssn — SHA-256 hex of original; use per-column salt for extra protection diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8ce59..93aacf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +### Added + +- **First-class strategies** `email`, `name`, `first_name`, `last_name`, and `phone` in config (same generators as `faker = "internet::SafeEmail"`, `name::Name`, `name::FirstName`, `name::LastName`, and locale-aware phone). Strategy names are normalized to lowercase at load. + +### Changed + +- **Random-path faker/phone/PII**: one reused `StdRng` on `AnonymizerRegistry` instead of re-seeding per cell. +- **`faker` locale resolution**: `resolved_locale_key` avoids allocating a `String` per faker call when locale is `en` or absent. + ## [0.4.3] - 2026-05-03 ### Fixed diff --git a/README.md b/README.md index 17a28e8..18bb3c3 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,8 @@ salt = "${DUMPLING_GLOBAL_SALT}" # Rules are keyed by either "table" or "schema.table" [rules."public.users"] -email = { strategy = "faker", faker = "internet::SafeEmail", domain = "customer_identity", unique_within_domain = true } -name = { strategy = "faker", faker = "name::Name", locale = "de_de" } # German-locale name +email = { strategy = "email", domain = "customer_identity", unique_within_domain = true } +name = { strategy = "name", locale = "de_de" } # German-locale name ssn = { strategy = "hash", salt = "${env:DUMPLING_USERS_SSN_SALT}", as_string = true } # SHA-256 of original (salted) age = { strategy = "int_range", min = 18, max = 90 } @@ -162,8 +162,12 @@ token = "high" | `redact` | Replace with `REDACTED` (string) | | `uuid` | Random UUIDv4-like string | | `hash` | SHA-256 hex of original value; supports per-column `salt` and global `salt` | -| `faker` | Values from the Rust [`fake`](https://crates.io/crates/fake) crate ([docs.rs](https://docs.rs/fake/latest/fake/), [`faker` modules](https://docs.rs/fake/latest/fake/faker/index.html)), chosen by a **string identifier** only (`faker = "module::Type"`, e.g. `internet::SafeEmail`). Config is **data only**: nothing from TOML is compiled or executed as Rust at runtime. Use `locale` for locale-aware generators; optional `min`/`max`, `length`, `format` as documented. Unsupported targets fail at config load. New generators require a **new Dumpling release** (or your own fork), not config-side code. | +| `email` | Safe email address (same generator as `faker = "internet::SafeEmail"`); supports `locale` | +| `name` | Full name (same as `faker = "name::Name"`); supports `locale` | +| `first_name` | First name (same as `faker = "name::FirstName"`); supports `locale` | +| `last_name` | Last name (same as `faker = "name::LastName"`); supports `locale` | | `phone` | Locale-aware fake phone number (configurable via `locale`); defaults to English format | +| `faker` | Values from the Rust [`fake`](https://crates.io/crates/fake) crate ([docs.rs](https://docs.rs/fake/latest/fake/), [`faker` modules](https://docs.rs/fake/latest/fake/faker/index.html)), chosen by a **string identifier** only (`faker = "module::Type"`, e.g. `internet::SafeEmail`). Config is **data only**: nothing from TOML is compiled or executed as Rust at runtime. Use `locale` for locale-aware generators; optional `min`/`max`, `length`, `format` as documented. Unsupported targets fail at config load. New generators require a **new Dumpling release** (or your own fork), not config-side code. | | `int_range` | Random integer in `[min, max]` | | `string` | Random alphanumeric string (`length = 12` by default) | | `date_fuzz` | Shifts a date by a random number of days in `[min_days, max_days]` (defaults: `-30..30`) | @@ -223,7 +227,7 @@ dumpling --security-profile hardened --input dump.sql --check - `unique_within_domain`: when true, different source values are assigned unique pseudonyms within the configured `domain`. NULL values are unaffected and always remain NULL. - `min_days` / `max_days`: used by `date_fuzz`. - `min_seconds` / `max_seconds`: used by `time_fuzz` and `datetime_fuzz`. -- `locale`: selects the language/regional format for the `faker` and `phone` strategies. Supported values: `en`, `fr_fr`, `de_de`, `it_it`, `pt_br`, `pt_pt`, `ar_sa`, `zh_cn`, `zh_tw`, `ja_jp`, `cy_gb`. Defaults to `en` when not specified. +- `locale`: selects the language/regional format for `email`, `name`, `first_name`, `last_name`, `faker`, and `phone`. Supported values: `en`, `fr_fr`, `de_de`, `it_it`, `pt_br`, `pt_pt`, `ar_sa`, `zh_cn`, `zh_tw`, `ja_jp`, `cy_gb`. Defaults to `en` when not specified. - `faker`: required when `strategy = "faker"`. A plain string `"module::Type"` (case-insensitive) that maps to a **built-in** generator compiled into Dumpling—not arbitrary Rust or expressions. Names follow [`fake::faker`](https://docs.rs/fake/latest/fake/faker/index.html) (e.g. `internet::SafeEmail` → `faker::internet::SafeEmail` in the crate). - `format`: used with `faker = "number::NumberWithFormat"`; pattern uses `#` (0–9) and `^` (1–9) per the [`fake` crate docs](https://docs.rs/fake/latest/fake/). @@ -394,7 +398,7 @@ Define default strategies in `rules.""` and add ordered per-column cases ```toml [rules."public.users"] email = { strategy = "hash", as_string = true } # default -name = { strategy = "faker", faker = "name::Name" } +name = { strategy = "name" } [[column_cases."public.users".email]] when.any = [{ column = "is_admin", op = "eq", value = "true" }] @@ -445,7 +449,7 @@ salt = "${DUMPLING_HMAC_KEY}" [rules."public.users"] ssn = { strategy = "hash", as_string = true } -email = { strategy = "faker", faker = "internet::SafeEmail", domain = "users" } +email = { strategy = "email", domain = "users" } ``` ```bash diff --git a/src/faker_dispatch.rs b/src/faker_dispatch.rs index cd78c36..1fd6ed8 100644 --- a/src/faker_dispatch.rs +++ b/src/faker_dispatch.rs @@ -89,6 +89,71 @@ pub fn parse_faker_path(faker: &str) -> Option<(&str, &str)> { Some((module, typ)) } +/// Normalized locale key for `faker`, `phone`, and built-in PII strategies (`email`, `name`, …). +/// Uses ASCII case-insensitive matching without allocating. +pub fn resolved_locale_key(spec: &AnonymizerSpec) -> &'static str { + let s = spec.locale.as_deref().map(str::trim).unwrap_or(""); + if s.is_empty() || s.eq_ignore_ascii_case("en") { + return "en"; + } + if s.eq_ignore_ascii_case("fr_fr") { + return "fr_fr"; + } + if s.eq_ignore_ascii_case("de_de") { + return "de_de"; + } + if s.eq_ignore_ascii_case("it_it") { + return "it_it"; + } + if s.eq_ignore_ascii_case("pt_br") { + return "pt_br"; + } + if s.eq_ignore_ascii_case("pt_pt") { + return "pt_pt"; + } + if s.eq_ignore_ascii_case("ar_sa") { + return "ar_sa"; + } + if s.eq_ignore_ascii_case("zh_cn") { + return "zh_cn"; + } + if s.eq_ignore_ascii_case("zh_tw") { + return "zh_tw"; + } + if s.eq_ignore_ascii_case("ja_jp") { + return "ja_jp"; + } + if s.eq_ignore_ascii_case("cy_gb") { + return "cy_gb"; + } + "en" +} + +/// Built-in `strategy = "email"` — same generator as `faker = "internet::SafeEmail"`. +pub fn pii_safe_email(loc: &str, rng: &mut StdRng) -> String { + fl!(loc, rng, SafeEmail) +} + +/// Built-in `strategy = "name"` — full name. +pub fn pii_full_name(loc: &str, rng: &mut StdRng) -> String { + fl!(loc, rng, Name) +} + +/// Built-in `strategy = "first_name"`. +pub fn pii_first_name(loc: &str, rng: &mut StdRng) -> String { + fl!(loc, rng, FirstName) +} + +/// Built-in `strategy = "last_name"`. +pub fn pii_last_name(loc: &str, rng: &mut StdRng) -> String { + fl!(loc, rng, LastName) +} + +/// Built-in `strategy = "phone"` — same generator as `faker` phone_number fakers. +pub fn pii_phone_number(loc: &str, rng: &mut StdRng) -> String { + fl!(loc, rng, PhoneNumber) +} + pub fn faker_string_with_rng(spec: &AnonymizerSpec, rng: &mut StdRng) -> Option { let faker = spec.faker.as_deref()?.trim(); if faker.is_empty() { @@ -97,12 +162,7 @@ pub fn faker_string_with_rng(spec: &AnonymizerSpec, rng: &mut StdRng) -> Option< let (module, typ) = parse_faker_path(faker)?; let module_lc = module.to_ascii_lowercase(); let typ_lc = typ.to_ascii_lowercase(); - let locale = spec - .locale - .as_deref() - .map(|l| l.trim().to_ascii_lowercase()) - .unwrap_or_else(|| "en".to_string()); - let loc = locale.as_str(); + let loc = resolved_locale_key(spec); let s: String = match (module_lc.as_str(), typ_lc.as_str()) { ("name", "firstname") => fl!(loc, rng, FirstName), diff --git a/src/filter.rs b/src/filter.rs index 119fe2b..8d15ffe 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -13,7 +13,7 @@ pub fn should_keep_row( schema: Option<&str>, table: &str, columns: &[String], - cells: &[Option], // unescaped strings; None for NULL + cells: &[Option<&str>], // unescaped strings; None for NULL ) -> bool { let set = match lookup_row_filters(cfg, schema, table) { Some(s) => s, @@ -43,7 +43,7 @@ pub fn should_keep_row( true } -fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option]) -> bool { +fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option<&str>]) -> bool { let targets = match extract_predicate_targets(pred, columns, cells) { Some(values) => values, None => return false, // top-level column missing -> does not match @@ -121,13 +121,14 @@ fn predicate_matches(pred: &Predicate, columns: &[String], cells: &[Option], + cells: &[Option<&str>], ) -> Option>> { if let Some(i) = columns .iter() .position(|c| c.eq_ignore_ascii_case(&pred.column)) { - return Some(vec![cells.get(i).cloned().unwrap_or(None)]); + let cell = cells.get(i).copied().flatten(); + return Some(vec![cell.map(|s| s.to_string())]); } let (base_column, path) = parse_json_column_key(&pred.column); @@ -413,7 +414,7 @@ where Ok(()) } -pub fn when_matches(when: &When, columns: &[String], cells: &[Option]) -> bool { +pub fn when_matches(when: &When, columns: &[String], cells: &[Option<&str>]) -> bool { // If any is non-empty, require at least one to match if !when.any.is_empty() { let mut matched_any = false; @@ -603,11 +604,7 @@ mod tests { Some("public"), "users", &cols, - &[ - Some("1".to_string()), - Some("alice@myco.com".to_string()), - Some("US".to_string()) - ] + &[Some("1"), Some("alice@myco.com"), Some("US")] )); // Case-insensitive keep (iregex) assert!(should_keep_row( @@ -615,11 +612,7 @@ mod tests { Some("public"), "users", &cols, - &[ - Some("2".to_string()), - Some("Carol@MYCO.COM".to_string()), - Some("GB".to_string()) - ] + &[Some("2"), Some("Carol@MYCO.COM"), Some("GB")] )); // Delete example.com assert!(!should_keep_row( @@ -627,11 +620,7 @@ mod tests { Some("public"), "users", &cols, - &[ - Some("3".to_string()), - Some("bob@example.com".to_string()), - Some("US".to_string()) - ] + &[Some("3"), Some("bob@example.com"), Some("US")] )); } @@ -668,14 +657,14 @@ mod tests { Some("public"), "events", &cols, - &[Some(r#"{"profile":{"tier":"gold"}}"#.to_string())] + &[Some(r#"{"profile":{"tier":"gold"}}"#)] )); assert!(!should_keep_row( &cfg, Some("public"), "events", &cols, - &[Some(r#"{"profile":{"tier":"silver"}}"#.to_string())] + &[Some(r#"{"profile":{"tier":"silver"}}"#)] )); } @@ -713,7 +702,7 @@ mod tests { "events", &cols, &[Some( - r#"{"items":[{"kind":"secondary"},{"kind":"primary"}]}"#.to_string() + r#"{"items":[{"kind":"secondary"},{"kind":"primary"}]}"# )] )); assert!(!should_keep_row( @@ -721,7 +710,7 @@ mod tests { Some("public"), "events", &cols, - &[Some(r#"{"items":[{"kind":"secondary"}]}"#.to_string())] + &[Some(r#"{"items":[{"kind":"secondary"}]}"#)] )); } diff --git a/src/main.rs b/src/main.rs index 4df9a5a..7194603 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,8 @@ use std::fs::File; use std::io::{self, BufRead, BufReader, BufWriter, Write}; + +/// Larger than default 8 KiB to reduce syscall overhead on big dumps. +const IO_BUF_CAPACITY: usize = 256 * 1024; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -343,7 +346,10 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { .take() .ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?; pg_restore_child = Some(child); - (Box::new(BufReader::new(stdout)), Some(archive_path.clone())) + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), + Some(archive_path.clone()), + ) } else { match &cli.input { Some(path) => { @@ -360,13 +366,19 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { ); } let f = File::open(path)?; - (Box::new(BufReader::new(f)), Some(path.clone())) + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), + Some(path.clone()), + ) } None => { if !cli.allow_ext.is_empty() { eprintln!("dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin"); } - (Box::new(BufReader::new(io::stdin())), None) + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, io::stdin())), + None, + ) } } }; @@ -380,9 +392,15 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { .ok_or_else(|| anyhow::anyhow!("--in-place requires an --input path"))?; let mut tmp = input_path.clone(); tmp.set_extension("sql.dumpling.tmp"); - Box::new(BufWriter::new(File::create(&tmp)?)) + Box::new(BufWriter::with_capacity( + IO_BUF_CAPACITY, + File::create(&tmp)?, + )) } else if let Some(path) = &cli.output { - Box::new(BufWriter::new(File::create(path)?)) + Box::new(BufWriter::with_capacity( + IO_BUF_CAPACITY, + File::create(path)?, + )) } else { Box::new(BufWriter::new(io::stdout())) }; diff --git a/src/settings.rs b/src/settings.rs index 3b06ca8..fed2f08 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -32,7 +32,7 @@ pub struct RawConfig { #[derive(Debug, Clone, Deserialize)] pub struct AnonymizerSpec { - /// Strategy name: redact|null|uuid|hash|faker|phone|int_range|string|date_fuzz|time_fuzz|datetime_fuzz + /// Strategy name: redact|null|uuid|hash|email|name|first_name|last_name|phone|faker|int_range|string|date_fuzz|time_fuzz|datetime_fuzz pub strategy: String, /// if strategy=hash: optional per-column salt override; otherwise ignored pub salt: Option, @@ -56,11 +56,12 @@ pub struct AnonymizerSpec { /// Force the replacement to be rendered as a SQL string literal /// If unset, we attempt to preserve the original quoting style. pub as_string: Option, - /// Locale for locale-aware strategies: `faker` (passed to the `fake` crate) and `phone`. + /// Locale for locale-aware strategies: built-in PII (`email`, `name`, `first_name`, `last_name`), `faker`, and `phone`. /// Supported values: en, fr_fr, de_de, it_it, pt_br, pt_pt, ar_sa, zh_cn, zh_tw, ja_jp, cy_gb. /// Defaults to "en" when not specified. pub locale: Option, /// When `strategy = "faker"`, selects the `fake` generator as `"module::Type"` (e.g. `internet::SafeEmail`, `name::FirstName`). + /// Only read when `strategy = "faker"` (other strategies must not set `faker`). /// See [`fake::faker`](https://docs.rs/fake/latest/fake/faker/index.html) and the [crate docs](https://docs.rs/fake/latest/fake/). #[serde(default)] pub faker: Option, @@ -448,31 +449,6 @@ fn is_simple_key(key: &str) -> bool { .all(|ch| ch.is_ascii_alphanumeric() || ch == '_') } -/// Map removed built-in strategies to `faker` + `faker` path for backwards compatibility. -fn normalize_anonymizer_spec(mut spec: AnonymizerSpec) -> AnonymizerSpec { - let s = spec.strategy.to_ascii_lowercase(); - match s.as_str() { - "email" => { - spec.strategy = "faker".to_string(); - spec.faker = Some("internet::SafeEmail".to_string()); - } - "name" => { - spec.strategy = "faker".to_string(); - spec.faker = Some("name::Name".to_string()); - } - "first_name" => { - spec.strategy = "faker".to_string(); - spec.faker = Some("name::FirstName".to_string()); - } - "last_name" => { - spec.strategy = "faker".to_string(); - spec.faker = Some("name::LastName".to_string()); - } - _ => {} - } - spec -} - fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { let RawConfig { salt, @@ -497,8 +473,9 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { for (table_key, cols) in rules.into_iter() { let table_key_norm = table_key.to_lowercase(); let mut col_map: HashMap = HashMap::new(); - for (col, spec) in cols.into_iter() { - col_map.insert(col.to_lowercase(), normalize_anonymizer_spec(spec)); + for (col, mut spec) in cols.into_iter() { + spec.strategy = spec.strategy.to_ascii_lowercase(); + col_map.insert(col.to_lowercase(), spec); } normalized_rules.insert(table_key_norm, col_map); } @@ -514,7 +491,7 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { let cases: Vec = cases .into_iter() .map(|mut c| { - c.strategy = normalize_anonymizer_spec(c.strategy.clone()); + c.strategy.strategy = c.strategy.strategy.to_ascii_lowercase(); c }) .collect(); @@ -570,8 +547,12 @@ const KNOWN_STRATEGIES: &[&str] = &[ "redact", "uuid", "hash", - "faker", + "email", + "name", + "first_name", + "last_name", "phone", + "faker", "int_range", "string", "date_fuzz", @@ -673,10 +654,8 @@ fn is_valid_severity(value: &str) -> bool { } fn validate_anonymizer_spec(spec: &AnonymizerSpec, path: &str) -> anyhow::Result<()> { - // Legacy strategy names (`email`, `name`, …) normalize to `faker` during `resolve()`; apply the - // same mapping here so file validation matches runtime behavior. - let spec = normalize_anonymizer_spec(spec.clone()); - let strategy = spec.strategy.as_str(); + let strategy = spec.strategy.to_ascii_lowercase(); + let strategy = strategy.as_str(); if !KNOWN_STRATEGIES.contains(&strategy) { anyhow::bail!( "{}.strategy has unknown strategy '{}'; expected one of {}", @@ -739,7 +718,12 @@ fn validate_anonymizer_spec(spec: &AnonymizerSpec, path: &str) -> anyhow::Result unsupported.push("max_seconds"); } } - if spec.locale.is_some() && !matches!(strategy, "faker" | "phone") { + if spec.locale.is_some() + && !matches!( + strategy, + "faker" | "phone" | "email" | "name" | "first_name" | "last_name" + ) + { unsupported.push("locale"); } @@ -843,7 +827,7 @@ fn validate_anonymizer_spec(spec: &AnonymizerSpec, path: &str) -> anyhow::Result faker ); } - if !crate::faker_dispatch::faker_path_supported(&spec) { + if !crate::faker_dispatch::faker_path_supported(spec) { anyhow::bail!( "{}.faker {:?} is not a supported generator; see README and \ https://docs.rs/fake/latest/fake/faker/index.html for upstream module names. \ @@ -853,6 +837,7 @@ fn validate_anonymizer_spec(spec: &AnonymizerSpec, path: &str) -> anyhow::Result ); } } + "email" | "name" | "first_name" | "last_name" | "phone" => {} _ => {} } @@ -1633,7 +1618,7 @@ salt = "${vault:secret/dumpling#key}" let path = write_temp_config( r#" [rules."public.users"] -full_name = { strategy = "faker", faker = "name::Name", locale = "de_de" } +full_name = { strategy = "name", locale = "de_de" } "#, ); let cfg = load_config(Some(&path), false).expect("locale=de_de should be valid"); @@ -1643,8 +1628,8 @@ full_name = { strategy = "faker", faker = "name::Name", locale = "de_de" } .and_then(|c| c.get("full_name")) .expect("expected full_name rule"); assert_eq!(spec.locale.as_deref(), Some("de_de")); - assert_eq!(spec.strategy, "faker"); - assert_eq!(spec.faker.as_deref(), Some("name::Name")); + assert_eq!(spec.strategy, "name"); + assert!(spec.faker.is_none()); let _ = fs::remove_file(path); } @@ -1682,7 +1667,7 @@ full_name = { strategy = "faker", faker = "name::Name", locale = "klingon" } } #[test] - fn locale_on_non_name_phone_strategy_fails_validation() { + fn locale_on_non_locale_strategy_fails_validation() { let path = write_temp_config( r#" [rules."public.users"] diff --git a/src/sql.rs b/src/sql.rs index 86019ba..8d40c21 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -216,22 +216,17 @@ impl SqlStreamProcessor { mode = Mode::Pass; } else { // data row - let fields: Vec<&str> = line.trim_end_matches('\n').split('\t').collect(); + let line_body = line.trim_end_matches(['\n', '\r']); + let fields: Vec<&str> = line_body.split('\t').collect(); if !*enabled { // passthrough unchanged writer.write_all(line.as_bytes())?; continue; } // Evaluate row filters - let unescaped: Vec> = fields + let unescaped: Vec> = fields .iter() - .map(|f| { - if *f == r"\N" { - None - } else { - Some((*f).to_string()) - } - }) + .map(|f| if *f == r"\N" { None } else { Some(*f) }) .collect(); let keep = should_keep_row( &self.config, @@ -353,16 +348,14 @@ impl SqlStreamProcessor { fn process_insert_statement(&mut self, stmt: &str) -> anyhow::Result { // Compact whitespace minimally for parsing while preserving output formatting by re-rendering. // Extract INSERT [OR REPLACE|OR IGNORE] INTO
(columns) VALUES ; - let s = stmt.trim().to_string(); + let s = stmt.trim(); // Ensure trailing semicolon present if !s.ends_with(';') { anyhow::bail!("INSERT without trailing semicolon"); } // Detect the INSERT variant keyword (SQLite supports OR REPLACE / OR IGNORE) - let up = s.to_uppercase(); - let (insert_keyword, keyword_len) = detect_insert_keyword(&up); - let idx_insert = up - .find(insert_keyword) + let (insert_keyword, keyword_len) = detect_insert_keyword(s); + let idx_insert = find_ignore_ascii_case(s, insert_keyword) .ok_or_else(|| anyhow::anyhow!("not an INSERT"))?; let after = &s[idx_insert + keyword_len..]; // Parse table ident then columns list @@ -377,27 +370,25 @@ impl SqlStreamProcessor { return Ok(stmt.to_string()); } // Expect VALUES - let rest_upper = rest_after_cols.to_uppercase(); - let idx_values = rest_upper - .find("VALUES") + let idx_values = find_ignore_ascii_case(rest_after_cols, "VALUES") .ok_or_else(|| anyhow::anyhow!("INSERT missing VALUES"))?; let after_values = &rest_after_cols[idx_values + "VALUES".len()..]; // Strip trailing semicolon let values_block = strip_trailing_semicolon(after_values.trim()); let rows = parse_values_rows(values_block)?; // Transform and filter rows - let mut out = String::new(); - out.push_str(&format!( - "{} {} ({}) VALUES ", - insert_keyword, - format_table_ident(schema.as_deref(), &table), - columns.join(", ") - )); + let mut out = String::with_capacity(stmt.len()); + out.push_str(insert_keyword); + out.push(' '); + out.push_str(&format_table_ident(schema.as_deref(), &table)); + out.push_str(" ("); + out.push_str(&columns.join(", ")); + out.push_str(") VALUES "); let mut first_row = true; for row in rows.into_iter() { // Row-level keep/drop - let cell_values: Vec> = - row.iter().map(|cell| cell.original.clone()).collect(); + let cell_values: Vec> = + row.iter().map(|cell| cell.original.as_deref()).collect(); let keep = should_keep_row( &self.config, schema.as_deref(), @@ -423,13 +414,13 @@ impl SqlStreamProcessor { } first_row = false; let mut rendered_cells: Vec = Vec::with_capacity(row.len()); - for (i, cell) in row.into_iter().enumerate() { + for (i, cell) in row.iter().enumerate() { let col = columns.get(i).map(|s| s.as_str()).unwrap_or(""); match self.apply_column_rules( schema.as_deref(), &table, &columns, - &cell_values, + cell_values.as_slice(), col, cell.original.as_deref(), ) { @@ -464,13 +455,18 @@ impl SqlStreamProcessor { } } } - rendered_cells.push(render_cell(&replacement, &cell)); + rendered_cells.push(render_cell(&replacement, cell)); } Err(e) => return Err(e), } } out.push('('); - out.push_str(&rendered_cells.join(", ")); + let mut sep = ""; + for part in &rendered_cells { + out.push_str(sep); + out.push_str(part); + sep = ", "; + } out.push(')'); } out.push_str(";\n"); @@ -537,7 +533,7 @@ impl SqlStreamProcessor { schema: Option<&str>, table: &str, columns: &[String], - row_cells: &[Option], + row_cells: &[Option<&str>], col: &str, cell_original: Option<&str>, ) -> anyhow::Result)>> { @@ -599,20 +595,20 @@ enum Mode { } fn starts_with_insert(line: &str) -> bool { - let upper = line.trim_start().to_uppercase(); - upper.starts_with("INSERT INTO") - || upper.starts_with("INSERT OR REPLACE INTO") - || upper.starts_with("INSERT OR IGNORE INTO") + let trimmed = line.trim_start(); + starts_with_ci(trimmed, "INSERT INTO") + || starts_with_ci(trimmed, "INSERT OR REPLACE INTO") + || starts_with_ci(trimmed, "INSERT OR IGNORE INTO") } /// Returns the INSERT keyword variant (uppercase) and its byte length. /// Handles standard INSERT INTO as well as SQLite's OR REPLACE / OR IGNORE forms. -fn detect_insert_keyword(stmt_upper: &str) -> (&'static str, usize) { +fn detect_insert_keyword(stmt: &str) -> (&'static str, usize) { // Search from the start of the (trimmed) statement for the first keyword - let trimmed = stmt_upper.trim_start(); - if trimmed.starts_with("INSERT OR REPLACE INTO") { + let trimmed = stmt.trim_start(); + if starts_with_ci(trimmed, "INSERT OR REPLACE INTO") { ("INSERT OR REPLACE INTO", "INSERT OR REPLACE INTO".len()) - } else if trimmed.starts_with("INSERT OR IGNORE INTO") { + } else if starts_with_ci(trimmed, "INSERT OR IGNORE INTO") { ("INSERT OR IGNORE INTO", "INSERT OR IGNORE INTO".len()) } else { ("INSERT INTO", "INSERT INTO".len()) @@ -621,23 +617,20 @@ fn detect_insert_keyword(stmt_upper: &str) -> (&'static str, usize) { fn starts_with_create_table(line: &str) -> bool { let trimmed = line.trim_start(); - let upper = trimmed.to_uppercase(); - upper.starts_with("CREATE TABLE") || upper.starts_with("CREATE UNLOGGED TABLE") + starts_with_ci(trimmed, "CREATE TABLE") || starts_with_ci(trimmed, "CREATE UNLOGGED TABLE") } fn statement_complete(buf: &str) -> bool { // Detect a semicolon that's not inside quotes or parentheses let mut depth: i32 = 0; let mut in_single = false; - let mut i = 0; - let chars: Vec = buf.chars().collect(); - while i < chars.len() { - let c = chars[i]; + let mut chars = buf.chars().peekable(); + while let Some(c) = chars.next() { if in_single { if c == '\'' { - // doubled single-quote escapes - if i + 1 < chars.len() && chars[i + 1] == '\'' { - i += 1; // skip escape + // doubled single-quote escapes (standard SQL) + if chars.peek() == Some(&'\'') { + let _ = chars.next(); } else { in_single = false; } @@ -651,7 +644,6 @@ fn statement_complete(buf: &str) -> bool { _ => {} } } - i += 1; } false } @@ -1155,6 +1147,27 @@ fn starts_with_ci(s: &str, prefix: &str) -> bool { .unwrap_or(false) } +/// Find `needle` in `haystack` using ASCII case-insensitive comparison (no full-string uppercase allocation). +fn find_ignore_ascii_case(haystack: &str, needle: &str) -> Option { + if needle.is_empty() { + return Some(0); + } + let hay = haystack.as_bytes(); + let nd = needle.as_bytes(); + if nd.len() > hay.len() { + return None; + } + 'outer: for start in 0..=(hay.len() - nd.len()) { + for j in 0..nd.len() { + if !hay[start + j].eq_ignore_ascii_case(&nd[j]) { + continue 'outer; + } + } + return Some(start); + } + None +} + #[derive(Clone, Debug)] struct Cell { original: Option, // None for NULL @@ -1268,91 +1281,81 @@ fn parse_parenthesized_values(s: &str) -> anyhow::Result<(Vec, usize)> { // Handles standard SQL '' escape doubling as well as MSSQL N'...' Unicode string literals // (and analogous E'...', B'...', X'...' prefixes used by other dialects). The one-character // prefix is silently stripped; the string content is preserved as-is. - let mut i = 0usize; - let chs: Vec = s.chars().collect(); - if chs.first() != Some(&'(') { + let mut it = s.char_indices().peekable(); + let (_, first) = it.next().ok_or_else(|| anyhow::anyhow!("expected '('"))?; + if first != '(' { anyhow::bail!("expected '('"); } - i += 1; // skip '(' let mut cells: Vec = Vec::new(); let mut in_single = false; let mut buf = String::new(); let mut trailing_expr = String::new(); let mut was_quoted = false; let mut closed_quoted_literal = false; - while i < chs.len() { - let c = chs[i]; + while let Some((_, c)) = it.peek().copied() { if in_single { if c == '\'' { - // doubled '' escape - if i + 1 < chs.len() && chs[i + 1] == '\'' { + let _ = it.next(); // consume this '\'' + if it.peek().map(|&(_, p)| p) == Some('\'') { + let _ = it.next(); buf.push('\''); - i += 2; - continue; } else { in_single = false; closed_quoted_literal = true; - i += 1; - continue; } - } else { - buf.push(c); - i += 1; continue; } - } else { - match c { - '\'' => { - // Strip a leading string-type prefix accumulated in buf when it is exactly - // one character: N (MSSQL Unicode), E (PostgreSQL escape), B/X (bit/hex). - if buf.len() == 1 - && matches!( - buf.chars().next(), - Some('N' | 'n' | 'E' | 'e' | 'B' | 'b' | 'X' | 'x') - ) - { + let _ = it.next(); + buf.push(c); + continue; + } + match c { + '\'' => { + // Strip a leading string-type prefix accumulated in buf when it is exactly + // one character: N (MSSQL Unicode), E (PostgreSQL escape), B/X (bit/hex). + if buf.len() == 1 { + let b0 = buf.as_bytes()[0]; + if matches!(b0, b'N' | b'n' | b'E' | b'e' | b'B' | b'b' | b'X' | b'x') { buf.clear(); } - in_single = true; - was_quoted = true; - i += 1; - } - ')' => { - // end cell, end row - let cell = finalize_cell(&buf, was_quoted, &trailing_expr); - cells.push(cell); - i += 1; - return Ok((cells, i)); } - ',' => { - // end cell - let cell = finalize_cell(&buf, was_quoted, &trailing_expr); - cells.push(cell); - buf.clear(); - trailing_expr.clear(); - was_quoted = false; - closed_quoted_literal = false; - i += 1; - // consume following spaces - while i < chs.len() && chs[i].is_whitespace() { - i += 1; + let _ = it.next(); + in_single = true; + was_quoted = true; + } + ')' => { + let (end_byte, _) = it.next().unwrap(); + let cell = finalize_cell(&buf, was_quoted, &trailing_expr); + cells.push(cell); + return Ok((cells, end_byte + ')'.len_utf8())); + } + ',' => { + let _ = it.next(); + let cell = finalize_cell(&buf, was_quoted, &trailing_expr); + cells.push(cell); + buf.clear(); + trailing_expr.clear(); + was_quoted = false; + closed_quoted_literal = false; + while let Some(&(_, w)) = it.peek() { + if !w.is_whitespace() { + break; } + let _ = it.next(); } - c if c.is_whitespace() => { - // Preserve whitespace after a quoted literal so explicit SQL casts stay intact. - if was_quoted && closed_quoted_literal { - trailing_expr.push(c); - } - // Skip insignificant whitespace between tokens when unquoted. - i += 1; + } + w if w.is_whitespace() => { + let _ = it.next(); + if was_quoted && closed_quoted_literal { + trailing_expr.push(w); } - other => { - if was_quoted && closed_quoted_literal { - trailing_expr.push(other); - } else { - buf.push(other); - } - i += 1; + } + other => { + let _ = it.next(); + if was_quoted && closed_quoted_literal { + trailing_expr.push(other); + } else { + buf.push(other); } } } @@ -1408,7 +1411,7 @@ fn select_strategy_for_cell( schema: Option<&str>, table: &str, columns: &[String], - row_cells: &[Option], + row_cells: &[Option<&str>], column: &str, ) -> Option { // First-match-wins on column_cases @@ -1461,20 +1464,20 @@ fn qualified_column_name(schema: Option<&str>, table: &str, column: &str) -> Str fn infer_auto_strategy(column: &str) -> Option { let normalized = column.to_ascii_lowercase().replace('-', "_"); let spec = if normalized.contains("email") { - faker_spec("internet::SafeEmail", Some(true)) + base_spec("email", Some(true)) } else if normalized.contains("first_name") || normalized == "fname" || normalized.contains("given_name") { - faker_spec("name::FirstName", Some(true)) + base_spec("first_name", Some(true)) } else if normalized.contains("last_name") || normalized.contains("surname") || normalized == "lname" || normalized.contains("family_name") { - faker_spec("name::LastName", Some(true)) + base_spec("last_name", Some(true)) } else if normalized.contains("name") { - faker_spec("name::Name", Some(true)) + base_spec("name", Some(true)) } else if normalized.contains("phone") || normalized.contains("mobile") || normalized.contains("cell") @@ -1514,26 +1517,6 @@ fn infer_auto_strategy(column: &str) -> Option { Some(spec) } -fn faker_spec(faker_path: &str, as_string: Option) -> AnonymizerSpec { - AnonymizerSpec { - strategy: "faker".to_string(), - salt: None, - min: None, - max: None, - length: None, - min_days: None, - max_days: None, - min_seconds: None, - max_seconds: None, - domain: None, - unique_within_domain: None, - as_string, - locale: None, - faker: Some(faker_path.to_string()), - format: None, - } -} - fn base_spec(strategy: &str, as_string: Option) -> AnonymizerSpec { AnonymizerSpec { strategy: strategy.to_string(), diff --git a/src/transform.rs b/src/transform.rs index 7731c05..871b8a9 100644 --- a/src/transform.rs +++ b/src/transform.rs @@ -1,10 +1,10 @@ -use crate::faker_dispatch::faker_string_with_rng; +use crate::faker_dispatch::{ + faker_string_with_rng, pii_first_name, pii_full_name, pii_last_name, pii_phone_number, + pii_safe_email, resolved_locale_key, +}; use crate::settings::{AnonymizerSpec, ResolvedConfig}; use chrono::Timelike; use chrono::{DateTime, Duration, NaiveDate, NaiveDateTime, NaiveTime}; -use fake::faker::phone_number::raw::PhoneNumber; -use fake::locales::{AR_SA, CY_GB, DE_DE, EN, FR_FR, IT_IT, JA_JP, PT_BR, PT_PT, ZH_CN, ZH_TW}; -use fake::Fake; use hmac::{Hmac, Mac}; use rand::rngs::StdRng; use rand::SeedableRng; @@ -36,6 +36,8 @@ pub struct AnonymizerRegistry { pub default_salt: Option, pub security_profile: SecurityProfile, domain_mappings: RefCell>, + /// Reused for `faker`, `phone`, and built-in PII strategies on the random (non-domain) path. + faker_rng: RefCell, } static mut RNG_SEED_OVERRIDE: Option = None; @@ -67,6 +69,7 @@ impl AnonymizerRegistry { default_salt: cfg.salt.clone(), security_profile: SecurityProfile::Standard, domain_mappings: RefCell::new(HashMap::new()), + faker_rng: RefCell::new(make_random_rng()), } } } @@ -230,21 +233,14 @@ fn make_deterministic_rng(stream: &mut DeterministicByteStream) -> StdRng { StdRng::from_seed(seed) } -/// Generate a localized phone number using a `rand` RNG. -fn fake_phone_with_rng(locale: &str, rng: &mut StdRng) -> String { - match locale { - "fr_fr" => PhoneNumber(FR_FR).fake_with_rng(rng), - "de_de" => PhoneNumber(DE_DE).fake_with_rng(rng), - "it_it" => PhoneNumber(IT_IT).fake_with_rng(rng), - "pt_br" => PhoneNumber(PT_BR).fake_with_rng(rng), - "pt_pt" => PhoneNumber(PT_PT).fake_with_rng(rng), - "ar_sa" => PhoneNumber(AR_SA).fake_with_rng(rng), - "zh_cn" => PhoneNumber(ZH_CN).fake_with_rng(rng), - "zh_tw" => PhoneNumber(ZH_TW).fake_with_rng(rng), - "ja_jp" => PhoneNumber(JA_JP).fake_with_rng(rng), - "cy_gb" => PhoneNumber(CY_GB).fake_with_rng(rng), - _ => PhoneNumber(EN).fake_with_rng(rng), - } +fn quoted_pii_string( + registry: &AnonymizerRegistry, + spec: &AnonymizerSpec, + f: impl FnOnce(&str, &mut StdRng) -> String, +) -> Replacement { + let loc = resolved_locale_key(spec); + let mut rng = registry.faker_rng.borrow_mut(); + Replacement::quoted(f(loc, &mut rng)) } fn apply_random_anonymizer( @@ -310,7 +306,7 @@ fn apply_random_anonymizer( } } "faker" => { - let mut rng = make_random_rng(); + let mut rng = registry.faker_rng.borrow_mut(); let value = faker_string_with_rng(spec, &mut rng).unwrap_or_else(|| { unreachable!( "faker strategy must be validated at config load; unsupported faker should never reach apply_random_anonymizer" @@ -318,15 +314,11 @@ fn apply_random_anonymizer( }); Replacement::quoted(value) } - "phone" => { - let locale = spec - .locale - .as_deref() - .map(|l| l.trim().to_ascii_lowercase()) - .unwrap_or_else(|| "en".to_string()); - let mut rng = make_random_rng(); - Replacement::quoted(fake_phone_with_rng(&locale, &mut rng)) - } + "email" => quoted_pii_string(registry, spec, pii_safe_email), + "name" => quoted_pii_string(registry, spec, pii_full_name), + "first_name" => quoted_pii_string(registry, spec, pii_first_name), + "last_name" => quoted_pii_string(registry, spec, pii_last_name), + "phone" => quoted_pii_string(registry, spec, pii_phone_number), "int_range" => { let min = spec.min.unwrap_or(0); let max = spec.max.unwrap_or(1_000_000); @@ -497,14 +489,30 @@ fn apply_deterministic_anonymizer( }); Replacement::quoted(value) } + "email" => { + let loc = resolved_locale_key(spec); + let mut rng = make_deterministic_rng(&mut stream); + Replacement::quoted(pii_safe_email(loc, &mut rng)) + } + "name" => { + let loc = resolved_locale_key(spec); + let mut rng = make_deterministic_rng(&mut stream); + Replacement::quoted(pii_full_name(loc, &mut rng)) + } + "first_name" => { + let loc = resolved_locale_key(spec); + let mut rng = make_deterministic_rng(&mut stream); + Replacement::quoted(pii_first_name(loc, &mut rng)) + } + "last_name" => { + let loc = resolved_locale_key(spec); + let mut rng = make_deterministic_rng(&mut stream); + Replacement::quoted(pii_last_name(loc, &mut rng)) + } "phone" => { - let locale = spec - .locale - .as_deref() - .map(|l| l.trim().to_ascii_lowercase()) - .unwrap_or_else(|| "en".to_string()); + let loc = resolved_locale_key(spec); let mut rng = make_deterministic_rng(&mut stream); - Replacement::quoted(fake_phone_with_rng(&locale, &mut rng)) + Replacement::quoted(pii_phone_number(loc, &mut rng)) } "int_range" => { let min = spec.min.unwrap_or(0); @@ -930,15 +938,8 @@ mod tests { use std::collections::HashMap; fn make_spec(strategy: &str, salt: Option<&str>, domain: Option<&str>) -> AnonymizerSpec { - let (strategy, faker) = match strategy { - "email" => ("faker".to_string(), Some("internet::SafeEmail".to_string())), - "name" => ("faker".to_string(), Some("name::Name".to_string())), - "first_name" => ("faker".to_string(), Some("name::FirstName".to_string())), - "last_name" => ("faker".to_string(), Some("name::LastName".to_string())), - _ => (strategy.to_string(), None), - }; AnonymizerSpec { - strategy, + strategy: strategy.to_string(), salt: salt.map(|s| s.to_string()), min: None, max: None, @@ -951,7 +952,7 @@ mod tests { unique_within_domain: None, as_string: None, locale: None, - faker, + faker: None, format: None, } } @@ -961,6 +962,7 @@ mod tests { default_salt: salt.map(|s| s.to_string()), security_profile: SecurityProfile::Standard, domain_mappings: RefCell::new(HashMap::new()), + faker_rng: RefCell::new(make_random_rng()), } } @@ -969,6 +971,7 @@ mod tests { default_salt: salt.map(|s| s.to_string()), security_profile: SecurityProfile::Hardened, domain_mappings: RefCell::new(HashMap::new()), + faker_rng: RefCell::new(make_random_rng()), } } @@ -1137,15 +1140,8 @@ mod tests { // --- Localized name and phone strategies --- fn make_spec_with_locale(strategy: &str, locale: Option<&str>) -> AnonymizerSpec { - let (strategy, faker) = match strategy { - "email" => ("faker".to_string(), Some("internet::SafeEmail".to_string())), - "name" => ("faker".to_string(), Some("name::Name".to_string())), - "first_name" => ("faker".to_string(), Some("name::FirstName".to_string())), - "last_name" => ("faker".to_string(), Some("name::LastName".to_string())), - _ => (strategy.to_string(), None), - }; AnonymizerSpec { - strategy, + strategy: strategy.to_string(), salt: None, min: None, max: None, @@ -1158,7 +1154,7 @@ mod tests { unique_within_domain: None, as_string: None, locale: locale.map(|l| l.to_string()), - faker, + faker: None, format: None, } }