diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e55a71..a66e659 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,21 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +## [0.7.0] - 2026-05-04 + +### Removed + +- **`--include-table` / `--exclude-table`**: these CLI flags and the associated per-table skip logic in the SQL stream processor are removed. Anonymize the full dump, or split/filter dumps outside Dumpling if you need a smaller input. + +### Changed + +- **Dump seal (`v=3`):** the fingerprint JSON no longer includes table-filter fields. Seals produced by Dumpling **0.6.x** (`v=2`) will not match **0.7.x**; the first line is treated as stale and the dump is re-processed. + ## [0.6.0] - 2026-05-03 ### Added -- **Dump seal** (leading `-- dumpling-seal:` SQL comment): records Dumpling version, security profile, a SHA-256 fingerprint of the resolved policy, and runtime CLI options that affect transforms (`--format`, sorted `--include-table` / `--exclude-table`, effective PRNG seed in standard profile). When the input already begins with a **matching** seal, the remainder is copied through unchanged; stale or unknown seal lines are stripped and the dump is re-processed. See README for full semantics ([#58](https://github.com/ababic/dumpling/pull/58)). +- **Dump seal** (leading `-- dumpling-seal:` SQL comment): records Dumpling version, security profile, a SHA-256 fingerprint of the resolved policy, and runtime CLI options that affect transforms (`--format` and the effective PRNG seed in standard profile; `null` in hardened, where seeds are ignored). When the input already begins with a **matching** seal, the remainder is copied through unchanged; stale or unknown seal lines are stripped and the dump is re-processed. See README for full semantics ([#58](https://github.com/ababic/dumpling/pull/58)). - **`--stats`**: prints `wall_ms` plus `domain_cache_hits` and `domain_cache_misses` for quick profiling of large runs ([#59](https://github.com/ababic/dumpling/pull/59)). - **`CONTRIBUTORS.md`** ([#59](https://github.com/ababic/dumpling/pull/59)). diff --git a/Cargo.lock b/Cargo.lock index 40274a5..97d18e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -262,7 +262,7 @@ dependencies = [ [[package]] name = "dumpling" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index bbea04a..f380314 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dumpling" -version = "0.6.0" +version = "0.7.0" edition = "2021" [dependencies] diff --git a/README.md b/README.md index 9092efa..420efe0 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,8 @@ Follow these steps once; you will have a working path from “raw dump” to “ **Draft policy generation (planned)** — A future command will stream a dump and emit a **draft** starter TOML so you spend less time hunting table and column names and basic DDL hints (for example `varchar(N)` lengths). Output will be explicitly **draft**: always review and edit before production or compliance workflows; it is a time-saver, not a full policy. +The same flow is spelled out in the docs: [Getting started](https://ababic.github.io/dumpling/getting-started.html). + --- ## Usage @@ -102,8 +104,6 @@ dumpling --report report.json -i dump.sql # write detailed JSON report of dumpling --strict-coverage --report report.json -i dump.sql --check # fail on uncovered sensitive columns dumpling --scan-output --report report.json -i dump.sql # scan transformed output for residual PII-like patterns dumpling --scan-output --fail-on-findings --report report.json -i dump.sql --check # fail if scan thresholds are exceeded -dumpling --include-table '^public\\.' -i dump.sql -o out.sql -dumpling --exclude-table '^audit\\.' -i dump.sql -o out.sql dumpling --allow-ext dmp -i data.dmp # restrict processing to specific extensions dumpling --allow-noop -i dump.sql -o out.sql # explicitly allow no-op when config is missing dumpling --format sqlite -i data.db.sql -o out.sql # process a SQLite .dump file diff --git a/docs/src/configuration.md b/docs/src/configuration.md index 88e0aa1..57a38df 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -59,9 +59,9 @@ run, pass `--allow-noop`. Every successful run that writes output prefixes the stream with a single-line SQL comment: -`-- dumpling-seal: v=2 version= profile= sha256=<64 hex chars>` +`-- dumpling-seal: v=3 version= profile= sha256=<64 hex chars>` -The `sha256` is over canonical JSON that includes the Dumpling version, the active security profile, a stable encoding of the resolved policy (rules, row filters, column cases, sensitive columns, output scan, global salt), and **runtime options** that affect transforms: `--format`, sorted `--include-table` / `--exclude-table` patterns, and the effective `--seed` / `DUMPLING_SEED` value in standard profile (`null` in hardened, where seeds are ignored). +The `sha256` is over canonical JSON that includes the Dumpling version, the active security profile, a stable encoding of the resolved policy (rules, row filters, column cases, sensitive columns, output scan, global salt), and **runtime options** that affect transforms: `--format` and the effective `--seed` / `DUMPLING_SEED` value in standard profile (`null` in hardened, where seeds are ignored). If the **input** already begins with a seal line and it **matches** the current run, Dumpling copies the rest of the file through unchanged. If the line looks like a seal but does **not** match (stale policy, different flags, or older `v=`), that line is **dropped** and the dump is re-processed so you do not end up with two seal lines. `--strict-coverage` cannot be combined with a matching seal (table definitions are not scanned in passthrough mode). `--check` writes no output and therefore emits no seal line. diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 3f713f4..e2e51dc 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -1,5 +1,7 @@ # Getting started +This page is the **shortest path** from zero to a first successful run. For strategy details, row filters, dump seals, and CI patterns, continue with the [configuration guide](configuration.md) and the repository `README.md`. + ## Prerequisites - Rust **stable** toolchain (`rustup` recommended). The repo includes `rust-toolchain.toml` (stable + `rustfmt` + `clippy`) so CI and local `cargo` stay aligned. @@ -14,20 +16,30 @@ cargo build --release ./target/release/dumpling --help ``` -## Test locally +### Python / pip (`dumpling-cli`) ```bash -cargo fmt --all -- --check -cargo clippy --all-targets --all-features -cargo test --all-targets --all-features +pip install dumpling-cli +dumpling --help ``` -## Run against a dump +## First anonymization (four steps) -```bash -dumpling -i dump.sql -o sanitized.sql -``` +1. **Start from the example policy** — In the repository, copy [`.dumplingconf.example`](https://github.com/ababic/dumpling/blob/main/.dumplingconf.example) to `.dumplingconf` in your project root (or merge the same keys under `[tool.dumpling]` in `pyproject.toml`). Set environment variables for `salt` and any `${…}` references so Dumpling can resolve secrets at startup. +2. **Name your tables and columns** — Open your dump next to the config. `CREATE TABLE`, `COPY … (…)` and `INSERT INTO … (…)` lines list the identifiers you need for `[rules."table"]` or `[rules."schema.table"]`. Trim the example rules to the tables you care about first, then add columns and strategies as you go. +3. **Run Dumpling** — `dumpling -i dump.sql -o sanitized.sql` (add `-c path` if the config is not in the default search path). Use `dumpling --check -i dump.sql` when you only want to know whether anything would change. +4. **Tighten the policy** — Run `dumpling lint-policy` on your config. When you are ready for stricter gates, add `[sensitive_columns]` and use `--strict-coverage`, `--report`, and `--scan-output` as described in the [configuration guide](configuration.md) and the repository `README.md`. + +**Draft policy generation (planned)** — A future command will stream a dump and emit a **draft** starter TOML (table/column names and basic DDL hints) so you spend less time spelunking large dumps by hand. Treat that output as **draft** only: review and edit before any production or compliance workflow. + +## PostgreSQL custom-format archives If your input is a PostgreSQL **custom-format** file (not plain SQL), decode and anonymize in one step with **`--dump-decode`** (needs `pg_restore` from PostgreSQL client tools). See [PostgreSQL custom-format archives](configuration.md#postgresql-custom-format-archives---dump-decode) in the configuration guide. -For full command examples and strategy options, see the repository `README.md`. +## Test locally (contributors) + +```bash +cargo fmt --all -- --check +cargo clippy --all-targets --all-features +cargo test --all-targets --all-features +``` diff --git a/docs/src/index.md b/docs/src/index.md index 54347bd..58f5339 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -2,6 +2,8 @@ Dumpling is a streaming anonymizer for plain SQL dumps. It supports PostgreSQL (`pg_dump` plain format), SQLite (`.dump`), and SQL Server / MSSQL (SSMS / mssql-scripter plain SQL output). For PostgreSQL **custom-format** archives (e.g. Heroku `pg:backups:download`), use **`--dump-decode`** so Dumpling invokes `pg_restore` and streams plain SQL—see [Dump format](configuration.html#postgresql-custom-format-archives---dump-decode) in the configuration guide. +**New here?** Start with [**Getting started**](getting-started.html): copy the example config, align rules with your dump, run Dumpling, then tighten with `lint-policy` and optional CI flags. + This documentation covers the operating model for day-to-day use: - how to build and run Dumpling locally, diff --git a/pyproject.toml b/pyproject.toml index 4a2c13a..a9eef9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "dumpling-cli" -version = "0.6.0" +version = "0.7.0" description = "Static anonymizer for plain SQL dumps (PostgreSQL, SQLite, SQL Server)." readme = "README.md" requires-python = ">=3.8" diff --git a/src/main.rs b/src/main.rs index 12d541e..fd01646 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,6 @@ mod sql; mod transform; use anyhow::Context; -use regex::Regex; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; use seal::{ @@ -91,14 +90,6 @@ struct Cli { #[arg(long = "fail-on-findings", action = ArgAction::SetTrue)] fail_on_findings: bool, - /// Include only tables matching these regex patterns (repeatable). If none provided, include all. - #[arg(long = "include-table")] - include_table: Vec, - - /// Exclude tables matching these regex patterns (repeatable). - #[arg(long = "exclude-table")] - exclude_table: Vec, - /// Only process input files with these extensions (repeatable), e.g. --allow-ext sql --allow-ext dmp /// Case-insensitive; leading dot optional. Ignored when reading from stdin. #[arg(long = "allow-ext")] @@ -296,16 +287,7 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { ); } - // Compile table include/exclude regex patterns - let include_res = compile_patterns(&cli.include_table)?; - let exclude_res = compile_patterns(&cli.exclude_table)?; - - let seal_runtime = SealRuntimeParams::new( - dump_format, - &cli.include_table, - &cli.exclude_table, - prng_seed_override_for_fingerprint(), - ); + let seal_runtime = SealRuntimeParams::new(dump_format, prng_seed_override_for_fingerprint()); // Determine IO (optional pg_restore child when --dump-decode) let mut pg_restore_child: Option = None; @@ -449,8 +431,6 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { let mut processor = SqlStreamProcessor::new( anonymizers, resolved_config, - include_res, - exclude_res, Some(&mut reporter), dump_format, ); @@ -640,14 +620,6 @@ fn remove_pg_archive(path: &Path) -> std::io::Result<()> { } } -fn compile_patterns(patterns: &[String]) -> anyhow::Result> { - let mut out = Vec::new(); - for p in patterns { - out.push(Regex::new(p)?); - } - Ok(out) -} - fn has_allowed_extension(path: &Path, allow_exts: &[String]) -> bool { if allow_exts.is_empty() { return true; diff --git a/src/seal.rs b/src/seal.rs index d6e81fe..3af44a4 100644 --- a/src/seal.rs +++ b/src/seal.rs @@ -13,38 +13,25 @@ use std::io::{self, BufRead, Read}; pub const SEAL_LINE_PREFIX: &str = "-- dumpling-seal:"; /// Bump when the JSON payload shape changes (parsers accept the `v=` field on the comment line). -pub const SEAL_PAYLOAD_VERSION: u32 = 2; +pub const SEAL_PAYLOAD_VERSION: u32 = 3; -#[derive(Debug, Serialize)] +#[derive(Debug, Clone, Serialize)] pub struct SealRuntimeParams { pub dump_format: String, - pub include_table: Vec, - pub exclude_table: Vec, /// `Some` when standard profile and `--seed` / `DUMPLING_SEED` is set; `None` otherwise. pub prng_seed: Option, } impl SealRuntimeParams { - pub fn new( - dump_format: DumpFormat, - include_table: &[String], - exclude_table: &[String], - prng_seed: Option, - ) -> Self { + pub fn new(dump_format: DumpFormat, prng_seed: Option) -> Self { let dump_format = match dump_format { DumpFormat::Postgres => "postgres", DumpFormat::Sqlite => "sqlite", DumpFormat::MsSql => "mssql", } .to_string(); - let mut include_table: Vec = include_table.to_vec(); - include_table.sort(); - let mut exclude_table: Vec = exclude_table.to_vec(); - exclude_table.sort(); Self { dump_format, - include_table, - exclude_table, prng_seed, } } @@ -183,12 +170,7 @@ pub fn compute_seal_digest( dumpling_version: env!("CARGO_PKG_VERSION"), security_profile: security_profile.to_string(), policy: resolved_to_raw_for_fingerprint(cfg), - runtime: SealRuntimeParams { - dump_format: runtime.dump_format.clone(), - include_table: runtime.include_table.clone(), - exclude_table: runtime.exclude_table.clone(), - prng_seed: runtime.prng_seed, - }, + runtime: runtime.clone(), }; let mut val = serde_json::to_value(&payload)?; sort_json_value(&mut val); @@ -388,7 +370,7 @@ mod tests { } fn default_runtime() -> SealRuntimeParams { - SealRuntimeParams::new(DumpFormat::Postgres, &[], &[], None) + SealRuntimeParams::new(DumpFormat::Postgres, None) } #[test] @@ -418,8 +400,8 @@ mod tests { #[test] fn runtime_options_change_digest() { let cfg = minimal_cfg(); - let rt1 = SealRuntimeParams::new(DumpFormat::Postgres, &[], &[], Some(42)); - let rt2 = SealRuntimeParams::new(DumpFormat::Postgres, &[], &[], Some(43)); + let rt1 = SealRuntimeParams::new(DumpFormat::Postgres, Some(42)); + let rt2 = SealRuntimeParams::new(DumpFormat::Postgres, Some(43)); assert_ne!( compute_seal_digest(&cfg, "standard", &rt1).unwrap(), compute_seal_digest(&cfg, "standard", &rt2).unwrap() diff --git a/src/sql.rs b/src/sql.rs index 1b79917..4a6ebdd 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -29,8 +29,6 @@ pub enum DumpFormat { pub struct SqlStreamProcessor { anonymizers: AnonymizerRegistry, config: ResolvedConfig, - include_tables: Vec, - exclude_tables: Vec, column_length_limits: HashMap>, reporter: Option<*mut Reporter>, // raw pointer to allow mutable borrow during process sensitive_columns_detected: HashSet, @@ -49,16 +47,12 @@ impl SqlStreamProcessor { pub fn new( anonymizers: AnonymizerRegistry, config: ResolvedConfig, - include_tables: Vec, - exclude_tables: Vec, reporter: Option<&mut Reporter>, format: DumpFormat, ) -> Self { Self { anonymizers, config, - include_tables, - exclude_tables, column_length_limits: HashMap::new(), reporter: reporter.map(|r| r as *mut Reporter), sensitive_columns_detected: HashSet::new(), @@ -154,13 +148,11 @@ impl SqlStreamProcessor { create_table_buf.push_str(&line); if statement_complete(&create_table_buf) { if let Some(parsed) = parse_create_table_details(&create_table_buf) { - if self.table_enabled(parsed.schema.as_deref(), &parsed.table) { - self.track_sensitive_coverage( - parsed.schema.as_deref(), - &parsed.table, - &parsed.columns, - ); - } + self.track_sensitive_coverage( + parsed.schema.as_deref(), + &parsed.table, + &parsed.columns, + ); if !parsed.lengths.is_empty() { self.register_column_lengths( parsed.schema.as_deref(), @@ -178,17 +170,13 @@ impl SqlStreamProcessor { // Begin COPY mode let (schema, table) = parse_table_ident(cap.get(1).unwrap().as_str()); let columns = split_ident_list(cap.get(2).unwrap().as_str()); - let enabled = self.table_enabled(schema.as_deref(), &table); - if enabled { - self.track_sensitive_coverage(schema.as_deref(), &table, &columns); - } + self.track_sensitive_coverage(schema.as_deref(), &table, &columns); // Emit the header intact writer.write_all(line.as_bytes())?; mode = Mode::InCopy { schema, table, columns, - enabled, }; } else { // passthrough @@ -217,7 +205,6 @@ impl SqlStreamProcessor { schema, table, columns, - enabled, } => { if line.trim_end() == "\\." { // end of copy @@ -227,11 +214,6 @@ impl SqlStreamProcessor { // data row let line_body = line.trim_end_matches(['\n', '\r']); let fields: Vec<&str> = line_body.split('\t').collect(); - if !*enabled { - // passthrough unchanged - writer.write_all(line.as_bytes())?; - continue; - } // Evaluate row filters let unescaped: Vec> = fields .iter() @@ -320,13 +302,11 @@ impl SqlStreamProcessor { create_table_buf.push_str(&line); if statement_complete(&create_table_buf) { if let Some(parsed) = parse_create_table_details(&create_table_buf) { - if self.table_enabled(parsed.schema.as_deref(), &parsed.table) { - self.track_sensitive_coverage( - parsed.schema.as_deref(), - &parsed.table, - &parsed.columns, - ); - } + self.track_sensitive_coverage( + parsed.schema.as_deref(), + &parsed.table, + &parsed.columns, + ); if !parsed.lengths.is_empty() { self.register_column_lengths( parsed.schema.as_deref(), @@ -371,14 +351,7 @@ impl SqlStreamProcessor { // Parse table ident then columns list let (schema, table, rest_after_table) = parse_table_and_rest(after)?; let (columns, rest_after_cols) = parse_parenthesized_ident_list(rest_after_table)?; - let table_enabled = self.table_enabled(schema.as_deref(), &table); - if table_enabled { - self.track_sensitive_coverage(schema.as_deref(), &table, &columns); - } - // If table is disabled by include/exclude, return original unchanged - if !table_enabled { - return Ok(stmt.to_string()); - } + self.track_sensitive_coverage(schema.as_deref(), &table, &columns); // Expect VALUES let idx_values = find_ignore_ascii_case(rest_after_cols, "VALUES") .ok_or_else(|| anyhow::anyhow!("INSERT missing VALUES"))?; @@ -483,22 +456,6 @@ impl SqlStreamProcessor { Ok(out) } - fn table_enabled(&self, schema: Option<&str>, table: &str) -> bool { - let q = format_table_ident(schema, table); - let include_ok = if self.include_tables.is_empty() { - true - } else { - self.include_tables.iter().any(|re| re.is_match(&q)) - }; - if !include_ok { - return false; - } - if self.exclude_tables.iter().any(|re| re.is_match(&q)) { - return false; - } - true - } - fn register_column_lengths( &mut self, schema: Option<&str>, @@ -600,7 +557,6 @@ enum Mode { schema: Option, table: String, columns: Vec, - enabled: bool, }, } @@ -1613,8 +1569,7 @@ mod tests { }; let reg = AnonymizerRegistry::from_config(&cfg); set_random_seed(42); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.events (id int, email text, the_date date); INSERT INTO public.events (id, email, the_date) VALUES @@ -1747,8 +1702,7 @@ COPY public.events (id, email, the_date) FROM stdin; }; let reg = AnonymizerRegistry::from_config(&cfg); set_random_seed(1); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.users (id int, email text, country text, is_admin bool); INSERT INTO public.users (id, email, country, is_admin) VALUES @@ -1807,8 +1761,7 @@ INSERT INTO public.users (id, email, country, is_admin) VALUES source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.customers (id int, email text); CREATE TABLE public.orders (id int, customer_email text); @@ -1891,8 +1844,7 @@ INSERT INTO public.orders (id, customer_email) VALUES source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.users (id int, email text); INSERT INTO public.users (id, email) VALUES @@ -1959,14 +1911,8 @@ INSERT INTO public.users (id, email) VALUES let reg = AnonymizerRegistry::from_config(&cfg); let mut reporter = crate::report::Reporter::new(false); { - let mut proc = SqlStreamProcessor::new( - reg, - cfg, - Vec::new(), - Vec::new(), - Some(&mut reporter), - DumpFormat::Postgres, - ); + let mut proc = + SqlStreamProcessor::new(reg, cfg, Some(&mut reporter), DumpFormat::Postgres); let input = r#" CREATE TABLE public.users (id int, email text); INSERT INTO public.users (id, email) VALUES (1, 'alice@myco.com'); @@ -2021,8 +1967,7 @@ INSERT INTO public.users (id, email) VALUES (1, 'alice@myco.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.users (id int, email text); INSERT INTO public.users (id, email) VALUES @@ -2108,8 +2053,7 @@ INSERT INTO public.users (id, email) VALUES source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); // tab-separated COPY rows: idemail let input = "COPY public.users (id, email) FROM stdin;\n\ 1\talice@myco.com\n\ @@ -2200,8 +2144,7 @@ INSERT INTO public.users (id, email) VALUES source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.customers (id int, email text); CREATE TABLE public.orders (id int, customer_email text); @@ -2274,8 +2217,7 @@ INSERT INTO public.orders (id, customer_email) VALUES }; let reg = AnonymizerRegistry::from_config(&cfg); set_random_seed(7); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" INSERT INTO public.users (id, email, first_name, password, dob, notes) VALUES (1, 'alice@myco.com', 'Alice', 's3cr3t', '1990-01-02', 'left alone'); @@ -2325,8 +2267,7 @@ INSERT INTO public.users (id, email, first_name, password, dob, notes) VALUES source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.events (id int, payload jsonb); INSERT INTO public.events (id, payload) VALUES @@ -2399,8 +2340,7 @@ COPY public.events (id, payload) FROM stdin; source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.events (id int, payload jsonb); INSERT INTO public.events (id, payload) VALUES @@ -2480,8 +2420,7 @@ COPY public.events (id, payload) FROM stdin; source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.events (id int, payload jsonb); INSERT INTO public.events (id, payload) VALUES @@ -2551,8 +2490,7 @@ COPY public.events (id, payload) FROM stdin; source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.events (id int, payload jsonb); INSERT INTO public.events (id, payload) VALUES @@ -2633,8 +2571,7 @@ COPY public.events (id, payload) FROM stdin; source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.payments (id int, pan text); INSERT INTO public.payments (id, pan) VALUES (1, '4111111111111111'); @@ -2722,8 +2659,7 @@ COPY public.payments (id, pan) FROM stdin; source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.events (id int, payload jsonb); INSERT INTO public.events (id, payload) VALUES @@ -2834,8 +2770,7 @@ INSERT INTO public.events (id, payload) VALUES }; set_random_seed(7); let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.users ( email varchar(12), @@ -2921,8 +2856,7 @@ old@example.com verylongname (000) 000-0000 source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = r#" CREATE TABLE public.users ( id int, @@ -2993,8 +2927,7 @@ CREATE TABLE public.users ( }; let reg = AnonymizerRegistry::from_config(&cfg); set_random_seed(1); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Sqlite); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Sqlite); let input = r#"CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT); INSERT OR REPLACE INTO users (id, email) VALUES (1, 'alice@example.com'); INSERT OR IGNORE INTO users (id, email) VALUES (2, 'bob@example.com'); @@ -3075,8 +3008,7 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Sqlite); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Sqlite); let input = "INSERT INTO \"users\" (\"id\", \"email\") VALUES (1, 'alice@example.com');\n"; let mut reader = std::io::BufReader::new(input.as_bytes()); let mut out = Vec::new(); @@ -3112,8 +3044,7 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Sqlite); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Sqlite); // A line that looks like a COPY header should just pass through let input = "COPY users (id, email) FROM stdin;\nalice@example.com\n\\.\n"; let mut reader = std::io::BufReader::new(input.as_bytes()); @@ -3166,8 +3097,7 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::MsSql); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::MsSql); let input = "INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, 'alice@example.com');\n"; let mut reader = std::io::BufReader::new(input.as_bytes()); let mut out = Vec::new(); @@ -3227,8 +3157,7 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::MsSql); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::MsSql); // N'...' is MSSQL Unicode notation; the N prefix should be transparently stripped let input = "INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com');\n"; let mut reader = std::io::BufReader::new(input.as_bytes()); @@ -3285,8 +3214,7 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); }; set_random_seed(42); let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::MsSql); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::MsSql); let input = r#"CREATE TABLE [dbo].[users] ( [id] int NOT NULL, [email] nvarchar(20) NOT NULL @@ -3329,8 +3257,7 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::MsSql); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::MsSql); let input = "COPY users (id, email) FROM stdin;\nalice@example.com\n\\.\n"; let mut reader = std::io::BufReader::new(input.as_bytes()); let mut out = Vec::new(); @@ -3379,8 +3306,7 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::MsSql); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::MsSql); let input = "INSERT INTO [dbo].[customers] ([id], [email]) VALUES (1, N'alice@corp.com'), (2, N'bob@corp.com');\n"; let mut reader = std::io::BufReader::new(input.as_bytes()); let mut out = Vec::new(); @@ -3463,8 +3389,7 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); - let mut proc = - SqlStreamProcessor::new(reg, cfg, Vec::new(), Vec::new(), None, DumpFormat::Postgres); + let mut proc = SqlStreamProcessor::new(reg, cfg, None, DumpFormat::Postgres); let input = "INSERT INTO public.contacts (id, full_name, phone) VALUES (1, 'Alice Müller', '+49 30 12345678');\n"; let mut reader = std::io::BufReader::new(input.as_bytes()); let mut out = Vec::new();