diff --git a/AGENTS.md b/AGENTS.md index 7ecbd27..f41df57 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,9 @@ src/ filter.rs — Row-filter predicate evaluation (eq/neq/like/regex/JSON-path/…) scan.rs — Post-transform residual PII scanner (email/SSN/PAN/token regex) report.rs — JSON report data structures and Reporter helper + compressed_input.rs — gzip/ZIP wrappers; streaming vs temp materialization + dump_input_resolve.rs — shared `--input` file resolution for anonymize + scaffold-config + dump_input_detect.rs — PGDMP / directory dumps / MSSQL sniff helpers docs/src/ — mdBook documentation source .github/ — CI/CD GitHub Actions workflows Cargo.toml — Rust package manifest diff --git a/CHANGELOG.md b/CHANGELOG.md index c53f2c5..2970b97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +### Added + +- **Gzip and ZIP inputs**: plain-SQL payloads inside **gzip** are decompressed **in-process** (streamed) when possible—no temporary file. Dumpling still materializes to the temp directory when required: **ZIP** archives (random-access central directory), **gzip wrapping `PGDMP`** or an inner **ZIP** (nested wrappers), or other cases where a filesystem path is needed for `pg_restore`. Temporary files are registered for removal when processing finishes. **`--in-place` is rejected** only when Dumpling had to write a **temporary** decompressed/extracted file (not when gzip plain-SQL streaming was used). Full multi-file ZIP packages (for example BACPAC) are still not supported as SQL input. + +### Changed + +- **CLI**: removed **`--dump-decode`**. PostgreSQL **custom-format** (`PGDMP`) and **directory-format** (`toc.dat`) inputs are auto-detected when `--format postgres` (default) and decoded with `pg_restore -f -`. Options renamed: **`--pg-restore-arg`** (repeatable, was `--dump-decode-arg`), **`--keep-original`** (was `--dump-decode-keep-input` / `--pg-restore-keep-input`). **`--keep-original` is incompatible with `--in-place`** (use `--output` or stdout). Optional `[pg_restore]` table in config for default path/args. + ## [0.7.0-alpha] - 2026-05-04 Pre-release toward **0.7.0** (stable **0.7.0** is not published yet; crates use the **0.7.0-alpha** prerelease identifier until then). diff --git a/Cargo.lock b/Cargo.lock index 4357fa2..95a6ed8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -76,6 +82,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -187,6 +202,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.7" @@ -231,6 +261,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "deunicode" version = "1.6.2" @@ -248,6 +289,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dummy" version = "0.11.0" @@ -268,6 +320,7 @@ dependencies = [ "chrono", "clap", "fake", + "flate2", "getrandom 0.2.17", "hmac", "lazy_static", @@ -276,8 +329,9 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror", + "thiserror 1.0.69", "toml", + "zip", ] [[package]] @@ -310,6 +364,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -456,6 +520,16 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -643,6 +717,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "strsim" version = "0.11.1" @@ -672,7 +752,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -686,6 +775,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "toml" version = "0.8.23" @@ -914,8 +1014,37 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.18", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 40c9b40..5dd206a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,5 @@ chrono = { version = "0.4" } lazy_static = "1" fake = { version = "4", features = ["derive"] } rand = "0.9" +flate2 = "1" +zip = { version = "2", default-features = false, features = ["deflate"] } diff --git a/README.md b/README.md index e81ae3e..4bb64ec 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ dumpling --help Follow these steps once; you will have a working path from “raw dump” to “first sanitized output,” then you can deepen coverage using the rest of this README and the [documentation site](https://ababic.github.io/dumpling/). -1. **Generate a draft policy (recommended)** — Run `dumpling scaffold-config -i dump.sql -o .dumplingconf` to emit a **beta** starter TOML with inferred `[rules]` from column names in `CREATE TABLE`, `INSERT`, and (PostgreSQL) `COPY` headers. Heuristics are **English-oriented**; treat the file as **draft only**—review every rule before production or compliance workflows. Add a global `salt` (for example `salt = "${DUMPLING_SALT}"`) and resolve `${…}` references before anonymizing. Optionally pass **`--infer-json-paths`** to sample up to **five rows per table** (reservoir) and suggest nested JSON keys as `column.path.to.leaf`; use **`--max-json-depth`** if you need a different walk depth (default 24). For PostgreSQL **custom-format** archives, add **`--dump-decode`** (requires **`--input`** and **`--format postgres`**). See `dumpling scaffold-config --help`. +1. **Generate a draft policy (recommended)** — Run `dumpling scaffold-config -i dump.sql -o .dumplingconf` to emit a **beta** starter TOML with inferred `[rules]` from column names in `CREATE TABLE`, `INSERT`, and (PostgreSQL) `COPY` headers. Heuristics are **English-oriented**; treat the file as **draft only**—review every rule before production or compliance workflows. Add a global `salt` (for example `salt = "${DUMPLING_SALT}"`) and resolve `${…}` references before anonymizing. Optionally pass **`--infer-json-paths`** to sample up to **five rows per table** (reservoir) and suggest nested JSON keys as `column.path.to.leaf`; use **`--max-json-depth`** if you need a different walk depth (default 24). For PostgreSQL **custom-format** or **directory-format** archives, pass **`--input`** pointing at the archive with **`--format postgres`** (default); Dumpling auto-detects and runs **`pg_restore`** (optional **`--pg-restore-path`** / **`--pg-restore-arg`**). See `dumpling scaffold-config --help`. 2. **Or start from the example policy** — Copy [`.dumplingconf.example`](.dumplingconf.example) to `.dumplingconf` (or merge under `[tool.dumpling]` in `pyproject.toml`) and edit `[rules]` by hand. Set environment variables for `salt` and any `${…}` references so Dumpling can resolve secrets at startup. 3. **Align rules with your dump** — If you did not use `scaffold-config`, open the dump beside the config: `CREATE TABLE`, `COPY … (…)`, and `INSERT INTO … (…)` lines list identifiers for `[rules."table"]` or `[rules."schema.table"]` (see [Configuration (TOML)](#configuration-toml)). Trim rules to the tables you care about first, then extend columns and strategies as you go. 4. **Run Dumpling** — `dumpling -i dump.sql -o sanitized.sql` (add `-c path` if the config is not in the default search path). Use `dumpling --check -i dump.sql` when you only want to know whether anything would change. @@ -473,12 +473,16 @@ Produced by `pg_dump --format=plain`. Handles: Binary, custom, and directory formats from `pg_dump` are not parsed directly — Dumpling’s SQL pipeline expects plain text. Use either: - **`pg_dump --format=plain`** when you control capture, or -- **`dumpling --dump-decode`** with `--input` set to a **custom-format** (`.dump`) or **directory-format** folder: Dumpling runs `pg_restore -f -` and streams the resulting SQL (same as a manual `pg_restore` “script” output, no database required). Requires PostgreSQL client tools on `PATH` (`pg_restore`), or set `--pg-restore-path`. Use `--dump-decode-arg` to pass extra flags (e.g. `--no-owner --no-acl`). **By default** the archive is removed after a fully successful run; pass **`--dump-decode-keep-input`** to retain it. **`--check`** requires **`--dump-decode-keep-input`** so the archive still exists if changes would be detected. +- **Auto-detected PostgreSQL archives** with `--format postgres` (default): if `--input` is a **custom-format** file (begins with `PGDMP`) or a **directory-format** dump (folder containing `toc.dat`), Dumpling runs **`pg_restore -f -`** and streams the resulting SQL (same as a manual `pg_restore` “script” output; no database required). Requires PostgreSQL client tools on **`PATH`** (`pg_restore`), or **`--pg-restore-path`**. Extra flags: **`--pg-restore-arg`** (repeatable), or defaults from **`[pg_restore]`** in `.dumplingconf` / `pyproject.toml` (CLI wins when set). + +**Compressed inputs:** **`.gz`** files whose payload is plain SQL are **decompressed in-process** (no temporary file). **ZIP** archives (and gzip wrapping `PGDMP` or an inner ZIP) are expanded under the system temp directory; those paths are removed when the run finishes. **`--in-place`** is rejected when Dumpling had to materialize a temp file for compression or when the input is a PostgreSQL archive path that must go through `pg_restore` (use **`--output`** or stdout instead). + +**Keeping archives:** **By default** the `--input` archive path (file or directory-format folder) is **removed** after a fully successful run. Pass **`--keep-original`** or set **`keep_original = true`** in config to retain it. **`--check`** against an archive requires an effective keep-original (CLI or config); **`--keep-original` cannot be combined with `--in-place`**. Example (e.g. after `heroku pg:backups:download`): ```bash -dumpling --dump-decode -i latest.dump -c .dumplingconf -o anonymized.sql +dumpling -i latest.dump -c .dumplingconf -o anonymized.sql ``` ### SQLite (`--format sqlite`) diff --git a/docs/src/configuration.md b/docs/src/configuration.md index 57a38df..59f4f2d 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -6,7 +6,7 @@ Use `--format` to declare the SQL dialect of your input file: | Value | Description | |---|---| -| `postgres` (default) | PostgreSQL `pg_dump` plain-text format. Supports `COPY … FROM stdin` blocks, `"double-quoted"` identifiers, `''`-escaped strings. Custom-format (`-Fc`) or directory dumps can be decoded on the fly with `dumpling --dump-decode` (wraps `pg_restore -f -`; requires client tools). By default the archive is deleted after success; use `--dump-decode-keep-input` to retain it. | +| `postgres` (default) | PostgreSQL `pg_dump` plain-text format. Supports `COPY … FROM stdin` blocks, `"double-quoted"` identifiers, `''`-escaped strings. **Custom-format** (`PGDMP`) and **directory-format** (`toc.dat`) dumps are **auto-detected** and decoded with `pg_restore -f -` (requires client tools). **Gzip** — wrapped plain SQL is decompressed in-process; **ZIP** (or gzip wrapping `PGDMP`/nested ZIP) uses a temp file that is removed after the run. By default the archive is deleted after success; use **`--keep-original`** or **`keep_original`** in config to retain it. | | `sqlite` | SQLite `.dump` format. Adds `INSERT OR REPLACE INTO` / `INSERT OR IGNORE INTO` support. No COPY blocks. | | `mssql` | SQL Server / MSSQL plain SQL. Adds `[bracket]` identifier quoting, `N'…'` Unicode string literals, and `nvarchar(n)` / `nchar(n)` length extraction. No COPY blocks. | @@ -17,26 +17,58 @@ dumpling --format sqlite -i data.db.sql -o anonymized.sql dumpling --format mssql -i backup.sql -o anonymized.sql ``` -### PostgreSQL custom-format archives (`--dump-decode`) +### PostgreSQL archives and compressed inputs -Heroku PGBackups and many pipelines ship **`pg_dump` custom format** (`-Fc`) or **directory-format** dumps to save bandwidth. Dumpling’s SQL engine still expects **plain text**; use **`--dump-decode`** so Dumpling runs **`pg_restore -f -`** (script to stdout, no database) and pipes the result through the same anonymizer as a normal plain-SQL file. +Heroku PGBackups and many pipelines ship **`pg_dump` custom format** (`-Fc`), **directory-format** dumps, or **gzip**/**ZIP**-wrapped files. Dumpling’s SQL engine still expects **plain text** at the parser; anything else is normalized first. -**Requirements:** PostgreSQL client tools on `PATH` (`pg_restore`), or set **`--pg-restore-path`**. Use **`--dump-decode-arg`** (repeatable) for extra `pg_restore` flags, e.g. `--dump-decode-arg=--no-owner --dump-decode-arg=--no-acl`. +#### Custom-format and directory dumps (auto-detected) -**Input deletion:** After a **fully successful** run, Dumpling **removes** the `--input` path (single file or directory-format folder) by default so only the anonymized output remains. Pass **`--dump-decode-keep-input`** to retain the archive. +With **`--format postgres`** (default), Dumpling detects: -**Check mode:** **`--check`** with **`--dump-decode`** requires **`--dump-decode-keep-input`**. Otherwise the default would delete the dump before you can iterate on config. +- **Custom-format** files (magic `PGDMP` at the start of the file), and +- **Directory-format** folders (a `toc.dat` beside table blobs), -Example (e.g. after `heroku pg:backups:download`): +then runs **`pg_restore -f -`** (script to stdout inside the process — no database) and pipes the result through the same anonymizer as a normal plain-SQL file. Detection from **`--input`** is automatic. + +**Requirements:** PostgreSQL client tools on **`PATH`** (`pg_restore`), or **`--pg-restore-path`**. + +**Extra `pg_restore` arguments:** + +- CLI: **`--pg-restore-arg`** (repeatable), e.g. `--pg-restore-arg=--no-owner --pg-restore-arg=--no-acl` +- Config (optional): **`[pg_restore]`** — CLI overrides these when you pass path or args: + +```toml +[pg_restore] +path = "/usr/bin/pg_restore" +args = ["--no-owner", "--no-acl"] +``` + +#### Gzip and ZIP wrappers + +- **Gzip (`.gz`)** whose decompressed payload is **plain SQL**: decompressed **in-process** (streamed); no temporary dump file. +- **ZIP** containing a single dump file (or a single `.sql` when multiple files exist), **gzip wrapping `PGDMP`**, or **gzip wrapping an inner ZIP**: Dumpling writes under the system temp directory and **removes** those paths when the run completes (including after errors — cleanup runs on drop). + +**`--in-place`** is **rejected** when Dumpling had to **materialize** a temp file for compression **or** when the resolved input is a PostgreSQL archive decoded via **`pg_restore`** (use **`--output`** or stdout). + +#### Keeping inputs and `--check` + +After a **fully successful** run, Dumpling **removes** the `--input` archive path (single file or directory-format folder) **by default**. To keep it: + +- **`--keep-original`**, or +- **`keep_original = true`** at the top level of `.dumplingconf` / `[tool.dumpling]` (merged with CLI; **`--keep-original` cannot be used with `--in-place`**). + +**`--check`** with a PostgreSQL archive requires an **effective** keep-original (CLI or config); otherwise the default deletion would remove the dump before you iterate on policy. + +Examples (e.g. after `heroku pg:backups:download`): ```bash -dumpling --dump-decode -i latest.dump -c .dumplingconf -o anonymized.sql +dumpling -i latest.dump -c .dumplingconf -o anonymized.sql ``` Dry run while keeping the downloaded file: ```bash -dumpling --dump-decode --dump-decode-keep-input --check -i latest.dump -c .dumplingconf +dumpling --keep-original --check -i latest.dump -c .dumplingconf ``` --- diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 4225fc5..4427492 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -38,7 +38,7 @@ dumpling --help - **`--infer-json-paths`** — Keep up to **five sampled rows per table** (reservoir) and suggest nested JSON rules as `column.path.leaf`. - **`--max-json-depth`** — Cap JSON walking depth when using `--infer-json-paths` (default 24). - **`--format`** — `postgres` (default), `sqlite`, or `mssql`. - - **`--dump-decode`** — Decode a PostgreSQL custom-format archive via `pg_restore` (requires **`--input`** and **`--format postgres`**); see [PostgreSQL custom-format archives](configuration.md#postgresql-custom-format-archives---dump-decode). + - **`--pg-restore-path`** / **`--pg-restore-arg`** — Optional **`pg_restore`** binary and extra arguments when **`--input`** is a PostgreSQL custom-format or directory-format archive (auto-detected with **`--format postgres`**); see [PostgreSQL archives and compressed inputs](configuration.md#postgresql-archives-and-compressed-inputs). Run `dumpling scaffold-config --help` for the full flag list. @@ -52,7 +52,7 @@ dumpling --help ## PostgreSQL custom-format archives -If your input is a PostgreSQL **custom-format** file (not plain SQL), decode and anonymize in one step with **`--dump-decode`** (needs `pg_restore` from PostgreSQL client tools). See [PostgreSQL custom-format archives](configuration.md#postgresql-custom-format-archives---dump-decode) in the configuration guide. +If your input is a PostgreSQL **custom-format** file or **directory-format** folder (not plain SQL), use **`--format postgres`** (default): Dumpling **auto-detects** the archive and runs **`pg_restore -f -`** (needs `pg_restore` from PostgreSQL client tools). Gzip-wrapped plain SQL is streamed without a temp file; ZIP (or gzip wrapping `PGDMP`) uses a temp extract that is cleaned up afterward. See [PostgreSQL archives and compressed inputs](configuration.md#postgresql-archives-and-compressed-inputs) in the configuration guide. ## Test locally (contributors) diff --git a/docs/src/index.md b/docs/src/index.md index 6d20a29..2380ed4 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,6 +1,6 @@ # Dumpling documentation -Dumpling is a streaming anonymizer for plain SQL dumps. It supports PostgreSQL (`pg_dump` plain format), SQLite (`.dump`), and SQL Server / MSSQL (SSMS / mssql-scripter plain SQL output). For PostgreSQL **custom-format** archives (e.g. Heroku `pg:backups:download`), use **`--dump-decode`** so Dumpling invokes `pg_restore` and streams plain SQL—see [Dump format](configuration.html#postgresql-custom-format-archives---dump-decode) in the configuration guide. +Dumpling is a streaming anonymizer for plain SQL dumps. It supports PostgreSQL (`pg_dump` plain format), SQLite (`.dump`), and SQL Server / MSSQL (SSMS / mssql-scripter plain SQL output). For PostgreSQL **custom-format** or **directory-format** archives (e.g. Heroku `pg:backups:download`), Dumpling **auto-detects** them when `--format postgres` (default) and invokes `pg_restore -f -`—see [PostgreSQL archives and compressed inputs](configuration.html#postgresql-archives-and-compressed-inputs) in the configuration guide. **New here?** Start with [**Getting started**](getting-started.html): generate a **draft** policy with `scaffold-config`, review and add secrets, run Dumpling, then tighten with `lint-policy` and optional CI flags. diff --git a/src/compressed_input.rs b/src/compressed_input.rs new file mode 100644 index 0000000..4f76d66 --- /dev/null +++ b/src/compressed_input.rs @@ -0,0 +1,340 @@ +//! Transparent gzip / ZIP resolution for `--input` file paths. +//! +//! When the payload is **plain SQL text**, gzip is decompressed **in-process** (streamed) so no +//! temporary file is created. When the inner payload must be **materialized** (PostgreSQL `PGDMP` +//! for `pg_restore`, nested ZIP after gzip, or any ZIP inner file — the `zip` crate needs random +//! access to the central directory), we write to the temp dir and register paths on +//! [`CompressionCleanup`] so they are always removed on drop. + +use crate::dump_input_detect::{read_file_prefix, PG_CUSTOM_FORMAT_MAGIC}; +use anyhow::Context; +use flate2::read::GzDecoder; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader, Cursor, Read}; +use std::path::{Path, PathBuf}; + +use crate::log_sanitize::path_basename_for_log; +use crate::IO_BUF_CAPACITY; + +/// Prefix size for gzip/ZIP sniffing (matches MSSQL sample size in `dump_input_detect`). +const SNIFF_PREFIX_LEN: usize = 4096; + +/// Paths and directories created while resolving compressed inputs; remove after a successful run. +#[derive(Default)] +pub(crate) struct CompressionCleanup { + paths: Vec, +} + +impl Drop for CompressionCleanup { + fn drop(&mut self) { + for p in self.paths.drain(..) { + let _ = if p.is_dir() { + fs::remove_dir_all(p) + } else { + fs::remove_file(p) + }; + } + } +} + +fn is_gzip_prefix(buf: &[u8]) -> bool { + buf.len() >= 2 && buf[0] == 0x1f && buf[1] == 0x8b +} + +fn is_zip_prefix(buf: &[u8]) -> bool { + buf.len() >= 4 && &buf[0..4] == b"PK\x03\x04" +} + +fn unique_temp_file(label: &str) -> PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + std::env::temp_dir().join(format!( + "dumpling_{}_{}_{}", + label, + std::process::id(), + nanos + )) +} + +/// Decompress gzip fully to a temp file (`pg_restore`, nested ZIP, or chained wrappers). +fn decompress_gzip_file(src: &Path, cleanup: &mut CompressionCleanup) -> anyhow::Result { + let out = unique_temp_file("ungz"); + let mut inp = File::open(src).with_context(|| format!("open {}", src.display()))?; + let mut dec = GzDecoder::new(&mut inp); + let mut tmp = File::create(&out).with_context(|| format!("create {}", out.display()))?; + std::io::copy(&mut dec, &mut tmp).with_context(|| format!("decompress {}", src.display()))?; + cleanup.paths.push(out.clone()); + Ok(out) +} + +/// Extract the sole file, or a single `.sql` when multiple entries exist, to a temp path. +fn extract_zip_inner_file(src: &Path, cleanup: &mut CompressionCleanup) -> anyhow::Result { + let file = File::open(src).with_context(|| format!("open {}", src.display()))?; + let mut archive = + zip::ZipArchive::new(file).with_context(|| format!("read ZIP {}", src.display()))?; + let mut file_indices: Vec = Vec::new(); + let mut sql_indices: Vec = Vec::new(); + for i in 0..archive.len() { + let ent = archive.by_index(i)?; + let name = ent.name(); + if name.ends_with('/') || name.ends_with('\\') { + continue; + } + file_indices.push(i); + if Path::new(name) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.eq_ignore_ascii_case("sql")) + .unwrap_or(false) + { + sql_indices.push(i); + } + } + let idx = if file_indices.len() == 1 { + file_indices[0] + } else if sql_indices.len() == 1 { + sql_indices[0] + } else if sql_indices.len() > 1 { + let names: Vec = sql_indices + .iter() + .map(|&i| archive.by_index(i).map(|e| e.name().to_string())) + .collect::>()?; + anyhow::bail!( + "ZIP `{}` contains multiple `.sql` files; Dumpling needs exactly one dump file. Found: {}", + src.display(), + names.join(", ") + ); + } else { + anyhow::bail!( + "ZIP `{}` contains {} file(s); Dumpling needs exactly one file (or exactly one `.sql`). \ + Repack or extract manually.", + src.display(), + file_indices.len() + ); + }; + + let mut ent = archive.by_index(idx).with_context(|| "ZIP entry")?; + let inner = Path::new(ent.name()); + let ext = inner.extension().and_then(|e| e.to_str()).unwrap_or("sql"); + let mut out = unique_temp_file("unzip"); + out.set_extension(ext); + let mut tmp = File::create(&out).with_context(|| format!("create {}", out.display()))?; + std::io::copy(&mut ent, &mut tmp).with_context(|| format!("extract {}", ent.name()))?; + cleanup.paths.push(out.clone()); + Ok(out) +} + +/// Result of walking gzip/ZIP wrappers on a file path. +pub(crate) enum ResolvedInput { + /// A filesystem path to open (plain file, directory dump, or materialized inner file). + Path { + path: PathBuf, + had_compression_wrapper: bool, + /// True when Dumpling wrote this path under the temp directory (gunzip / unzip extract). + materialized_temp_file: bool, + }, + /// Gzip was applied and the inner payload is plain SQL: decompress in memory without a temp file. + PlainSqlStream { + reader: Box, + had_compression_wrapper: bool, + /// First bytes after gzip (used for `--format mssql` classification). + sniff_prefix: Vec, + }, +} + +const MAX_WRAP_DEPTH: u32 = 16; + +/// Follow gzip and/or ZIP wrappers so downstream logic can open a path or use a stream. +/// +/// Temporary paths are registered on `cleanup` and deleted when `cleanup` is dropped. +pub(crate) fn resolve_compressed_wrappers( + original: &Path, + cleanup: &mut CompressionCleanup, +) -> anyhow::Result { + resolve_compressed_wrappers_inner(original, cleanup, 0, false, false) +} + +fn resolve_compressed_wrappers_inner( + original: &Path, + cleanup: &mut CompressionCleanup, + depth: u32, + materialized_temp_file: bool, + compressed_so_far: bool, +) -> anyhow::Result { + if depth > MAX_WRAP_DEPTH { + anyhow::bail!( + "nested gzip/ZIP wrappers exceed maximum depth ({MAX_WRAP_DEPTH}); simplify the archive chain" + ); + } + + if !original.is_file() { + return Ok(ResolvedInput::Path { + path: original.to_path_buf(), + had_compression_wrapper: compressed_so_far, + materialized_temp_file: false, + }); + } + + let mut current = original.to_path_buf(); + + let prefix = read_file_prefix(¤t, 4) + .map_err(|e| anyhow::anyhow!("read {}: {}", current.display(), e))?; + + if is_gzip_prefix(&prefix) { + eprintln!( + "dumpling: decompressing gzip input {} in-process (streamed when inner is plain SQL)", + path_basename_for_log(¤t) + ); + let inp = File::open(¤t).with_context(|| format!("open {}", current.display()))?; + let mut dec = GzDecoder::new(inp); + let mut sniff = vec![0u8; SNIFF_PREFIX_LEN]; + let n = dec + .read(&mut sniff) + .with_context(|| format!("read gzip stream {}", current.display()))?; + sniff.truncate(n); + + if sniff.starts_with(PG_CUSTOM_FORMAT_MAGIC) { + // pg_restore needs a path; materialize full decompress. + let temp = decompress_gzip_file(¤t, cleanup)?; + return resolve_compressed_wrappers_inner(&temp, cleanup, depth + 1, true, true); + } + if is_zip_prefix(&sniff) { + // Inner ZIP: materialize gunzip result then recurse (ZIP uses central directory). + let temp = decompress_gzip_file(¤t, cleanup)?; + return resolve_compressed_wrappers_inner(&temp, cleanup, depth + 1, true, true); + } + + // Plain SQL (or other format we read as UTF-8 bytes): chain prefix + remainder, no temp file. + let sniff_prefix = sniff.clone(); + let chained: Box = Box::new(Cursor::new(sniff).chain(dec)); + let reader = Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, chained)); + return Ok(ResolvedInput::PlainSqlStream { + reader, + had_compression_wrapper: true, + sniff_prefix, + }); + } + + let prefix = read_file_prefix(¤t, 4) + .map_err(|e| anyhow::anyhow!("read {}: {}", current.display(), e))?; + if is_zip_prefix(&prefix) { + eprintln!( + "dumpling: extracting inner file from ZIP {} (materialized for zip directory access)", + path_basename_for_log(¤t) + ); + current = extract_zip_inner_file(¤t, cleanup)?; + return resolve_compressed_wrappers_inner(¤t, cleanup, depth + 1, true, true); + } + + Ok(ResolvedInput::Path { + path: current, + had_compression_wrapper: compressed_so_far, + materialized_temp_file, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + use zip::write::SimpleFileOptions; + + #[test] + fn gzip_plain_sql_streams_without_materializing_path_branch() { + let dir = std::env::temp_dir(); + let plain = dir.join(format!("dumpling_plain_{}.sql", std::process::id())); + fs::write(&plain, b"SELECT 1;\n").unwrap(); + + let gz_path = dir.join(format!("dumpling_test_{}.sql.gz", std::process::id())); + let raw = fs::read(&plain).unwrap(); + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(&raw).unwrap(); + let compressed = enc.finish().unwrap(); + fs::write(&gz_path, &compressed).unwrap(); + + let mut cleanup = CompressionCleanup::default(); + let resolved = resolve_compressed_wrappers(&gz_path, &mut cleanup).unwrap(); + match resolved { + ResolvedInput::PlainSqlStream { mut reader, .. } => { + let mut s = String::new(); + reader.read_to_string(&mut s).unwrap(); + assert_eq!(s, "SELECT 1;\n"); + } + ResolvedInput::Path { .. } => panic!("expected PlainSqlStream"), + } + assert!(cleanup.paths.is_empty()); + + drop(cleanup); + let _ = fs::remove_file(&plain); + let _ = fs::remove_file(&gz_path); + } + + fn gzip_bytes(data: &[u8]) -> Vec { + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(data).unwrap(); + enc.finish().unwrap() + } + + #[test] + fn gzip_nested_zip_materializes_then_cleans() { + let dir = std::env::temp_dir(); + let zip_path = dir.join(format!("dumpling_inner_{}.zip", std::process::id())); + let file = File::create(&zip_path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let opts = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + zip.start_file("dump.sql", opts).unwrap(); + zip.write_all(b"INSERT INTO t VALUES (1);\n").unwrap(); + zip.finish().unwrap(); + + let gz_path = dir.join(format!("dumpling_nested_{}.gz", std::process::id())); + let gz_inner = fs::read(&zip_path).unwrap(); + fs::write(&gz_path, gzip_bytes(&gz_inner)).unwrap(); + + let mut cleanup = CompressionCleanup::default(); + let resolved = resolve_compressed_wrappers(&gz_path, &mut cleanup).unwrap(); + match resolved { + ResolvedInput::Path { path, .. } => { + let text = fs::read_to_string(&path).unwrap(); + assert!(text.contains("INSERT INTO")); + } + ResolvedInput::PlainSqlStream { .. } => panic!("expected Path after nested zip"), + } + assert!( + !cleanup.paths.is_empty(), + "nested gzip->zip should register temp paths" + ); + + drop(cleanup); + let _ = fs::remove_file(&zip_path); + let _ = fs::remove_file(&gz_path); + } + + #[test] + fn zip_single_sql_extracts() { + let dir = std::env::temp_dir(); + let zip_path = dir.join(format!("dumpling_zipt_{}.zip", std::process::id())); + let file = File::create(&zip_path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let opts = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + zip.start_file("dump.sql", opts).unwrap(); + zip.write_all(b"INSERT INTO t VALUES (1);\n").unwrap(); + zip.finish().unwrap(); + + let mut cleanup = CompressionCleanup::default(); + let resolved = resolve_compressed_wrappers(&zip_path, &mut cleanup).unwrap(); + match resolved { + ResolvedInput::Path { path, .. } => { + let text = fs::read_to_string(&path).unwrap(); + assert!(text.contains("INSERT INTO")); + } + ResolvedInput::PlainSqlStream { .. } => panic!("expected Path for zip"), + } + + drop(cleanup); + let _ = fs::remove_file(&zip_path); + } +} diff --git a/src/dump_input_detect.rs b/src/dump_input_detect.rs new file mode 100644 index 0000000..650ffc8 --- /dev/null +++ b/src/dump_input_detect.rs @@ -0,0 +1,171 @@ +//! Heuristic detection of non-plain-SQL dump inputs (archives, native backups). + +use std::fs::File; +use std::io::{self, Read}; +use std::path::Path; + +/// `pg_dump -Fc` custom format files begin with this magic (see PostgreSQL `pg_backup_archiver`). +pub(crate) const PG_CUSTOM_FORMAT_MAGIC: &[u8] = b"PGDMP"; + +pub(crate) fn read_file_prefix(path: &Path, max: usize) -> io::Result> { + let mut f = File::open(path)?; + let mut buf = vec![0u8; max]; + let n = f.read(&mut buf)?; + buf.truncate(n); + Ok(buf) +} + +pub(crate) fn is_pg_custom_format_file(path: &Path) -> io::Result { + let buf = read_file_prefix(path, PG_CUSTOM_FORMAT_MAGIC.len())?; + Ok(buf.starts_with(PG_CUSTOM_FORMAT_MAGIC)) +} + +/// Directory-format dumps include a `toc.dat` table-of-contents file at the root. +pub(crate) fn is_pg_directory_format_dump(path: &Path) -> bool { + path.is_dir() && path.join("toc.dat").is_file() +} + +/// Custom-format file (`PGDMP` prefix) or directory dump (`toc.dat`). +pub(crate) fn postgres_input_needs_pg_restore(path: &Path) -> io::Result { + if is_pg_directory_format_dump(path) { + return Ok(true); + } + if path.is_file() { + return is_pg_custom_format_file(path); + } + Ok(false) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MssqlFileKind { + PlainSqlText, + PostgresCustomArchiveWrongDialect, + ZipArchive, + Utf16EncodedSql, + LikelyBinaryBackup, +} + +pub(crate) fn classify_mssql_dump_file(path: &Path) -> io::Result { + const SAMPLE: usize = 4096; + let buf = read_file_prefix(path, SAMPLE)?; + Ok(classify_mssql_prefix(&buf)) +} + +/// Classify the start of a UTF-8 SQL script for `--format mssql` without reading from a path. +pub(crate) fn classify_mssql_prefix(buf: &[u8]) -> MssqlFileKind { + if buf.starts_with(PG_CUSTOM_FORMAT_MAGIC) { + return MssqlFileKind::PostgresCustomArchiveWrongDialect; + } + // BACPAC / DACPAC / generic zip package + if buf.len() >= 4 && &buf[0..4] == b"PK\x03\x04" { + return MssqlFileKind::ZipArchive; + } + // UTF-16 BOM — Dumpling expects UTF-8 plain SQL. + if buf.starts_with(&[0xFF, 0xFE]) || buf.starts_with(&[0xFE, 0xFF]) { + return MssqlFileKind::Utf16EncodedSql; + } + if buf.is_empty() { + return MssqlFileKind::PlainSqlText; + } + let mut nul = 0usize; + let mut control = 0usize; + for &b in buf { + if b == 0 { + nul += 1; + } + if b < 0x09 || (b > 0x0d && b < 0x20) { + control += 1; + } + } + let sample = buf.len(); + if nul > 0 + || (sample >= 64 && (nul * 100 / sample) >= 1) + || (sample >= 64 && (control * 100 / sample) > 15) + { + return MssqlFileKind::LikelyBinaryBackup; + } + MssqlFileKind::PlainSqlText +} + +pub(crate) const MSSQL_BACPAC_HINT: &str = "This looks like a ZIP package (for example a BACPAC/DACPAC). \ +Dumpling only processes plain UTF-8 SQL text. Export scripts from SSMS, use sqlpackage to produce SQL where applicable, \ +or use a tool that emits `.sql` text before running dumpling."; + +pub(crate) const MSSQL_BINARY_HINT: &str = "This file does not look like UTF-8 plain SQL (many binary or NUL bytes). \ +If this is a native SQL Server backup (`.bak`), a detached database (`.mdf`), or another binary format, Dumpling cannot read it. \ +Restore or script the database to plain SQL (for example via SSMS Generate Scripts, sqlpackage, or mssql-scripter), then pass that `.sql` file."; + +pub(crate) const MSSQL_UTF16_HINT: &str = + "The file begins with a UTF-16 byte order mark. Dumpling expects UTF-8 plain SQL; \ +re-export or convert the script to UTF-8 without BOM."; + +pub(crate) const MSSQL_WRONG_POSTGRES_ARCHIVE: &str = "This file is a PostgreSQL custom-format archive (`pg_dump -Fc`), not SQL Server text. \ +Use `--format postgres` (or omit `--format` — postgres is the default); Dumpling auto-detects custom-format (`PGDMP`) and directory dumps (`toc.dat`)."; + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn pg_custom_magic_detected() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "dumpling_pg_magic_{}_{}.dump", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let mut f = File::create(&path).unwrap(); + f.write_all(PG_CUSTOM_FORMAT_MAGIC).unwrap(); + f.write_all(&[1, 2, 3]).unwrap(); + drop(f); + assert!(is_pg_custom_format_file(&path).unwrap()); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn mssql_classifies_zip_prefix() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "dumpling_zip_{}_{}.bacpac", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let mut f = File::create(&path).unwrap(); + f.write_all(b"PK\x03\x04").unwrap(); + f.write_all(b"fake zip content").unwrap(); + drop(f); + assert_eq!( + classify_mssql_dump_file(&path).unwrap(), + MssqlFileKind::ZipArchive + ); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn mssql_classifies_pg_archive_under_mssql() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "dumpling_wrong_{}_{}.dump", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let mut f = File::create(&path).unwrap(); + f.write_all(PG_CUSTOM_FORMAT_MAGIC).unwrap(); + drop(f); + assert_eq!( + classify_mssql_dump_file(&path).unwrap(), + MssqlFileKind::PostgresCustomArchiveWrongDialect + ); + let _ = std::fs::remove_file(&path); + } +} diff --git a/src/dump_input_resolve.rs b/src/dump_input_resolve.rs new file mode 100644 index 0000000..ba7f8cf --- /dev/null +++ b/src/dump_input_resolve.rs @@ -0,0 +1,224 @@ +//! Shared `--input` file resolution for `run_anonymize` and `scaffold-config`. +//! +//! Keeps gzip/ZIP handling, format checks, `pg_restore`, and archive cleanup aligned between commands. + +use crate::compressed_input::{resolve_compressed_wrappers, CompressionCleanup, ResolvedInput}; +use crate::dump_input_detect::{ + classify_mssql_dump_file, classify_mssql_prefix, postgres_input_needs_pg_restore, + MssqlFileKind, MSSQL_BACPAC_HINT, MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, + MSSQL_WRONG_POSTGRES_ARCHIVE, +}; +use crate::log_sanitize::path_basename_for_log; +use crate::pg_restore_decode; +use crate::sql::DumpFormat; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; + +use crate::IO_BUF_CAPACITY; + +pub(crate) struct ResolveDumpInputParams<'a> { + pub user_input_path: &'a Path, + pub dump_format: DumpFormat, + pub compression_cleanup: &'a mut CompressionCleanup, + pub pg_restore_path: &'a Path, + pub pg_restore_arg: &'a [String], + pub keep_original: bool, + pub in_place: bool, +} + +pub(crate) struct ResolvedDumpInput { + pub reader: Box, + /// User's `--input` path (preserved for `--in-place` and logging). + pub original_input_path: PathBuf, + pub pg_restore_child: Option, + pub path_to_remove_pg_archive: Option, +} + +pub(crate) fn resolve_dump_input_from_path( + p: ResolveDumpInputParams<'_>, +) -> anyhow::Result { + let ResolveDumpInputParams { + user_input_path: path, + dump_format, + compression_cleanup, + pg_restore_path, + pg_restore_arg, + keep_original, + in_place, + } = p; + + if !path.exists() { + anyhow::bail!("input path does not exist: {}", path.display()); + } + + let resolved_input = if path.is_dir() { + ResolvedInput::Path { + path: path.to_path_buf(), + had_compression_wrapper: false, + materialized_temp_file: false, + } + } else { + resolve_compressed_wrappers(path, compression_cleanup)? + }; + + let ( + inner_reader_opt, + inner_path, + had_compression_wrapper, + materialized_compression_temp, + mssql_sniff_prefix, + ) = match resolved_input { + ResolvedInput::Path { + path: ip, + had_compression_wrapper, + materialized_temp_file, + } => ( + None, + ip, + had_compression_wrapper, + materialized_temp_file, + None::>, + ), + ResolvedInput::PlainSqlStream { + reader, + had_compression_wrapper, + sniff_prefix, + .. + } => ( + Some(reader), + path.to_path_buf(), + had_compression_wrapper, + false, + Some(sniff_prefix), + ), + }; + + if inner_path.is_dir() && dump_format != DumpFormat::Postgres { + anyhow::bail!( + "input `{}` is a directory; Dumpling expects a single plain-SQL file for this `--format`. \ + For PostgreSQL directory-format dumps (folder containing `toc.dat`), use `--format postgres`.", + inner_path.display() + ); + } + + if dump_format == DumpFormat::Sqlite && postgres_input_needs_pg_restore(&inner_path)? { + anyhow::bail!( + "input `{}` looks like a PostgreSQL custom-format or directory-format archive, not a SQLite `.dump`. \ + Use `--format postgres` (default) so Dumpling can decode it with pg_restore.", + inner_path.display() + ); + } + + if dump_format == DumpFormat::MsSql { + let kind = if let Some(ref pref) = mssql_sniff_prefix { + classify_mssql_prefix(pref) + } else if inner_path.is_file() { + classify_mssql_dump_file(&inner_path)? + } else { + MssqlFileKind::PlainSqlText + }; + match kind { + MssqlFileKind::PlainSqlText => {} + MssqlFileKind::PostgresCustomArchiveWrongDialect => { + anyhow::bail!( + "input `{}` is not plain SQL Server text.\n\n{}", + inner_path.display(), + MSSQL_WRONG_POSTGRES_ARCHIVE + ); + } + MssqlFileKind::ZipArchive => { + anyhow::bail!( + "input `{}` is not plain UTF-8 SQL text.\n\n{}", + inner_path.display(), + MSSQL_BACPAC_HINT + ); + } + MssqlFileKind::Utf16EncodedSql => { + anyhow::bail!( + "input `{}` is not UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_UTF16_HINT + ); + } + MssqlFileKind::LikelyBinaryBackup => { + anyhow::bail!( + "input `{}` does not look like UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_BINARY_HINT + ); + } + } + } + + if had_compression_wrapper && in_place && materialized_compression_temp { + anyhow::bail!( + "this input was gzip- and/or ZIP-wrapped and Dumpling wrote a temporary decompressed file; \ + --in-place cannot safely replace the original path with anonymized SQL. Use --output (or stdout) instead" + ); + } + + let auto_pg_restore = dump_format == DumpFormat::Postgres + && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { + anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) + })?; + + if auto_pg_restore && in_place { + anyhow::bail!( + "this input is a PostgreSQL custom-format or directory-format archive (decoded via pg_restore). \ + --in-place cannot replace the archive with plain SQL while atomically preserving the path; \ + write to --output (or stdout) instead, or decode manually with pg_restore -f -" + ); + } + + let use_pg_restore = auto_pg_restore; + + let mut pg_restore_child = None; + let mut path_to_remove_pg_archive = None; + + let reader: Box = if use_pg_restore { + if dump_format != DumpFormat::Postgres { + anyhow::bail!( + "PostgreSQL archive decoding only applies when --format postgres (default)" + ); + } + if auto_pg_restore { + eprintln!( + "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {} (input file)", + path_basename_for_log(pg_restore_path), + path_basename_for_log(&inner_path) + ); + } else { + eprintln!( + "dumpling: decoding PostgreSQL archive via {} -f - {} (input file)", + path_basename_for_log(pg_restore_path), + path_basename_for_log(&inner_path) + ); + } + let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( + pg_restore_path, + pg_restore_arg, + &inner_path, + )?; + pg_restore_child = Some(pg); + if !keep_original { + let tmp_root = std::env::temp_dir(); + if !inner_path.starts_with(&tmp_root) { + path_to_remove_pg_archive = Some(inner_path.clone()); + } + } + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)) + } else if let Some(r) = inner_reader_opt { + r + } else { + let f = File::open(&inner_path)?; + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)) + }; + + Ok(ResolvedDumpInput { + reader, + original_input_path: path.to_path_buf(), + pg_restore_child, + path_to_remove_pg_archive, + }) +} diff --git a/src/filter.rs b/src/filter.rs index 56e9df2..679ec2f 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -595,6 +595,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let cols = vec!["id".to_string(), "email".to_string(), "country".to_string()]; @@ -649,6 +651,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let cols = vec!["payload".to_string()]; @@ -693,6 +697,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let cols = vec!["payload".to_string()]; @@ -743,6 +749,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let registry = AnonymizerRegistry::from_config(&cfg); @@ -778,6 +786,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let registry = AnonymizerRegistry::from_config(&cfg); @@ -863,6 +873,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let registry = AnonymizerRegistry::from_config(&cfg); diff --git a/src/lint.rs b/src/lint.rs index 8ec87db..adaca2b 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -336,6 +336,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, } } diff --git a/src/log_sanitize.rs b/src/log_sanitize.rs new file mode 100644 index 0000000..a987e8c --- /dev/null +++ b/src/log_sanitize.rs @@ -0,0 +1,29 @@ +//! Avoid logging full filesystem paths on stderr (directory names may include PII; CodeQL flags this). + +use std::path::Path; + +/// Last path segment for user-visible logs, or a generic placeholder when absent. +pub(crate) fn path_basename_for_log(path: &Path) -> String { + path.file_name() + .and_then(|s| s.to_str()) + .map(str::to_string) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "input".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn basename_prefers_final_component() { + let p = PathBuf::from("/tmp/secrets/my.dump"); + assert_eq!(path_basename_for_log(&p), "my.dump"); + } + + #[test] + fn basename_fallback_for_empty_file_name() { + assert_eq!(path_basename_for_log(Path::new("/")), "input"); + } +} diff --git a/src/main.rs b/src/main.rs index f07c290..c2cdcc5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,6 @@ use std::fs::File; use std::io::{self, BufRead, BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use std::sync::atomic::Ordering; use std::sync::mpsc::sync_channel; use std::thread::JoinHandle; @@ -9,9 +8,14 @@ use std::time::Instant; use clap::{ArgAction, Parser, Subcommand}; +mod compressed_input; +mod dump_input_detect; +mod dump_input_resolve; mod faker_dispatch; mod filter; mod lint; +mod log_sanitize; +mod pg_restore_decode; mod report; mod scaffold; mod scan; @@ -28,14 +32,16 @@ const OUTPUT_PIPE_CHUNK: usize = IO_BUF_CAPACITY; /// Number of full chunks allowed in flight (transform can run ahead of disk this far). const OUTPUT_PIPE_DEPTH: usize = 8; -use anyhow::Context; +use compressed_input::CompressionCleanup; +use dump_input_resolve::{resolve_dump_input_from_path, ResolveDumpInputParams}; +use log_sanitize::path_basename_for_log; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; use seal::{ compute_seal_digest, format_seal_line, read_first_line_for_seal, FirstLineReplayBufRead, SealFirstLine, SealRuntimeParams, }; -use settings::ResolvedConfig; +use settings::{merge_keep_original, ResolvedConfig}; use sql::{DumpFormat, SqlStreamProcessor}; use transform::{ prng_seed_override_for_fingerprint, set_hardened_profile, set_random_seed, AnonymizerRegistry, @@ -119,25 +125,19 @@ struct Cli { #[arg(long = "security-profile", default_value = "standard")] security_profile: String, - /// Decode PostgreSQL custom-format or directory-format dumps via `pg_restore -f -` before anonymizing. - /// Requires `--input` pointing at the archive file or directory and `--format postgres`. Requires a - /// PostgreSQL client install (`pg_restore` on PATH unless overridden by `--pg-restore-path`). - #[arg(long = "dump-decode", action = ArgAction::SetTrue)] - dump_decode: bool, + /// Keep original PostgreSQL archive inputs after decode (ignored with `--check`; incompatible with `--in-place`). + /// Also set `keep_original = true` in `.dumplingconf`. + #[arg(long = "keep-original", action = ArgAction::SetTrue)] + keep_original: bool, - /// Keep the input archive after `--dump-decode` (default: delete file or directory after a fully - /// successful run). Cannot retain the archive with `--check` (would delete before verifying changes). - #[arg(long = "dump-decode-keep-input", action = ArgAction::SetTrue)] - dump_decode_keep_input: bool, - - /// `pg_restore` executable to use with `--dump-decode` (default: `pg_restore` on PATH). - #[arg(long = "pg-restore-path", default_value = "pg_restore")] - pg_restore_path: PathBuf, + /// `pg_restore` executable (optional; default: `pg_restore` on PATH or `[pg_restore] path` in config). + #[arg(long = "pg-restore-path")] + pg_restore_path: Option, /// Extra arguments forwarded to `pg_restore` before the archive path (repeatable). Example: - /// `--dump-decode-arg=--no-owner` `--dump-decode-arg=--no-acl` - #[arg(long = "dump-decode-arg")] - dump_decode_arg: Vec, + /// `--pg-restore-arg=--no-owner` `--pg-restore-arg=--no-acl` + #[arg(long = "pg-restore-arg")] + pg_restore_arg: Vec, #[command(subcommand)] command: Option, @@ -183,21 +183,17 @@ enum Commands { #[arg(long = "allow-ext")] allow_ext: Vec, - /// Decode a PostgreSQL custom- or directory-format dump via `pg_restore -f -` before scanning. Requires `--input` and `--format postgres`. - #[arg(long = "dump-decode", action = ArgAction::SetTrue)] - dump_decode: bool, - - /// Keep the input archive after `--dump-decode` (default: delete on success). - #[arg(long = "dump-decode-keep-input", action = ArgAction::SetTrue)] - dump_decode_keep_input: bool, + /// Keep original inputs (see main `dumpling --keep-original`). + #[arg(long = "keep-original", action = ArgAction::SetTrue)] + keep_original: bool, - /// `pg_restore` for `--dump-decode` (default: `pg_restore` on PATH). - #[arg(long = "pg-restore-path", default_value = "pg_restore")] - pg_restore_path: PathBuf, + /// `pg_restore` for custom- or directory-format archives (optional; see main `dumpling` help). + #[arg(long = "pg-restore-path")] + pg_restore_path: Option, /// Extra args for `pg_restore` before the archive path (repeatable). - #[arg(long = "dump-decode-arg")] - dump_decode_arg: Vec, + #[arg(long = "pg-restore-arg")] + pg_restore_arg: Vec, /// Sample JSON path hints: keep 5 rows per table (reservoir) from INSERT/COPY and infer nested `column.path` rules. #[arg(long = "infer-json-paths", action = ArgAction::SetTrue)] @@ -220,10 +216,9 @@ fn main() -> anyhow::Result<()> { output, format, allow_ext, - dump_decode, - dump_decode_keep_input, + keep_original, pg_restore_path, - dump_decode_arg, + pg_restore_arg, infer_json_paths, max_json_depth, }) = &cli.command @@ -237,20 +232,21 @@ fn main() -> anyhow::Result<()> { other ), }; - if *dump_decode && dump_format != DumpFormat::Postgres { - anyhow::bail!( - "--dump-decode only applies to PostgreSQL dumps; use --format postgres (default)" - ); - } + let resolved_for_pg = settings::load_config(cli.config.as_ref(), false)?; + let (pg_restore_path_eff, pg_restore_arg_eff) = settings::merge_pg_restore_cli( + &resolved_for_pg.pg_restore, + pg_restore_path.clone(), + pg_restore_arg.as_slice(), + ); + let keep_original_eff = merge_keep_original(*keep_original, resolved_for_pg.keep_original); return scaffold::run_scaffold_config(scaffold::ScaffoldConfigOptions { - input: input.as_ref(), - output: output.as_ref(), + input: input.clone(), + output: output.clone(), dump_format, - allow_ext, - dump_decode: *dump_decode, - dump_decode_keep_input: *dump_decode_keep_input, - pg_restore_path, - dump_decode_arg, + allow_ext: allow_ext.clone(), + keep_original: keep_original_eff, + pg_restore_path: pg_restore_path_eff, + pg_restore_arg: pg_restore_arg_eff, infer_json_paths: *infer_json_paths, max_json_depth: *max_json_depth, }); @@ -262,7 +258,10 @@ fn main() -> anyhow::Result<()> { fn run_lint_policy(config: Option<&PathBuf>, allow_noop: bool) -> anyhow::Result<()> { let resolved_config: ResolvedConfig = settings::load_config(config, allow_noop)?; if let Some(path) = resolved_config.source_path.as_ref() { - eprintln!("dumpling: using config source {}", path.display()); + eprintln!( + "dumpling: using config source {}", + path_basename_for_log(path.as_path()) + ); } else if allow_noop { eprintln!("dumpling: no config discovered; continuing because --allow-noop was set"); } @@ -399,26 +398,22 @@ impl Write for AnonWriter { } fn run_anonymize(cli: Cli) -> anyhow::Result<()> { + let mut compression_cleanup = CompressionCleanup::default(); if cli.in_place && cli.output.is_some() { anyhow::bail!("--in-place cannot be used together with --output"); } if cli.check && (cli.in_place || cli.output.is_some()) { anyhow::bail!("--check cannot be used together with --output or --in-place"); } - if cli.dump_decode && !cli.dump_decode_keep_input && cli.check { - anyhow::bail!( - "--dump-decode removes the input archive on success by default; use --dump-decode-keep-input with --check" - ); - } - if cli.dump_decode && cli.in_place { - anyhow::bail!("--dump-decode cannot be used with --in-place"); - } // Resolve config from provided path or discover in CWD let resolved_config: ResolvedConfig = settings::load_config(cli.config.as_ref(), cli.allow_noop)?; if let Some(path) = resolved_config.source_path.as_ref() { - eprintln!("dumpling: using config source {}", path.display()); + eprintln!( + "dumpling: using config source {}", + path_basename_for_log(path.as_path()) + ); } else if cli.allow_noop { eprintln!("dumpling: no config discovered; continuing because --allow-noop was set"); } @@ -476,104 +471,69 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { other ), }; - if cli.dump_decode && dump_format != DumpFormat::Postgres { + + let seal_runtime = SealRuntimeParams::new(dump_format, prng_seed_override_for_fingerprint()); + + let (pg_restore_path_eff, pg_restore_arg_eff) = settings::merge_pg_restore_cli( + &resolved_config.pg_restore, + cli.pg_restore_path.clone(), + &cli.pg_restore_arg, + ); + let keep_original_eff = merge_keep_original(cli.keep_original, resolved_config.keep_original); + + if cli.in_place && keep_original_eff { anyhow::bail!( - "--dump-decode only applies to PostgreSQL dumps; use --format postgres (default)" + "`--keep-original` (or `keep_original = true` in config) cannot be used with `--in-place`; \ + keeping the original path while overwriting it in place is ambiguous. Use `--output` (or stdout) instead, \ + or omit `--keep-original` for in-place overwrite." ); } - let seal_runtime = SealRuntimeParams::new(dump_format, prng_seed_override_for_fingerprint()); + if !keep_original_eff && cli.check { + anyhow::bail!( + "PostgreSQL archive decoding removes the `--input` archive on success by default; use `--keep-original` or `keep_original = true` in config with --check" + ); + } - // Determine IO (optional pg_restore child when --dump-decode) - let mut pg_restore_child: Option = None; - let (mut reader, input_path_for_inplace): (Box, Option) = if cli - .dump_decode + // Determine IO (optional pg_restore child for PostgreSQL custom/directory archives) + let mut pg_restore_child: Option = None; + let mut path_to_remove_pg_archive: Option = None; + let (mut reader, input_path_for_inplace): (Box, Option) = match &cli.input { - let archive_path = cli - .input - .as_ref() - .ok_or_else(|| { - anyhow::anyhow!( - "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" - ) - })?; - if !cli.allow_ext.is_empty() && !has_allowed_extension(archive_path, &cli.allow_ext) { - let actual = archive_path - .extension() - .and_then(|s| s.to_str()) - .unwrap_or("") - .to_string(); - anyhow::bail!( - "input file extension '{}' is not in allowed set {:?}", - actual, - cli.allow_ext - ); - } - if !archive_path.exists() { - anyhow::bail!( - "--dump-decode input path does not exist: {}", - archive_path.display() - ); - } - eprintln!( - "dumpling: decoding PostgreSQL archive via {} -f - {}", - cli.pg_restore_path.display(), - archive_path.display() - ); - let mut cmd = Command::new(&cli.pg_restore_path); - for a in &cli.dump_decode_arg { - cmd.arg(a); - } - cmd.arg("-f") - .arg("-") - .arg(archive_path) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()); - let mut child = cmd.spawn().with_context(|| { - format!( - "failed to spawn `{}`; install PostgreSQL client tools or set --pg-restore-path", - cli.pg_restore_path.display() - ) - })?; - let stdout = child - .stdout - .take() - .ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?; - pg_restore_child = Some(child); - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), - Some(archive_path.clone()), - ) - } else { - match &cli.input { - Some(path) => { - if !cli.allow_ext.is_empty() && !has_allowed_extension(path, &cli.allow_ext) { - let actual = path - .extension() - .and_then(|s| s.to_str()) - .unwrap_or("") - .to_string(); - anyhow::bail!( - "input file extension '{}' is not in allowed set {:?}", - actual, - cli.allow_ext - ); - } - let f = File::open(path)?; - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), - Some(path.clone()), - ) + None => { + if !cli.allow_ext.is_empty() { + eprintln!("dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin"); } - None => { - if !cli.allow_ext.is_empty() { - eprintln!("dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin"); - } - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, io::stdin())), - None, - ) + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, io::stdin())), + None, + ) + } + Some(path) => { + if !cli.allow_ext.is_empty() && !has_allowed_extension(path, &cli.allow_ext) { + let actual = path + .extension() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(); + anyhow::bail!( + "input file extension '{}' is not in allowed set {:?}", + actual, + cli.allow_ext + ); } + let resolved = resolve_dump_input_from_path(ResolveDumpInputParams { + user_input_path: path, + dump_format, + compression_cleanup: &mut compression_cleanup, + pg_restore_path: &pg_restore_path_eff, + pg_restore_arg: &pg_restore_arg_eff, + keep_original: keep_original_eff, + in_place: cli.in_place, + })?; + pg_restore_child = resolved.pg_restore_child; + path_to_remove_pg_archive = resolved.path_to_remove_pg_archive; + (resolved.reader, Some(resolved.original_input_path)) } }; @@ -688,20 +648,8 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { } }; - if let Some(mut child) = pg_restore_child { - if proc_res.is_err() { - let _ = child.kill(); - } - let status = child - .wait() - .with_context(|| format!("waiting for `{}`", cli.pg_restore_path.display()))?; - if proc_res.is_ok() && !status.success() { - anyhow::bail!( - "`{}` exited with status {}", - cli.pg_restore_path.display(), - status - ); - } + if let Some(pg_child) = pg_restore_child { + pg_child.finish(proc_res.is_ok())?; } proc_res?; @@ -779,9 +727,11 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { } if strict_coverage_failed { + drop(compression_cleanup); std::process::exit(2); } if scan_failed { + drop(compression_cleanup); std::process::exit(3); } @@ -789,19 +739,21 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { if cli.check && (reporter.report.total_cells_changed > 0 || reporter.report.total_rows_dropped > 0) { + drop(compression_cleanup); std::process::exit(1); } - if cli.dump_decode && !cli.dump_decode_keep_input { - if let Some(ref p) = input_path_for_inplace { - match remove_pg_archive(p) { - Ok(()) => eprintln!("dumpling: removed input archive {}", p.display()), - Err(e) => eprintln!( - "dumpling: warning: could not remove input archive {}: {}", - p.display(), - e - ), - } + if let Some(ref p) = path_to_remove_pg_archive { + match remove_pg_archive(p) { + Ok(()) => eprintln!( + "dumpling: removed input archive {}", + path_basename_for_log(p) + ), + Err(e) => eprintln!( + "dumpling: warning: could not remove input archive {}: {}", + path_basename_for_log(p), + e + ), } } @@ -996,21 +948,22 @@ email = { strategy = "email" } } #[test] - fn test_dump_decode_flags_parse() { + fn test_pg_restore_flags_parse() { let cli = Cli::parse_from([ "dumpling", - "--dump-decode", - "--dump-decode-keep-input", + "--keep-original", "--pg-restore-path", "/usr/bin/pg_restore", - "--dump-decode-arg=--no-owner", + "--pg-restore-arg=--no-owner", "-i", "/tmp/latest.dump", ]); - assert!(cli.dump_decode); - assert!(cli.dump_decode_keep_input); - assert_eq!(cli.pg_restore_path, PathBuf::from("/usr/bin/pg_restore")); - assert_eq!(cli.dump_decode_arg, vec!["--no-owner"]); + assert!(cli.keep_original); + assert_eq!( + cli.pg_restore_path, + Some(PathBuf::from("/usr/bin/pg_restore")) + ); + assert_eq!(cli.pg_restore_arg, vec!["--no-owner"]); } #[test] diff --git a/src/pg_restore_decode.rs b/src/pg_restore_decode.rs new file mode 100644 index 0000000..22d55cb --- /dev/null +++ b/src/pg_restore_decode.rs @@ -0,0 +1,221 @@ +//! Helpers for PostgreSQL custom/directory archives decoded via `pg_restore`. + +use anyhow::Context; +use std::io::Write; +use std::path::Path; +use std::process::{Child, ChildStdout, Command, Stdio}; +use std::thread::JoinHandle; + +/// Shown when `pg_restore` cannot be run or fails without stderr output. +pub(crate) const PG_RESTORE_MISSING_HINT: &str = "\ +PostgreSQL client tools are required when Dumpling decodes a custom or directory-format archive. Install them (for example the \ +`postgresql-client` package on Debian/Ubuntu, `postgresql` via Homebrew on macOS, or the \ +official PostgreSQL installer on Windows), ensure `pg_restore` is on your PATH, or pass \ +`--pg-restore-path` pointing at the `pg_restore` executable."; + +/// When `--pg-restore-path` is a concrete filesystem path, verify it exists and can be executed +/// before starting a long anonymize run. A bare `pg_restore` relies on the system PATH at spawn +/// time (the same install hint is used if `spawn` fails). +pub(crate) fn ensure_pg_restore_available(pg_restore_path: &Path) -> anyhow::Result<()> { + if pg_restore_path.as_os_str().is_empty() { + anyhow::bail!("--pg-restore-path is empty"); + } + if pg_restore_path == Path::new("pg_restore") { + return Ok(()); + } + if !pg_restore_path.exists() { + anyhow::bail!( + "pg_restore not found at `{}`\n\n{}", + pg_restore_path.display(), + PG_RESTORE_MISSING_HINT + ); + } + check_executable_file(pg_restore_path)?; + Ok(()) +} + +#[cfg(unix)] +fn check_executable_file(path: &Path) -> anyhow::Result<()> { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let meta = fs::metadata(path).with_context(|| format!("stat `{}`", path.display()))?; + if !meta.is_file() { + anyhow::bail!( + "`{}` is not a regular file\n\n{}", + path.display(), + PG_RESTORE_MISSING_HINT + ); + } + let mode = meta.permissions().mode(); + if mode & 0o111 == 0 { + anyhow::bail!( + "`{}` is not executable\n\n{}", + path.display(), + PG_RESTORE_MISSING_HINT + ); + } + Ok(()) +} + +#[cfg(not(unix))] +fn check_executable_file(path: &Path) -> anyhow::Result<()> { + use std::fs; + + let meta = fs::metadata(path).with_context(|| format!("stat `{}`", path.display()))?; + if !meta.is_file() { + anyhow::bail!( + "`{}` is not a file\n\n{}", + path.display(), + PG_RESTORE_MISSING_HINT + ); + } + Ok(()) +} + +pub(crate) struct PgRestoreDecodeProcess { + child: Child, + stderr_join: JoinHandle>>, + program: std::path::PathBuf, +} + +/// Spawn `pg_restore -f - ` with stdout/stderr piped. Stderr is collected so we can +/// attach it to failure messages; join it in [`PgRestoreDecodeProcess::finish`]. +/// +/// Returns stdout for piping into the anonymizer; wait with [`PgRestoreDecodeProcess::finish`]. +pub(crate) fn spawn_pg_restore_decode( + pg_restore_path: &Path, + pg_restore_arg: &[String], + archive_path: &Path, +) -> anyhow::Result<(ChildStdout, PgRestoreDecodeProcess)> { + ensure_pg_restore_available(pg_restore_path)?; + + let mut cmd = Command::new(pg_restore_path); + for a in pg_restore_arg { + cmd.arg(a); + } + cmd.arg("-f") + .arg("-") + .arg(archive_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let program = pg_restore_path.to_path_buf(); + let mut child = cmd + .spawn() + .with_context(|| spawn_failed_context(&program))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| anyhow::anyhow!("pg_restore stderr missing"))?; + + let stderr_join = std::thread::spawn(move || { + use std::io::{BufRead, BufReader}; + let mut captured = Vec::new(); + let mut reader = BufReader::new(stderr); + let mut line = Vec::new(); + loop { + line.clear(); + let n = reader.read_until(b'\n', &mut line)?; + if n == 0 { + break; + } + captured.extend_from_slice(&line); + // Mirror pg_restore diagnostics to the user's terminal while buffering for errors. + std::io::stderr().write_all(&line)?; + } + Ok(captured) + }); + + Ok(( + stdout, + PgRestoreDecodeProcess { + child, + stderr_join, + program, + }, + )) +} + +fn spawn_failed_context(program: &Path) -> String { + format!( + "failed to run `{}`\n\n{}", + program.display(), + PG_RESTORE_MISSING_HINT + ) +} + +impl PgRestoreDecodeProcess { + /// Wait for `pg_restore` after the pipeline has finished reading its stdout. If the pipeline + /// failed, the child is killed before waiting. + pub(crate) fn finish(mut self, pipeline_ok: bool) -> anyhow::Result<()> { + if !pipeline_ok { + let _ = self.child.kill(); + } + let status = self + .child + .wait() + .with_context(|| format!("waiting for `{}`", self.program.display()))?; + + let stderr_bytes = match self.stderr_join.join() { + Ok(Ok(buf)) => buf, + Ok(Err(e)) => { + return Err(e).context(format!("reading `{}` stderr", self.program.display())); + } + Err(_) => anyhow::bail!( + "internal error: stderr reader for `{}` panicked", + self.program.display() + ), + }; + let stderr_text = String::from_utf8_lossy(&stderr_bytes); + let stderr_trimmed = stderr_text.trim(); + + if pipeline_ok && !status.success() { + let mut msg = format!( + "`{}` failed (exit status: {})", + self.program.display(), + status + ); + if !stderr_trimmed.is_empty() { + msg.push_str("\n\n"); + msg.push_str(stderr_trimmed); + } else { + msg.push_str("\n\n"); + msg.push_str(PG_RESTORE_MISSING_HINT); + } + anyhow::bail!(msg); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ensure_rejects_missing_explicit_path() { + let p = Path::new("/this/path/does/not/exist/pg_restore_dumpling_test"); + let err = ensure_pg_restore_available(p).unwrap_err(); + let s = format!("{:#}", err); + assert!( + s.contains("not found") || s.contains("pg_restore"), + "unexpected message: {}", + s + ); + assert!( + s.contains("PostgreSQL client") || s.contains("PATH"), + "missing install hint: {}", + s + ); + } + + #[test] + fn ensure_accepts_bare_pg_restore() { + ensure_pg_restore_available(Path::new("pg_restore")).unwrap(); + } +} diff --git a/src/scaffold.rs b/src/scaffold.rs index 025f787..df10fd2 100644 --- a/src/scaffold.rs +++ b/src/scaffold.rs @@ -1,5 +1,9 @@ //! Draft `[rules]` generation from column names in a SQL dump (starter / beta). +use crate::compressed_input::CompressionCleanup; +use crate::dump_input_resolve::{resolve_dump_input_from_path, ResolveDumpInputParams}; +use crate::log_sanitize::path_basename_for_log; +use crate::pg_restore_decode; use crate::settings::{validate_raw_config, RawConfig}; use crate::sql::{ discover_scaffold_column_rules, discover_scaffold_rules, DumpFormat, ScaffoldDiscoverOptions, @@ -7,38 +11,35 @@ use crate::sql::{ use anyhow::Context; use serde_json::Value as JsonValue; use std::collections::HashMap; -use std::fs::File; use std::io::{BufRead, BufReader, Write}; use std::path::PathBuf; -use std::process::{Command, Stdio}; use crate::IO_BUF_CAPACITY; /// CLI-backed options for [`run_scaffold_config`]. -pub struct ScaffoldConfigOptions<'a> { - pub input: Option<&'a PathBuf>, - pub output: Option<&'a PathBuf>, +pub struct ScaffoldConfigOptions { + pub input: Option, + pub output: Option, pub dump_format: DumpFormat, - pub allow_ext: &'a [String], - pub dump_decode: bool, - pub dump_decode_keep_input: bool, - pub pg_restore_path: &'a PathBuf, - pub dump_decode_arg: &'a [String], + pub allow_ext: Vec, + pub keep_original: bool, + pub pg_restore_path: PathBuf, + pub pg_restore_arg: Vec, pub infer_json_paths: bool, pub max_json_depth: usize, } /// Emit header comments, stderr notice, and TOML body for a starter config. -pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<()> { +pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { + let mut compression_cleanup = CompressionCleanup::default(); let ScaffoldConfigOptions { input, output, dump_format, allow_ext, - dump_decode, - dump_decode_keep_input, + keep_original, pg_restore_path, - dump_decode_arg, + pg_restore_arg, infer_json_paths, max_json_depth, } = opts; @@ -53,146 +54,120 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() } ); - let mut pg_restore_child = None; - let (mut reader, _input_path): (Box, Option) = if dump_decode { - let archive_path = input.ok_or_else(|| { - anyhow::anyhow!( - "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" + let mut pg_restore_child: Option = None; + let mut path_to_remove_pg_archive: Option = None; + let (mut reader, _input_path): (Box, Option) = match input { + None => { + if !allow_ext.is_empty() { + eprintln!( + "dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin" + ); + } + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, std::io::stdin())), + None, ) - })?; - if !allow_ext.is_empty() && !crate::has_allowed_extension(archive_path, allow_ext) { - anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); - } - if !archive_path.exists() { - anyhow::bail!( - "--dump-decode input path does not exist: {}", - archive_path.display() - ); } - eprintln!( - "dumpling: decoding PostgreSQL archive via {} -f - {}", - pg_restore_path.display(), - archive_path.display() - ); - let mut cmd = Command::new(pg_restore_path); - for a in dump_decode_arg { - cmd.arg(a); - } - cmd.arg("-f") - .arg("-") - .arg(archive_path) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()); - let mut child = cmd.spawn().with_context(|| { - format!( - "failed to spawn `{}`; install PostgreSQL client tools or set --pg-restore-path", - pg_restore_path.display() - ) - })?; - let stdout = child - .stdout - .take() - .ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?; - pg_restore_child = Some((child, archive_path.clone())); - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), - Some(archive_path.clone()), - ) - } else { - match input { - Some(path) => { - if !allow_ext.is_empty() && !crate::has_allowed_extension(path, allow_ext) { - anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); - } - let f = File::open(path).with_context(|| format!("open {}", path.display()))?; - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), - Some(path.clone()), - ) - } - None => { - if !allow_ext.is_empty() { - eprintln!( - "dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin" - ); - } - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, std::io::stdin())), - None, - ) + Some(path) => { + if !allow_ext.is_empty() && !crate::has_allowed_extension(&path, &allow_ext) { + anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); } + let resolved = resolve_dump_input_from_path(ResolveDumpInputParams { + user_input_path: &path, + dump_format, + compression_cleanup: &mut compression_cleanup, + pg_restore_path: &pg_restore_path, + pg_restore_arg: &pg_restore_arg, + keep_original, + in_place: false, + })?; + pg_restore_child = resolved.pg_restore_child; + path_to_remove_pg_archive = resolved.path_to_remove_pg_archive; + (resolved.reader, Some(path)) } }; - let rules = if !infer_json_paths { - discover_scaffold_column_rules(&mut *reader, dump_format) - .context("scanning dump for scaffold rules")? - } else { - let discover_opts = ScaffoldDiscoverOptions { - infer_json_paths: true, - max_json_depth, + let discovery_res = (|| -> anyhow::Result<()> { + let rules = if !infer_json_paths { + discover_scaffold_column_rules(&mut *reader, dump_format) + .context("scanning dump for scaffold rules")? + } else { + let discover_opts = ScaffoldDiscoverOptions { + infer_json_paths: true, + max_json_depth, + }; + discover_scaffold_rules(&mut *reader, dump_format, &discover_opts) + .context("scanning dump for scaffold rules")? }; - discover_scaffold_rules(&mut *reader, dump_format, &discover_opts) - .context("scanning dump for scaffold rules")? - }; - - if rules.is_empty() { - eprintln!( - "dumpling scaffold-config: warning: no rules inferred; emitted file contains header only" - ); - } else if infer_json_paths { - eprintln!( - "dumpling scaffold-config: reservoir sample ({} rows max per table) for JSON path hints", - crate::sql::SCAFFOLD_JSON_RESERVOIR_SIZE - ); - } - let raw = RawConfig { - salt: None, - rules, - row_filters: HashMap::new(), - column_cases: HashMap::new(), - table_options: HashMap::new(), - sensitive_columns: HashMap::new(), - output_scan: crate::settings::OutputScanConfig::default(), - }; - validate_raw_config(&raw).context("internal error: scaffold rules failed validation")?; + if rules.is_empty() { + eprintln!( + "dumpling scaffold-config: warning: no rules inferred; emitted file contains header only" + ); + } else if infer_json_paths { + eprintln!( + "dumpling scaffold-config: reservoir sample ({} rows max per table) for JSON path hints", + crate::sql::SCAFFOLD_JSON_RESERVOIR_SIZE + ); + } - let body = raw_config_to_toml(&raw)?; - let mut text = SCAFFOLD_HEADER.to_string(); - text.push_str(&body); + let raw = RawConfig { + salt: None, + rules, + row_filters: HashMap::new(), + column_cases: HashMap::new(), + table_options: HashMap::new(), + sensitive_columns: HashMap::new(), + output_scan: crate::settings::OutputScanConfig::default(), + keep_original: None, + pg_restore: crate::settings::PgRestoreRawConfig::default(), + }; + validate_raw_config(&raw).context("internal error: scaffold rules failed validation")?; - if let Some(path) = output { - std::fs::write(path, &text).with_context(|| format!("write {}", path.display()))?; - eprintln!("dumpling scaffold-config: wrote {}", path.display()); - } else { - std::io::stdout().write_all(text.as_bytes())?; - } + let body = raw_config_to_toml(&raw)?; + let mut text = SCAFFOLD_HEADER.to_string(); + text.push_str(&body); - if let Some((mut child, archive_path)) = pg_restore_child { - let status = child - .wait() - .with_context(|| format!("waiting for `{}`", pg_restore_path.display()))?; - if !status.success() { - anyhow::bail!( - "`{}` exited with status {}", - pg_restore_path.display(), - status + if let Some(ref path) = output { + std::fs::write(path, &text).with_context(|| format!("write {}", path.display()))?; + eprintln!( + "dumpling scaffold-config: wrote {}", + path_basename_for_log(path.as_path()) ); + } else { + std::io::stdout().write_all(text.as_bytes())?; } - if dump_decode && !dump_decode_keep_input { - match crate::remove_pg_archive(&archive_path) { - Ok(()) => eprintln!("dumpling: removed input archive {}", archive_path.display()), + Ok(()) + })(); + + let pipeline_ok = discovery_res.is_ok(); + if let Some(pg_child) = pg_restore_child { + pg_child.finish(pipeline_ok).with_context(|| { + format!( + "`{}` failed while decoding the PostgreSQL archive", + path_basename_for_log(pg_restore_path.as_path()) + ) + })?; + } + if pipeline_ok { + if let Some(ref p) = path_to_remove_pg_archive { + match crate::remove_pg_archive(p) { + Ok(()) => eprintln!( + "dumpling: removed input archive {}", + path_basename_for_log(p.as_path()) + ), Err(e) => eprintln!( "dumpling: warning: could not remove input archive {}: {}", - archive_path.display(), + path_basename_for_log(p.as_path()), e ), } } } - Ok(()) + let out = discovery_res; + drop(compression_cleanup); + out } const SCAFFOLD_HEADER: &str = r#"# Dumpling starter config (beta) — generated by `dumpling scaffold-config`. diff --git a/src/seal.rs b/src/seal.rs index 3af44a4..4e5b69c 100644 --- a/src/seal.rs +++ b/src/seal.rs @@ -2,7 +2,9 @@ //! fingerprint of the resolved policy, and runtime CLI options that affect transforms. When the //! first line matches, the remainder of the dump is copied through unchanged. -use crate::settings::{AnonymizerSpec, ColumnCase, RawConfig, ResolvedConfig, RowFilterSet}; +use crate::settings::{ + AnonymizerSpec, ColumnCase, PgRestoreRawConfig, RawConfig, ResolvedConfig, RowFilterSet, +}; use crate::sql::DumpFormat; use serde::Serialize; use serde_json::Value; @@ -134,6 +136,15 @@ fn resolved_to_raw_for_fingerprint(cfg: &ResolvedConfig) -> RawConfig { table_options: HashMap::new(), sensitive_columns, output_scan: cfg.output_scan.clone(), + keep_original: cfg.keep_original, + pg_restore: PgRestoreRawConfig { + path: cfg + .pg_restore + .path + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + args: cfg.pg_restore.args.clone(), + }, } } @@ -365,6 +376,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, } } diff --git a/src/settings.rs b/src/settings.rs index 6c02af6..5239ab7 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -4,6 +4,8 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; +use crate::log_sanitize::path_basename_for_log; + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct RawConfig { /// Optional default salt used by certain anonymizers (e.g., hash) @@ -28,6 +30,50 @@ pub struct RawConfig { /// Post-transform output scanning config for residual sensitive patterns. #[serde(default)] pub output_scan: OutputScanConfig, + /// When true, keep original PostgreSQL archive inputs after decode. Incompatible with `--in-place` (use `--output` instead). CLI `--keep-original` also sets this. + #[serde(default)] + pub keep_original: Option, + /// Optional defaults for `pg_restore` when decoding PostgreSQL archives (CLI overrides when set). + #[serde(default)] + pub pg_restore: PgRestoreRawConfig, +} + +/// Raw `[pg_restore]` table (optional). +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct PgRestoreRawConfig { + /// Path to the `pg_restore` executable. + pub path: Option, + /// Extra arguments before the archive path (e.g. `"--no-owner"`). + #[serde(default)] + pub args: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct PgRestoreConfig { + pub path: Option, + pub args: Vec, +} + +/// Merge `[pg_restore]` path and args with CLI; CLI wins when provided. +pub fn merge_pg_restore_cli( + cfg: &PgRestoreConfig, + cli_path: Option, + cli_args: &[String], +) -> (PathBuf, Vec) { + let path = cli_path + .or_else(|| cfg.path.clone()) + .unwrap_or_else(|| PathBuf::from("pg_restore")); + let args = if !cli_args.is_empty() { + cli_args.to_vec() + } else { + cfg.args.clone() + }; + (path, args) +} + +/// Effective `--keep-original`: CLI or config, default false. +pub fn merge_keep_original(cli: bool, cfg: Option) -> bool { + cli || cfg.unwrap_or(false) } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -72,7 +118,7 @@ pub struct AnonymizerSpec { pub format: Option, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct ResolvedConfig { pub salt: Option, /// Normalized rule map: lowercase keys for table and column names @@ -85,6 +131,10 @@ pub struct ResolvedConfig { pub sensitive_columns: HashMap>, /// Resolved output scan config pub output_scan: OutputScanConfig, + /// Defaults for PostgreSQL archive decoding (`pg_restore`); merged with CLI in `merge_pg_restore_cli`. + pub pg_restore: PgRestoreConfig, + /// Whether to keep original archive inputs. Merged with CLI in `merge_keep_original` (incompatible with `--in-place`). + pub keep_original: Option, /// For debugging/trace pub source_path: Option, } @@ -241,7 +291,7 @@ fn resolve_raw_config_value( eprintln!( "dumpling: warning: insecure plaintext secret at config path '{}' in {}; use ${{ENV_VAR}} or ${{env:ENV_VAR}}", secret_path, - source_path.display() + path_basename_for_log(source_path) ); } raw_value.try_into().with_context(|| { @@ -460,6 +510,8 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { table_options: _, sensitive_columns, output_scan, + keep_original, + pg_restore: pg_raw, } = raw; let OutputScanConfig { enabled_categories, @@ -525,6 +577,10 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { .into_iter() .map(|value| value.to_ascii_lowercase()) .collect::>(); + let pg_restore = PgRestoreConfig { + path: pg_raw.path.map(PathBuf::from), + args: pg_raw.args, + }; ResolvedConfig { salt, rules: normalized_rules, @@ -540,6 +596,8 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { fail_on_severity: fail_on_severity.to_ascii_lowercase(), sample_limit_per_category, }, + pg_restore, + keep_original, source_path, } } @@ -1137,6 +1195,8 @@ fn empty_config(source_path: Option) -> ResolvedConfig { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: OutputScanConfig::default(), + pg_restore: PgRestoreConfig::default(), + keep_original: None, source_path, } } diff --git a/src/sql.rs b/src/sql.rs index 00522ae..9e38c4c 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -2164,6 +2164,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2297,6 +2299,8 @@ COPY public.events (id, email, the_date) FROM stdin; column_cases, sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2357,6 +2361,8 @@ INSERT INTO public.users (id, email, country, is_admin) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2440,6 +2446,8 @@ INSERT INTO public.orders (id, customer_email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2505,6 +2513,8 @@ INSERT INTO public.users (id, email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2563,6 +2573,8 @@ INSERT INTO public.users (id, email) VALUES (1, 'alice@myco.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2649,6 +2661,8 @@ INSERT INTO public.users (id, email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2740,6 +2754,8 @@ INSERT INTO public.users (id, email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2812,6 +2828,8 @@ INSERT INTO public.orders (id, customer_email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2863,6 +2881,8 @@ INSERT INTO public.users (id, email, first_name, password, dob, notes) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2936,6 +2956,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3016,6 +3038,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3086,6 +3110,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3167,6 +3193,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3255,6 +3283,8 @@ COPY public.payments (id, pan) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3365,6 +3395,8 @@ INSERT INTO public.events (id, payload) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; set_random_seed(7); @@ -3452,6 +3484,8 @@ old@example.com verylongname (000) 000-0000 column_cases: HashMap::new(), sensitive_columns, output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3522,6 +3556,8 @@ CREATE TABLE public.users ( column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3604,6 +3640,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3640,6 +3678,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3693,6 +3733,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3753,6 +3795,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3809,6 +3853,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; set_random_seed(42); @@ -3853,6 +3899,8 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3902,6 +3950,8 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3985,6 +4035,8 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg);