From 7b6a8acb0eeaeb381dc1125ab08acd3ead2d36af Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 4 May 2026 21:23:50 +0000 Subject: [PATCH 1/9] feat: friendlier --dump-decode errors for missing or failed pg_restore - Add pg_restore_decode module: preflight when --pg-restore-path is a concrete path, improved spawn/exit errors with install hint, capture pg_restore stderr (tee to stderr) and include on non-zero exit - Reap pg_restore in scaffold-config when discovery fails; only remove archive after full success - Remove unused Context import from main Co-authored-by: Andy Babic --- src/main.rs | 47 ++------- src/pg_restore_decode.rs | 221 +++++++++++++++++++++++++++++++++++++++ src/scaffold.rs | 134 +++++++++++------------- 3 files changed, 291 insertions(+), 111 deletions(-) create mode 100644 src/pg_restore_decode.rs diff --git a/src/main.rs b/src/main.rs index f07c290..d82c7af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,6 @@ use std::fs::File; use std::io::{self, BufRead, BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use std::sync::atomic::Ordering; use std::sync::mpsc::sync_channel; use std::thread::JoinHandle; @@ -12,6 +11,7 @@ use clap::{ArgAction, Parser, Subcommand}; mod faker_dispatch; mod filter; mod lint; +mod pg_restore_decode; mod report; mod scaffold; mod scan; @@ -28,7 +28,6 @@ const OUTPUT_PIPE_CHUNK: usize = IO_BUF_CAPACITY; /// Number of full chunks allowed in flight (transform can run ahead of disk this far). const OUTPUT_PIPE_DEPTH: usize = 8; -use anyhow::Context; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; use seal::{ @@ -485,7 +484,7 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { let seal_runtime = SealRuntimeParams::new(dump_format, prng_seed_override_for_fingerprint()); // Determine IO (optional pg_restore child when --dump-decode) - let mut pg_restore_child: Option = None; + let mut pg_restore_child: Option = None; let (mut reader, input_path_for_inplace): (Box, Option) = if cli .dump_decode { @@ -520,26 +519,12 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { cli.pg_restore_path.display(), archive_path.display() ); - let mut cmd = Command::new(&cli.pg_restore_path); - for a in &cli.dump_decode_arg { - cmd.arg(a); - } - cmd.arg("-f") - .arg("-") - .arg(archive_path) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()); - let mut child = cmd.spawn().with_context(|| { - format!( - "failed to spawn `{}`; install PostgreSQL client tools or set --pg-restore-path", - cli.pg_restore_path.display() - ) - })?; - let stdout = child - .stdout - .take() - .ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?; - pg_restore_child = Some(child); + let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( + &cli.pg_restore_path, + &cli.dump_decode_arg, + archive_path, + )?; + pg_restore_child = Some(pg); ( Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), Some(archive_path.clone()), @@ -688,20 +673,8 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { } }; - if let Some(mut child) = pg_restore_child { - if proc_res.is_err() { - let _ = child.kill(); - } - let status = child - .wait() - .with_context(|| format!("waiting for `{}`", cli.pg_restore_path.display()))?; - if proc_res.is_ok() && !status.success() { - anyhow::bail!( - "`{}` exited with status {}", - cli.pg_restore_path.display(), - status - ); - } + if let Some(pg_child) = pg_restore_child { + pg_child.finish(proc_res.is_ok())?; } proc_res?; diff --git a/src/pg_restore_decode.rs b/src/pg_restore_decode.rs new file mode 100644 index 0000000..93ff207 --- /dev/null +++ b/src/pg_restore_decode.rs @@ -0,0 +1,221 @@ +//! Helpers for `--dump-decode` (PostgreSQL custom/directory archives via `pg_restore`). + +use anyhow::Context; +use std::io::Write; +use std::path::Path; +use std::process::{Child, ChildStdout, Command, Stdio}; +use std::thread::JoinHandle; + +/// Shown when `pg_restore` cannot be run or fails without stderr output. +pub(crate) const PG_RESTORE_MISSING_HINT: &str = "\ +PostgreSQL client tools are required for --dump-decode. Install them (for example the \ +`postgresql-client` package on Debian/Ubuntu, `postgresql` via Homebrew on macOS, or the \ +official PostgreSQL installer on Windows), ensure `pg_restore` is on your PATH, or pass \ +`--pg-restore-path` pointing at the `pg_restore` executable."; + +/// When `--pg-restore-path` is a concrete filesystem path, verify it exists and can be executed +/// before starting a long anonymize run. A bare `pg_restore` relies on the system PATH at spawn +/// time (the same install hint is used if `spawn` fails). +pub(crate) fn ensure_pg_restore_available(pg_restore_path: &Path) -> anyhow::Result<()> { + if pg_restore_path.as_os_str().is_empty() { + anyhow::bail!("--pg-restore-path is empty"); + } + if pg_restore_path == Path::new("pg_restore") { + return Ok(()); + } + if !pg_restore_path.exists() { + anyhow::bail!( + "pg_restore not found at `{}`\n\n{}", + pg_restore_path.display(), + PG_RESTORE_MISSING_HINT + ); + } + check_executable_file(pg_restore_path)?; + Ok(()) +} + +#[cfg(unix)] +fn check_executable_file(path: &Path) -> anyhow::Result<()> { + use std::fs; + use std::os::unix::fs::PermissionsExt; + + let meta = fs::metadata(path).with_context(|| format!("stat `{}`", path.display()))?; + if !meta.is_file() { + anyhow::bail!( + "`{}` is not a regular file\n\n{}", + path.display(), + PG_RESTORE_MISSING_HINT + ); + } + let mode = meta.permissions().mode(); + if mode & 0o111 == 0 { + anyhow::bail!( + "`{}` is not executable\n\n{}", + path.display(), + PG_RESTORE_MISSING_HINT + ); + } + Ok(()) +} + +#[cfg(not(unix))] +fn check_executable_file(path: &Path) -> anyhow::Result<()> { + use std::fs; + + let meta = fs::metadata(path).with_context(|| format!("stat `{}`", path.display()))?; + if !meta.is_file() { + anyhow::bail!( + "`{}` is not a file\n\n{}", + path.display(), + PG_RESTORE_MISSING_HINT + ); + } + Ok(()) +} + +pub(crate) struct PgRestoreDecodeProcess { + child: Child, + stderr_join: JoinHandle>>, + program: std::path::PathBuf, +} + +/// Spawn `pg_restore -f - ` with stdout/stderr piped. Stderr is collected so we can +/// attach it to failure messages; join it in [`PgRestoreDecodeProcess::finish`]. +/// +/// Returns stdout for piping into the anonymizer; wait with [`PgRestoreDecodeProcess::finish`]. +pub(crate) fn spawn_pg_restore_decode( + pg_restore_path: &Path, + dump_decode_arg: &[String], + archive_path: &Path, +) -> anyhow::Result<(ChildStdout, PgRestoreDecodeProcess)> { + ensure_pg_restore_available(pg_restore_path)?; + + let mut cmd = Command::new(pg_restore_path); + for a in dump_decode_arg { + cmd.arg(a); + } + cmd.arg("-f") + .arg("-") + .arg(archive_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let program = pg_restore_path.to_path_buf(); + let mut child = cmd + .spawn() + .with_context(|| spawn_failed_context(&program))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?; + let stderr = child + .stderr + .take() + .ok_or_else(|| anyhow::anyhow!("pg_restore stderr missing"))?; + + let stderr_join = std::thread::spawn(move || { + use std::io::{BufRead, BufReader}; + let mut captured = Vec::new(); + let mut reader = BufReader::new(stderr); + let mut line = Vec::new(); + loop { + line.clear(); + let n = reader.read_until(b'\n', &mut line)?; + if n == 0 { + break; + } + captured.extend_from_slice(&line); + // Mirror pg_restore diagnostics to the user's terminal while buffering for errors. + std::io::stderr().write_all(&line)?; + } + Ok(captured) + }); + + Ok(( + stdout, + PgRestoreDecodeProcess { + child, + stderr_join, + program, + }, + )) +} + +fn spawn_failed_context(program: &Path) -> String { + format!( + "failed to run `{}`\n\n{}", + program.display(), + PG_RESTORE_MISSING_HINT + ) +} + +impl PgRestoreDecodeProcess { + /// Wait for `pg_restore` after the pipeline has finished reading its stdout. If the pipeline + /// failed, the child is killed before waiting. + pub(crate) fn finish(mut self, pipeline_ok: bool) -> anyhow::Result<()> { + if !pipeline_ok { + let _ = self.child.kill(); + } + let status = self + .child + .wait() + .with_context(|| format!("waiting for `{}`", self.program.display()))?; + + let stderr_bytes = match self.stderr_join.join() { + Ok(Ok(buf)) => buf, + Ok(Err(e)) => { + return Err(e).context(format!("reading `{}` stderr", self.program.display())); + } + Err(_) => anyhow::bail!( + "internal error: stderr reader for `{}` panicked", + self.program.display() + ), + }; + let stderr_text = String::from_utf8_lossy(&stderr_bytes); + let stderr_trimmed = stderr_text.trim(); + + if pipeline_ok && !status.success() { + let mut msg = format!( + "`{}` failed (exit status: {})", + self.program.display(), + status + ); + if !stderr_trimmed.is_empty() { + msg.push_str("\n\n"); + msg.push_str(stderr_trimmed); + } else { + msg.push_str("\n\n"); + msg.push_str(PG_RESTORE_MISSING_HINT); + } + anyhow::bail!(msg); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ensure_rejects_missing_explicit_path() { + let p = Path::new("/this/path/does/not/exist/pg_restore_dumpling_test"); + let err = ensure_pg_restore_available(p).unwrap_err(); + let s = format!("{:#}", err); + assert!( + s.contains("not found") || s.contains("pg_restore"), + "unexpected message: {}", + s + ); + assert!( + s.contains("PostgreSQL client") || s.contains("PATH"), + "missing install hint: {}", + s + ); + } + + #[test] + fn ensure_accepts_bare_pg_restore() { + ensure_pg_restore_available(Path::new("pg_restore")).unwrap(); + } +} diff --git a/src/scaffold.rs b/src/scaffold.rs index 025f787..fdff7df 100644 --- a/src/scaffold.rs +++ b/src/scaffold.rs @@ -1,5 +1,6 @@ //! Draft `[rules]` generation from column names in a SQL dump (starter / beta). +use crate::pg_restore_decode; use crate::settings::{validate_raw_config, RawConfig}; use crate::sql::{ discover_scaffold_column_rules, discover_scaffold_rules, DumpFormat, ScaffoldDiscoverOptions, @@ -10,7 +11,6 @@ use std::collections::HashMap; use std::fs::File; use std::io::{BufRead, BufReader, Write}; use std::path::PathBuf; -use std::process::{Command, Stdio}; use crate::IO_BUF_CAPACITY; @@ -53,7 +53,7 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() } ); - let mut pg_restore_child = None; + let mut pg_restore_child: Option<(pg_restore_decode::PgRestoreDecodeProcess, PathBuf)> = None; let (mut reader, _input_path): (Box, Option) = if dump_decode { let archive_path = input.ok_or_else(|| { anyhow::anyhow!( @@ -74,26 +74,12 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() pg_restore_path.display(), archive_path.display() ); - let mut cmd = Command::new(pg_restore_path); - for a in dump_decode_arg { - cmd.arg(a); - } - cmd.arg("-f") - .arg("-") - .arg(archive_path) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()); - let mut child = cmd.spawn().with_context(|| { - format!( - "failed to spawn `{}`; install PostgreSQL client tools or set --pg-restore-path", - pg_restore_path.display() - ) - })?; - let stdout = child - .stdout - .take() - .ok_or_else(|| anyhow::anyhow!("pg_restore stdout missing"))?; - pg_restore_child = Some((child, archive_path.clone())); + let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( + pg_restore_path, + dump_decode_arg, + archive_path, + )?; + pg_restore_child = Some((pg, archive_path.clone())); ( Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), Some(archive_path.clone()), @@ -124,63 +110,63 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() } }; - let rules = if !infer_json_paths { - discover_scaffold_column_rules(&mut *reader, dump_format) - .context("scanning dump for scaffold rules")? - } else { - let discover_opts = ScaffoldDiscoverOptions { - infer_json_paths: true, - max_json_depth, + let discovery_res = (|| -> anyhow::Result<()> { + let rules = if !infer_json_paths { + discover_scaffold_column_rules(&mut *reader, dump_format) + .context("scanning dump for scaffold rules")? + } else { + let discover_opts = ScaffoldDiscoverOptions { + infer_json_paths: true, + max_json_depth, + }; + discover_scaffold_rules(&mut *reader, dump_format, &discover_opts) + .context("scanning dump for scaffold rules")? }; - discover_scaffold_rules(&mut *reader, dump_format, &discover_opts) - .context("scanning dump for scaffold rules")? - }; - if rules.is_empty() { - eprintln!( - "dumpling scaffold-config: warning: no rules inferred; emitted file contains header only" - ); - } else if infer_json_paths { - eprintln!( - "dumpling scaffold-config: reservoir sample ({} rows max per table) for JSON path hints", - crate::sql::SCAFFOLD_JSON_RESERVOIR_SIZE - ); - } - - let raw = RawConfig { - salt: None, - rules, - row_filters: HashMap::new(), - column_cases: HashMap::new(), - table_options: HashMap::new(), - sensitive_columns: HashMap::new(), - output_scan: crate::settings::OutputScanConfig::default(), - }; - validate_raw_config(&raw).context("internal error: scaffold rules failed validation")?; + if rules.is_empty() { + eprintln!( + "dumpling scaffold-config: warning: no rules inferred; emitted file contains header only" + ); + } else if infer_json_paths { + eprintln!( + "dumpling scaffold-config: reservoir sample ({} rows max per table) for JSON path hints", + crate::sql::SCAFFOLD_JSON_RESERVOIR_SIZE + ); + } - let body = raw_config_to_toml(&raw)?; - let mut text = SCAFFOLD_HEADER.to_string(); - text.push_str(&body); + let raw = RawConfig { + salt: None, + rules, + row_filters: HashMap::new(), + column_cases: HashMap::new(), + table_options: HashMap::new(), + sensitive_columns: HashMap::new(), + output_scan: crate::settings::OutputScanConfig::default(), + }; + validate_raw_config(&raw).context("internal error: scaffold rules failed validation")?; - if let Some(path) = output { - std::fs::write(path, &text).with_context(|| format!("write {}", path.display()))?; - eprintln!("dumpling scaffold-config: wrote {}", path.display()); - } else { - std::io::stdout().write_all(text.as_bytes())?; - } + let body = raw_config_to_toml(&raw)?; + let mut text = SCAFFOLD_HEADER.to_string(); + text.push_str(&body); - if let Some((mut child, archive_path)) = pg_restore_child { - let status = child - .wait() - .with_context(|| format!("waiting for `{}`", pg_restore_path.display()))?; - if !status.success() { - anyhow::bail!( - "`{}` exited with status {}", - pg_restore_path.display(), - status - ); + if let Some(path) = output { + std::fs::write(path, &text).with_context(|| format!("write {}", path.display()))?; + eprintln!("dumpling scaffold-config: wrote {}", path.display()); + } else { + std::io::stdout().write_all(text.as_bytes())?; } - if dump_decode && !dump_decode_keep_input { + Ok(()) + })(); + + let pipeline_ok = discovery_res.is_ok(); + if let Some((pg_child, archive_path)) = pg_restore_child { + pg_child.finish(pipeline_ok).with_context(|| { + format!( + "`{}` failed while decoding the PostgreSQL archive", + pg_restore_path.display() + ) + })?; + if pipeline_ok && dump_decode && !dump_decode_keep_input { match crate::remove_pg_archive(&archive_path) { Ok(()) => eprintln!("dumpling: removed input archive {}", archive_path.display()), Err(e) => eprintln!( @@ -192,7 +178,7 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() } } - Ok(()) + discovery_res } const SCAFFOLD_HEADER: &str = r#"# Dumpling starter config (beta) — generated by `dumpling scaffold-config`. From 0c6a198ceaf67a4a739c21a2b93cfc5cb410706b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 4 May 2026 21:46:36 +0000 Subject: [PATCH 2/9] feat: decompress gzip and ZIP inputs before anonymizing - Add compressed_input: sniff gzip (1f 8b) and ZIP (PK), decompress/extract to temp files with Drop cleanup; combined gzip-then-ZIP supported - ZIP: single member wins; else exactly one .sql file among multiple files - Fail --in-place when input used gzip/ZIP wrappers (unsafe path semantics) - pg_restore archive removal skips paths under temp dir (avoid double-delete) - Dependencies: flate2, zip (deflate only) - scaffold-config mirrors main pipeline; changelog [Unreleased] Co-authored-by: Andy Babic --- CHANGELOG.md | 4 + Cargo.lock | 133 +++++++++++++++++++- Cargo.toml | 2 + src/compressed_input.rs | 218 +++++++++++++++++++++++++++++++++ src/dump_input_detect.rs | 166 +++++++++++++++++++++++++ src/main.rs | 254 +++++++++++++++++++++++++++------------ src/scaffold.rs | 196 ++++++++++++++++++++++-------- 7 files changed, 841 insertions(+), 132 deletions(-) create mode 100644 src/compressed_input.rs create mode 100644 src/dump_input_detect.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c53f2c5..08fbdaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ## [Unreleased] +### Added + +- **Gzip and ZIP inputs**: a file `--input` that is gzip-compressed and/or a ZIP archive containing a single dump file (or a single `.sql` when multiple files are present) is decompressed to a temporary file before processing, then temp files are removed. For PostgreSQL, a `.dump.gz` that wraps a `PGDMP` custom-format file is decompressed so `pg_restore` can read it. **`--in-place` is not allowed** when a gzip or ZIP wrapper was used (use `--output` or stdout). Full multi-file ZIP packages (for example BACPAC) are still not supported as SQL input. + ## [0.7.0-alpha] - 2026-05-04 Pre-release toward **0.7.0** (stable **0.7.0** is not published yet; crates use the **0.7.0-alpha** prerelease identifier until then). diff --git a/Cargo.lock b/Cargo.lock index 4357fa2..95a6ed8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.4" @@ -76,6 +82,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -187,6 +202,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.7" @@ -231,6 +261,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "deunicode" version = "1.6.2" @@ -248,6 +289,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dummy" version = "0.11.0" @@ -268,6 +320,7 @@ dependencies = [ "chrono", "clap", "fake", + "flate2", "getrandom 0.2.17", "hmac", "lazy_static", @@ -276,8 +329,9 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror", + "thiserror 1.0.69", "toml", + "zip", ] [[package]] @@ -310,6 +364,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -456,6 +520,16 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -643,6 +717,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "strsim" version = "0.11.1" @@ -672,7 +752,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -686,6 +775,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "toml" version = "0.8.23" @@ -914,8 +1014,37 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.18", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 40c9b40..5dd206a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,5 @@ chrono = { version = "0.4" } lazy_static = "1" fake = { version = "4", features = ["derive"] } rand = "0.9" +flate2 = "1" +zip = { version = "2", default-features = false, features = ["deflate"] } diff --git a/src/compressed_input.rs b/src/compressed_input.rs new file mode 100644 index 0000000..207d139 --- /dev/null +++ b/src/compressed_input.rs @@ -0,0 +1,218 @@ +//! Transparent gzip / ZIP resolution for `--input` file paths. + +use crate::dump_input_detect::read_file_prefix; +use anyhow::Context; +use flate2::read::GzDecoder; +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; + +/// Paths and directories created while resolving compressed inputs; remove after a successful run. +#[derive(Default)] +pub(crate) struct CompressionCleanup { + paths: Vec, +} + +impl Drop for CompressionCleanup { + fn drop(&mut self) { + for p in self.paths.drain(..) { + let _ = if p.is_dir() { + fs::remove_dir_all(&p) + } else { + fs::remove_file(&p) + }; + } + } +} + +fn is_gzip_prefix(buf: &[u8]) -> bool { + buf.len() >= 2 && buf[0] == 0x1f && buf[1] == 0x8b +} + +fn is_zip_prefix(buf: &[u8]) -> bool { + buf.len() >= 4 && &buf[0..4] == b"PK\x03\x04" +} + +fn unique_temp_file(label: &str) -> PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + std::env::temp_dir().join(format!( + "dumpling_{}_{}_{}", + label, + std::process::id(), + nanos + )) +} + +/// Decompress gzip fully to a temp file (needed so `pg_restore` can read PGDMP or we can re-sniff). +fn decompress_gzip_file(src: &Path, cleanup: &mut CompressionCleanup) -> anyhow::Result { + let out = unique_temp_file("ungz"); + let mut inp = File::open(src).with_context(|| format!("open {}", src.display()))?; + let mut dec = GzDecoder::new(&mut inp); + let mut tmp = File::create(&out).with_context(|| format!("create {}", out.display()))?; + std::io::copy(&mut dec, &mut tmp).with_context(|| format!("decompress {}", src.display()))?; + cleanup.paths.push(out.clone()); + Ok(out) +} + +/// Extract the sole file, or a single `.sql` when multiple entries exist, to a temp path. +fn extract_zip_inner_file(src: &Path, cleanup: &mut CompressionCleanup) -> anyhow::Result { + let file = File::open(src).with_context(|| format!("open {}", src.display()))?; + let mut archive = + zip::ZipArchive::new(file).with_context(|| format!("read ZIP {}", src.display()))?; + let mut file_indices: Vec = Vec::new(); + let mut sql_indices: Vec = Vec::new(); + for i in 0..archive.len() { + let ent = archive.by_index(i)?; + let name = ent.name(); + if name.ends_with('/') || name.ends_with('\\') { + continue; + } + file_indices.push(i); + if Path::new(name) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.eq_ignore_ascii_case("sql")) + .unwrap_or(false) + { + sql_indices.push(i); + } + } + let idx = if file_indices.len() == 1 { + file_indices[0] + } else if sql_indices.len() == 1 { + sql_indices[0] + } else if sql_indices.len() > 1 { + let names: Vec = sql_indices + .iter() + .map(|&i| archive.by_index(i).map(|e| e.name().to_string())) + .collect::>()?; + anyhow::bail!( + "ZIP `{}` contains multiple `.sql` files; Dumpling needs exactly one dump file. Found: {}", + src.display(), + names.join(", ") + ); + } else { + anyhow::bail!( + "ZIP `{}` contains {} file(s); Dumpling needs exactly one file (or exactly one `.sql`). \ + Repack or extract manually.", + src.display(), + file_indices.len() + ); + }; + + let mut ent = archive.by_index(idx).with_context(|| "ZIP entry")?; + let inner = Path::new(ent.name()); + let ext = inner.extension().and_then(|e| e.to_str()).unwrap_or("sql"); + let mut out = unique_temp_file("unzip"); + out.set_extension(ext); + let mut tmp = File::create(&out).with_context(|| format!("create {}", out.display()))?; + std::io::copy(&mut ent, &mut tmp).with_context(|| format!("extract {}", ent.name()))?; + cleanup.paths.push(out.clone()); + Ok(out) +} + +pub(crate) struct ResolvedInput { + pub path: PathBuf, + /// True if the input was gzip and/or ZIP (temporary files may exist until `cleanup` drops). + pub had_compression_wrapper: bool, +} + +/// Follow gzip and/or ZIP wrappers so downstream logic sees a concrete file or directory path. +/// +/// Any temporary paths are registered on `cleanup` and deleted when `cleanup` is dropped. +pub(crate) fn resolve_compressed_wrappers( + original: &Path, + cleanup: &mut CompressionCleanup, +) -> anyhow::Result { + if !original.is_file() { + return Ok(ResolvedInput { + path: original.to_path_buf(), + had_compression_wrapper: false, + }); + } + + let mut current = original.to_path_buf(); + let mut had_wrap = false; + let prefix = read_file_prefix(¤t, 4) + .map_err(|e| anyhow::anyhow!("read {}: {}", current.display(), e))?; + + if is_gzip_prefix(&prefix) { + had_wrap = true; + eprintln!( + "dumpling: decompressing gzip input {} to a temporary file", + original.display() + ); + current = decompress_gzip_file(¤t, cleanup)?; + } + + let prefix = read_file_prefix(¤t, 4) + .map_err(|e| anyhow::anyhow!("read {}: {}", current.display(), e))?; + if is_zip_prefix(&prefix) { + had_wrap = true; + eprintln!( + "dumpling: extracting inner file from ZIP {} (from {})", + current.display(), + original.display() + ); + current = extract_zip_inner_file(¤t, cleanup)?; + } + + Ok(ResolvedInput { + path: current, + had_compression_wrapper: had_wrap, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + use zip::write::SimpleFileOptions; + + #[test] + fn gzip_then_plain_roundtrip_path() { + let dir = std::env::temp_dir(); + let plain = dir.join(format!("dumpling_plain_{}.sql", std::process::id())); + fs::write(&plain, b"SELECT 1;\n").unwrap(); + + let gz_path = dir.join(format!("dumpling_test_{}.sql.gz", std::process::id())); + let raw = fs::read(&plain).unwrap(); + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(&raw).unwrap(); + let compressed = enc.finish().unwrap(); + fs::write(&gz_path, &compressed).unwrap(); + + let mut cleanup = CompressionCleanup::default(); + let resolved = resolve_compressed_wrappers(&gz_path, &mut cleanup).unwrap(); + let text = fs::read_to_string(&resolved.path).unwrap(); + assert_eq!(text, "SELECT 1;\n"); + + drop(cleanup); + let _ = fs::remove_file(&plain); + let _ = fs::remove_file(&gz_path); + } + + #[test] + fn zip_single_sql_extracts() { + let dir = std::env::temp_dir(); + let zip_path = dir.join(format!("dumpling_zipt_{}.zip", std::process::id())); + let file = File::create(&zip_path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let opts = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + zip.start_file("dump.sql", opts).unwrap(); + zip.write_all(b"INSERT INTO t VALUES (1);\n").unwrap(); + zip.finish().unwrap(); + + let mut cleanup = CompressionCleanup::default(); + let resolved = resolve_compressed_wrappers(&zip_path, &mut cleanup).unwrap(); + let text = fs::read_to_string(&resolved.path).unwrap(); + assert!(text.contains("INSERT INTO")); + + drop(cleanup); + let _ = fs::remove_file(&zip_path); + } +} diff --git a/src/dump_input_detect.rs b/src/dump_input_detect.rs new file mode 100644 index 0000000..afc32a6 --- /dev/null +++ b/src/dump_input_detect.rs @@ -0,0 +1,166 @@ +//! Heuristic detection of non-plain-SQL dump inputs (archives, native backups). + +use std::fs::File; +use std::io::{self, Read}; +use std::path::Path; + +/// `pg_dump -Fc` custom format files begin with this magic (see PostgreSQL `pg_backup_archiver`). +pub(crate) const PG_CUSTOM_FORMAT_MAGIC: &[u8] = b"PGDMP"; + +pub(crate) fn read_file_prefix(path: &Path, max: usize) -> io::Result> { + let mut f = File::open(path)?; + let mut buf = vec![0u8; max]; + let n = f.read(&mut buf)?; + buf.truncate(n); + Ok(buf) +} + +pub(crate) fn is_pg_custom_format_file(path: &Path) -> io::Result { + let buf = read_file_prefix(path, PG_CUSTOM_FORMAT_MAGIC.len())?; + Ok(buf.starts_with(PG_CUSTOM_FORMAT_MAGIC)) +} + +/// Directory-format dumps include a `toc.dat` table-of-contents file at the root. +pub(crate) fn is_pg_directory_format_dump(path: &Path) -> bool { + path.is_dir() && path.join("toc.dat").is_file() +} + +/// Custom-format file (`PGDMP` prefix) or directory dump (`toc.dat`). +pub(crate) fn postgres_input_needs_pg_restore(path: &Path) -> io::Result { + if is_pg_directory_format_dump(path) { + return Ok(true); + } + if path.is_file() { + return is_pg_custom_format_file(path); + } + Ok(false) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MssqlFileKind { + PlainSqlText, + PostgresCustomArchiveWrongDialect, + ZipArchive, + Utf16EncodedSql, + LikelyBinaryBackup, +} + +pub(crate) fn classify_mssql_dump_file(path: &Path) -> io::Result { + const SAMPLE: usize = 4096; + let buf = read_file_prefix(path, SAMPLE)?; + if buf.starts_with(PG_CUSTOM_FORMAT_MAGIC) { + return Ok(MssqlFileKind::PostgresCustomArchiveWrongDialect); + } + // BACPAC / DACPAC / generic zip package + if buf.len() >= 4 && &buf[0..4] == b"PK\x03\x04" { + return Ok(MssqlFileKind::ZipArchive); + } + // UTF-16 BOM — Dumpling expects UTF-8 plain SQL. + if buf.starts_with(&[0xFF, 0xFE]) || buf.starts_with(&[0xFE, 0xFF]) { + return Ok(MssqlFileKind::Utf16EncodedSql); + } + if buf.is_empty() { + return Ok(MssqlFileKind::PlainSqlText); + } + let mut nul = 0usize; + let mut control = 0usize; + for &b in &buf { + if b == 0 { + nul += 1; + } + if b < 0x09 || (b > 0x0d && b < 0x20) { + control += 1; + } + } + let sample = buf.len(); + if nul > 0 + || (sample >= 64 && (nul * 100 / sample) >= 1) + || (sample >= 64 && (control * 100 / sample) > 15) + { + return Ok(MssqlFileKind::LikelyBinaryBackup); + } + Ok(MssqlFileKind::PlainSqlText) +} + +pub(crate) const MSSQL_BACPAC_HINT: &str = "This looks like a ZIP package (for example a BACPAC/DACPAC). \ +Dumpling only processes plain UTF-8 SQL text. Export scripts from SSMS, use sqlpackage to produce SQL where applicable, \ +or use a tool that emits `.sql` text before running dumpling."; + +pub(crate) const MSSQL_BINARY_HINT: &str = "This file does not look like UTF-8 plain SQL (many binary or NUL bytes). \ +If this is a native SQL Server backup (`.bak`), a detached database (`.mdf`), or another binary format, Dumpling cannot read it. \ +Restore or script the database to plain SQL (for example via SSMS Generate Scripts, sqlpackage, or mssql-scripter), then pass that `.sql` file."; + +pub(crate) const MSSQL_UTF16_HINT: &str = + "The file begins with a UTF-16 byte order mark. Dumpling expects UTF-8 plain SQL; \ +re-export or convert the script to UTF-8 without BOM."; + +pub(crate) const MSSQL_WRONG_POSTGRES_ARCHIVE: &str = "This file is a PostgreSQL custom-format archive (`pg_dump -Fc`), not SQL Server text. \ +Use `--format postgres` (or omit `--format` — postgres is the default), optionally with `--dump-decode` if you prefer to be explicit."; + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn pg_custom_magic_detected() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "dumpling_pg_magic_{}_{}.dump", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let mut f = File::create(&path).unwrap(); + f.write_all(PG_CUSTOM_FORMAT_MAGIC).unwrap(); + f.write_all(&[1, 2, 3]).unwrap(); + drop(f); + assert!(is_pg_custom_format_file(&path).unwrap()); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn mssql_classifies_zip_prefix() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "dumpling_zip_{}_{}.bacpac", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let mut f = File::create(&path).unwrap(); + f.write_all(b"PK\x03\x04").unwrap(); + f.write_all(b"fake zip content").unwrap(); + drop(f); + assert_eq!( + classify_mssql_dump_file(&path).unwrap(), + MssqlFileKind::ZipArchive + ); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn mssql_classifies_pg_archive_under_mssql() { + let dir = std::env::temp_dir(); + let path = dir.join(format!( + "dumpling_wrong_{}_{}.dump", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let mut f = File::create(&path).unwrap(); + f.write_all(PG_CUSTOM_FORMAT_MAGIC).unwrap(); + drop(f); + assert_eq!( + classify_mssql_dump_file(&path).unwrap(), + MssqlFileKind::PostgresCustomArchiveWrongDialect + ); + let _ = std::fs::remove_file(&path); + } +} diff --git a/src/main.rs b/src/main.rs index d82c7af..abd1a59 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,8 @@ use std::time::Instant; use clap::{ArgAction, Parser, Subcommand}; +mod compressed_input; +mod dump_input_detect; mod faker_dispatch; mod filter; mod lint; @@ -28,6 +30,11 @@ const OUTPUT_PIPE_CHUNK: usize = IO_BUF_CAPACITY; /// Number of full chunks allowed in flight (transform can run ahead of disk this far). const OUTPUT_PIPE_DEPTH: usize = 8; +use compressed_input::{resolve_compressed_wrappers, CompressionCleanup}; +use dump_input_detect::{ + classify_mssql_dump_file, postgres_input_needs_pg_restore, MssqlFileKind, MSSQL_BACPAC_HINT, + MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, MSSQL_WRONG_POSTGRES_ARCHIVE, +}; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; use seal::{ @@ -119,13 +126,14 @@ struct Cli { security_profile: String, /// Decode PostgreSQL custom-format or directory-format dumps via `pg_restore -f -` before anonymizing. - /// Requires `--input` pointing at the archive file or directory and `--format postgres`. Requires a - /// PostgreSQL client install (`pg_restore` on PATH unless overridden by `--pg-restore-path`). + /// When omitted, Dumpling auto-detects these archives for `--format postgres` (same behavior). + /// Requires `--input` pointing at the archive file or directory. Requires PostgreSQL client tools + /// (`pg_restore` on PATH unless overridden by `--pg-restore-path`). #[arg(long = "dump-decode", action = ArgAction::SetTrue)] dump_decode: bool, - /// Keep the input archive after `--dump-decode` (default: delete file or directory after a fully - /// successful run). Cannot retain the archive with `--check` (would delete before verifying changes). + /// Keep the input archive after decoding a PostgreSQL archive (default: delete file or directory after a fully + /// successful run). Applies to `--dump-decode` and auto-detected archives. Cannot retain the archive with `--check`. #[arg(long = "dump-decode-keep-input", action = ArgAction::SetTrue)] dump_decode_keep_input: bool, @@ -182,7 +190,7 @@ enum Commands { #[arg(long = "allow-ext")] allow_ext: Vec, - /// Decode a PostgreSQL custom- or directory-format dump via `pg_restore -f -` before scanning. Requires `--input` and `--format postgres`. + /// Decode a PostgreSQL custom- or directory-format dump via `pg_restore -f -` before scanning. When omitted, custom/directory archives are still auto-detected for `--format postgres`. Requires `--input` and `--format postgres`. #[arg(long = "dump-decode", action = ArgAction::SetTrue)] dump_decode: bool, @@ -398,6 +406,7 @@ impl Write for AnonWriter { } fn run_anonymize(cli: Cli) -> anyhow::Result<()> { + let mut compression_cleanup = CompressionCleanup::default(); if cli.in_place && cli.output.is_some() { anyhow::bail!("--in-place cannot be used together with --output"); } @@ -406,7 +415,7 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { } if cli.dump_decode && !cli.dump_decode_keep_input && cli.check { anyhow::bail!( - "--dump-decode removes the input archive on success by default; use --dump-decode-keep-input with --check" + "decoding a PostgreSQL archive removes the input archive on success by default; use --dump-decode-keep-input with --check" ); } if cli.dump_decode && cli.in_place { @@ -483,80 +492,164 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { let seal_runtime = SealRuntimeParams::new(dump_format, prng_seed_override_for_fingerprint()); - // Determine IO (optional pg_restore child when --dump-decode) + // Determine IO (optional pg_restore child for PostgreSQL custom/directory archives) let mut pg_restore_child: Option = None; - let (mut reader, input_path_for_inplace): (Box, Option) = if cli - .dump_decode + let mut path_to_remove_pg_archive: Option = None; + let (mut reader, input_path_for_inplace): (Box, Option) = match &cli.input { - let archive_path = cli - .input - .as_ref() - .ok_or_else(|| { - anyhow::anyhow!( - "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" - ) - })?; - if !cli.allow_ext.is_empty() && !has_allowed_extension(archive_path, &cli.allow_ext) { - let actual = archive_path - .extension() - .and_then(|s| s.to_str()) - .unwrap_or("") - .to_string(); - anyhow::bail!( - "input file extension '{}' is not in allowed set {:?}", - actual, - cli.allow_ext - ); - } - if !archive_path.exists() { - anyhow::bail!( - "--dump-decode input path does not exist: {}", - archive_path.display() - ); + None => { + if cli.dump_decode { + anyhow::bail!( + "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" + ); + } + if !cli.allow_ext.is_empty() { + eprintln!("dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin"); + } + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, io::stdin())), + None, + ) } - eprintln!( - "dumpling: decoding PostgreSQL archive via {} -f - {}", - cli.pg_restore_path.display(), - archive_path.display() - ); - let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( - &cli.pg_restore_path, - &cli.dump_decode_arg, - archive_path, - )?; - pg_restore_child = Some(pg); - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), - Some(archive_path.clone()), - ) - } else { - match &cli.input { - Some(path) => { - if !cli.allow_ext.is_empty() && !has_allowed_extension(path, &cli.allow_ext) { - let actual = path - .extension() - .and_then(|s| s.to_str()) - .unwrap_or("") - .to_string(); + Some(path) => { + if !cli.allow_ext.is_empty() && !has_allowed_extension(path, &cli.allow_ext) { + let actual = path + .extension() + .and_then(|s| s.to_str()) + .unwrap_or("") + .to_string(); + anyhow::bail!( + "input file extension '{}' is not in allowed set {:?}", + actual, + cli.allow_ext + ); + } + if !path.exists() { + anyhow::bail!("input path does not exist: {}", path.display()); + } + + let (inner_path, had_compression_wrapper) = if path.is_dir() { + (path.clone(), false) + } else { + let r = resolve_compressed_wrappers(path, &mut compression_cleanup)?; + (r.path, r.had_compression_wrapper) + }; + + if inner_path.is_dir() && dump_format != DumpFormat::Postgres { + anyhow::bail!( + "input `{}` is a directory; Dumpling expects a single plain-SQL file for this `--format`. \ + For PostgreSQL directory-format dumps (folder containing `toc.dat`), use `--format postgres`.", + inner_path.display() + ); + } + + if dump_format == DumpFormat::Sqlite && postgres_input_needs_pg_restore(&inner_path)? { + anyhow::bail!( + "input `{}` looks like a PostgreSQL custom-format or directory-format archive, not a SQLite `.dump`. \ + Use `--format postgres` (default) so Dumpling can decode it with pg_restore.", + inner_path.display() + ); + } + + if dump_format == DumpFormat::MsSql && inner_path.is_file() { + match classify_mssql_dump_file(&inner_path) { + Ok(MssqlFileKind::PlainSqlText) => {} + Ok(MssqlFileKind::PostgresCustomArchiveWrongDialect) => { + anyhow::bail!( + "input `{}` is not plain SQL Server text.\n\n{}", + inner_path.display(), + MSSQL_WRONG_POSTGRES_ARCHIVE + ); + } + Ok(MssqlFileKind::ZipArchive) => { + anyhow::bail!( + "input `{}` is not plain UTF-8 SQL text.\n\n{}", + inner_path.display(), + MSSQL_BACPAC_HINT + ); + } + Ok(MssqlFileKind::Utf16EncodedSql) => { + anyhow::bail!( + "input `{}` is not UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_UTF16_HINT + ); + } + Ok(MssqlFileKind::LikelyBinaryBackup) => { + anyhow::bail!( + "input `{}` does not look like UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_BINARY_HINT + ); + } + Err(e) => { + anyhow::bail!("read `{}`: {}", inner_path.display(), e); + } + } + } + + if had_compression_wrapper && cli.in_place { + anyhow::bail!( + "this input was gzip- and/or ZIP-wrapped; Dumpling wrote a temporary decompressed file. \ + --in-place cannot safely replace the original path with anonymized SQL; use --output (or stdout) instead" + ); + } + + let auto_pg_restore = dump_format == DumpFormat::Postgres + && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { + anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) + })?; + + if auto_pg_restore && cli.in_place { + anyhow::bail!( + "this input is a PostgreSQL custom-format or directory-format archive (decoded via pg_restore). \ + --in-place cannot replace the archive with plain SQL while atomically preserving the path; \ + write to --output (or stdout) instead, or decode manually with pg_restore -f -" + ); + } + + let use_pg_restore = cli.dump_decode || auto_pg_restore; + + if use_pg_restore { + if dump_format != DumpFormat::Postgres { anyhow::bail!( - "input file extension '{}' is not in allowed set {:?}", - actual, - cli.allow_ext + "PostgreSQL archive decoding only applies when --format postgres (default)" ); } - let f = File::open(path)?; + if auto_pg_restore { + eprintln!( + "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {}", + cli.pg_restore_path.display(), + inner_path.display() + ); + } else { + eprintln!( + "dumpling: decoding PostgreSQL archive via {} -f - {}", + cli.pg_restore_path.display(), + inner_path.display() + ); + } + let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( + &cli.pg_restore_path, + &cli.dump_decode_arg, + &inner_path, + )?; + pg_restore_child = Some(pg); + if !cli.dump_decode_keep_input { + let tmp_root = std::env::temp_dir(); + if !inner_path.starts_with(&tmp_root) { + path_to_remove_pg_archive = Some(inner_path.clone()); + } + } ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), Some(path.clone()), ) - } - None => { - if !cli.allow_ext.is_empty() { - eprintln!("dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin"); - } + } else { + let f = File::open(&inner_path)?; ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, io::stdin())), - None, + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), + Some(path.clone()), ) } } @@ -752,9 +845,11 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { } if strict_coverage_failed { + drop(compression_cleanup); std::process::exit(2); } if scan_failed { + drop(compression_cleanup); std::process::exit(3); } @@ -762,19 +857,18 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { if cli.check && (reporter.report.total_cells_changed > 0 || reporter.report.total_rows_dropped > 0) { + drop(compression_cleanup); std::process::exit(1); } - if cli.dump_decode && !cli.dump_decode_keep_input { - if let Some(ref p) = input_path_for_inplace { - match remove_pg_archive(p) { - Ok(()) => eprintln!("dumpling: removed input archive {}", p.display()), - Err(e) => eprintln!( - "dumpling: warning: could not remove input archive {}: {}", - p.display(), - e - ), - } + if let Some(ref p) = path_to_remove_pg_archive { + match remove_pg_archive(p) { + Ok(()) => eprintln!("dumpling: removed input archive {}", p.display()), + Err(e) => eprintln!( + "dumpling: warning: could not remove input archive {}: {}", + p.display(), + e + ), } } diff --git a/src/scaffold.rs b/src/scaffold.rs index fdff7df..4a1b5a8 100644 --- a/src/scaffold.rs +++ b/src/scaffold.rs @@ -1,5 +1,10 @@ //! Draft `[rules]` generation from column names in a SQL dump (starter / beta). +use crate::compressed_input::{resolve_compressed_wrappers, CompressionCleanup}; +use crate::dump_input_detect::{ + classify_mssql_dump_file, postgres_input_needs_pg_restore, MssqlFileKind, MSSQL_BACPAC_HINT, + MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, MSSQL_WRONG_POSTGRES_ARCHIVE, +}; use crate::pg_restore_decode; use crate::settings::{validate_raw_config, RawConfig}; use crate::sql::{ @@ -30,6 +35,7 @@ pub struct ScaffoldConfigOptions<'a> { /// Emit header comments, stderr notice, and TOML body for a starter config. pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<()> { + let mut compression_cleanup = CompressionCleanup::default(); let ScaffoldConfigOptions { input, output, @@ -53,58 +59,144 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() } ); - let mut pg_restore_child: Option<(pg_restore_decode::PgRestoreDecodeProcess, PathBuf)> = None; - let (mut reader, _input_path): (Box, Option) = if dump_decode { - let archive_path = input.ok_or_else(|| { - anyhow::anyhow!( - "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" + let mut pg_restore_child: Option = None; + let mut path_to_remove_pg_archive: Option = None; + let (mut reader, _input_path): (Box, Option) = match input { + None => { + if dump_decode { + anyhow::bail!( + "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" + ); + } + if !allow_ext.is_empty() { + eprintln!( + "dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin" + ); + } + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, std::io::stdin())), + None, ) - })?; - if !allow_ext.is_empty() && !crate::has_allowed_extension(archive_path, allow_ext) { - anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); } - if !archive_path.exists() { - anyhow::bail!( - "--dump-decode input path does not exist: {}", - archive_path.display() - ); - } - eprintln!( - "dumpling: decoding PostgreSQL archive via {} -f - {}", - pg_restore_path.display(), - archive_path.display() - ); - let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( - pg_restore_path, - dump_decode_arg, - archive_path, - )?; - pg_restore_child = Some((pg, archive_path.clone())); - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), - Some(archive_path.clone()), - ) - } else { - match input { - Some(path) => { - if !allow_ext.is_empty() && !crate::has_allowed_extension(path, allow_ext) { - anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); + Some(path) => { + if !allow_ext.is_empty() && !crate::has_allowed_extension(path, allow_ext) { + anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); + } + if !path.exists() { + anyhow::bail!("input path does not exist: {}", path.display()); + } + + let (inner_path, _had_wrap) = if path.is_dir() { + (path.clone(), false) + } else { + let r = resolve_compressed_wrappers(path, &mut compression_cleanup)?; + (r.path, r.had_compression_wrapper) + }; + + if inner_path.is_dir() && dump_format != crate::sql::DumpFormat::Postgres { + anyhow::bail!( + "input `{}` is a directory; this dialect expects a single SQL file. \ + For PostgreSQL directory-format dumps (folder containing `toc.dat`), use `--format postgres`.", + inner_path.display() + ); + } + + if dump_format == crate::sql::DumpFormat::Sqlite + && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { + anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) + })? + { + anyhow::bail!( + "input `{}` looks like a PostgreSQL custom-format or directory-format archive, not a SQLite `.dump`. \ + Use `--format postgres` (default) so Dumpling can decode it with pg_restore.", + inner_path.display() + ); + } + + if dump_format == crate::sql::DumpFormat::MsSql && inner_path.is_file() { + match classify_mssql_dump_file(&inner_path) { + Ok(MssqlFileKind::PlainSqlText) => {} + Ok(MssqlFileKind::PostgresCustomArchiveWrongDialect) => { + anyhow::bail!( + "input `{}` is not plain SQL Server text.\n\n{}", + inner_path.display(), + MSSQL_WRONG_POSTGRES_ARCHIVE + ); + } + Ok(MssqlFileKind::ZipArchive) => { + anyhow::bail!( + "input `{}` is not plain UTF-8 SQL text.\n\n{}", + inner_path.display(), + MSSQL_BACPAC_HINT + ); + } + Ok(MssqlFileKind::Utf16EncodedSql) => { + anyhow::bail!( + "input `{}` is not UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_UTF16_HINT + ); + } + Ok(MssqlFileKind::LikelyBinaryBackup) => { + anyhow::bail!( + "input `{}` does not look like UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_BINARY_HINT + ); + } + Err(e) => { + anyhow::bail!("read `{}`: {}", inner_path.display(), e); + } } - let f = File::open(path).with_context(|| format!("open {}", path.display()))?; - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), - Some(path.clone()), - ) } - None => { - if !allow_ext.is_empty() { + + let auto_pg_restore = dump_format == crate::sql::DumpFormat::Postgres + && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { + anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) + })?; + let use_pg_restore = dump_decode || auto_pg_restore; + + if use_pg_restore { + if dump_format != crate::sql::DumpFormat::Postgres { + anyhow::bail!( + "PostgreSQL archive decoding only applies when --format postgres (default)" + ); + } + if auto_pg_restore { eprintln!( - "dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin" + "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {}", + pg_restore_path.display(), + inner_path.display() + ); + } else { + eprintln!( + "dumpling: decoding PostgreSQL archive via {} -f - {}", + pg_restore_path.display(), + inner_path.display() ); } + let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( + pg_restore_path, + dump_decode_arg, + &inner_path, + )?; + pg_restore_child = Some(pg); + if !dump_decode_keep_input { + let tmp_root = std::env::temp_dir(); + if !inner_path.starts_with(&tmp_root) { + path_to_remove_pg_archive = Some(inner_path.clone()); + } + } ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, std::io::stdin())), - None, + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), + Some(inner_path.clone()), + ) + } else { + let f = File::open(&inner_path) + .with_context(|| format!("open {}", inner_path.display()))?; + ( + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), + Some(inner_path.clone()), ) } } @@ -159,26 +251,30 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() })(); let pipeline_ok = discovery_res.is_ok(); - if let Some((pg_child, archive_path)) = pg_restore_child { + if let Some(pg_child) = pg_restore_child { pg_child.finish(pipeline_ok).with_context(|| { format!( "`{}` failed while decoding the PostgreSQL archive", pg_restore_path.display() ) })?; - if pipeline_ok && dump_decode && !dump_decode_keep_input { - match crate::remove_pg_archive(&archive_path) { - Ok(()) => eprintln!("dumpling: removed input archive {}", archive_path.display()), + } + if pipeline_ok { + if let Some(ref p) = path_to_remove_pg_archive { + match crate::remove_pg_archive(p) { + Ok(()) => eprintln!("dumpling: removed input archive {}", p.display()), Err(e) => eprintln!( "dumpling: warning: could not remove input archive {}: {}", - archive_path.display(), + p.display(), e ), } } } - discovery_res + let out = discovery_res; + drop(compression_cleanup); + out } const SCAFFOLD_HEADER: &str = r#"# Dumpling starter config (beta) — generated by `dumpling scaffold-config`. From b4a551e7760058b3cfd4a9f1a7372881ecb0d9a3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 05:57:06 +0000 Subject: [PATCH 3/9] Reject --keep-original with --in-place and fix scaffold/tests --keep-original combined with --in-place now fails fast with a clear error, matching gzip/ZIP and pg_restore archive restrictions. Updated scaffold path, ResolvedConfig test helpers, and changelog for unified keep-original/pg_restore settings. Co-authored-by: Andy Babic --- CHANGELOG.md | 4 ++ src/dump_input_detect.rs | 2 +- src/filter.rs | 12 ++++ src/lint.rs | 2 + src/main.rs | 143 +++++++++++++++++++-------------------- src/pg_restore_decode.rs | 8 +-- src/scaffold.rs | 43 ++++++------ src/seal.rs | 15 +++- src/settings.rs | 60 +++++++++++++++- src/sql.rs | 52 ++++++++++++++ 10 files changed, 235 insertions(+), 106 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08fbdaf..0e5af7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht - **Gzip and ZIP inputs**: a file `--input` that is gzip-compressed and/or a ZIP archive containing a single dump file (or a single `.sql` when multiple files are present) is decompressed to a temporary file before processing, then temp files are removed. For PostgreSQL, a `.dump.gz` that wraps a `PGDMP` custom-format file is decompressed so `pg_restore` can read it. **`--in-place` is not allowed** when a gzip or ZIP wrapper was used (use `--output` or stdout). Full multi-file ZIP packages (for example BACPAC) are still not supported as SQL input. +### Changed + +- **CLI**: removed **`--dump-decode`**. PostgreSQL **custom-format** (`PGDMP`) and **directory-format** (`toc.dat`) inputs are auto-detected when `--format postgres` (default) and decoded with `pg_restore -f -`. Options renamed: **`--pg-restore-arg`** (repeatable, was `--dump-decode-arg`), **`--keep-original`** (was `--dump-decode-keep-input` / `--pg-restore-keep-input`). **`--keep-original` is incompatible with `--in-place`** (use `--output` or stdout). Optional `[pg_restore]` table in config for default path/args. + ## [0.7.0-alpha] - 2026-05-04 Pre-release toward **0.7.0** (stable **0.7.0** is not published yet; crates use the **0.7.0-alpha** prerelease identifier until then). diff --git a/src/dump_input_detect.rs b/src/dump_input_detect.rs index afc32a6..8c6538a 100644 --- a/src/dump_input_detect.rs +++ b/src/dump_input_detect.rs @@ -95,7 +95,7 @@ pub(crate) const MSSQL_UTF16_HINT: &str = re-export or convert the script to UTF-8 without BOM."; pub(crate) const MSSQL_WRONG_POSTGRES_ARCHIVE: &str = "This file is a PostgreSQL custom-format archive (`pg_dump -Fc`), not SQL Server text. \ -Use `--format postgres` (or omit `--format` — postgres is the default), optionally with `--dump-decode` if you prefer to be explicit."; +Use `--format postgres` (or omit `--format` — postgres is the default); Dumpling auto-detects custom-format (`PGDMP`) and directory dumps (`toc.dat`)."; #[cfg(test)] mod tests { diff --git a/src/filter.rs b/src/filter.rs index 56e9df2..679ec2f 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -595,6 +595,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let cols = vec!["id".to_string(), "email".to_string(), "country".to_string()]; @@ -649,6 +651,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let cols = vec!["payload".to_string()]; @@ -693,6 +697,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let cols = vec!["payload".to_string()]; @@ -743,6 +749,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let registry = AnonymizerRegistry::from_config(&cfg); @@ -778,6 +786,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let registry = AnonymizerRegistry::from_config(&cfg); @@ -863,6 +873,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let registry = AnonymizerRegistry::from_config(&cfg); diff --git a/src/lint.rs b/src/lint.rs index 8ec87db..adaca2b 100644 --- a/src/lint.rs +++ b/src/lint.rs @@ -336,6 +336,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, } } diff --git a/src/main.rs b/src/main.rs index abd1a59..12306cb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,7 +41,7 @@ use seal::{ compute_seal_digest, format_seal_line, read_first_line_for_seal, FirstLineReplayBufRead, SealFirstLine, SealRuntimeParams, }; -use settings::ResolvedConfig; +use settings::{merge_keep_original, ResolvedConfig}; use sql::{DumpFormat, SqlStreamProcessor}; use transform::{ prng_seed_override_for_fingerprint, set_hardened_profile, set_random_seed, AnonymizerRegistry, @@ -125,26 +125,19 @@ struct Cli { #[arg(long = "security-profile", default_value = "standard")] security_profile: String, - /// Decode PostgreSQL custom-format or directory-format dumps via `pg_restore -f -` before anonymizing. - /// When omitted, Dumpling auto-detects these archives for `--format postgres` (same behavior). - /// Requires `--input` pointing at the archive file or directory. Requires PostgreSQL client tools - /// (`pg_restore` on PATH unless overridden by `--pg-restore-path`). - #[arg(long = "dump-decode", action = ArgAction::SetTrue)] - dump_decode: bool, + /// Keep original PostgreSQL archive inputs after decode (ignored with `--check`; incompatible with `--in-place`). + /// Also set `keep_original = true` in `.dumplingconf`. + #[arg(long = "keep-original", action = ArgAction::SetTrue)] + keep_original: bool, - /// Keep the input archive after decoding a PostgreSQL archive (default: delete file or directory after a fully - /// successful run). Applies to `--dump-decode` and auto-detected archives. Cannot retain the archive with `--check`. - #[arg(long = "dump-decode-keep-input", action = ArgAction::SetTrue)] - dump_decode_keep_input: bool, - - /// `pg_restore` executable to use with `--dump-decode` (default: `pg_restore` on PATH). - #[arg(long = "pg-restore-path", default_value = "pg_restore")] - pg_restore_path: PathBuf, + /// `pg_restore` executable (optional; default: `pg_restore` on PATH or `[pg_restore] path` in config). + #[arg(long = "pg-restore-path")] + pg_restore_path: Option, /// Extra arguments forwarded to `pg_restore` before the archive path (repeatable). Example: - /// `--dump-decode-arg=--no-owner` `--dump-decode-arg=--no-acl` - #[arg(long = "dump-decode-arg")] - dump_decode_arg: Vec, + /// `--pg-restore-arg=--no-owner` `--pg-restore-arg=--no-acl` + #[arg(long = "pg-restore-arg")] + pg_restore_arg: Vec, #[command(subcommand)] command: Option, @@ -190,21 +183,17 @@ enum Commands { #[arg(long = "allow-ext")] allow_ext: Vec, - /// Decode a PostgreSQL custom- or directory-format dump via `pg_restore -f -` before scanning. When omitted, custom/directory archives are still auto-detected for `--format postgres`. Requires `--input` and `--format postgres`. - #[arg(long = "dump-decode", action = ArgAction::SetTrue)] - dump_decode: bool, - - /// Keep the input archive after `--dump-decode` (default: delete on success). - #[arg(long = "dump-decode-keep-input", action = ArgAction::SetTrue)] - dump_decode_keep_input: bool, + /// Keep original inputs (see main `dumpling --keep-original`). + #[arg(long = "keep-original", action = ArgAction::SetTrue)] + keep_original: bool, - /// `pg_restore` for `--dump-decode` (default: `pg_restore` on PATH). - #[arg(long = "pg-restore-path", default_value = "pg_restore")] - pg_restore_path: PathBuf, + /// `pg_restore` for custom- or directory-format archives (optional; see main `dumpling` help). + #[arg(long = "pg-restore-path")] + pg_restore_path: Option, /// Extra args for `pg_restore` before the archive path (repeatable). - #[arg(long = "dump-decode-arg")] - dump_decode_arg: Vec, + #[arg(long = "pg-restore-arg")] + pg_restore_arg: Vec, /// Sample JSON path hints: keep 5 rows per table (reservoir) from INSERT/COPY and infer nested `column.path` rules. #[arg(long = "infer-json-paths", action = ArgAction::SetTrue)] @@ -227,10 +216,9 @@ fn main() -> anyhow::Result<()> { output, format, allow_ext, - dump_decode, - dump_decode_keep_input, + keep_original, pg_restore_path, - dump_decode_arg, + pg_restore_arg, infer_json_paths, max_json_depth, }) = &cli.command @@ -244,20 +232,21 @@ fn main() -> anyhow::Result<()> { other ), }; - if *dump_decode && dump_format != DumpFormat::Postgres { - anyhow::bail!( - "--dump-decode only applies to PostgreSQL dumps; use --format postgres (default)" - ); - } + let resolved_for_pg = settings::load_config(cli.config.as_ref(), false)?; + let (pg_restore_path_eff, pg_restore_arg_eff) = settings::merge_pg_restore_cli( + &resolved_for_pg.pg_restore, + pg_restore_path.clone(), + pg_restore_arg.as_slice(), + ); + let keep_original_eff = merge_keep_original(*keep_original, resolved_for_pg.keep_original); return scaffold::run_scaffold_config(scaffold::ScaffoldConfigOptions { - input: input.as_ref(), - output: output.as_ref(), + input: input.clone(), + output: output.clone(), dump_format, - allow_ext, - dump_decode: *dump_decode, - dump_decode_keep_input: *dump_decode_keep_input, - pg_restore_path, - dump_decode_arg, + allow_ext: allow_ext.clone(), + keep_original: keep_original_eff, + pg_restore_path: pg_restore_path_eff, + pg_restore_arg: pg_restore_arg_eff, infer_json_paths: *infer_json_paths, max_json_depth: *max_json_depth, }); @@ -413,14 +402,6 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { if cli.check && (cli.in_place || cli.output.is_some()) { anyhow::bail!("--check cannot be used together with --output or --in-place"); } - if cli.dump_decode && !cli.dump_decode_keep_input && cli.check { - anyhow::bail!( - "decoding a PostgreSQL archive removes the input archive on success by default; use --dump-decode-keep-input with --check" - ); - } - if cli.dump_decode && cli.in_place { - anyhow::bail!("--dump-decode cannot be used with --in-place"); - } // Resolve config from provided path or discover in CWD let resolved_config: ResolvedConfig = @@ -484,13 +465,29 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { other ), }; - if cli.dump_decode && dump_format != DumpFormat::Postgres { + + let seal_runtime = SealRuntimeParams::new(dump_format, prng_seed_override_for_fingerprint()); + + let (pg_restore_path_eff, pg_restore_arg_eff) = settings::merge_pg_restore_cli( + &resolved_config.pg_restore, + cli.pg_restore_path.clone(), + &cli.pg_restore_arg, + ); + let keep_original_eff = merge_keep_original(cli.keep_original, resolved_config.keep_original); + + if cli.in_place && keep_original_eff { anyhow::bail!( - "--dump-decode only applies to PostgreSQL dumps; use --format postgres (default)" + "`--keep-original` (or `keep_original = true` in config) cannot be used with `--in-place`; \ + keeping the original path while overwriting it in place is ambiguous. Use `--output` (or stdout) instead, \ + or omit `--keep-original` for in-place overwrite." ); } - let seal_runtime = SealRuntimeParams::new(dump_format, prng_seed_override_for_fingerprint()); + if !keep_original_eff && cli.check { + anyhow::bail!( + "PostgreSQL archive decoding removes the `--input` archive on success by default; use `--keep-original` or `keep_original = true` in config with --check" + ); + } // Determine IO (optional pg_restore child for PostgreSQL custom/directory archives) let mut pg_restore_child: Option = None; @@ -498,11 +495,6 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { let (mut reader, input_path_for_inplace): (Box, Option) = match &cli.input { None => { - if cli.dump_decode { - anyhow::bail!( - "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" - ); - } if !cli.allow_ext.is_empty() { eprintln!("dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin"); } @@ -608,7 +600,7 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { ); } - let use_pg_restore = cli.dump_decode || auto_pg_restore; + let use_pg_restore = auto_pg_restore; if use_pg_restore { if dump_format != DumpFormat::Postgres { @@ -619,23 +611,23 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { if auto_pg_restore { eprintln!( "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {}", - cli.pg_restore_path.display(), + pg_restore_path_eff.display(), inner_path.display() ); } else { eprintln!( "dumpling: decoding PostgreSQL archive via {} -f - {}", - cli.pg_restore_path.display(), + pg_restore_path_eff.display(), inner_path.display() ); } let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( - &cli.pg_restore_path, - &cli.dump_decode_arg, + &pg_restore_path_eff, + &pg_restore_arg_eff, &inner_path, )?; pg_restore_child = Some(pg); - if !cli.dump_decode_keep_input { + if !keep_original_eff { let tmp_root = std::env::temp_dir(); if !inner_path.starts_with(&tmp_root) { path_to_remove_pg_archive = Some(inner_path.clone()); @@ -1063,21 +1055,22 @@ email = { strategy = "email" } } #[test] - fn test_dump_decode_flags_parse() { + fn test_pg_restore_flags_parse() { let cli = Cli::parse_from([ "dumpling", - "--dump-decode", - "--dump-decode-keep-input", + "--keep-original", "--pg-restore-path", "/usr/bin/pg_restore", - "--dump-decode-arg=--no-owner", + "--pg-restore-arg=--no-owner", "-i", "/tmp/latest.dump", ]); - assert!(cli.dump_decode); - assert!(cli.dump_decode_keep_input); - assert_eq!(cli.pg_restore_path, PathBuf::from("/usr/bin/pg_restore")); - assert_eq!(cli.dump_decode_arg, vec!["--no-owner"]); + assert!(cli.keep_original); + assert_eq!( + cli.pg_restore_path, + Some(PathBuf::from("/usr/bin/pg_restore")) + ); + assert_eq!(cli.pg_restore_arg, vec!["--no-owner"]); } #[test] diff --git a/src/pg_restore_decode.rs b/src/pg_restore_decode.rs index 93ff207..22d55cb 100644 --- a/src/pg_restore_decode.rs +++ b/src/pg_restore_decode.rs @@ -1,4 +1,4 @@ -//! Helpers for `--dump-decode` (PostgreSQL custom/directory archives via `pg_restore`). +//! Helpers for PostgreSQL custom/directory archives decoded via `pg_restore`. use anyhow::Context; use std::io::Write; @@ -8,7 +8,7 @@ use std::thread::JoinHandle; /// Shown when `pg_restore` cannot be run or fails without stderr output. pub(crate) const PG_RESTORE_MISSING_HINT: &str = "\ -PostgreSQL client tools are required for --dump-decode. Install them (for example the \ +PostgreSQL client tools are required when Dumpling decodes a custom or directory-format archive. Install them (for example the \ `postgresql-client` package on Debian/Ubuntu, `postgresql` via Homebrew on macOS, or the \ official PostgreSQL installer on Windows), ensure `pg_restore` is on your PATH, or pass \ `--pg-restore-path` pointing at the `pg_restore` executable."; @@ -85,13 +85,13 @@ pub(crate) struct PgRestoreDecodeProcess { /// Returns stdout for piping into the anonymizer; wait with [`PgRestoreDecodeProcess::finish`]. pub(crate) fn spawn_pg_restore_decode( pg_restore_path: &Path, - dump_decode_arg: &[String], + pg_restore_arg: &[String], archive_path: &Path, ) -> anyhow::Result<(ChildStdout, PgRestoreDecodeProcess)> { ensure_pg_restore_available(pg_restore_path)?; let mut cmd = Command::new(pg_restore_path); - for a in dump_decode_arg { + for a in pg_restore_arg { cmd.arg(a); } cmd.arg("-f") diff --git a/src/scaffold.rs b/src/scaffold.rs index 4a1b5a8..28de861 100644 --- a/src/scaffold.rs +++ b/src/scaffold.rs @@ -20,31 +20,29 @@ use std::path::PathBuf; use crate::IO_BUF_CAPACITY; /// CLI-backed options for [`run_scaffold_config`]. -pub struct ScaffoldConfigOptions<'a> { - pub input: Option<&'a PathBuf>, - pub output: Option<&'a PathBuf>, +pub struct ScaffoldConfigOptions { + pub input: Option, + pub output: Option, pub dump_format: DumpFormat, - pub allow_ext: &'a [String], - pub dump_decode: bool, - pub dump_decode_keep_input: bool, - pub pg_restore_path: &'a PathBuf, - pub dump_decode_arg: &'a [String], + pub allow_ext: Vec, + pub keep_original: bool, + pub pg_restore_path: PathBuf, + pub pg_restore_arg: Vec, pub infer_json_paths: bool, pub max_json_depth: usize, } /// Emit header comments, stderr notice, and TOML body for a starter config. -pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<()> { +pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { let mut compression_cleanup = CompressionCleanup::default(); let ScaffoldConfigOptions { input, output, dump_format, allow_ext, - dump_decode, - dump_decode_keep_input, + keep_original, pg_restore_path, - dump_decode_arg, + pg_restore_arg, infer_json_paths, max_json_depth, } = opts; @@ -63,11 +61,6 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() let mut path_to_remove_pg_archive: Option = None; let (mut reader, _input_path): (Box, Option) = match input { None => { - if dump_decode { - anyhow::bail!( - "--dump-decode requires --input pointing at a pg_dump custom-format file or directory-format directory" - ); - } if !allow_ext.is_empty() { eprintln!( "dumpling: --allow-ext provided but no --input file; extension check is ignored for stdin" @@ -79,7 +72,7 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() ) } Some(path) => { - if !allow_ext.is_empty() && !crate::has_allowed_extension(path, allow_ext) { + if !allow_ext.is_empty() && !crate::has_allowed_extension(&path, &allow_ext) { anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); } if !path.exists() { @@ -89,7 +82,7 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() let (inner_path, _had_wrap) = if path.is_dir() { (path.clone(), false) } else { - let r = resolve_compressed_wrappers(path, &mut compression_cleanup)?; + let r = resolve_compressed_wrappers(&path, &mut compression_cleanup)?; (r.path, r.had_compression_wrapper) }; @@ -154,7 +147,7 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) })?; - let use_pg_restore = dump_decode || auto_pg_restore; + let use_pg_restore = auto_pg_restore; if use_pg_restore { if dump_format != crate::sql::DumpFormat::Postgres { @@ -176,12 +169,12 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() ); } let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( - pg_restore_path, - dump_decode_arg, + &pg_restore_path, + &pg_restore_arg, &inner_path, )?; pg_restore_child = Some(pg); - if !dump_decode_keep_input { + if !keep_original { let tmp_root = std::env::temp_dir(); if !inner_path.starts_with(&tmp_root) { path_to_remove_pg_archive = Some(inner_path.clone()); @@ -234,6 +227,8 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() table_options: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + keep_original: None, + pg_restore: crate::settings::PgRestoreRawConfig::default(), }; validate_raw_config(&raw).context("internal error: scaffold rules failed validation")?; @@ -241,7 +236,7 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions<'_>) -> anyhow::Result<() let mut text = SCAFFOLD_HEADER.to_string(); text.push_str(&body); - if let Some(path) = output { + if let Some(ref path) = output { std::fs::write(path, &text).with_context(|| format!("write {}", path.display()))?; eprintln!("dumpling scaffold-config: wrote {}", path.display()); } else { diff --git a/src/seal.rs b/src/seal.rs index 3af44a4..4e5b69c 100644 --- a/src/seal.rs +++ b/src/seal.rs @@ -2,7 +2,9 @@ //! fingerprint of the resolved policy, and runtime CLI options that affect transforms. When the //! first line matches, the remainder of the dump is copied through unchanged. -use crate::settings::{AnonymizerSpec, ColumnCase, RawConfig, ResolvedConfig, RowFilterSet}; +use crate::settings::{ + AnonymizerSpec, ColumnCase, PgRestoreRawConfig, RawConfig, ResolvedConfig, RowFilterSet, +}; use crate::sql::DumpFormat; use serde::Serialize; use serde_json::Value; @@ -134,6 +136,15 @@ fn resolved_to_raw_for_fingerprint(cfg: &ResolvedConfig) -> RawConfig { table_options: HashMap::new(), sensitive_columns, output_scan: cfg.output_scan.clone(), + keep_original: cfg.keep_original, + pg_restore: PgRestoreRawConfig { + path: cfg + .pg_restore + .path + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + args: cfg.pg_restore.args.clone(), + }, } } @@ -365,6 +376,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, } } diff --git a/src/settings.rs b/src/settings.rs index 6c02af6..9dcb3af 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -28,6 +28,50 @@ pub struct RawConfig { /// Post-transform output scanning config for residual sensitive patterns. #[serde(default)] pub output_scan: OutputScanConfig, + /// When true, keep original PostgreSQL archive inputs after decode. Incompatible with `--in-place` (use `--output` instead). CLI `--keep-original` also sets this. + #[serde(default)] + pub keep_original: Option, + /// Optional defaults for `pg_restore` when decoding PostgreSQL archives (CLI overrides when set). + #[serde(default)] + pub pg_restore: PgRestoreRawConfig, +} + +/// Raw `[pg_restore]` table (optional). +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct PgRestoreRawConfig { + /// Path to the `pg_restore` executable. + pub path: Option, + /// Extra arguments before the archive path (e.g. `"--no-owner"`). + #[serde(default)] + pub args: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct PgRestoreConfig { + pub path: Option, + pub args: Vec, +} + +/// Merge `[pg_restore]` path and args with CLI; CLI wins when provided. +pub fn merge_pg_restore_cli( + cfg: &PgRestoreConfig, + cli_path: Option, + cli_args: &[String], +) -> (PathBuf, Vec) { + let path = cli_path + .or_else(|| cfg.path.clone()) + .unwrap_or_else(|| PathBuf::from("pg_restore")); + let args = if !cli_args.is_empty() { + cli_args.to_vec() + } else { + cfg.args.clone() + }; + (path, args) +} + +/// Effective `--keep-original`: CLI or config, default false. +pub fn merge_keep_original(cli: bool, cfg: Option) -> bool { + cli || cfg.unwrap_or(false) } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -72,7 +116,7 @@ pub struct AnonymizerSpec { pub format: Option, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct ResolvedConfig { pub salt: Option, /// Normalized rule map: lowercase keys for table and column names @@ -85,6 +129,10 @@ pub struct ResolvedConfig { pub sensitive_columns: HashMap>, /// Resolved output scan config pub output_scan: OutputScanConfig, + /// Defaults for PostgreSQL archive decoding (`pg_restore`); merged with CLI in `merge_pg_restore_cli`. + pub pg_restore: PgRestoreConfig, + /// Whether to keep original archive inputs. Merged with CLI in `merge_keep_original` (incompatible with `--in-place`). + pub keep_original: Option, /// For debugging/trace pub source_path: Option, } @@ -460,6 +508,8 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { table_options: _, sensitive_columns, output_scan, + keep_original, + pg_restore: pg_raw, } = raw; let OutputScanConfig { enabled_categories, @@ -525,6 +575,10 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { .into_iter() .map(|value| value.to_ascii_lowercase()) .collect::>(); + let pg_restore = PgRestoreConfig { + path: pg_raw.path.map(PathBuf::from), + args: pg_raw.args, + }; ResolvedConfig { salt, rules: normalized_rules, @@ -540,6 +594,8 @@ fn resolve(raw: RawConfig, source_path: Option) -> ResolvedConfig { fail_on_severity: fail_on_severity.to_ascii_lowercase(), sample_limit_per_category, }, + pg_restore, + keep_original, source_path, } } @@ -1137,6 +1193,8 @@ fn empty_config(source_path: Option) -> ResolvedConfig { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: OutputScanConfig::default(), + pg_restore: PgRestoreConfig::default(), + keep_original: None, source_path, } } diff --git a/src/sql.rs b/src/sql.rs index 00522ae..9e38c4c 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -2164,6 +2164,8 @@ mod tests { column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2297,6 +2299,8 @@ COPY public.events (id, email, the_date) FROM stdin; column_cases, sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2357,6 +2361,8 @@ INSERT INTO public.users (id, email, country, is_admin) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2440,6 +2446,8 @@ INSERT INTO public.orders (id, customer_email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2505,6 +2513,8 @@ INSERT INTO public.users (id, email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2563,6 +2573,8 @@ INSERT INTO public.users (id, email) VALUES (1, 'alice@myco.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2649,6 +2661,8 @@ INSERT INTO public.users (id, email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2740,6 +2754,8 @@ INSERT INTO public.users (id, email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2812,6 +2828,8 @@ INSERT INTO public.orders (id, customer_email) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2863,6 +2881,8 @@ INSERT INTO public.users (id, email, first_name, password, dob, notes) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -2936,6 +2956,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3016,6 +3038,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3086,6 +3110,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3167,6 +3193,8 @@ COPY public.events (id, payload) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3255,6 +3283,8 @@ COPY public.payments (id, pan) FROM stdin; column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3365,6 +3395,8 @@ INSERT INTO public.events (id, payload) VALUES column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; set_random_seed(7); @@ -3452,6 +3484,8 @@ old@example.com verylongname (000) 000-0000 column_cases: HashMap::new(), sensitive_columns, output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3522,6 +3556,8 @@ CREATE TABLE public.users ( column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3604,6 +3640,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3640,6 +3678,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3693,6 +3733,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3753,6 +3795,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3809,6 +3853,8 @@ INSERT INTO users (id, email) VALUES (3, 'carol@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; set_random_seed(42); @@ -3853,6 +3899,8 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3902,6 +3950,8 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); @@ -3985,6 +4035,8 @@ INSERT INTO [dbo].[users] ([id], [email]) VALUES (1, N'alice@example.com'); column_cases: HashMap::new(), sensitive_columns: HashMap::new(), output_scan: crate::settings::OutputScanConfig::default(), + pg_restore: crate::settings::PgRestoreConfig::default(), + keep_original: None, source_path: None, }; let reg = AnonymizerRegistry::from_config(&cfg); From 4bd6965fcafb76e59c793cb6657661182d429f20 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 06:08:08 +0000 Subject: [PATCH 4/9] Stream gzip plain-SQL inputs; materialize only when required Gzip-wrapped plain SQL is decompressed with flate2 and fed into the pipeline without a temp file. PGDMP, nested gzip->ZIP, and ZIP extraction still use temp paths registered on CompressionCleanup. --in-place now rejects only inputs that required a materialized temp file from compression handling. Refactors MSSQL classification with classify_mssql_prefix for sniff buffers. Co-authored-by: Andy Babic --- CHANGELOG.md | 2 +- src/compressed_input.rs | 177 ++++++++++++++++++++++++++++++++------- src/dump_input_detect.rs | 19 +++-- src/main.rs | 81 +++++++++++++----- src/scaffold.rs | 53 ++++++++---- 5 files changed, 259 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5af7c..2970b97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht ### Added -- **Gzip and ZIP inputs**: a file `--input` that is gzip-compressed and/or a ZIP archive containing a single dump file (or a single `.sql` when multiple files are present) is decompressed to a temporary file before processing, then temp files are removed. For PostgreSQL, a `.dump.gz` that wraps a `PGDMP` custom-format file is decompressed so `pg_restore` can read it. **`--in-place` is not allowed** when a gzip or ZIP wrapper was used (use `--output` or stdout). Full multi-file ZIP packages (for example BACPAC) are still not supported as SQL input. +- **Gzip and ZIP inputs**: plain-SQL payloads inside **gzip** are decompressed **in-process** (streamed) when possible—no temporary file. Dumpling still materializes to the temp directory when required: **ZIP** archives (random-access central directory), **gzip wrapping `PGDMP`** or an inner **ZIP** (nested wrappers), or other cases where a filesystem path is needed for `pg_restore`. Temporary files are registered for removal when processing finishes. **`--in-place` is rejected** only when Dumpling had to write a **temporary** decompressed/extracted file (not when gzip plain-SQL streaming was used). Full multi-file ZIP packages (for example BACPAC) are still not supported as SQL input. ### Changed diff --git a/src/compressed_input.rs b/src/compressed_input.rs index 207d139..18c2527 100644 --- a/src/compressed_input.rs +++ b/src/compressed_input.rs @@ -1,11 +1,23 @@ //! Transparent gzip / ZIP resolution for `--input` file paths. +//! +//! When the payload is **plain SQL text**, gzip is decompressed **in-process** (streamed) so no +//! temporary file is created. When the inner payload must be **materialized** (PostgreSQL `PGDMP` +//! for `pg_restore`, nested ZIP after gzip, or any ZIP inner file — the `zip` crate needs random +//! access to the central directory), we write to the temp dir and register paths on +//! [`CompressionCleanup`] so they are always removed on drop. -use crate::dump_input_detect::read_file_prefix; +use crate::dump_input_detect::{read_file_prefix, PG_CUSTOM_FORMAT_MAGIC}; use anyhow::Context; use flate2::read::GzDecoder; use std::fs::{self, File}; +use std::io::{BufRead, BufReader, Cursor, Read}; use std::path::{Path, PathBuf}; +use crate::IO_BUF_CAPACITY; + +/// Prefix size for gzip/ZIP sniffing (matches MSSQL sample size in `dump_input_detect`). +const SNIFF_PREFIX_LEN: usize = 4096; + /// Paths and directories created while resolving compressed inputs; remove after a successful run. #[derive(Default)] pub(crate) struct CompressionCleanup { @@ -16,9 +28,9 @@ impl Drop for CompressionCleanup { fn drop(&mut self) { for p in self.paths.drain(..) { let _ = if p.is_dir() { - fs::remove_dir_all(&p) + fs::remove_dir_all(p) } else { - fs::remove_file(&p) + fs::remove_file(p) }; } } @@ -45,7 +57,7 @@ fn unique_temp_file(label: &str) -> PathBuf { )) } -/// Decompress gzip fully to a temp file (needed so `pg_restore` can read PGDMP or we can re-sniff). +/// Decompress gzip fully to a temp file (`pg_restore`, nested ZIP, or chained wrappers). fn decompress_gzip_file(src: &Path, cleanup: &mut CompressionCleanup) -> anyhow::Result { let out = unique_temp_file("ungz"); let mut inp = File::open(src).with_context(|| format!("open {}", src.display()))?; @@ -113,55 +125,112 @@ fn extract_zip_inner_file(src: &Path, cleanup: &mut CompressionCleanup) -> anyho Ok(out) } -pub(crate) struct ResolvedInput { - pub path: PathBuf, - /// True if the input was gzip and/or ZIP (temporary files may exist until `cleanup` drops). - pub had_compression_wrapper: bool, +/// Result of walking gzip/ZIP wrappers on a file path. +pub(crate) enum ResolvedInput { + /// A filesystem path to open (plain file, directory dump, or materialized inner file). + Path { + path: PathBuf, + had_compression_wrapper: bool, + /// True when Dumpling wrote this path under the temp directory (gunzip / unzip extract). + materialized_temp_file: bool, + }, + /// Gzip was applied and the inner payload is plain SQL: decompress in memory without a temp file. + PlainSqlStream { + reader: Box, + had_compression_wrapper: bool, + /// First bytes after gzip (used for `--format mssql` classification). + sniff_prefix: Vec, + }, } -/// Follow gzip and/or ZIP wrappers so downstream logic sees a concrete file or directory path. +const MAX_WRAP_DEPTH: u32 = 16; + +/// Follow gzip and/or ZIP wrappers so downstream logic can open a path or use a stream. /// -/// Any temporary paths are registered on `cleanup` and deleted when `cleanup` is dropped. +/// Temporary paths are registered on `cleanup` and deleted when `cleanup` is dropped. pub(crate) fn resolve_compressed_wrappers( original: &Path, cleanup: &mut CompressionCleanup, ) -> anyhow::Result { + resolve_compressed_wrappers_inner(original, cleanup, 0, false, false) +} + +fn resolve_compressed_wrappers_inner( + original: &Path, + cleanup: &mut CompressionCleanup, + depth: u32, + materialized_temp_file: bool, + compressed_so_far: bool, +) -> anyhow::Result { + if depth > MAX_WRAP_DEPTH { + anyhow::bail!( + "nested gzip/ZIP wrappers exceed maximum depth ({MAX_WRAP_DEPTH}); simplify the archive chain" + ); + } + if !original.is_file() { - return Ok(ResolvedInput { + return Ok(ResolvedInput::Path { path: original.to_path_buf(), - had_compression_wrapper: false, + had_compression_wrapper: compressed_so_far, + materialized_temp_file: false, }); } let mut current = original.to_path_buf(); - let mut had_wrap = false; + let prefix = read_file_prefix(¤t, 4) .map_err(|e| anyhow::anyhow!("read {}: {}", current.display(), e))?; if is_gzip_prefix(&prefix) { - had_wrap = true; eprintln!( - "dumpling: decompressing gzip input {} to a temporary file", - original.display() + "dumpling: decompressing gzip input {} in-process (streamed when inner is plain SQL)", + current.display() ); - current = decompress_gzip_file(¤t, cleanup)?; + let inp = File::open(¤t).with_context(|| format!("open {}", current.display()))?; + let mut dec = GzDecoder::new(inp); + let mut sniff = vec![0u8; SNIFF_PREFIX_LEN]; + let n = dec + .read(&mut sniff) + .with_context(|| format!("read gzip stream {}", current.display()))?; + sniff.truncate(n); + + if sniff.starts_with(PG_CUSTOM_FORMAT_MAGIC) { + // pg_restore needs a path; materialize full decompress. + let temp = decompress_gzip_file(¤t, cleanup)?; + return resolve_compressed_wrappers_inner(&temp, cleanup, depth + 1, true, true); + } + if is_zip_prefix(&sniff) { + // Inner ZIP: materialize gunzip result then recurse (ZIP uses central directory). + let temp = decompress_gzip_file(¤t, cleanup)?; + return resolve_compressed_wrappers_inner(&temp, cleanup, depth + 1, true, true); + } + + // Plain SQL (or other format we read as UTF-8 bytes): chain prefix + remainder, no temp file. + let sniff_prefix = sniff.clone(); + let chained: Box = Box::new(Cursor::new(sniff).chain(dec)); + let reader = Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, chained)); + return Ok(ResolvedInput::PlainSqlStream { + reader, + had_compression_wrapper: true, + sniff_prefix, + }); } let prefix = read_file_prefix(¤t, 4) .map_err(|e| anyhow::anyhow!("read {}: {}", current.display(), e))?; if is_zip_prefix(&prefix) { - had_wrap = true; eprintln!( - "dumpling: extracting inner file from ZIP {} (from {})", - current.display(), - original.display() + "dumpling: extracting inner file from ZIP {} (materialized for zip directory access)", + current.display() ); current = extract_zip_inner_file(¤t, cleanup)?; + return resolve_compressed_wrappers_inner(¤t, cleanup, depth + 1, true, true); } - Ok(ResolvedInput { + Ok(ResolvedInput::Path { path: current, - had_compression_wrapper: had_wrap, + had_compression_wrapper: compressed_so_far, + materialized_temp_file, }) } @@ -174,7 +243,7 @@ mod tests { use zip::write::SimpleFileOptions; #[test] - fn gzip_then_plain_roundtrip_path() { + fn gzip_plain_sql_streams_without_materializing_path_branch() { let dir = std::env::temp_dir(); let plain = dir.join(format!("dumpling_plain_{}.sql", std::process::id())); fs::write(&plain, b"SELECT 1;\n").unwrap(); @@ -188,14 +257,61 @@ mod tests { let mut cleanup = CompressionCleanup::default(); let resolved = resolve_compressed_wrappers(&gz_path, &mut cleanup).unwrap(); - let text = fs::read_to_string(&resolved.path).unwrap(); - assert_eq!(text, "SELECT 1;\n"); + match resolved { + ResolvedInput::PlainSqlStream { mut reader, .. } => { + let mut s = String::new(); + reader.read_to_string(&mut s).unwrap(); + assert_eq!(s, "SELECT 1;\n"); + } + ResolvedInput::Path { .. } => panic!("expected PlainSqlStream"), + } + assert!(cleanup.paths.is_empty()); drop(cleanup); let _ = fs::remove_file(&plain); let _ = fs::remove_file(&gz_path); } + fn gzip_bytes(data: &[u8]) -> Vec { + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(data).unwrap(); + enc.finish().unwrap() + } + + #[test] + fn gzip_nested_zip_materializes_then_cleans() { + let dir = std::env::temp_dir(); + let zip_path = dir.join(format!("dumpling_inner_{}.zip", std::process::id())); + let file = File::create(&zip_path).unwrap(); + let mut zip = zip::ZipWriter::new(file); + let opts = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + zip.start_file("dump.sql", opts).unwrap(); + zip.write_all(b"INSERT INTO t VALUES (1);\n").unwrap(); + zip.finish().unwrap(); + + let gz_path = dir.join(format!("dumpling_nested_{}.gz", std::process::id())); + let gz_inner = fs::read(&zip_path).unwrap(); + fs::write(&gz_path, gzip_bytes(&gz_inner)).unwrap(); + + let mut cleanup = CompressionCleanup::default(); + let resolved = resolve_compressed_wrappers(&gz_path, &mut cleanup).unwrap(); + match resolved { + ResolvedInput::Path { path, .. } => { + let text = fs::read_to_string(&path).unwrap(); + assert!(text.contains("INSERT INTO")); + } + ResolvedInput::PlainSqlStream { .. } => panic!("expected Path after nested zip"), + } + assert!( + !cleanup.paths.is_empty(), + "nested gzip->zip should register temp paths" + ); + + drop(cleanup); + let _ = fs::remove_file(&zip_path); + let _ = fs::remove_file(&gz_path); + } + #[test] fn zip_single_sql_extracts() { let dir = std::env::temp_dir(); @@ -209,8 +325,13 @@ mod tests { let mut cleanup = CompressionCleanup::default(); let resolved = resolve_compressed_wrappers(&zip_path, &mut cleanup).unwrap(); - let text = fs::read_to_string(&resolved.path).unwrap(); - assert!(text.contains("INSERT INTO")); + match resolved { + ResolvedInput::Path { path, .. } => { + let text = fs::read_to_string(&path).unwrap(); + assert!(text.contains("INSERT INTO")); + } + ResolvedInput::PlainSqlStream { .. } => panic!("expected Path for zip"), + } drop(cleanup); let _ = fs::remove_file(&zip_path); diff --git a/src/dump_input_detect.rs b/src/dump_input_detect.rs index 8c6538a..650ffc8 100644 --- a/src/dump_input_detect.rs +++ b/src/dump_input_detect.rs @@ -48,23 +48,28 @@ pub(crate) enum MssqlFileKind { pub(crate) fn classify_mssql_dump_file(path: &Path) -> io::Result { const SAMPLE: usize = 4096; let buf = read_file_prefix(path, SAMPLE)?; + Ok(classify_mssql_prefix(&buf)) +} + +/// Classify the start of a UTF-8 SQL script for `--format mssql` without reading from a path. +pub(crate) fn classify_mssql_prefix(buf: &[u8]) -> MssqlFileKind { if buf.starts_with(PG_CUSTOM_FORMAT_MAGIC) { - return Ok(MssqlFileKind::PostgresCustomArchiveWrongDialect); + return MssqlFileKind::PostgresCustomArchiveWrongDialect; } // BACPAC / DACPAC / generic zip package if buf.len() >= 4 && &buf[0..4] == b"PK\x03\x04" { - return Ok(MssqlFileKind::ZipArchive); + return MssqlFileKind::ZipArchive; } // UTF-16 BOM — Dumpling expects UTF-8 plain SQL. if buf.starts_with(&[0xFF, 0xFE]) || buf.starts_with(&[0xFE, 0xFF]) { - return Ok(MssqlFileKind::Utf16EncodedSql); + return MssqlFileKind::Utf16EncodedSql; } if buf.is_empty() { - return Ok(MssqlFileKind::PlainSqlText); + return MssqlFileKind::PlainSqlText; } let mut nul = 0usize; let mut control = 0usize; - for &b in &buf { + for &b in buf { if b == 0 { nul += 1; } @@ -77,9 +82,9 @@ pub(crate) fn classify_mssql_dump_file(path: &Path) -> io::Result || (sample >= 64 && (nul * 100 / sample) >= 1) || (sample >= 64 && (control * 100 / sample) > 15) { - return Ok(MssqlFileKind::LikelyBinaryBackup); + return MssqlFileKind::LikelyBinaryBackup; } - Ok(MssqlFileKind::PlainSqlText) + MssqlFileKind::PlainSqlText } pub(crate) const MSSQL_BACPAC_HINT: &str = "This looks like a ZIP package (for example a BACPAC/DACPAC). \ diff --git a/src/main.rs b/src/main.rs index 12306cb..70d2587 100644 --- a/src/main.rs +++ b/src/main.rs @@ -30,10 +30,11 @@ const OUTPUT_PIPE_CHUNK: usize = IO_BUF_CAPACITY; /// Number of full chunks allowed in flight (transform can run ahead of disk this far). const OUTPUT_PIPE_DEPTH: usize = 8; -use compressed_input::{resolve_compressed_wrappers, CompressionCleanup}; +use compressed_input::{resolve_compressed_wrappers, CompressionCleanup, ResolvedInput}; use dump_input_detect::{ - classify_mssql_dump_file, postgres_input_needs_pg_restore, MssqlFileKind, MSSQL_BACPAC_HINT, - MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, MSSQL_WRONG_POSTGRES_ARCHIVE, + classify_mssql_dump_file, classify_mssql_prefix, postgres_input_needs_pg_restore, + MssqlFileKind, MSSQL_BACPAC_HINT, MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, + MSSQL_WRONG_POSTGRES_ARCHIVE, }; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; @@ -520,11 +521,45 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { anyhow::bail!("input path does not exist: {}", path.display()); } - let (inner_path, had_compression_wrapper) = if path.is_dir() { - (path.clone(), false) + let resolved_input = if path.is_dir() { + ResolvedInput::Path { + path: path.clone(), + had_compression_wrapper: false, + materialized_temp_file: false, + } } else { - let r = resolve_compressed_wrappers(path, &mut compression_cleanup)?; - (r.path, r.had_compression_wrapper) + resolve_compressed_wrappers(path, &mut compression_cleanup)? + }; + + let ( + inner_reader_opt, + inner_path, + had_compression_wrapper, + materialized_compression_temp, + mssql_sniff_prefix, + ) = match resolved_input { + ResolvedInput::Path { + path: p, + had_compression_wrapper, + materialized_temp_file, + } => ( + None, + p, + had_compression_wrapper, + materialized_temp_file, + None::>, + ), + ResolvedInput::PlainSqlStream { + reader, + had_compression_wrapper, + sniff_prefix, + } => ( + Some(reader), + path.clone(), + had_compression_wrapper, + false, + Some(sniff_prefix), + ), }; if inner_path.is_dir() && dump_format != DumpFormat::Postgres { @@ -543,47 +578,51 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { ); } - if dump_format == DumpFormat::MsSql && inner_path.is_file() { - match classify_mssql_dump_file(&inner_path) { - Ok(MssqlFileKind::PlainSqlText) => {} - Ok(MssqlFileKind::PostgresCustomArchiveWrongDialect) => { + if dump_format == DumpFormat::MsSql { + let kind = if let Some(ref pref) = mssql_sniff_prefix { + classify_mssql_prefix(pref) + } else if inner_path.is_file() { + classify_mssql_dump_file(&inner_path)? + } else { + MssqlFileKind::PlainSqlText + }; + match kind { + MssqlFileKind::PlainSqlText => {} + MssqlFileKind::PostgresCustomArchiveWrongDialect => { anyhow::bail!( "input `{}` is not plain SQL Server text.\n\n{}", inner_path.display(), MSSQL_WRONG_POSTGRES_ARCHIVE ); } - Ok(MssqlFileKind::ZipArchive) => { + MssqlFileKind::ZipArchive => { anyhow::bail!( "input `{}` is not plain UTF-8 SQL text.\n\n{}", inner_path.display(), MSSQL_BACPAC_HINT ); } - Ok(MssqlFileKind::Utf16EncodedSql) => { + MssqlFileKind::Utf16EncodedSql => { anyhow::bail!( "input `{}` is not UTF-8 plain SQL.\n\n{}", inner_path.display(), MSSQL_UTF16_HINT ); } - Ok(MssqlFileKind::LikelyBinaryBackup) => { + MssqlFileKind::LikelyBinaryBackup => { anyhow::bail!( "input `{}` does not look like UTF-8 plain SQL.\n\n{}", inner_path.display(), MSSQL_BINARY_HINT ); } - Err(e) => { - anyhow::bail!("read `{}`: {}", inner_path.display(), e); - } } } - if had_compression_wrapper && cli.in_place { + if had_compression_wrapper && cli.in_place && materialized_compression_temp { anyhow::bail!( - "this input was gzip- and/or ZIP-wrapped; Dumpling wrote a temporary decompressed file. \ - --in-place cannot safely replace the original path with anonymized SQL; use --output (or stdout) instead" + "this input was gzip- and/or ZIP-wrapped and Dumpling wrote a temporary decompressed file; \ + --in-place cannot safely replace the original path with anonymized SQL. Use --output (or stdout) instead" ); } @@ -637,6 +676,8 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), Some(path.clone()), ) + } else if let Some(r) = inner_reader_opt { + (r, Some(path.clone())) } else { let f = File::open(&inner_path)?; ( diff --git a/src/scaffold.rs b/src/scaffold.rs index 28de861..647ccfa 100644 --- a/src/scaffold.rs +++ b/src/scaffold.rs @@ -1,9 +1,10 @@ //! Draft `[rules]` generation from column names in a SQL dump (starter / beta). -use crate::compressed_input::{resolve_compressed_wrappers, CompressionCleanup}; +use crate::compressed_input::{resolve_compressed_wrappers, CompressionCleanup, ResolvedInput}; use crate::dump_input_detect::{ - classify_mssql_dump_file, postgres_input_needs_pg_restore, MssqlFileKind, MSSQL_BACPAC_HINT, - MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, MSSQL_WRONG_POSTGRES_ARCHIVE, + classify_mssql_dump_file, classify_mssql_prefix, postgres_input_needs_pg_restore, + MssqlFileKind, MSSQL_BACPAC_HINT, MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, + MSSQL_WRONG_POSTGRES_ARCHIVE, }; use crate::pg_restore_decode; use crate::settings::{validate_raw_config, RawConfig}; @@ -79,11 +80,23 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { anyhow::bail!("input path does not exist: {}", path.display()); } - let (inner_path, _had_wrap) = if path.is_dir() { - (path.clone(), false) + let resolved_input = if path.is_dir() { + ResolvedInput::Path { + path: path.clone(), + had_compression_wrapper: false, + materialized_temp_file: false, + } } else { - let r = resolve_compressed_wrappers(&path, &mut compression_cleanup)?; - (r.path, r.had_compression_wrapper) + resolve_compressed_wrappers(&path, &mut compression_cleanup)? + }; + + let (inner_reader_opt, inner_path, mssql_sniff_prefix) = match resolved_input { + ResolvedInput::Path { path: p, .. } => (None, p, None::>), + ResolvedInput::PlainSqlStream { + reader, + sniff_prefix, + .. + } => (Some(reader), path.clone(), Some(sniff_prefix)), }; if inner_path.is_dir() && dump_format != crate::sql::DumpFormat::Postgres { @@ -106,40 +119,44 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { ); } - if dump_format == crate::sql::DumpFormat::MsSql && inner_path.is_file() { - match classify_mssql_dump_file(&inner_path) { - Ok(MssqlFileKind::PlainSqlText) => {} - Ok(MssqlFileKind::PostgresCustomArchiveWrongDialect) => { + if dump_format == crate::sql::DumpFormat::MsSql { + let kind = if let Some(ref pref) = mssql_sniff_prefix { + classify_mssql_prefix(pref) + } else if inner_path.is_file() { + classify_mssql_dump_file(&inner_path)? + } else { + MssqlFileKind::PlainSqlText + }; + match kind { + MssqlFileKind::PlainSqlText => {} + MssqlFileKind::PostgresCustomArchiveWrongDialect => { anyhow::bail!( "input `{}` is not plain SQL Server text.\n\n{}", inner_path.display(), MSSQL_WRONG_POSTGRES_ARCHIVE ); } - Ok(MssqlFileKind::ZipArchive) => { + MssqlFileKind::ZipArchive => { anyhow::bail!( "input `{}` is not plain UTF-8 SQL text.\n\n{}", inner_path.display(), MSSQL_BACPAC_HINT ); } - Ok(MssqlFileKind::Utf16EncodedSql) => { + MssqlFileKind::Utf16EncodedSql => { anyhow::bail!( "input `{}` is not UTF-8 plain SQL.\n\n{}", inner_path.display(), MSSQL_UTF16_HINT ); } - Ok(MssqlFileKind::LikelyBinaryBackup) => { + MssqlFileKind::LikelyBinaryBackup => { anyhow::bail!( "input `{}` does not look like UTF-8 plain SQL.\n\n{}", inner_path.display(), MSSQL_BINARY_HINT ); } - Err(e) => { - anyhow::bail!("read `{}`: {}", inner_path.display(), e); - } } } @@ -184,6 +201,8 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), Some(inner_path.clone()), ) + } else if let Some(r) = inner_reader_opt { + (r, Some(inner_path.clone())) } else { let f = File::open(&inner_path) .with_context(|| format!("open {}", inner_path.display()))?; From f31d6de76a7e9c97026266ecd496296916dae358 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 06:53:04 +0000 Subject: [PATCH 5/9] Docs: sync PostgreSQL archives, gzip/ZIP, pg_restore config, keep-original Replace deprecated --dump-decode guidance with auto-detection, streaming gzip, temp ZIP/PGDMP behavior, [pg_restore] TOML, --keep-original vs --in-place, and --check with archives. Update README and mdBook configuration/getting-started/index. Co-authored-by: Andy Babic --- README.md | 8 ++++-- docs/src/configuration.md | 50 ++++++++++++++++++++++++++++++------- docs/src/getting-started.md | 2 +- docs/src/index.md | 2 +- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e81ae3e..f681666 100644 --- a/README.md +++ b/README.md @@ -473,12 +473,16 @@ Produced by `pg_dump --format=plain`. Handles: Binary, custom, and directory formats from `pg_dump` are not parsed directly — Dumpling’s SQL pipeline expects plain text. Use either: - **`pg_dump --format=plain`** when you control capture, or -- **`dumpling --dump-decode`** with `--input` set to a **custom-format** (`.dump`) or **directory-format** folder: Dumpling runs `pg_restore -f -` and streams the resulting SQL (same as a manual `pg_restore` “script” output, no database required). Requires PostgreSQL client tools on `PATH` (`pg_restore`), or set `--pg-restore-path`. Use `--dump-decode-arg` to pass extra flags (e.g. `--no-owner --no-acl`). **By default** the archive is removed after a fully successful run; pass **`--dump-decode-keep-input`** to retain it. **`--check`** requires **`--dump-decode-keep-input`** so the archive still exists if changes would be detected. +- **Auto-detected PostgreSQL archives** with `--format postgres` (default): if `--input` is a **custom-format** file (begins with `PGDMP`) or a **directory-format** dump (folder containing `toc.dat`), Dumpling runs **`pg_restore -f -`** and streams the resulting SQL (same as a manual `pg_restore` “script” output; no database required). Requires PostgreSQL client tools on **`PATH`** (`pg_restore`), or **`--pg-restore-path`**. Extra flags: **`--pg-restore-arg`** (repeatable), or defaults from **`[pg_restore]`** in `.dumplingconf` / `pyproject.toml` (CLI wins when set). + +**Compressed inputs:** **`.gz`** files whose payload is plain SQL are **decompressed in-process** (no temporary file). **ZIP** archives (and gzip wrapping `PGDMP` or an inner ZIP) are expanded under the system temp directory; those paths are removed when the run finishes. **`--in-place`** is rejected when Dumpling had to materialize a temp file for compression or when the input is a PostgreSQL archive path that must go through `pg_restore` (use **`--output`** or stdout instead). + +**Keeping archives:** **By default** the `--input` archive path (file or directory-format folder) is **removed** after a fully successful run. Pass **`--keep-original`** or set **`keep_original = true`** in config to retain it. **`--check`** against an archive requires an effective keep-original (CLI or config); **`--keep-original` cannot be combined with `--in-place`**. Example (e.g. after `heroku pg:backups:download`): ```bash -dumpling --dump-decode -i latest.dump -c .dumplingconf -o anonymized.sql +dumpling -i latest.dump -c .dumplingconf -o anonymized.sql ``` ### SQLite (`--format sqlite`) diff --git a/docs/src/configuration.md b/docs/src/configuration.md index 57a38df..0fcdb01 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -6,7 +6,7 @@ Use `--format` to declare the SQL dialect of your input file: | Value | Description | |---|---| -| `postgres` (default) | PostgreSQL `pg_dump` plain-text format. Supports `COPY … FROM stdin` blocks, `"double-quoted"` identifiers, `''`-escaped strings. Custom-format (`-Fc`) or directory dumps can be decoded on the fly with `dumpling --dump-decode` (wraps `pg_restore -f -`; requires client tools). By default the archive is deleted after success; use `--dump-decode-keep-input` to retain it. | +| `postgres` (default) | PostgreSQL `pg_dump` plain-text format. Supports `COPY … FROM stdin` blocks, `"double-quoted"` identifiers, `''`-escaped strings. **Custom-format** (`PGDMP`) and **directory-format** (`toc.dat`) dumps are **auto-detected** and decoded with `pg_restore -f -` (requires client tools). **Gzip** — wrapped plain SQL is decompressed in-process; **ZIP** (or gzip wrapping `PGDMP`/nested ZIP) uses a temp file that is removed after the run. By default the archive is deleted after success; use **`--keep-original`** or **`keep_original`** in config to retain it. | | `sqlite` | SQLite `.dump` format. Adds `INSERT OR REPLACE INTO` / `INSERT OR IGNORE INTO` support. No COPY blocks. | | `mssql` | SQL Server / MSSQL plain SQL. Adds `[bracket]` identifier quoting, `N'…'` Unicode string literals, and `nvarchar(n)` / `nchar(n)` length extraction. No COPY blocks. | @@ -17,26 +17,58 @@ dumpling --format sqlite -i data.db.sql -o anonymized.sql dumpling --format mssql -i backup.sql -o anonymized.sql ``` -### PostgreSQL custom-format archives (`--dump-decode`) +### PostgreSQL archives and compressed inputs -Heroku PGBackups and many pipelines ship **`pg_dump` custom format** (`-Fc`) or **directory-format** dumps to save bandwidth. Dumpling’s SQL engine still expects **plain text**; use **`--dump-decode`** so Dumpling runs **`pg_restore -f -`** (script to stdout, no database) and pipes the result through the same anonymizer as a normal plain-SQL file. +Heroku PGBackups and many pipelines ship **`pg_dump` custom format** (`-Fc`), **directory-format** dumps, or **gzip**/**ZIP**-wrapped files. Dumpling’s SQL engine still expects **plain text** at the parser; anything else is normalized first. -**Requirements:** PostgreSQL client tools on `PATH` (`pg_restore`), or set **`--pg-restore-path`**. Use **`--dump-decode-arg`** (repeatable) for extra `pg_restore` flags, e.g. `--dump-decode-arg=--no-owner --dump-decode-arg=--no-acl`. +#### Custom-format and directory dumps (auto-detected) -**Input deletion:** After a **fully successful** run, Dumpling **removes** the `--input` path (single file or directory-format folder) by default so only the anonymized output remains. Pass **`--dump-decode-keep-input`** to retain the archive. +With **`--format postgres`** (default), Dumpling detects: -**Check mode:** **`--check`** with **`--dump-decode`** requires **`--dump-decode-keep-input`**. Otherwise the default would delete the dump before you can iterate on config. +- **Custom-format** files (magic `PGDMP` at the start of the file), and +- **Directory-format** folders (a `toc.dat` beside table blobs), -Example (e.g. after `heroku pg:backups:download`): +then runs **`pg_restore -f -`** (script to stdout inside the process — no database) and pipes the result through the same anonymizer as a normal plain-SQL file. There is **no** `--dump-decode` flag; detection is automatic. + +**Requirements:** PostgreSQL client tools on **`PATH`** (`pg_restore`), or **`--pg-restore-path`**. + +**Extra `pg_restore` arguments:** + +- CLI: **`--pg-restore-arg`** (repeatable), e.g. `--pg-restore-arg=--no-owner --pg-restore-arg=--no-acl` +- Config (optional): **`[pg_restore]`** — CLI overrides these when you pass path or args: + +```toml +[pg_restore] +path = "/usr/bin/pg_restore" +args = ["--no-owner", "--no-acl"] +``` + +#### Gzip and ZIP wrappers + +- **Gzip (`.gz`)** whose decompressed payload is **plain SQL**: decompressed **in-process** (streamed); no temporary dump file. +- **ZIP** containing a single dump file (or a single `.sql` when multiple files exist), **gzip wrapping `PGDMP`**, or **gzip wrapping an inner ZIP**: Dumpling writes under the system temp directory and **removes** those paths when the run completes (including after errors — cleanup runs on drop). + +**`--in-place`** is **rejected** when Dumpling had to **materialize** a temp file for compression **or** when the resolved input is a PostgreSQL archive decoded via **`pg_restore`** (use **`--output`** or stdout). + +#### Keeping inputs and `--check` + +After a **fully successful** run, Dumpling **removes** the `--input` archive path (single file or directory-format folder) **by default**. To keep it: + +- **`--keep-original`**, or +- **`keep_original = true`** at the top level of `.dumplingconf` / `[tool.dumpling]` (merged with CLI; **`--keep-original` cannot be used with `--in-place`**). + +**`--check`** with a PostgreSQL archive requires an **effective** keep-original (CLI or config); otherwise the default deletion would remove the dump before you iterate on policy. + +Examples (e.g. after `heroku pg:backups:download`): ```bash -dumpling --dump-decode -i latest.dump -c .dumplingconf -o anonymized.sql +dumpling -i latest.dump -c .dumplingconf -o anonymized.sql ``` Dry run while keeping the downloaded file: ```bash -dumpling --dump-decode --dump-decode-keep-input --check -i latest.dump -c .dumplingconf +dumpling --keep-original --check -i latest.dump -c .dumplingconf ``` --- diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 4225fc5..52b3719 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -52,7 +52,7 @@ dumpling --help ## PostgreSQL custom-format archives -If your input is a PostgreSQL **custom-format** file (not plain SQL), decode and anonymize in one step with **`--dump-decode`** (needs `pg_restore` from PostgreSQL client tools). See [PostgreSQL custom-format archives](configuration.md#postgresql-custom-format-archives---dump-decode) in the configuration guide. +If your input is a PostgreSQL **custom-format** file or **directory-format** folder (not plain SQL), use **`--format postgres`** (default): Dumpling **auto-detects** the archive and runs **`pg_restore -f -`** (needs `pg_restore` from PostgreSQL client tools). Gzip-wrapped plain SQL is streamed without a temp file; ZIP (or gzip wrapping `PGDMP`) uses a temp extract that is cleaned up afterward. See [PostgreSQL archives and compressed inputs](configuration.md#postgresql-archives-and-compressed-inputs) in the configuration guide. ## Test locally (contributors) diff --git a/docs/src/index.md b/docs/src/index.md index 6d20a29..2380ed4 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,6 +1,6 @@ # Dumpling documentation -Dumpling is a streaming anonymizer for plain SQL dumps. It supports PostgreSQL (`pg_dump` plain format), SQLite (`.dump`), and SQL Server / MSSQL (SSMS / mssql-scripter plain SQL output). For PostgreSQL **custom-format** archives (e.g. Heroku `pg:backups:download`), use **`--dump-decode`** so Dumpling invokes `pg_restore` and streams plain SQL—see [Dump format](configuration.html#postgresql-custom-format-archives---dump-decode) in the configuration guide. +Dumpling is a streaming anonymizer for plain SQL dumps. It supports PostgreSQL (`pg_dump` plain format), SQLite (`.dump`), and SQL Server / MSSQL (SSMS / mssql-scripter plain SQL output). For PostgreSQL **custom-format** or **directory-format** archives (e.g. Heroku `pg:backups:download`), Dumpling **auto-detects** them when `--format postgres` (default) and invokes `pg_restore -f -`—see [PostgreSQL archives and compressed inputs](configuration.html#postgresql-archives-and-compressed-inputs) in the configuration guide. **New here?** Start with [**Getting started**](getting-started.html): generate a **draft** policy with `scaffold-config`, review and add secrets, run Dumpling, then tighten with `lint-policy` and optional CI flags. From 0b2689c2ce8f0b8d4dcd73d46edd80f99bed9e1f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 07:00:53 +0000 Subject: [PATCH 6/9] docs: drop negated dump-decode wording; state auto-detection only Co-authored-by: Andy Babic --- docs/src/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/configuration.md b/docs/src/configuration.md index 0fcdb01..59f4f2d 100644 --- a/docs/src/configuration.md +++ b/docs/src/configuration.md @@ -28,7 +28,7 @@ With **`--format postgres`** (default), Dumpling detects: - **Custom-format** files (magic `PGDMP` at the start of the file), and - **Directory-format** folders (a `toc.dat` beside table blobs), -then runs **`pg_restore -f -`** (script to stdout inside the process — no database) and pipes the result through the same anonymizer as a normal plain-SQL file. There is **no** `--dump-decode` flag; detection is automatic. +then runs **`pg_restore -f -`** (script to stdout inside the process — no database) and pipes the result through the same anonymizer as a normal plain-SQL file. Detection from **`--input`** is automatic. **Requirements:** PostgreSQL client tools on **`PATH`** (`pg_restore`), or **`--pg-restore-path`**. From aa137e48463f1e0b9f004eb976cc7047d2bf3ce0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 07:03:26 +0000 Subject: [PATCH 7/9] docs: remove stale --dump-decode scaffold hints; fix pg_restore links Co-authored-by: Andy Babic --- README.md | 2 +- docs/src/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f681666..4bb64ec 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ dumpling --help Follow these steps once; you will have a working path from “raw dump” to “first sanitized output,” then you can deepen coverage using the rest of this README and the [documentation site](https://ababic.github.io/dumpling/). -1. **Generate a draft policy (recommended)** — Run `dumpling scaffold-config -i dump.sql -o .dumplingconf` to emit a **beta** starter TOML with inferred `[rules]` from column names in `CREATE TABLE`, `INSERT`, and (PostgreSQL) `COPY` headers. Heuristics are **English-oriented**; treat the file as **draft only**—review every rule before production or compliance workflows. Add a global `salt` (for example `salt = "${DUMPLING_SALT}"`) and resolve `${…}` references before anonymizing. Optionally pass **`--infer-json-paths`** to sample up to **five rows per table** (reservoir) and suggest nested JSON keys as `column.path.to.leaf`; use **`--max-json-depth`** if you need a different walk depth (default 24). For PostgreSQL **custom-format** archives, add **`--dump-decode`** (requires **`--input`** and **`--format postgres`**). See `dumpling scaffold-config --help`. +1. **Generate a draft policy (recommended)** — Run `dumpling scaffold-config -i dump.sql -o .dumplingconf` to emit a **beta** starter TOML with inferred `[rules]` from column names in `CREATE TABLE`, `INSERT`, and (PostgreSQL) `COPY` headers. Heuristics are **English-oriented**; treat the file as **draft only**—review every rule before production or compliance workflows. Add a global `salt` (for example `salt = "${DUMPLING_SALT}"`) and resolve `${…}` references before anonymizing. Optionally pass **`--infer-json-paths`** to sample up to **five rows per table** (reservoir) and suggest nested JSON keys as `column.path.to.leaf`; use **`--max-json-depth`** if you need a different walk depth (default 24). For PostgreSQL **custom-format** or **directory-format** archives, pass **`--input`** pointing at the archive with **`--format postgres`** (default); Dumpling auto-detects and runs **`pg_restore`** (optional **`--pg-restore-path`** / **`--pg-restore-arg`**). See `dumpling scaffold-config --help`. 2. **Or start from the example policy** — Copy [`.dumplingconf.example`](.dumplingconf.example) to `.dumplingconf` (or merge under `[tool.dumpling]` in `pyproject.toml`) and edit `[rules]` by hand. Set environment variables for `salt` and any `${…}` references so Dumpling can resolve secrets at startup. 3. **Align rules with your dump** — If you did not use `scaffold-config`, open the dump beside the config: `CREATE TABLE`, `COPY … (…)`, and `INSERT INTO … (…)` lines list identifiers for `[rules."table"]` or `[rules."schema.table"]` (see [Configuration (TOML)](#configuration-toml)). Trim rules to the tables you care about first, then extend columns and strategies as you go. 4. **Run Dumpling** — `dumpling -i dump.sql -o sanitized.sql` (add `-c path` if the config is not in the default search path). Use `dumpling --check -i dump.sql` when you only want to know whether anything would change. diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 52b3719..4427492 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -38,7 +38,7 @@ dumpling --help - **`--infer-json-paths`** — Keep up to **five sampled rows per table** (reservoir) and suggest nested JSON rules as `column.path.leaf`. - **`--max-json-depth`** — Cap JSON walking depth when using `--infer-json-paths` (default 24). - **`--format`** — `postgres` (default), `sqlite`, or `mssql`. - - **`--dump-decode`** — Decode a PostgreSQL custom-format archive via `pg_restore` (requires **`--input`** and **`--format postgres`**); see [PostgreSQL custom-format archives](configuration.md#postgresql-custom-format-archives---dump-decode). + - **`--pg-restore-path`** / **`--pg-restore-arg`** — Optional **`pg_restore`** binary and extra arguments when **`--input`** is a PostgreSQL custom-format or directory-format archive (auto-detected with **`--format postgres`**); see [PostgreSQL archives and compressed inputs](configuration.md#postgresql-archives-and-compressed-inputs). Run `dumpling scaffold-config --help` for the full flag list. From 969d3c8eae05ecc8299769690f0fd69d6732ceb6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 07:15:50 +0000 Subject: [PATCH 8/9] Share dump input resolution between anonymize and scaffold-config Extract resolve_dump_input_from_path into dump_input_resolve.rs so scaffold-config uses the same gzip/ZIP, MSSQL classification, pg_restore, and archive-removal logic as run_anonymize. Document new modules in AGENTS.md. Co-authored-by: Andy Babic --- AGENTS.md | 3 + src/dump_input_resolve.rs | 223 ++++++++++++++++++++++++++++++++++++++ src/main.rs | 189 +++----------------------------- src/scaffold.rs | 156 +++----------------------- 4 files changed, 255 insertions(+), 316 deletions(-) create mode 100644 src/dump_input_resolve.rs diff --git a/AGENTS.md b/AGENTS.md index 7ecbd27..f41df57 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,9 @@ src/ filter.rs — Row-filter predicate evaluation (eq/neq/like/regex/JSON-path/…) scan.rs — Post-transform residual PII scanner (email/SSN/PAN/token regex) report.rs — JSON report data structures and Reporter helper + compressed_input.rs — gzip/ZIP wrappers; streaming vs temp materialization + dump_input_resolve.rs — shared `--input` file resolution for anonymize + scaffold-config + dump_input_detect.rs — PGDMP / directory dumps / MSSQL sniff helpers docs/src/ — mdBook documentation source .github/ — CI/CD GitHub Actions workflows Cargo.toml — Rust package manifest diff --git a/src/dump_input_resolve.rs b/src/dump_input_resolve.rs new file mode 100644 index 0000000..c4926e0 --- /dev/null +++ b/src/dump_input_resolve.rs @@ -0,0 +1,223 @@ +//! Shared `--input` file resolution for `run_anonymize` and `scaffold-config`. +//! +//! Keeps gzip/ZIP handling, format checks, `pg_restore`, and archive cleanup aligned between commands. + +use crate::compressed_input::{resolve_compressed_wrappers, CompressionCleanup, ResolvedInput}; +use crate::dump_input_detect::{ + classify_mssql_dump_file, classify_mssql_prefix, postgres_input_needs_pg_restore, + MssqlFileKind, MSSQL_BACPAC_HINT, MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, + MSSQL_WRONG_POSTGRES_ARCHIVE, +}; +use crate::pg_restore_decode; +use crate::sql::DumpFormat; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; + +use crate::IO_BUF_CAPACITY; + +pub(crate) struct ResolveDumpInputParams<'a> { + pub user_input_path: &'a Path, + pub dump_format: DumpFormat, + pub compression_cleanup: &'a mut CompressionCleanup, + pub pg_restore_path: &'a Path, + pub pg_restore_arg: &'a [String], + pub keep_original: bool, + pub in_place: bool, +} + +pub(crate) struct ResolvedDumpInput { + pub reader: Box, + /// User's `--input` path (preserved for `--in-place` and logging). + pub original_input_path: PathBuf, + pub pg_restore_child: Option, + pub path_to_remove_pg_archive: Option, +} + +pub(crate) fn resolve_dump_input_from_path( + p: ResolveDumpInputParams<'_>, +) -> anyhow::Result { + let ResolveDumpInputParams { + user_input_path: path, + dump_format, + compression_cleanup, + pg_restore_path, + pg_restore_arg, + keep_original, + in_place, + } = p; + + if !path.exists() { + anyhow::bail!("input path does not exist: {}", path.display()); + } + + let resolved_input = if path.is_dir() { + ResolvedInput::Path { + path: path.to_path_buf(), + had_compression_wrapper: false, + materialized_temp_file: false, + } + } else { + resolve_compressed_wrappers(path, compression_cleanup)? + }; + + let ( + inner_reader_opt, + inner_path, + had_compression_wrapper, + materialized_compression_temp, + mssql_sniff_prefix, + ) = match resolved_input { + ResolvedInput::Path { + path: ip, + had_compression_wrapper, + materialized_temp_file, + } => ( + None, + ip, + had_compression_wrapper, + materialized_temp_file, + None::>, + ), + ResolvedInput::PlainSqlStream { + reader, + had_compression_wrapper, + sniff_prefix, + .. + } => ( + Some(reader), + path.to_path_buf(), + had_compression_wrapper, + false, + Some(sniff_prefix), + ), + }; + + if inner_path.is_dir() && dump_format != DumpFormat::Postgres { + anyhow::bail!( + "input `{}` is a directory; Dumpling expects a single plain-SQL file for this `--format`. \ + For PostgreSQL directory-format dumps (folder containing `toc.dat`), use `--format postgres`.", + inner_path.display() + ); + } + + if dump_format == DumpFormat::Sqlite && postgres_input_needs_pg_restore(&inner_path)? { + anyhow::bail!( + "input `{}` looks like a PostgreSQL custom-format or directory-format archive, not a SQLite `.dump`. \ + Use `--format postgres` (default) so Dumpling can decode it with pg_restore.", + inner_path.display() + ); + } + + if dump_format == DumpFormat::MsSql { + let kind = if let Some(ref pref) = mssql_sniff_prefix { + classify_mssql_prefix(pref) + } else if inner_path.is_file() { + classify_mssql_dump_file(&inner_path)? + } else { + MssqlFileKind::PlainSqlText + }; + match kind { + MssqlFileKind::PlainSqlText => {} + MssqlFileKind::PostgresCustomArchiveWrongDialect => { + anyhow::bail!( + "input `{}` is not plain SQL Server text.\n\n{}", + inner_path.display(), + MSSQL_WRONG_POSTGRES_ARCHIVE + ); + } + MssqlFileKind::ZipArchive => { + anyhow::bail!( + "input `{}` is not plain UTF-8 SQL text.\n\n{}", + inner_path.display(), + MSSQL_BACPAC_HINT + ); + } + MssqlFileKind::Utf16EncodedSql => { + anyhow::bail!( + "input `{}` is not UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_UTF16_HINT + ); + } + MssqlFileKind::LikelyBinaryBackup => { + anyhow::bail!( + "input `{}` does not look like UTF-8 plain SQL.\n\n{}", + inner_path.display(), + MSSQL_BINARY_HINT + ); + } + } + } + + if had_compression_wrapper && in_place && materialized_compression_temp { + anyhow::bail!( + "this input was gzip- and/or ZIP-wrapped and Dumpling wrote a temporary decompressed file; \ + --in-place cannot safely replace the original path with anonymized SQL. Use --output (or stdout) instead" + ); + } + + let auto_pg_restore = dump_format == DumpFormat::Postgres + && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { + anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) + })?; + + if auto_pg_restore && in_place { + anyhow::bail!( + "this input is a PostgreSQL custom-format or directory-format archive (decoded via pg_restore). \ + --in-place cannot replace the archive with plain SQL while atomically preserving the path; \ + write to --output (or stdout) instead, or decode manually with pg_restore -f -" + ); + } + + let use_pg_restore = auto_pg_restore; + + let mut pg_restore_child = None; + let mut path_to_remove_pg_archive = None; + + let reader: Box = if use_pg_restore { + if dump_format != DumpFormat::Postgres { + anyhow::bail!( + "PostgreSQL archive decoding only applies when --format postgres (default)" + ); + } + if auto_pg_restore { + eprintln!( + "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {}", + pg_restore_path.display(), + inner_path.display() + ); + } else { + eprintln!( + "dumpling: decoding PostgreSQL archive via {} -f - {}", + pg_restore_path.display(), + inner_path.display() + ); + } + let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( + pg_restore_path, + pg_restore_arg, + &inner_path, + )?; + pg_restore_child = Some(pg); + if !keep_original { + let tmp_root = std::env::temp_dir(); + if !inner_path.starts_with(&tmp_root) { + path_to_remove_pg_archive = Some(inner_path.clone()); + } + } + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)) + } else if let Some(r) = inner_reader_opt { + r + } else { + let f = File::open(&inner_path)?; + Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)) + }; + + Ok(ResolvedDumpInput { + reader, + original_input_path: path.to_path_buf(), + pg_restore_child, + path_to_remove_pg_archive, + }) +} diff --git a/src/main.rs b/src/main.rs index 70d2587..37ef7cd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ use clap::{ArgAction, Parser, Subcommand}; mod compressed_input; mod dump_input_detect; +mod dump_input_resolve; mod faker_dispatch; mod filter; mod lint; @@ -30,12 +31,8 @@ const OUTPUT_PIPE_CHUNK: usize = IO_BUF_CAPACITY; /// Number of full chunks allowed in flight (transform can run ahead of disk this far). const OUTPUT_PIPE_DEPTH: usize = 8; -use compressed_input::{resolve_compressed_wrappers, CompressionCleanup, ResolvedInput}; -use dump_input_detect::{ - classify_mssql_dump_file, classify_mssql_prefix, postgres_input_needs_pg_restore, - MssqlFileKind, MSSQL_BACPAC_HINT, MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, - MSSQL_WRONG_POSTGRES_ARCHIVE, -}; +use compressed_input::CompressionCleanup; +use dump_input_resolve::{resolve_dump_input_from_path, ResolveDumpInputParams}; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; use seal::{ @@ -517,174 +514,18 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { cli.allow_ext ); } - if !path.exists() { - anyhow::bail!("input path does not exist: {}", path.display()); - } - - let resolved_input = if path.is_dir() { - ResolvedInput::Path { - path: path.clone(), - had_compression_wrapper: false, - materialized_temp_file: false, - } - } else { - resolve_compressed_wrappers(path, &mut compression_cleanup)? - }; - - let ( - inner_reader_opt, - inner_path, - had_compression_wrapper, - materialized_compression_temp, - mssql_sniff_prefix, - ) = match resolved_input { - ResolvedInput::Path { - path: p, - had_compression_wrapper, - materialized_temp_file, - } => ( - None, - p, - had_compression_wrapper, - materialized_temp_file, - None::>, - ), - ResolvedInput::PlainSqlStream { - reader, - had_compression_wrapper, - sniff_prefix, - } => ( - Some(reader), - path.clone(), - had_compression_wrapper, - false, - Some(sniff_prefix), - ), - }; - - if inner_path.is_dir() && dump_format != DumpFormat::Postgres { - anyhow::bail!( - "input `{}` is a directory; Dumpling expects a single plain-SQL file for this `--format`. \ - For PostgreSQL directory-format dumps (folder containing `toc.dat`), use `--format postgres`.", - inner_path.display() - ); - } - - if dump_format == DumpFormat::Sqlite && postgres_input_needs_pg_restore(&inner_path)? { - anyhow::bail!( - "input `{}` looks like a PostgreSQL custom-format or directory-format archive, not a SQLite `.dump`. \ - Use `--format postgres` (default) so Dumpling can decode it with pg_restore.", - inner_path.display() - ); - } - - if dump_format == DumpFormat::MsSql { - let kind = if let Some(ref pref) = mssql_sniff_prefix { - classify_mssql_prefix(pref) - } else if inner_path.is_file() { - classify_mssql_dump_file(&inner_path)? - } else { - MssqlFileKind::PlainSqlText - }; - match kind { - MssqlFileKind::PlainSqlText => {} - MssqlFileKind::PostgresCustomArchiveWrongDialect => { - anyhow::bail!( - "input `{}` is not plain SQL Server text.\n\n{}", - inner_path.display(), - MSSQL_WRONG_POSTGRES_ARCHIVE - ); - } - MssqlFileKind::ZipArchive => { - anyhow::bail!( - "input `{}` is not plain UTF-8 SQL text.\n\n{}", - inner_path.display(), - MSSQL_BACPAC_HINT - ); - } - MssqlFileKind::Utf16EncodedSql => { - anyhow::bail!( - "input `{}` is not UTF-8 plain SQL.\n\n{}", - inner_path.display(), - MSSQL_UTF16_HINT - ); - } - MssqlFileKind::LikelyBinaryBackup => { - anyhow::bail!( - "input `{}` does not look like UTF-8 plain SQL.\n\n{}", - inner_path.display(), - MSSQL_BINARY_HINT - ); - } - } - } - - if had_compression_wrapper && cli.in_place && materialized_compression_temp { - anyhow::bail!( - "this input was gzip- and/or ZIP-wrapped and Dumpling wrote a temporary decompressed file; \ - --in-place cannot safely replace the original path with anonymized SQL. Use --output (or stdout) instead" - ); - } - - let auto_pg_restore = dump_format == DumpFormat::Postgres - && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { - anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) - })?; - - if auto_pg_restore && cli.in_place { - anyhow::bail!( - "this input is a PostgreSQL custom-format or directory-format archive (decoded via pg_restore). \ - --in-place cannot replace the archive with plain SQL while atomically preserving the path; \ - write to --output (or stdout) instead, or decode manually with pg_restore -f -" - ); - } - - let use_pg_restore = auto_pg_restore; - - if use_pg_restore { - if dump_format != DumpFormat::Postgres { - anyhow::bail!( - "PostgreSQL archive decoding only applies when --format postgres (default)" - ); - } - if auto_pg_restore { - eprintln!( - "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {}", - pg_restore_path_eff.display(), - inner_path.display() - ); - } else { - eprintln!( - "dumpling: decoding PostgreSQL archive via {} -f - {}", - pg_restore_path_eff.display(), - inner_path.display() - ); - } - let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( - &pg_restore_path_eff, - &pg_restore_arg_eff, - &inner_path, - )?; - pg_restore_child = Some(pg); - if !keep_original_eff { - let tmp_root = std::env::temp_dir(); - if !inner_path.starts_with(&tmp_root) { - path_to_remove_pg_archive = Some(inner_path.clone()); - } - } - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), - Some(path.clone()), - ) - } else if let Some(r) = inner_reader_opt { - (r, Some(path.clone())) - } else { - let f = File::open(&inner_path)?; - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), - Some(path.clone()), - ) - } + let resolved = resolve_dump_input_from_path(ResolveDumpInputParams { + user_input_path: path, + dump_format, + compression_cleanup: &mut compression_cleanup, + pg_restore_path: &pg_restore_path_eff, + pg_restore_arg: &pg_restore_arg_eff, + keep_original: keep_original_eff, + in_place: cli.in_place, + })?; + pg_restore_child = resolved.pg_restore_child; + path_to_remove_pg_archive = resolved.path_to_remove_pg_archive; + (resolved.reader, Some(resolved.original_input_path)) } }; diff --git a/src/scaffold.rs b/src/scaffold.rs index 647ccfa..11190b1 100644 --- a/src/scaffold.rs +++ b/src/scaffold.rs @@ -1,11 +1,7 @@ //! Draft `[rules]` generation from column names in a SQL dump (starter / beta). -use crate::compressed_input::{resolve_compressed_wrappers, CompressionCleanup, ResolvedInput}; -use crate::dump_input_detect::{ - classify_mssql_dump_file, classify_mssql_prefix, postgres_input_needs_pg_restore, - MssqlFileKind, MSSQL_BACPAC_HINT, MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, - MSSQL_WRONG_POSTGRES_ARCHIVE, -}; +use crate::compressed_input::CompressionCleanup; +use crate::dump_input_resolve::{resolve_dump_input_from_path, ResolveDumpInputParams}; use crate::pg_restore_decode; use crate::settings::{validate_raw_config, RawConfig}; use crate::sql::{ @@ -14,7 +10,6 @@ use crate::sql::{ use anyhow::Context; use serde_json::Value as JsonValue; use std::collections::HashMap; -use std::fs::File; use std::io::{BufRead, BufReader, Write}; use std::path::PathBuf; @@ -76,141 +71,18 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { if !allow_ext.is_empty() && !crate::has_allowed_extension(&path, &allow_ext) { anyhow::bail!("input file extension is not in allowed set {:?}", allow_ext); } - if !path.exists() { - anyhow::bail!("input path does not exist: {}", path.display()); - } - - let resolved_input = if path.is_dir() { - ResolvedInput::Path { - path: path.clone(), - had_compression_wrapper: false, - materialized_temp_file: false, - } - } else { - resolve_compressed_wrappers(&path, &mut compression_cleanup)? - }; - - let (inner_reader_opt, inner_path, mssql_sniff_prefix) = match resolved_input { - ResolvedInput::Path { path: p, .. } => (None, p, None::>), - ResolvedInput::PlainSqlStream { - reader, - sniff_prefix, - .. - } => (Some(reader), path.clone(), Some(sniff_prefix)), - }; - - if inner_path.is_dir() && dump_format != crate::sql::DumpFormat::Postgres { - anyhow::bail!( - "input `{}` is a directory; this dialect expects a single SQL file. \ - For PostgreSQL directory-format dumps (folder containing `toc.dat`), use `--format postgres`.", - inner_path.display() - ); - } - - if dump_format == crate::sql::DumpFormat::Sqlite - && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { - anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) - })? - { - anyhow::bail!( - "input `{}` looks like a PostgreSQL custom-format or directory-format archive, not a SQLite `.dump`. \ - Use `--format postgres` (default) so Dumpling can decode it with pg_restore.", - inner_path.display() - ); - } - - if dump_format == crate::sql::DumpFormat::MsSql { - let kind = if let Some(ref pref) = mssql_sniff_prefix { - classify_mssql_prefix(pref) - } else if inner_path.is_file() { - classify_mssql_dump_file(&inner_path)? - } else { - MssqlFileKind::PlainSqlText - }; - match kind { - MssqlFileKind::PlainSqlText => {} - MssqlFileKind::PostgresCustomArchiveWrongDialect => { - anyhow::bail!( - "input `{}` is not plain SQL Server text.\n\n{}", - inner_path.display(), - MSSQL_WRONG_POSTGRES_ARCHIVE - ); - } - MssqlFileKind::ZipArchive => { - anyhow::bail!( - "input `{}` is not plain UTF-8 SQL text.\n\n{}", - inner_path.display(), - MSSQL_BACPAC_HINT - ); - } - MssqlFileKind::Utf16EncodedSql => { - anyhow::bail!( - "input `{}` is not UTF-8 plain SQL.\n\n{}", - inner_path.display(), - MSSQL_UTF16_HINT - ); - } - MssqlFileKind::LikelyBinaryBackup => { - anyhow::bail!( - "input `{}` does not look like UTF-8 plain SQL.\n\n{}", - inner_path.display(), - MSSQL_BINARY_HINT - ); - } - } - } - - let auto_pg_restore = dump_format == crate::sql::DumpFormat::Postgres - && postgres_input_needs_pg_restore(&inner_path).map_err(|e| { - anyhow::anyhow!("could not inspect input `{}`: {}", inner_path.display(), e) - })?; - let use_pg_restore = auto_pg_restore; - - if use_pg_restore { - if dump_format != crate::sql::DumpFormat::Postgres { - anyhow::bail!( - "PostgreSQL archive decoding only applies when --format postgres (default)" - ); - } - if auto_pg_restore { - eprintln!( - "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {}", - pg_restore_path.display(), - inner_path.display() - ); - } else { - eprintln!( - "dumpling: decoding PostgreSQL archive via {} -f - {}", - pg_restore_path.display(), - inner_path.display() - ); - } - let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( - &pg_restore_path, - &pg_restore_arg, - &inner_path, - )?; - pg_restore_child = Some(pg); - if !keep_original { - let tmp_root = std::env::temp_dir(); - if !inner_path.starts_with(&tmp_root) { - path_to_remove_pg_archive = Some(inner_path.clone()); - } - } - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, stdout)), - Some(inner_path.clone()), - ) - } else if let Some(r) = inner_reader_opt { - (r, Some(inner_path.clone())) - } else { - let f = File::open(&inner_path) - .with_context(|| format!("open {}", inner_path.display()))?; - ( - Box::new(BufReader::with_capacity(IO_BUF_CAPACITY, f)), - Some(inner_path.clone()), - ) - } + let resolved = resolve_dump_input_from_path(ResolveDumpInputParams { + user_input_path: &path, + dump_format, + compression_cleanup: &mut compression_cleanup, + pg_restore_path: &pg_restore_path, + pg_restore_arg: &pg_restore_arg, + keep_original, + in_place: false, + })?; + pg_restore_child = resolved.pg_restore_child; + path_to_remove_pg_archive = resolved.path_to_remove_pg_archive; + (resolved.reader, Some(path)) } }; From e4204ccf9f8878307755278604227770c4302193 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 5 May 2026 07:57:51 +0000 Subject: [PATCH 9/9] Redact path prefixes from stderr logs for CodeQL cleartext-logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add log_sanitize::path_basename_for_log and use it for pg_restore progress lines, gzip/ZIP notices, archive removal, scaffold write/remove messages, and plaintext salt warnings — avoids emitting full filesystem paths that may contain sensitive directory names. Context/full paths remain in anyhow messages where appropriate. Co-authored-by: Andy Babic --- src/compressed_input.rs | 5 +++-- src/dump_input_resolve.rs | 13 +++++++------ src/log_sanitize.rs | 29 +++++++++++++++++++++++++++++ src/main.rs | 19 +++++++++++++++---- src/scaffold.rs | 15 +++++++++++---- src/settings.rs | 4 +++- 6 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 src/log_sanitize.rs diff --git a/src/compressed_input.rs b/src/compressed_input.rs index 18c2527..4f76d66 100644 --- a/src/compressed_input.rs +++ b/src/compressed_input.rs @@ -13,6 +13,7 @@ use std::fs::{self, File}; use std::io::{BufRead, BufReader, Cursor, Read}; use std::path::{Path, PathBuf}; +use crate::log_sanitize::path_basename_for_log; use crate::IO_BUF_CAPACITY; /// Prefix size for gzip/ZIP sniffing (matches MSSQL sample size in `dump_input_detect`). @@ -184,7 +185,7 @@ fn resolve_compressed_wrappers_inner( if is_gzip_prefix(&prefix) { eprintln!( "dumpling: decompressing gzip input {} in-process (streamed when inner is plain SQL)", - current.display() + path_basename_for_log(¤t) ); let inp = File::open(¤t).with_context(|| format!("open {}", current.display()))?; let mut dec = GzDecoder::new(inp); @@ -221,7 +222,7 @@ fn resolve_compressed_wrappers_inner( if is_zip_prefix(&prefix) { eprintln!( "dumpling: extracting inner file from ZIP {} (materialized for zip directory access)", - current.display() + path_basename_for_log(¤t) ); current = extract_zip_inner_file(¤t, cleanup)?; return resolve_compressed_wrappers_inner(¤t, cleanup, depth + 1, true, true); diff --git a/src/dump_input_resolve.rs b/src/dump_input_resolve.rs index c4926e0..ba7f8cf 100644 --- a/src/dump_input_resolve.rs +++ b/src/dump_input_resolve.rs @@ -8,6 +8,7 @@ use crate::dump_input_detect::{ MssqlFileKind, MSSQL_BACPAC_HINT, MSSQL_BINARY_HINT, MSSQL_UTF16_HINT, MSSQL_WRONG_POSTGRES_ARCHIVE, }; +use crate::log_sanitize::path_basename_for_log; use crate::pg_restore_decode; use crate::sql::DumpFormat; use std::fs::File; @@ -183,15 +184,15 @@ pub(crate) fn resolve_dump_input_from_path( } if auto_pg_restore { eprintln!( - "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {}", - pg_restore_path.display(), - inner_path.display() + "dumpling: detected PostgreSQL custom or directory-format archive; decoding via {} -f - {} (input file)", + path_basename_for_log(pg_restore_path), + path_basename_for_log(&inner_path) ); } else { eprintln!( - "dumpling: decoding PostgreSQL archive via {} -f - {}", - pg_restore_path.display(), - inner_path.display() + "dumpling: decoding PostgreSQL archive via {} -f - {} (input file)", + path_basename_for_log(pg_restore_path), + path_basename_for_log(&inner_path) ); } let (stdout, pg) = pg_restore_decode::spawn_pg_restore_decode( diff --git a/src/log_sanitize.rs b/src/log_sanitize.rs new file mode 100644 index 0000000..a987e8c --- /dev/null +++ b/src/log_sanitize.rs @@ -0,0 +1,29 @@ +//! Avoid logging full filesystem paths on stderr (directory names may include PII; CodeQL flags this). + +use std::path::Path; + +/// Last path segment for user-visible logs, or a generic placeholder when absent. +pub(crate) fn path_basename_for_log(path: &Path) -> String { + path.file_name() + .and_then(|s| s.to_str()) + .map(str::to_string) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "input".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn basename_prefers_final_component() { + let p = PathBuf::from("/tmp/secrets/my.dump"); + assert_eq!(path_basename_for_log(&p), "my.dump"); + } + + #[test] + fn basename_fallback_for_empty_file_name() { + assert_eq!(path_basename_for_log(Path::new("/")), "input"); + } +} diff --git a/src/main.rs b/src/main.rs index 37ef7cd..c2cdcc5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,7 @@ mod dump_input_resolve; mod faker_dispatch; mod filter; mod lint; +mod log_sanitize; mod pg_restore_decode; mod report; mod scaffold; @@ -33,6 +34,7 @@ const OUTPUT_PIPE_DEPTH: usize = 8; use compressed_input::CompressionCleanup; use dump_input_resolve::{resolve_dump_input_from_path, ResolveDumpInputParams}; +use log_sanitize::path_basename_for_log; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; use seal::{ @@ -256,7 +258,10 @@ fn main() -> anyhow::Result<()> { fn run_lint_policy(config: Option<&PathBuf>, allow_noop: bool) -> anyhow::Result<()> { let resolved_config: ResolvedConfig = settings::load_config(config, allow_noop)?; if let Some(path) = resolved_config.source_path.as_ref() { - eprintln!("dumpling: using config source {}", path.display()); + eprintln!( + "dumpling: using config source {}", + path_basename_for_log(path.as_path()) + ); } else if allow_noop { eprintln!("dumpling: no config discovered; continuing because --allow-noop was set"); } @@ -405,7 +410,10 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { let resolved_config: ResolvedConfig = settings::load_config(cli.config.as_ref(), cli.allow_noop)?; if let Some(path) = resolved_config.source_path.as_ref() { - eprintln!("dumpling: using config source {}", path.display()); + eprintln!( + "dumpling: using config source {}", + path_basename_for_log(path.as_path()) + ); } else if cli.allow_noop { eprintln!("dumpling: no config discovered; continuing because --allow-noop was set"); } @@ -737,10 +745,13 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { if let Some(ref p) = path_to_remove_pg_archive { match remove_pg_archive(p) { - Ok(()) => eprintln!("dumpling: removed input archive {}", p.display()), + Ok(()) => eprintln!( + "dumpling: removed input archive {}", + path_basename_for_log(p) + ), Err(e) => eprintln!( "dumpling: warning: could not remove input archive {}: {}", - p.display(), + path_basename_for_log(p), e ), } diff --git a/src/scaffold.rs b/src/scaffold.rs index 11190b1..df10fd2 100644 --- a/src/scaffold.rs +++ b/src/scaffold.rs @@ -2,6 +2,7 @@ use crate::compressed_input::CompressionCleanup; use crate::dump_input_resolve::{resolve_dump_input_from_path, ResolveDumpInputParams}; +use crate::log_sanitize::path_basename_for_log; use crate::pg_restore_decode; use crate::settings::{validate_raw_config, RawConfig}; use crate::sql::{ @@ -129,7 +130,10 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { if let Some(ref path) = output { std::fs::write(path, &text).with_context(|| format!("write {}", path.display()))?; - eprintln!("dumpling scaffold-config: wrote {}", path.display()); + eprintln!( + "dumpling scaffold-config: wrote {}", + path_basename_for_log(path.as_path()) + ); } else { std::io::stdout().write_all(text.as_bytes())?; } @@ -141,17 +145,20 @@ pub fn run_scaffold_config(opts: ScaffoldConfigOptions) -> anyhow::Result<()> { pg_child.finish(pipeline_ok).with_context(|| { format!( "`{}` failed while decoding the PostgreSQL archive", - pg_restore_path.display() + path_basename_for_log(pg_restore_path.as_path()) ) })?; } if pipeline_ok { if let Some(ref p) = path_to_remove_pg_archive { match crate::remove_pg_archive(p) { - Ok(()) => eprintln!("dumpling: removed input archive {}", p.display()), + Ok(()) => eprintln!( + "dumpling: removed input archive {}", + path_basename_for_log(p.as_path()) + ), Err(e) => eprintln!( "dumpling: warning: could not remove input archive {}: {}", - p.display(), + path_basename_for_log(p.as_path()), e ), } diff --git a/src/settings.rs b/src/settings.rs index 9dcb3af..5239ab7 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -4,6 +4,8 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; +use crate::log_sanitize::path_basename_for_log; + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct RawConfig { /// Optional default salt used by certain anonymizers (e.g., hash) @@ -289,7 +291,7 @@ fn resolve_raw_config_value( eprintln!( "dumpling: warning: insecure plaintext secret at config path '{}' in {}; use ${{ENV_VAR}} or ${{env:ENV_VAR}}", secret_path, - source_path.display() + path_basename_for_log(source_path) ); } raw_value.try_into().with_context(|| {