From 2e9486258b03cb0368009a4e9b2ad5c466dd990d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 3 May 2026 16:36:34 +0000 Subject: [PATCH 1/2] feat: sealed dump markers with version and policy fingerprint Add --emit-seal-line to prefix output with a SQL comment encoding format version, Dumpling semver, security profile, and SHA-256 of canonical resolved policy JSON. Add --trust-sealed-dumps to pass the remainder of the file through unchanged when the first line matches the current binary, profile, and fingerprint. Reject --strict-coverage with a matching seal (no CREATE TABLE scan). Document in README and add an integration test. Co-authored-by: Andy Babic --- README.md | 12 ++ src/main.rs | 183 +++++++++++++++++++++- src/seal.rs | 406 ++++++++++++++++++++++++++++++++++++++++++++++++ src/settings.rs | 18 +-- src/sql.rs | 5 + 5 files changed, 609 insertions(+), 15 deletions(-) create mode 100644 src/seal.rs diff --git a/README.md b/README.md index 18bb3c3..f852b2a 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,8 @@ dumpling --allow-noop -i dump.sql -o out.sql # explicitly allow no-op when co dumpling --format sqlite -i data.db.sql -o out.sql # process a SQLite .dump file dumpling --format mssql -i backup.sql -o out.sql # process a SQL Server plain-SQL dump dumpling --security-profile hardened -i dump.sql -o sanitized.sql # hardened CSPRNG + HMAC mode +dumpling --emit-seal-line -i dump.sql -o sealed.sql # prefix output with policy/version fingerprint +dumpling --trust-sealed-dumps -i sealed.sql -o out.sql # pass through when leading seal matches dumpling lint-policy # lint the anonymization policy config dumpling lint-policy --config .dumplingconf # lint with explicit config path ``` @@ -108,6 +110,16 @@ If no configuration is found, Dumpling fails closed by default and exits non-zer The error output lists every checked location. Use `--allow-noop` to explicitly permit no-op behavior. +### Sealed dumps (`--emit-seal-line` / `--trust-sealed-dumps`) + +`--emit-seal-line` writes a first-line SQL comment after any replayed input prefix: + +`-- dumpling-seal: v=1 version= profile= sha256=<64 hex chars>` + +The `sha256` value is over canonical JSON that includes the Dumpling version, the active security profile, and a stable encoding of the resolved policy (rules, row filters, column cases, sensitive columns, output scan, and global salt). + +With `--trust-sealed-dumps`, if the input begins with a seal line that matches the current binary version, security profile, and policy fingerprint, Dumpling copies the remainder of the file through unchanged (no INSERT/COPY rewriting). This is opt-in and intended as a pipeline hint, not cryptographic proof. `--strict-coverage` cannot be combined with a matching trusted seal (table definitions are not scanned). `--emit-seal-line` is incompatible with `--check`. + --- ## Configuration (TOML) diff --git a/src/main.rs b/src/main.rs index 7194603..9e5be55 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ mod filter; mod lint; mod report; mod scan; +mod seal; mod settings; mod sql; mod transform; @@ -21,6 +22,9 @@ use anyhow::Context; use regex::Regex; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; +use seal::{ + compute_policy_sha256, format_seal_line, read_first_line_for_seal, FirstLineReplayBufRead, +}; use settings::ResolvedConfig; use sql::{DumpFormat, SqlStreamProcessor}; use transform::{set_hardened_profile, set_random_seed, AnonymizerRegistry, SecurityProfile}; @@ -110,6 +114,18 @@ struct Cli { #[arg(long = "security-profile", default_value = "standard")] security_profile: String, + /// Trust a leading `-- dumpling-seal:` comment when it matches this binary's version, the active + /// `--security-profile`, and a fingerprint of the loaded policy; the remainder of the dump is + /// copied through unchanged (no row/cell transforms). Fail-closed unless this flag is set. + #[arg(long = "trust-sealed-dumps", action = ArgAction::SetTrue)] + trust_sealed_dumps: bool, + + /// Write a fresh `-- dumpling-seal:` line at the start of output (after any replayed input prefix). + /// The seal encodes the Dumpling version, active security profile, and a SHA-256 fingerprint of + /// the resolved policy. Incompatible with `--check`. + #[arg(long = "emit-seal-line", action = ArgAction::SetTrue)] + emit_seal_line: bool, + /// Decode PostgreSQL custom-format or directory-format dumps via `pg_restore -f -` before anonymizing. /// Requires `--input` pointing at the archive file or directory and `--format postgres`. Requires a /// PostgreSQL client install (`pg_restore` on PATH unless overridden by `--pg-restore-path`). @@ -217,6 +233,9 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { if cli.dump_decode && cli.in_place { anyhow::bail!("--dump-decode cannot be used with --in-place"); } + if cli.emit_seal_line && cli.check { + anyhow::bail!("--emit-seal-line cannot be used with --check"); + } // Resolve config from provided path or discover in CWD let resolved_config: ResolvedConfig = @@ -429,7 +448,17 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { .unwrap_or_else(|| Reporter::new(false)); reporter.report.security_profile = security_profile_name.to_string(); - // Process SQL stream + let policy_digest = if cli.emit_seal_line && !cli.check { + Some(compute_policy_sha256( + &resolved_config, + security_profile_name, + )?) + } else { + None + }; + + let mut writer = output; + let mut processor = SqlStreamProcessor::new( anonymizers, resolved_config, @@ -438,12 +467,52 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { Some(&mut reporter), dump_format, ); - let mut writer = output; - let proc_res = if let Some(scanner) = output_scanner.as_mut() { - let mut scanning_writer = ScanningWriter::new(&mut writer, scanner); - processor.process(&mut reader, &mut scanning_writer) + + let (first_line, seal_skip) = read_first_line_for_seal( + reader.as_mut(), + processor.config_snapshot(), + security_profile_name, + cli.trust_sealed_dumps, + )?; + + if seal_skip && cli.strict_coverage { + anyhow::bail!( + "--strict-coverage cannot be used when the input begins with a matching trusted seal; \ + the dump is passed through without parsing table definitions" + ); + } + + let replay_first = if seal_skip || first_line.is_empty() { + None + } else { + Some(first_line.into_bytes()) + }; + let mut adapted_reader = FirstLineReplayBufRead::new(reader.as_mut(), replay_first); + + let proc_res: anyhow::Result<()> = if seal_skip { + if let Some(ref digest) = policy_digest { + writer.write_all(format_seal_line(security_profile_name, digest).as_bytes())?; + } + if let Some(scanner) = output_scanner.as_mut() { + let mut scanning_writer = ScanningWriter::new(&mut writer, scanner); + std::io::copy(&mut adapted_reader, &mut scanning_writer) + .map(|_| ()) + .map_err(anyhow::Error::from) + } else { + std::io::copy(&mut adapted_reader, &mut writer) + .map(|_| ()) + .map_err(anyhow::Error::from) + } } else { - processor.process(&mut reader, &mut writer) + if let Some(ref digest) = policy_digest { + writer.write_all(format_seal_line(security_profile_name, digest).as_bytes())?; + } + if let Some(scanner) = output_scanner.as_mut() { + let mut scanning_writer = ScanningWriter::new(&mut writer, scanner); + processor.process(&mut adapted_reader, &mut scanning_writer) + } else { + processor.process(&mut adapted_reader, &mut writer) + } }; if let Some(mut child) = pg_restore_child { @@ -591,7 +660,109 @@ fn has_allowed_extension(path: &Path, allow_exts: &[String]) -> bool { mod tests_main { use super::{has_allowed_extension, Cli, Commands}; use clap::Parser; + use std::fs; + use std::io::Read; use std::path::PathBuf; + use std::process::Command; + + #[test] + fn seal_emit_then_trust_roundtrip() { + let exe = match option_env!("CARGO_BIN_EXE_dumpling") { + Some(p) => PathBuf::from(p), + None => return, + }; + let base = + std::env::temp_dir().join(format!("dumpling_seal_integration_{}", std::process::id())); + let conf = base.with_extension("toml"); + let pass1_in = base.with_extension("p1.sql"); + let pass1_out = base.with_extension("p2.sql"); + let pass2_out = base.with_extension("p3.sql"); + + fs::write( + &conf, + r#" +[rules."public.users"] +email = { strategy = "email" } +"#, + ) + .unwrap(); + fs::write( + &pass1_in, + "INSERT INTO public.users (email) VALUES ('alice@example.com');\n", + ) + .unwrap(); + + let s1 = Command::new(&exe) + .args([ + "-c", + conf.to_str().unwrap(), + "-i", + pass1_in.to_str().unwrap(), + "-o", + pass1_out.to_str().unwrap(), + "--emit-seal-line", + "--seed", + "42", + ]) + .output() + .unwrap(); + assert!( + s1.status.success(), + "pass1 stderr={}", + String::from_utf8_lossy(&s1.stderr) + ); + + let mut sealed = String::new(); + fs::File::open(&pass1_out) + .unwrap() + .read_to_string(&mut sealed) + .unwrap(); + let first = sealed.lines().next().unwrap_or(""); + assert!( + first.starts_with("-- dumpling-seal:"), + "expected seal prefix, got: {first:?}" + ); + assert!( + !sealed.contains("alice@example.com"), + "expected anonymization in pass1" + ); + + let s2 = Command::new(&exe) + .args([ + "-c", + conf.to_str().unwrap(), + "-i", + pass1_out.to_str().unwrap(), + "-o", + pass2_out.to_str().unwrap(), + "--trust-sealed-dumps", + "--emit-seal-line", + ]) + .output() + .unwrap(); + assert!( + s2.status.success(), + "pass2 stderr={}", + String::from_utf8_lossy(&s2.stderr) + ); + + let mut final_out = String::new(); + fs::File::open(&pass2_out) + .unwrap() + .read_to_string(&mut final_out) + .unwrap(); + let rest_mid: String = sealed.lines().skip(1).collect::>().join("\n"); + let rest_out: String = final_out.lines().skip(1).collect::>().join("\n"); + assert_eq!( + rest_mid, rest_out, + "trusted pass-through should preserve dump body after seal line" + ); + + let _ = fs::remove_file(&conf); + let _ = fs::remove_file(&pass1_in); + let _ = fs::remove_file(&pass1_out); + let _ = fs::remove_file(&pass2_out); + } #[test] fn test_allowed_extensions() { diff --git a/src/seal.rs b/src/seal.rs new file mode 100644 index 0000000..af5d36e --- /dev/null +++ b/src/seal.rs @@ -0,0 +1,406 @@ +//! Optional dump seal: a leading SQL comment records the Dumpling version, security profile, +//! and a SHA-256 fingerprint of the resolved policy. With `--trust-sealed-dumps`, a matching +//! seal causes the rest of the dump to pass through unchanged. + +use crate::settings::{AnonymizerSpec, ColumnCase, RawConfig, ResolvedConfig, RowFilterSet}; +use serde::Serialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, HashMap}; +use std::io::{self, BufRead, Read}; + +pub const SEAL_LINE_PREFIX: &str = "-- dumpling-seal:"; + +/// JSON object key order is stabilized recursively so the fingerprint is deterministic. +#[derive(Debug, Serialize)] +struct SealFingerprintPayload { + format_version: u32, + dumpling_version: &'static str, + security_profile: String, + policy: RawConfig, +} + +/// Hex-encode a 32-byte digest (lowercase, no `0x` prefix). +pub fn sha256_hex_32(digest: &[u8; 32]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut s = String::with_capacity(64); + for b in digest { + s.push(HEX[(b >> 4) as usize] as char); + s.push(HEX[(b & 0xf) as usize] as char); + } + s +} + +fn parse_hex_32(s: &str) -> Option<[u8; 32]> { + if s.len() != 64 { + return None; + } + let mut out = [0u8; 32]; + for (i, chunk) in s.as_bytes().chunks(2).enumerate() { + let hi = hex_val(chunk[0])?; + let lo = hex_val(chunk.get(1).copied().unwrap_or(0))?; + out[i] = (hi << 4) | lo; + } + Some(out) +} + +fn hex_val(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(10 + (b - b'a')), + b'A'..=b'F' => Some(10 + (b - b'A')), + _ => None, + } +} + +fn resolved_to_raw_for_fingerprint(cfg: &ResolvedConfig) -> RawConfig { + let mut rules: HashMap> = HashMap::new(); + let mut table_keys: Vec = cfg.rules.keys().cloned().collect(); + table_keys.sort(); + for tk in table_keys { + let col_map = cfg.rules.get(&tk).unwrap(); + let mut col_keys: Vec = col_map.keys().cloned().collect(); + col_keys.sort(); + let mut inner = HashMap::new(); + for ck in col_keys { + inner.insert(ck.clone(), col_map.get(&ck).unwrap().clone()); + } + rules.insert(tk, inner); + } + + let mut row_filters: HashMap = HashMap::new(); + let mut rf_keys: Vec = cfg.row_filters.keys().cloned().collect(); + rf_keys.sort(); + for k in rf_keys { + row_filters.insert(k.clone(), cfg.row_filters.get(&k).unwrap().clone()); + } + + let mut column_cases: HashMap>> = HashMap::new(); + let mut case_table_keys: Vec = cfg.column_cases.keys().cloned().collect(); + case_table_keys.sort(); + for tk in case_table_keys { + let col_map = cfg.column_cases.get(&tk).unwrap(); + let mut col_keys: Vec = col_map.keys().cloned().collect(); + col_keys.sort(); + let mut inner: HashMap> = HashMap::new(); + for ck in col_keys { + inner.insert(ck.clone(), col_map.get(&ck).unwrap().clone()); + } + column_cases.insert(tk, inner); + } + + let mut sensitive_columns: HashMap> = HashMap::new(); + let mut sens_keys: Vec = cfg.sensitive_columns.keys().cloned().collect(); + sens_keys.sort(); + for tk in sens_keys { + let set = cfg.sensitive_columns.get(&tk).unwrap(); + let mut cols: Vec = set.iter().cloned().collect(); + cols.sort(); + sensitive_columns.insert(tk, cols); + } + + RawConfig { + salt: cfg.salt.clone(), + rules, + row_filters, + column_cases, + table_options: HashMap::new(), + sensitive_columns, + output_scan: cfg.output_scan.clone(), + } +} + +fn sort_json_value(v: &mut Value) { + match v { + Value::Object(map) => { + let keys: Vec = map.keys().cloned().collect(); + let mut sorted: BTreeMap = BTreeMap::new(); + for k in keys { + if let Some(mut child) = map.remove(&k) { + sort_json_value(&mut child); + sorted.insert(k, child); + } + } + *map = sorted.into_iter().collect(); + } + Value::Array(arr) => { + for item in arr { + sort_json_value(item); + } + } + _ => {} + } +} + +/// SHA-256 of stable JSON for the policy plus format version, crate version, and security profile. +pub fn compute_policy_sha256( + cfg: &ResolvedConfig, + security_profile: &str, +) -> anyhow::Result<[u8; 32]> { + let payload = SealFingerprintPayload { + format_version: 1, + dumpling_version: env!("CARGO_PKG_VERSION"), + security_profile: security_profile.to_string(), + policy: resolved_to_raw_for_fingerprint(cfg), + }; + let mut val = serde_json::to_value(&payload)?; + sort_json_value(&mut val); + let bytes = serde_json::to_vec(&val)?; + let digest = Sha256::digest(&bytes); + Ok(digest.into()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedSeal { + pub format_version: u32, + pub dumpling_version: String, + pub security_profile: String, + pub sha256: [u8; 32], +} + +/// Parse a single-line SQL comment body after `-- dumpling-seal:` (caller passes full line). +pub fn parse_seal_line(line: &str) -> Option { + let trimmed = line.trim_end_matches(['\r', '\n', ' ']); + let rest = trimmed.strip_prefix(SEAL_LINE_PREFIX)?.trim_start(); + let mut format_version: Option = None; + let mut dumpling_version: Option = None; + let mut security_profile: Option = None; + let mut sha256: Option<[u8; 32]> = None; + + for token in rest.split_whitespace() { + let (k, v) = token.split_once('=')?; + match k { + "v" => { + format_version = Some(v.parse().ok()?); + } + "version" => { + dumpling_version = Some(v.to_string()); + } + "profile" => { + security_profile = Some(v.to_ascii_lowercase()); + } + "sha256" => { + sha256 = parse_hex_32(v); + } + _ => {} + } + } + + Some(ParsedSeal { + format_version: format_version?, + dumpling_version: dumpling_version?, + security_profile: security_profile?, + sha256: sha256?, + }) +} + +pub fn format_seal_line(security_profile: &str, digest: &[u8; 32]) -> String { + format!( + "{} v=1 version={} profile={} sha256={}\n", + SEAL_LINE_PREFIX, + env!("CARGO_PKG_VERSION"), + security_profile.to_ascii_lowercase(), + sha256_hex_32(digest) + ) +} + +pub fn seal_matches_current( + parsed: &ParsedSeal, + cfg: &ResolvedConfig, + security_profile: &str, +) -> anyhow::Result { + if parsed.format_version != 1 { + return Ok(false); + } + if parsed.dumpling_version != env!("CARGO_PKG_VERSION") { + return Ok(false); + } + if parsed.security_profile != security_profile.to_ascii_lowercase() { + return Ok(false); + } + let expected = compute_policy_sha256(cfg, security_profile)?; + Ok(parsed.sha256 == expected) +} + +/// Wraps a `BufRead` and optionally replays the first line (bytes) before delegating. +pub struct FirstLineReplayBufRead<'a> { + inner: &'a mut dyn BufRead, + replay: Option>, + pos: usize, +} + +impl<'a> FirstLineReplayBufRead<'a> { + pub fn new(inner: &'a mut dyn BufRead, replay: Option>) -> Self { + Self { + inner, + replay, + pos: 0, + } + } +} + +impl Read for FirstLineReplayBufRead<'_> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if let Some(ref r) = self.replay { + if self.pos < r.len() { + let n = (r.len() - self.pos).min(buf.len()); + buf[..n].copy_from_slice(&r[self.pos..self.pos + n]); + self.pos += n; + if self.pos >= r.len() { + self.replay = None; + self.pos = 0; + } + return Ok(n); + } + } + self.inner.read(buf) + } +} + +impl BufRead for FirstLineReplayBufRead<'_> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + if let Some(ref r) = self.replay { + if self.pos < r.len() { + return Ok(&r[self.pos..]); + } + } + self.inner.fill_buf() + } + + fn consume(&mut self, amt: usize) { + if let Some(ref r) = self.replay { + if self.pos < r.len() { + self.pos += amt; + if self.pos >= r.len() { + self.replay = None; + self.pos = 0; + } + return; + } + } + self.inner.consume(amt); + } +} + +/// Read the first line from `reader` and return `(first_line, skip_first_line)`. +/// If `skip_first_line` is true, the first line matched a trusted seal and must not be replayed. +pub fn read_first_line_for_seal( + reader: &mut dyn BufRead, + cfg: &ResolvedConfig, + security_profile: &str, + trust_sealed: bool, +) -> anyhow::Result<(String, bool)> { + let mut first = String::new(); + let n = reader.read_line(&mut first)?; + if n == 0 { + return Ok((String::new(), false)); + } + + let skip_first = if trust_sealed && first.trim_start().starts_with(SEAL_LINE_PREFIX) { + match parse_seal_line(&first) { + Some(p) => seal_matches_current(&p, cfg, security_profile)?, + None => false, + } + } else { + false + }; + + if skip_first { + eprintln!("dumpling: sealed dump header matches current version, profile, and policy; passing through unchanged"); + } + + Ok((first, skip_first)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::settings::OutputScanConfig; + + fn minimal_cfg() -> ResolvedConfig { + ResolvedConfig { + salt: None, + rules: HashMap::new(), + row_filters: HashMap::new(), + column_cases: HashMap::new(), + sensitive_columns: HashMap::new(), + output_scan: OutputScanConfig::default(), + source_path: None, + } + } + + #[test] + fn seal_line_round_trip() { + let cfg = minimal_cfg(); + let digest = compute_policy_sha256(&cfg, "standard").unwrap(); + let line = format_seal_line("standard", &digest); + let parsed = parse_seal_line(&line).expect("parse"); + assert!(seal_matches_current(&parsed, &cfg, "standard").unwrap()); + } + + #[test] + fn wrong_version_does_not_match() { + let cfg = minimal_cfg(); + let digest = compute_policy_sha256(&cfg, "standard").unwrap(); + let mut line = format_seal_line("standard", &digest); + line = line.replace( + &format!("version={}", env!("CARGO_PKG_VERSION")), + "version=0.0.0-wrong", + ); + let parsed = parse_seal_line(&line).expect("parse"); + assert!(!seal_matches_current(&parsed, &cfg, "standard").unwrap()); + } + + #[test] + fn fingerprint_stable_across_map_insert_order() { + let mut cfg1 = minimal_cfg(); + let mut cfg2 = minimal_cfg(); + let mut r1 = HashMap::new(); + r1.insert( + "email".into(), + AnonymizerSpec { + strategy: "email".into(), + salt: None, + min: None, + max: None, + length: None, + min_days: None, + max_days: None, + min_seconds: None, + max_seconds: None, + domain: None, + unique_within_domain: None, + as_string: None, + locale: None, + faker: None, + format: None, + }, + ); + cfg1.rules.insert("users".into(), r1.clone()); + let mut r2 = HashMap::new(); + r2.insert( + "email".into(), + AnonymizerSpec { + strategy: "email".into(), + salt: None, + min: None, + max: None, + length: None, + min_days: None, + max_days: None, + min_seconds: None, + max_seconds: None, + domain: None, + unique_within_domain: None, + as_string: None, + locale: None, + faker: None, + format: None, + }, + ); + cfg2.rules.insert("users".into(), r2); + let h1 = compute_policy_sha256(&cfg1, "standard").unwrap(); + let h2 = compute_policy_sha256(&cfg2, "standard").unwrap(); + assert_eq!(h1, h2); + } +} diff --git a/src/settings.rs b/src/settings.rs index fed2f08..79ebf1f 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1,10 +1,10 @@ use anyhow::Context; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct RawConfig { /// Optional default salt used by certain anonymizers (e.g., hash) pub salt: Option, @@ -30,7 +30,7 @@ pub struct RawConfig { pub output_scan: OutputScanConfig, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct AnonymizerSpec { /// Strategy name: redact|null|uuid|hash|email|name|first_name|last_name|phone|faker|int_range|string|date_fuzz|time_fuzz|datetime_fuzz pub strategy: String, @@ -87,7 +87,7 @@ pub struct ResolvedConfig { pub source_path: Option, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct OutputScanConfig { /// Optional category allowlist; default enables all built-ins. #[serde(default)] @@ -126,7 +126,7 @@ impl Default for OutputScanConfig { } } -#[derive(Debug, Clone, Deserialize, Default)] +#[derive(Debug, Clone, Deserialize, Serialize, Default)] pub struct TableOptions { /// Deprecated: retained for config parsing so we can fail with a targeted message. #[serde(default, rename = "auto")] @@ -990,7 +990,7 @@ pub fn lookup_column_rule<'a>( .and_then(|cols| lookup_whole_column_rule_in_map(cols, &column_norm)) } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct RowFilterSet { /// Keep a row if at least one predicate matches (when non-empty) /// Preferred name: retain. Back-compat alias: include_any. @@ -1002,7 +1002,7 @@ pub struct RowFilterSet { pub delete: Vec, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct Predicate { /// Column name to check (case-insensitive, unquoted). /// Supports nested JSON path targeting via: @@ -1038,7 +1038,7 @@ pub fn lookup_row_filters<'a>( cfg.row_filters.get(&key) } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct ColumnCase { /// Conditions for this case #[serde(default)] @@ -1047,7 +1047,7 @@ pub struct ColumnCase { pub strategy: AnonymizerSpec, } -#[derive(Debug, Clone, Default, Deserialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct When { /// Any-of predicates (OR) #[serde(default)] diff --git a/src/sql.rs b/src/sql.rs index 8d40c21..8584f39 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -67,6 +67,11 @@ impl SqlStreamProcessor { } } + /// Borrow the resolved config for fingerprinting (e.g. sealed-dump checks) without cloning. + pub fn config_snapshot(&self) -> &ResolvedConfig { + &self.config + } + pub fn sensitive_coverage_summary(&self) -> SensitiveCoverageSummary { let mut detected = self .sensitive_columns_detected From 80da65a14d3d82924a53113dd11a86aac7174961 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 3 May 2026 17:37:08 +0000 Subject: [PATCH 2/2] feat(seal): always-on v2 fingerprint with runtime options - Remove --emit-seal-line and --trust-sealed-dumps; seal is written on every non-check output and matching seals always enable passthrough - Bump seal payload to v=2; hash includes policy plus runtime params (format, sorted include/exclude patterns, PRNG seed override in standard) - Strip a non-matching leading seal so re-runs after config/flag changes do not duplicate seal lines - Add prng_seed_override_for_fingerprint() for seal vs hardened behavior Co-authored-by: Andy Babic --- README.md | 12 ++--- src/main.rs | 80 +++++++++++++-------------- src/seal.rs | 138 ++++++++++++++++++++++++++++++++++++----------- src/transform.rs | 10 ++++ 4 files changed, 160 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index f852b2a..301ff22 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,6 @@ dumpling --allow-noop -i dump.sql -o out.sql # explicitly allow no-op when co dumpling --format sqlite -i data.db.sql -o out.sql # process a SQLite .dump file dumpling --format mssql -i backup.sql -o out.sql # process a SQL Server plain-SQL dump dumpling --security-profile hardened -i dump.sql -o sanitized.sql # hardened CSPRNG + HMAC mode -dumpling --emit-seal-line -i dump.sql -o sealed.sql # prefix output with policy/version fingerprint -dumpling --trust-sealed-dumps -i sealed.sql -o out.sql # pass through when leading seal matches dumpling lint-policy # lint the anonymization policy config dumpling lint-policy --config .dumplingconf # lint with explicit config path ``` @@ -110,15 +108,15 @@ If no configuration is found, Dumpling fails closed by default and exits non-zer The error output lists every checked location. Use `--allow-noop` to explicitly permit no-op behavior. -### Sealed dumps (`--emit-seal-line` / `--trust-sealed-dumps`) +### Dump seal (always on) -`--emit-seal-line` writes a first-line SQL comment after any replayed input prefix: +Every successful run that writes output prefixes the stream with a single-line SQL comment: -`-- dumpling-seal: v=1 version= profile= sha256=<64 hex chars>` +`-- dumpling-seal: v=2 version= profile= sha256=<64 hex chars>` -The `sha256` value is over canonical JSON that includes the Dumpling version, the active security profile, and a stable encoding of the resolved policy (rules, row filters, column cases, sensitive columns, output scan, and global salt). +The `sha256` is over canonical JSON that includes the Dumpling version, the active security profile, a stable encoding of the resolved policy (rules, row filters, column cases, sensitive columns, output scan, global salt), and **runtime options** that affect transforms: `--format`, sorted `--include-table` / `--exclude-table` patterns, and the effective `--seed` / `DUMPLING_SEED` value in standard profile (`null` in hardened, where seeds are ignored). -With `--trust-sealed-dumps`, if the input begins with a seal line that matches the current binary version, security profile, and policy fingerprint, Dumpling copies the remainder of the file through unchanged (no INSERT/COPY rewriting). This is opt-in and intended as a pipeline hint, not cryptographic proof. `--strict-coverage` cannot be combined with a matching trusted seal (table definitions are not scanned). `--emit-seal-line` is incompatible with `--check`. +If the **input** already begins with a seal line and it **matches** the current run, Dumpling copies the rest of the file through unchanged. If the line looks like a seal but does **not** match (stale policy, different flags, or older `v=`), that line is **dropped** and the dump is re-processed so you do not end up with two seal lines. `--strict-coverage` cannot be combined with a matching seal (table definitions are not scanned in passthrough mode). `--check` writes no output and therefore emits no seal line. --- diff --git a/src/main.rs b/src/main.rs index 9e5be55..2ac3d23 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,11 +23,15 @@ use regex::Regex; use report::Reporter; use scan::{OutputScanner, ScanningWriter}; use seal::{ - compute_policy_sha256, format_seal_line, read_first_line_for_seal, FirstLineReplayBufRead, + compute_seal_digest, format_seal_line, read_first_line_for_seal, FirstLineReplayBufRead, + SealFirstLine, SealRuntimeParams, }; use settings::ResolvedConfig; use sql::{DumpFormat, SqlStreamProcessor}; -use transform::{set_hardened_profile, set_random_seed, AnonymizerRegistry, SecurityProfile}; +use transform::{ + prng_seed_override_for_fingerprint, set_hardened_profile, set_random_seed, AnonymizerRegistry, + SecurityProfile, +}; #[derive(Parser, Debug)] #[command( @@ -114,18 +118,6 @@ struct Cli { #[arg(long = "security-profile", default_value = "standard")] security_profile: String, - /// Trust a leading `-- dumpling-seal:` comment when it matches this binary's version, the active - /// `--security-profile`, and a fingerprint of the loaded policy; the remainder of the dump is - /// copied through unchanged (no row/cell transforms). Fail-closed unless this flag is set. - #[arg(long = "trust-sealed-dumps", action = ArgAction::SetTrue)] - trust_sealed_dumps: bool, - - /// Write a fresh `-- dumpling-seal:` line at the start of output (after any replayed input prefix). - /// The seal encodes the Dumpling version, active security profile, and a SHA-256 fingerprint of - /// the resolved policy. Incompatible with `--check`. - #[arg(long = "emit-seal-line", action = ArgAction::SetTrue)] - emit_seal_line: bool, - /// Decode PostgreSQL custom-format or directory-format dumps via `pg_restore -f -` before anonymizing. /// Requires `--input` pointing at the archive file or directory and `--format postgres`. Requires a /// PostgreSQL client install (`pg_restore` on PATH unless overridden by `--pg-restore-path`). @@ -233,9 +225,6 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { if cli.dump_decode && cli.in_place { anyhow::bail!("--dump-decode cannot be used with --in-place"); } - if cli.emit_seal_line && cli.check { - anyhow::bail!("--emit-seal-line cannot be used with --check"); - } // Resolve config from provided path or discover in CWD let resolved_config: ResolvedConfig = @@ -309,6 +298,13 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { let include_res = compile_patterns(&cli.include_table)?; let exclude_res = compile_patterns(&cli.exclude_table)?; + let seal_runtime = SealRuntimeParams::new( + dump_format, + &cli.include_table, + &cli.exclude_table, + prng_seed_override_for_fingerprint(), + ); + // Determine IO (optional pg_restore child when --dump-decode) let mut pg_restore_child: Option = None; let (mut reader, input_path_for_inplace): (Box, Option) = if cli @@ -448,17 +444,6 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { .unwrap_or_else(|| Reporter::new(false)); reporter.report.security_profile = security_profile_name.to_string(); - let policy_digest = if cli.emit_seal_line && !cli.check { - Some(compute_policy_sha256( - &resolved_config, - security_profile_name, - )?) - } else { - None - }; - - let mut writer = output; - let mut processor = SqlStreamProcessor::new( anonymizers, resolved_config, @@ -468,29 +453,41 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { dump_format, ); - let (first_line, seal_skip) = read_first_line_for_seal( + let seal_digest = if cli.check { + None + } else { + Some(compute_seal_digest( + processor.config_snapshot(), + security_profile_name, + &seal_runtime, + )?) + }; + + let mut writer = output; + + let seal_first = read_first_line_for_seal( reader.as_mut(), processor.config_snapshot(), security_profile_name, - cli.trust_sealed_dumps, + &seal_runtime, )?; - if seal_skip && cli.strict_coverage { + if matches!(seal_first, SealFirstLine::TrustedPassthrough) && cli.strict_coverage { anyhow::bail!( - "--strict-coverage cannot be used when the input begins with a matching trusted seal; \ + "--strict-coverage cannot be used when the input begins with a matching seal; \ the dump is passed through without parsing table definitions" ); } - let replay_first = if seal_skip || first_line.is_empty() { - None - } else { - Some(first_line.into_bytes()) + let replay_first = match &seal_first { + SealFirstLine::TrustedPassthrough | SealFirstLine::StaleSealStripped => None, + SealFirstLine::Replay(v) if v.is_empty() => None, + SealFirstLine::Replay(v) => Some(v.clone()), }; let mut adapted_reader = FirstLineReplayBufRead::new(reader.as_mut(), replay_first); - let proc_res: anyhow::Result<()> = if seal_skip { - if let Some(ref digest) = policy_digest { + let proc_res: anyhow::Result<()> = if matches!(seal_first, SealFirstLine::TrustedPassthrough) { + if let Some(ref digest) = seal_digest { writer.write_all(format_seal_line(security_profile_name, digest).as_bytes())?; } if let Some(scanner) = output_scanner.as_mut() { @@ -504,7 +501,7 @@ fn run_anonymize(cli: Cli) -> anyhow::Result<()> { .map_err(anyhow::Error::from) } } else { - if let Some(ref digest) = policy_digest { + if let Some(ref digest) = seal_digest { writer.write_all(format_seal_line(security_profile_name, digest).as_bytes())?; } if let Some(scanner) = output_scanner.as_mut() { @@ -700,7 +697,6 @@ email = { strategy = "email" } pass1_in.to_str().unwrap(), "-o", pass1_out.to_str().unwrap(), - "--emit-seal-line", "--seed", "42", ]) @@ -735,8 +731,8 @@ email = { strategy = "email" } pass1_out.to_str().unwrap(), "-o", pass2_out.to_str().unwrap(), - "--trust-sealed-dumps", - "--emit-seal-line", + "--seed", + "42", ]) .output() .unwrap(); diff --git a/src/seal.rs b/src/seal.rs index af5d36e..af3c3a3 100644 --- a/src/seal.rs +++ b/src/seal.rs @@ -1,8 +1,9 @@ -//! Optional dump seal: a leading SQL comment records the Dumpling version, security profile, -//! and a SHA-256 fingerprint of the resolved policy. With `--trust-sealed-dumps`, a matching -//! seal causes the rest of the dump to pass through unchanged. +//! Dump seal: a leading SQL comment records the Dumpling version, security profile, a SHA-256 +//! fingerprint of the resolved policy, and runtime CLI options that affect transforms. When the +//! first line matches, the remainder of the dump is copied through unchanged. use crate::settings::{AnonymizerSpec, ColumnCase, RawConfig, ResolvedConfig, RowFilterSet}; +use crate::sql::DumpFormat; use serde::Serialize; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -11,6 +12,44 @@ use std::io::{self, BufRead, Read}; pub const SEAL_LINE_PREFIX: &str = "-- dumpling-seal:"; +/// Bump when the JSON payload shape changes (parsers accept the `v=` field on the comment line). +pub const SEAL_PAYLOAD_VERSION: u32 = 2; + +#[derive(Debug, Serialize)] +pub struct SealRuntimeParams { + pub dump_format: String, + pub include_table: Vec, + pub exclude_table: Vec, + /// `Some` when standard profile and `--seed` / `DUMPLING_SEED` is set; `None` otherwise. + pub prng_seed: Option, +} + +impl SealRuntimeParams { + pub fn new( + dump_format: DumpFormat, + include_table: &[String], + exclude_table: &[String], + prng_seed: Option, + ) -> Self { + let dump_format = match dump_format { + DumpFormat::Postgres => "postgres", + DumpFormat::Sqlite => "sqlite", + DumpFormat::MsSql => "mssql", + } + .to_string(); + let mut include_table: Vec = include_table.to_vec(); + include_table.sort(); + let mut exclude_table: Vec = exclude_table.to_vec(); + exclude_table.sort(); + Self { + dump_format, + include_table, + exclude_table, + prng_seed, + } + } +} + /// JSON object key order is stabilized recursively so the fingerprint is deterministic. #[derive(Debug, Serialize)] struct SealFingerprintPayload { @@ -18,6 +57,7 @@ struct SealFingerprintPayload { dumpling_version: &'static str, security_profile: String, policy: RawConfig, + runtime: SealRuntimeParams, } /// Hex-encode a 32-byte digest (lowercase, no `0x` prefix). @@ -132,16 +172,23 @@ fn sort_json_value(v: &mut Value) { } } -/// SHA-256 of stable JSON for the policy plus format version, crate version, and security profile. -pub fn compute_policy_sha256( +/// SHA-256 of stable JSON: version, crate semver, security profile, resolved policy, and runtime params. +pub fn compute_seal_digest( cfg: &ResolvedConfig, security_profile: &str, + runtime: &SealRuntimeParams, ) -> anyhow::Result<[u8; 32]> { let payload = SealFingerprintPayload { - format_version: 1, + format_version: SEAL_PAYLOAD_VERSION, dumpling_version: env!("CARGO_PKG_VERSION"), security_profile: security_profile.to_string(), policy: resolved_to_raw_for_fingerprint(cfg), + runtime: SealRuntimeParams { + dump_format: runtime.dump_format.clone(), + include_table: runtime.include_table.clone(), + exclude_table: runtime.exclude_table.clone(), + prng_seed: runtime.prng_seed, + }, }; let mut val = serde_json::to_value(&payload)?; sort_json_value(&mut val); @@ -196,8 +243,9 @@ pub fn parse_seal_line(line: &str) -> Option { pub fn format_seal_line(security_profile: &str, digest: &[u8; 32]) -> String { format!( - "{} v=1 version={} profile={} sha256={}\n", + "{} v={} version={} profile={} sha256={}\n", SEAL_LINE_PREFIX, + SEAL_PAYLOAD_VERSION, env!("CARGO_PKG_VERSION"), security_profile.to_ascii_lowercase(), sha256_hex_32(digest) @@ -208,8 +256,9 @@ pub fn seal_matches_current( parsed: &ParsedSeal, cfg: &ResolvedConfig, security_profile: &str, + runtime: &SealRuntimeParams, ) -> anyhow::Result { - if parsed.format_version != 1 { + if parsed.format_version != SEAL_PAYLOAD_VERSION { return Ok(false); } if parsed.dumpling_version != env!("CARGO_PKG_VERSION") { @@ -218,11 +267,21 @@ pub fn seal_matches_current( if parsed.security_profile != security_profile.to_ascii_lowercase() { return Ok(false); } - let expected = compute_policy_sha256(cfg, security_profile)?; + let expected = compute_seal_digest(cfg, security_profile, runtime)?; Ok(parsed.sha256 == expected) } -/// Wraps a `BufRead` and optionally replays the first line (bytes) before delegating. +/// Outcome of reading the first line for seal handling. +pub enum SealFirstLine { + /// Seal matched: first line consumed; stream continues at byte after the seal line. + TrustedPassthrough, + /// First line was a stale seal (wrong fingerprint/version/etc.): dropped; stream continues after it. + StaleSealStripped, + /// Replay these bytes (UTF-8 first line including newline) before the rest of the stream. + Replay(Vec), +} + +/// Wraps a `BufRead` and optionally replays bytes before delegating to `inner`. pub struct FirstLineReplayBufRead<'a> { inner: &'a mut dyn BufRead, replay: Option>, @@ -282,34 +341,33 @@ impl BufRead for FirstLineReplayBufRead<'_> { } } -/// Read the first line from `reader` and return `(first_line, skip_first_line)`. -/// If `skip_first_line` is true, the first line matched a trusted seal and must not be replayed. +/// Read the first line and decide trusted passthrough, stale seal strip, or replay of the first line. pub fn read_first_line_for_seal( reader: &mut dyn BufRead, cfg: &ResolvedConfig, security_profile: &str, - trust_sealed: bool, -) -> anyhow::Result<(String, bool)> { + runtime: &SealRuntimeParams, +) -> anyhow::Result { let mut first = String::new(); let n = reader.read_line(&mut first)?; if n == 0 { - return Ok((String::new(), false)); + return Ok(SealFirstLine::Replay(Vec::new())); } - let skip_first = if trust_sealed && first.trim_start().starts_with(SEAL_LINE_PREFIX) { - match parse_seal_line(&first) { - Some(p) => seal_matches_current(&p, cfg, security_profile)?, + if first.trim_start().starts_with(SEAL_LINE_PREFIX) { + let skip_first = match parse_seal_line(&first) { + Some(p) => seal_matches_current(&p, cfg, security_profile, runtime)?, None => false, + }; + if skip_first { + eprintln!("dumpling: sealed dump header matches current version, profile, policy, and runtime options; passing through unchanged"); + return Ok(SealFirstLine::TrustedPassthrough); } - } else { - false - }; - - if skip_first { - eprintln!("dumpling: sealed dump header matches current version, profile, and policy; passing through unchanged"); + eprintln!("dumpling: ignoring stale leading seal line (config, version, profile, or runtime options differ)"); + return Ok(SealFirstLine::StaleSealStripped); } - Ok((first, skip_first)) + Ok(SealFirstLine::Replay(first.into_bytes())) } #[cfg(test)] @@ -329,26 +387,43 @@ mod tests { } } + fn default_runtime() -> SealRuntimeParams { + SealRuntimeParams::new(DumpFormat::Postgres, &[], &[], None) + } + #[test] fn seal_line_round_trip() { let cfg = minimal_cfg(); - let digest = compute_policy_sha256(&cfg, "standard").unwrap(); + let rt = default_runtime(); + let digest = compute_seal_digest(&cfg, "standard", &rt).unwrap(); let line = format_seal_line("standard", &digest); let parsed = parse_seal_line(&line).expect("parse"); - assert!(seal_matches_current(&parsed, &cfg, "standard").unwrap()); + assert!(seal_matches_current(&parsed, &cfg, "standard", &rt).unwrap()); } #[test] fn wrong_version_does_not_match() { let cfg = minimal_cfg(); - let digest = compute_policy_sha256(&cfg, "standard").unwrap(); + let rt = default_runtime(); + let digest = compute_seal_digest(&cfg, "standard", &rt).unwrap(); let mut line = format_seal_line("standard", &digest); line = line.replace( &format!("version={}", env!("CARGO_PKG_VERSION")), "version=0.0.0-wrong", ); let parsed = parse_seal_line(&line).expect("parse"); - assert!(!seal_matches_current(&parsed, &cfg, "standard").unwrap()); + assert!(!seal_matches_current(&parsed, &cfg, "standard", &rt).unwrap()); + } + + #[test] + fn runtime_options_change_digest() { + let cfg = minimal_cfg(); + let rt1 = SealRuntimeParams::new(DumpFormat::Postgres, &[], &[], Some(42)); + let rt2 = SealRuntimeParams::new(DumpFormat::Postgres, &[], &[], Some(43)); + assert_ne!( + compute_seal_digest(&cfg, "standard", &rt1).unwrap(), + compute_seal_digest(&cfg, "standard", &rt2).unwrap() + ); } #[test] @@ -399,8 +474,9 @@ mod tests { }, ); cfg2.rules.insert("users".into(), r2); - let h1 = compute_policy_sha256(&cfg1, "standard").unwrap(); - let h2 = compute_policy_sha256(&cfg2, "standard").unwrap(); + let rt = default_runtime(); + let h1 = compute_seal_digest(&cfg1, "standard", &rt).unwrap(); + let h2 = compute_seal_digest(&cfg2, "standard", &rt).unwrap(); assert_eq!(h1, h2); } } diff --git a/src/transform.rs b/src/transform.rs index 871b8a9..0b3231b 100644 --- a/src/transform.rs +++ b/src/transform.rs @@ -860,6 +860,16 @@ pub fn set_random_seed(seed: u64) { STATE.store(seed | 1, Ordering::Relaxed); } +/// Active `--seed` / `DUMPLING_SEED` override for the standard profile, if any (`None` means +/// non-reproducible system-time seeding). Used for dump seal fingerprints; hardened mode ignores +/// CLI/env seeds at runtime, so this returns `None` when the hardened profile is active. +pub fn prng_seed_override_for_fingerprint() -> Option { + if is_hardened_profile() { + return None; + } + unsafe { RNG_SEED_OVERRIDE } +} + fn fuzz_date(input: &str, shift_days: i64) -> Option { // Try parse as YYYY-MM-DD let trimmed = input.trim();