diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de757..2911ecb 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -1,14 +1,17 @@ //! GET /ipfs/{cid} — content-addressed retrieval of git objects by CIDv1. //! -//! Every git object stored on this node is addressable by its IPFS CIDv1. +//! Every git object pinned on this node is addressable by its IPFS CIDv1. //! The CID is computed as: //! //! CIDv1(codec=raw, multihash=sha2-256(content_bytes)) //! //! where `content_bytes` is the raw object content as returned by -//! `git cat-file ` (i.e. without the git framing header). -//! This is consistent with how `gitlawb_core::cid::Cid::from_git_object_bytes` -//! computes CIDs when objects are pushed. +//! `git cat-file ` (i.e. without the git framing header) — the +//! same bytes `gitlawb_core::cid::Cid::from_git_object_bytes` hashes when the +//! object is pinned. That digest is NOT the object's git oid: git frames the +//! content with a `" \0"` header before hashing, so `sha2-256(content)` +//! and the git oid differ. The handler therefore maps the CID back to its oid via +//! the `pinned_cids` table rather than treating the digest as an oid (#173). //! //! Serving is access-controlled: an object is returned only from a repo row the //! requesting caller is permitted to read (per-caller path-scoped visibility, @@ -27,14 +30,17 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{ + allowed_blob_set_for_caller, allowed_tree_set_for_caller, has_path_scoped_rule, +}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; /// GET /ipfs/{cid} /// -/// Search all repos on the node for a git object whose SHA-256 hash matches -/// the given CIDv1, returning its raw content if the caller may read it. +/// Resolve the CIDv1 to its git oid via the `pinned_cids` table, then search all +/// repos on the node for that object, returning its raw content if the caller may +/// read it. /// /// Visibility (#110, #126): the object is served only from a repo row the /// caller passes. For each iterated row we gate against that row's OWN rules @@ -43,13 +49,15 @@ use crate::visibility::{visibility_check, Decision}; /// row than the one read (KTD2a). We check object existence via /// `store::object_type` *before* the expensive reachability walk so random-CID /// spray cannot trigger full-history git walks on repos that don't carry the -/// object. When the row carries path-scoped rules (KTD4) the served object -/// must be either a non-blob (trees/commits are structural; KTD3) OR a blob -/// in the caller's *reachable* allowed-set (`allowed_blob_set_for_caller`). -/// The reachable allowed-set excludes dangling blobs — a blob written via -/// `git hash-object -w` and never committed has no path to gate, so it is -/// fail-closed 404'd under path-scoped rules (#126). Denial and genuine -/// not-found both fall through to an opaque 404. +/// object. When the row carries path-scoped rules (KTD4) the served object must +/// be either a `commit`/`tag` (root-level metadata the caller already cleared the +/// `"/"` gate for) OR a `blob`/`tree` in the caller's *reachable* allowed-set +/// (`allowed_blob_set_for_caller` / `allowed_tree_set_for_caller`). A withheld +/// subtree's tree object is denied here exactly as `get_tree` denies its path, so +/// its child names and oids cannot leak by CID (#135). The reachable allowed-sets +/// exclude dangling objects — a blob or tree written via plumbing and never +/// committed has no path to gate, so it is fail-closed 404'd under path-scoped +/// rules (#126). Denial and genuine not-found both fall through to an opaque 404. /// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked @@ -57,9 +65,12 @@ use crate::visibility::{visibility_check, Decision}; pub async fn get_by_cid( Path(cid_str): Path, State(state): State, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: HeaderMap, auth: Option>, ) -> Result { - // 1. Decode the CID and extract the SHA-256 digest + // 1. Decode and validate the CID (uniform 400 on a malformed / non-sha2-256 + // CID, before any DB or git work). let cid = CidGeneric::<64>::from_str(&cid_str) .map_err(|e| AppError::BadRequest(format!("invalid CID: {e}")))?; @@ -72,7 +83,34 @@ pub async fn get_by_cid( )); } - let sha256_hex = hex::encode(mh.digest()); + // Canonicalize the CID for the pinned_cids lookup. Pins are stored under the + // canonical base32 `cid.to_string()`, but a client may send any equivalent + // multibase spelling (base58/base64) of the same CID; those parse and pass + // the sha2-256 check yet miss the canonical key, so they must be normalized + // before the DB lookup (#173). Response headers and error messages still echo + // the original `cid_str` the client sent. + let canonical_cid = cid.to_string(); + + // Resolve the content-addressed CID to the object's git oid(s). A real pin + // CID digests the raw object content (`Cid::from_git_object_bytes`), NOT the + // git oid (git frames content with a `" \0"` header first), so we + // map it back through `pinned_cids` rather than treating the digest as an oid + // (#173). The cid index is non-unique, so one CID can map to several oids (a + // tree and a blob whose raw bytes collide, or content pinned under two oids); + // we try each candidate below rather than pick one arbitrarily and false-404 + // when the chosen one is withheld or absent while another is readable (#173). + // An empty result is an opaque 404, uniform with a genuine not-found and a + // visibility denial. + let oids = state + .db + .oids_for_cid(&canonical_cid) + .await + .map_err(AppError::Internal)?; + if oids.is_empty() { + return Err(AppError::RepoNotFound(format!( + "no git object found for CID {cid_str}" + ))); + } let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); @@ -102,107 +140,172 @@ pub async fn get_by_cid( // (`allowed_blob_set_for_caller`) so dangling blobs — never enumerated by // the reachable walk — fail closed instead of slipping through an empty // deny entry (#126). - let mut allowed_memo: HashMap> = HashMap::new(); - - for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). - let rules: &[crate::db::VisibilityRule] = rules_by_repo - .get(&repo.id) - .map(Vec::as_slice) - .unwrap_or(&[]); - if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") == Decision::Deny { - continue; - } + let mut allowed_blob_memo: HashMap> = HashMap::new(); + // The tree analog (#135): a withheld subtree's tree object is gated the same way + // a withheld blob is, so its structure cannot leak by CID where get_tree protects + // it. Built lazily and only for a tree fetch (a request is one CID = one object + // type), so one request builds exactly one of the two sets — no double walk. + let mut allowed_tree_memo: HashMap> = HashMap::new(); - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, - }; - - // Check whether the object exists in this repo before any expensive - // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); - continue; - } - }; - - // Per-blob gating only applies when a path-scoped rule exists (KTD4). - // Without any path-scoped rule, the "/" gate above is the whole story. - // Trees/commits are always served under path-scoped rules (KTD3). - let path_scoped = has_path_scoped_rule(rules); - if path_scoped && obj_type == "blob" { - if !allowed_memo.contains_key(&repo.id) { - let rp = repo_path.clone(); - let r = rules.to_vec(); - let is_public = repo.is_public; - let owner = repo.owner_did.clone(); - let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. - let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( - &rp, - &r, - is_public, - &owner, - caller_for_walk.as_deref(), - ) - }) - .await; - // Fail closed on EITHER a task panic (JoinError) or a walk error: - // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. - let set = match walk { - Ok(Ok(set)) => set, - Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); - continue; - } - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); - continue; - } - }; - allowed_memo.insert(repo.id.clone(), set); - } - let in_allowed = allowed_memo + // The `/ipfs/{cid}` route is anonymous and a valid tree CID makes each + // path-scoped repo pay a fresh full-history allowed-set walk, memoized only + // per request — repeat requests are unbounded amplification (INV-10, #173). + // We brake that walk on the non-farmable source IP, checked ONCE per request + // right before the first walk actually spawns: a memo hit or a cheap non-walk + // fetch never trips it, and a request that never reaches a walk is never + // rate-limited. The allowed-set is per repo/caller and independent of which + // candidate oid we probe, so the memos live outside the oid loop and are + // reused across candidates. + let mut walk_rate_checked = false; + + // Outer loop over every oid the CID maps to (#173): the cid index is + // non-unique, so try each candidate and serve the first readable one rather + // than false-404 when an arbitrary pick is withheld/absent while another is + // readable. The common single-oid case runs the inner loop exactly once. + for sha256_hex in &oids { + for repo in &repos { + // Repo-level read gate against THIS row's own rules (KTD2a). + let rules: &[crate::db::VisibilityRule] = rules_by_repo .get(&repo.id) - .is_some_and(|set| set.contains(&sha256_hex)); - if !in_allowed { + .map(Vec::as_slice) + .unwrap_or(&[]); + if visibility_check(rules, repo.is_public, &repo.owner_did, caller, "/") + == Decision::Deny + { continue; } - } - // Now that we've passed the gate, read the content. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); - continue; + let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { + Ok(p) => p, + Err(_) => continue, + }; + + // Check whether the object exists in this repo before any expensive + // reachability walk. This prevents random-CID spray from triggering + // full-history git walks on repos that don't carry the object. + let obj_type = match store::object_type(&repo_path, sha256_hex) { + Ok(Some(t)) => t, + Ok(None) => continue, + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + continue; + } + }; + + // Per-object gating applies only when a path-scoped rule exists (KTD4); + // without one, the "/" gate above is the whole story. Under a path-scoped + // rule a `blob` is gated against the caller's allowed-blob-set and a `tree` + // against the allowed-tree-set (#135 — a withheld subtree's tree structure + // must not leak by CID where get_tree protects it). A `commit`/`tag` exposes + // only "/"-level metadata the caller already cleared the "/" gate for, so it + // falls through to serve. + let path_scoped = has_path_scoped_rule(rules); + if path_scoped && (obj_type == "blob" || obj_type == "tree") { + let is_blob = obj_type == "blob"; + let memo = if is_blob { + &mut allowed_blob_memo + } else { + &mut allowed_tree_memo + }; + if !memo.contains_key(&repo.id) { + // First actual walk of this request: brake it on the source IP + // once. A memo hit below never reaches here, so a same-request + // re-check is impossible. When no key can be resolved (e.g. a + // test `oneshot` with no peer and no trusted header) the limiter + // is simply skipped, exactly as the other IP brakes behave. + if !walk_rate_checked { + walk_rate_checked = true; + if let Some(key) = + crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) + { + if !state.ipfs_rate_limiter.check(&key).await { + return Err(AppError::TooManyRequests( + "ipfs retrieval rate limit exceeded — try again later".into(), + )); + } + } + } + + let rp = repo_path.clone(); + let r = rules.to_vec(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = caller_owned.clone(); + // Full-history walk shells out to git — keep it off the async runtime. + // Only the fetched object's type is walked (blob XOR tree), so a tree + // fetch never pays the blob walk and vice-versa. + let walk = tokio::task::spawn_blocking(move || { + if is_blob { + allowed_blob_set_for_caller( + &rp, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ) + } else { + allowed_tree_set_for_caller( + &rp, + &r, + is_public, + &owner, + caller_for_walk.as_deref(), + ) + } + }) + .await; + // Fail closed on EITHER a task panic (JoinError) or a walk error: + // we cannot prove the caller may read here, so skip this repo and + // let a public copy (if any) serve. Never serve on an unproven gate. + let set = match walk { + Ok(Ok(set)) => set, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk failed; skipping repo"); + continue; + } + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "allowed-set walk task panicked; skipping repo"); + continue; + } + }; + memo.insert(repo.id.clone(), set); + } + let in_allowed = memo + .get(&repo.id) + .is_some_and(|set| set.contains(sha256_hex)); + if !in_allowed { + continue; + } } - }; - - // 3. Return the content with IPFS-style headers - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/octet-stream"), - ); - headers.insert( - HeaderName::from_static("x-content-cid"), - HeaderValue::from_str(&cid_str).unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - headers.insert( - HeaderName::from_static("x-git-hash"), - HeaderValue::from_str(&sha256_hex) - .unwrap_or_else(|_| HeaderValue::from_static("invalid")), - ); - - return Ok((StatusCode::OK, headers, content).into_response()); + + // Now that we've passed the gate, read the content. + let content = match store::read_object_content(&repo_path, sha256_hex, &obj_type) { + Ok(c) => c, + Err(e) => { + tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + continue; + } + }; + + // 3. Return the content with IPFS-style headers + let mut resp_headers = HeaderMap::new(); + resp_headers.insert( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/octet-stream"), + ); + resp_headers.insert( + HeaderName::from_static("x-content-cid"), + HeaderValue::from_str(&cid_str) + .unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + resp_headers.insert( + HeaderName::from_static("x-git-hash"), + HeaderValue::from_str(sha256_hex) + .unwrap_or_else(|_| HeaderValue::from_static("invalid")), + ); + + return Ok((StatusCode::OK, resp_headers, content).into_response()); + } } // Not found in any repo diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3a..0886010 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -516,6 +516,7 @@ mod tests { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5324a4b..e9b6d24 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -872,6 +872,19 @@ const MIGRATIONS: &[Migration] = &[ "CREATE UNIQUE INDEX IF NOT EXISTS idx_ref_certs_repo_ref ON ref_certificates(repo_id, ref_name)", ], }, + Migration { + version: 11, + name: "pinned_cids_cid_index", + stmts: &[ + // GET /ipfs/{cid} resolves an incoming CID -> git oid via pinned_cids.cid + // (#173); index it so the per-request lookup is not a table scan. This is + // a NEW versioned migration (not appended to the applied v1 bundle) so a + // node already past v1 actually gets the index. Non-unique on purpose: cid + // is a function of raw content, so a UNIQUE index could reject a legitimate + // record_pinned_cid insert, and colliding rows serve byte-identical content. + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_cid ON pinned_cids(cid)", + ], + }, ]; // ── Repos ───────────────────────────────────────────────────────────────────── @@ -2166,6 +2179,27 @@ impl Db { Ok(row.get::("cnt") > 0) } + /// Every git oid a pinned CID maps to (`pinned_cids.cid` -> `sha256_hex`). + /// `GET /ipfs/{cid}` resolves the content-addressed CID a client sends back to + /// the object's git oid this way: a real pin CID digests the raw object + /// content, not the git oid, so the digest cannot be `git cat-file`d directly + /// (#173). The index is unique on the git oid but NON-unique on cid, so two + /// distinct oids can share one content-CID (a tree and a blob whose raw bytes + /// collide, or byte-identical content pinned under two oids). Returning every + /// candidate lets the handler try each rather than pick one arbitrarily and + /// false-404 when the chosen one is withheld or absent while another is + /// readable (#173). Empty when the CID was never pinned on this node. + pub async fn oids_for_cid(&self, cid: &str) -> Result> { + let rows = sqlx::query("SELECT sha256_hex FROM pinned_cids WHERE cid = $1") + .bind(cid) + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|r| r.get::("sha256_hex")) + .collect()) + } + pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { sqlx::query( "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) @@ -5037,6 +5071,51 @@ mod ref_certificate_tests { ); } + /// INV-7: upgrade-path test — an existing node already past v1 must still get + /// the `pinned_cids.cid` index. It ships as its OWN v11 migration (not appended + /// to the applied v1 bundle), so dropping the index + its `schema_migrations` + /// row and re-running migrations must recreate it, exercising the real code + /// path rather than hand-copying the SQL. + #[sqlx::test] + async fn v11_pinned_cids_cid_index_applies_on_upgrade(pool: PgPool) { + async fn index_exists(pool: &PgPool) -> bool { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_indexes WHERE indexname = 'idx_pinned_cids_cid')", + ) + .fetch_one(pool) + .await + .unwrap() + } + + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "fresh migration chain creates the index" + ); + + // Simulate a node at v10 (pre-v11): drop the index and its migration record. + sqlx::query("DROP INDEX IF EXISTS idx_pinned_cids_cid") + .execute(&pool) + .await + .unwrap(); + sqlx::query("DELETE FROM schema_migrations WHERE version = 11") + .execute(&pool) + .await + .unwrap(); + assert!( + !index_exists(&pool).await, + "precondition: index and its migration record removed" + ); + + // Re-run migrations: v11 re-applies and recreates the index on the upgrade. + db.run_migrations().await.unwrap(); + assert!( + index_exists(&pool).await, + "v11 must recreate idx_pinned_cids_cid on an upgrading node" + ); + } + /// INV-7: upgrade-path test — seed a database at v9 with duplicate /// ref_certificates, then let the real v10 migration fire via /// run_migrations(). This exercises the migration code path rather than diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39..2031c67 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -1,7 +1,11 @@ -//! Resolve which blob OIDs must be withheld from a caller because every path -//! at which the blob appears is denied by the repo's visibility rules. Trees -//! and commits are never withheld (mode B keeps SHAs intact); only blob -//! content is held back. +//! Resolve which git objects must be withheld from a caller under the repo's +//! visibility rules. A blob is withheld when every path at which it appears is +//! denied. Since #135 a tree is likewise withheld on the CID serve surface +//! (`GET /ipfs/{cid}`) when every path at which it appears is denied, via the +//! parallel `allowed_tree_set_for_caller` — so a withheld subtree's structure does +//! not leak by content address. Commits and tags are never withheld here, and +//! trees still replicate freely on the pin path (the withheld-tree pin gap is +//! tracked as #172). Mode B keeps all object SHAs intact; only readability is gated. use crate::db::VisibilityRule; use crate::git::store; @@ -125,73 +129,79 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { Ok(()) } -/// List every (blob_oid, "/repo/relative/path") pair reachable from any commit in -/// `repo_path` — every ref *and* every historical commit those refs reach, not just -/// the ref tips. `git upload-pack` (serve) and the whole-repo pin fallback -/// (`git cat-file --batch-all-objects`) expose the full reachable object graph, -/// including a blob that only ever existed -/// in an older commit (a since-deleted file, a rotated secret whose previous version -/// is still in history). Classifying only ref-tip trees would leave those blobs -/// unwithheld while pin/serve still hand them out in cleartext, so we enumerate all -/// reachable commits and walk each commit's tree. -/// -/// `--all` covers every ref namespace (a blob reachable only through `refs/notes/*` -/// must not escape withholding); HEAD is added explicitly for the detached case, -/// where HEAD reaches commits that no ref does. `git ls-tree -rz ` per commit -/// keeps every path a blob lives at (the same blob content can appear at several -/// paths, and the per-path visibility check needs all of them). This is why it is -/// not `git rev-list --objects`, which reports only one path per object. Pairs are -/// de-duplicated across commits. Paths carry a leading "/" to match the glob form -/// used by visibility rules ("/secret/**"). -/// -/// Fails closed: if commit enumeration or any tree walk fails, returns an error so -/// the caller aborts the serve/pin rather than producing a partial (under-withheld) -/// set. -fn blob_paths(repo_path: &Path) -> Result> { +/// Every reachable commit oid: `git rev-list --all` plus HEAD (the detached case, +/// where HEAD reaches commits no ref does). Empty on an unborn branch. The single +/// commit-enumeration seam shared by `object_paths` and the root-tree derivation, so +/// the tree walk and the root-tree inclusion can never disagree on which commits are +/// reachable — callers compute the set ONCE and pass it to both. Runs the fail-closed +/// ref guard first (a ref pointing at a non-commit aborts), then enumerates. Fails +/// closed on a rev-list error. +fn reachable_commits(repo_path: &Path) -> Result> { assert_all_refs_are_commits(repo_path)?; - - // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; - // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no - // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. let head = store::head_commit(repo_path).context("resolve HEAD failed")?; let mut rev_args = vec!["rev-list", "--all"]; if head.is_some() { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") + let out = std::process::Command::new("git") .args(&rev_args) .current_dir(repo_path) .output() .context("git rev-list --all failed")?; - if !commits.status.success() { + if !out.status.success() { anyhow::bail!( "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) + String::from_utf8_lossy(&out.stderr) ); } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); - let mut out: HashSet<(String, String)> = HashSet::new(); - for commit in commits_stdout.lines() { - let commit = commit.trim(); - if commit.is_empty() { - continue; - } + Ok(String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + +/// Every `(oid, "/repo/relative/path", kind)` triple reachable from any commit in +/// `repo_path` — the single walk seam that `blob_paths` (`kind == "blob"`) and +/// `tree_paths` (`kind == "tree"`) both filter, so the blob and tree visibility +/// gates cannot drift. One `git ls-tree -rzt` per reachable commit: `-rzt` is +/// byte-identical to `-rz` for blob records and additionally emits the tree object +/// for each directory at its own path. `kind` is git's object-type string +/// ("blob", "tree", or "commit" for a gitlink). The commit's ROOT tree is not +/// emitted by `ls-tree` (it lists entries *under* a tree); `tree_paths` adds it. +/// +/// Enumerates every reachable commit, not just ref tips: `git upload-pack` (serve) +/// and the whole-repo pin fallback (`git cat-file --batch-all-objects`) expose the +/// full reachable object graph, including a blob or tree that only ever existed in +/// an older commit. `--all` covers every ref namespace (`refs/notes/*` included); +/// HEAD is added explicitly for the detached case, where HEAD reaches commits no +/// ref does. When HEAD does not resolve (unborn branch on an empty repo) `--all` +/// alone yields nothing, which is correct. Triples are de-duplicated across commits +/// and paths carry a leading "/" to match the glob form of visibility rules +/// ("/secret/**"). +/// +/// Fails closed: if any tree walk fails — or a path is not valid UTF-8 — it returns +/// an error so the caller aborts the serve/pin rather than producing a partial +/// (under-withheld) set. Takes the `commits` from [`reachable_commits`] (which runs +/// the ref guard) so blob and tree walks share one commit enumeration. +fn object_paths(repo_path: &Path, commits: &[String]) -> Result> { + let mut out: HashSet<(String, String, String)> = HashSet::new(); + for commit in commits { let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) + .args(["ls-tree", "-rzt", commit]) .current_dir(repo_path) .output() - .context("git ls-tree -rz failed")?; + .context("git ls-tree -rzt failed")?; if !listing.status.success() { anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", + "git ls-tree -rzt {commit} failed: {}", String::from_utf8_lossy(&listing.stderr) ); } // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a - // visibility rule like "/secret/**", under-withholding the blob. The TAB + // visibility rule like "/secret/**", under-withholding the object. The TAB // field separator survives `-z`, so the per-record parse is unchanged. // // Parse strictly: a lossy decode would replace an invalid byte in a denied @@ -200,12 +210,12 @@ fn blob_paths(repo_path: &Path) -> Result> { // layer down. Fail closed instead so the caller aborts rather than leaks. let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { anyhow::bail!( - "git ls-tree -rz {commit} returned a non-UTF-8 path; \ + "git ls-tree -rzt {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" ); }; for record in listing_stdout.split('\0') { - // " blob \t" + // " \t" let Some((meta, path)) = record.split_once('\t') else { continue; }; @@ -213,14 +223,26 @@ fn blob_paths(repo_path: &Path) -> Result> { let _mode = parts.next(); let kind = parts.next(); let oid = parts.next(); - if kind == Some("blob") { - if let Some(oid) = oid { - out.insert((oid.to_string(), format!("/{path}"))); - } + if let (Some(kind), Some(oid)) = (kind, oid) { + out.insert((oid.to_string(), format!("/{path}"), kind.to_string())); } } } - Ok(out.into_iter().collect()) + Ok(out) +} + +/// `(blob_oid, "/path")` pairs reachable from any commit — the `kind == "blob"` +/// slice of [`object_paths`]. Output is byte-identical to the pre-refactor direct +/// walk (blobs only, deduped, leading-slash paths), so every caller +/// (`withheld_blob_oids`, `replicable_blob_set`, `allowed_blob_set_for_caller`) is +/// unaffected. See [`object_paths`] for the walk and fail-closed semantics. +fn blob_paths(repo_path: &Path) -> Result> { + let commits = reachable_commits(repo_path)?; + Ok(object_paths(repo_path, &commits)? + .into_iter() + .filter(|(_, _, kind)| kind == "blob") + .map(|(oid, path, _)| (oid, path)) + .collect()) } /// Blob OIDs the caller may not read. A blob is withheld only if visibility @@ -322,9 +344,28 @@ pub fn replicable_blob_set( /// to authorize the blob at. /// /// A blob reachable at an allowed path is included even when also denied -/// elsewhere (its content is readable to this caller elsewhere). Trees and -/// commits are NOT included here; the caller decides per object type whether -/// the allow-set applies (it does not for trees/commits — KTD3). +/// elsewhere (its content is readable to this caller elsewhere). This set is +/// blobs only; trees have the parallel `allowed_tree_set_for_caller` (#135), and +/// commits/tags are served as root-level structure once the `"/"` gate passes. +/// The OIDs from a `(oid, "/path")` listing that visibility ALLOWS `caller` at some +/// path — the shared inner loop of the blob and tree allowed-sets. An oid reachable +/// at an allowed path is kept even when also reachable at a denied one. +fn allowed_set_from_pairs<'a>( + pairs: impl IntoIterator, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> HashSet { + pairs + .into_iter() + .filter(|(_, path)| { + visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow + }) + .map(|(oid, _)| oid.clone()) + .collect() +} + pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -332,14 +373,116 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; - let mut allowed = HashSet::new(); - for (oid, path) in &pairs { - if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { - allowed.insert(oid.clone()); + Ok(allowed_set_from_pairs( + &blob_paths(repo_path)?, + rules, + is_public, + owner_did, + caller, + )) +} + +/// Root tree oid of every reachable commit, at "/". `ls-tree` never emits a commit's +/// own root tree (it lists entries *under* a tree), so it is added explicitly here. +/// Resolved in ONE `git log --no-walk --format=%T --stdin` pass over the shared commit +/// set — not a per-commit `rev-parse` — so a tree-set walk costs the same subprocess +/// order as the blob walk. The commit oids go on STDIN, not argv: a long history has +/// tens of thousands of reachable commits, and passing them all as arguments overflows +/// ARG_MAX so `git log` fails to spawn — which the caller treats as a walk error and +/// fail-closed 404s an authorized reader of a reachable/root tree (#173 P2). A commit +/// whose root tree git cannot resolve fails the pass (bail), failing closed. +fn root_tree_pairs(repo_path: &Path, commits: &[String]) -> Result> { + if commits.is_empty() { + return Ok(HashSet::new()); + } + use std::io::Write; + let mut child = std::process::Command::new("git") + .args(["log", "--no-walk=unsorted", "--format=%T", "--stdin"]) + .current_dir(repo_path) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("spawn git log --no-walk --format=%T --stdin")?; + // Write the oids from a separate thread while this one drains stdout: on a long + // history the output is large, and a single thread that writes all of stdin before + // reading stdout would deadlock once git's stdout pipe fills. + let mut stdin = child.stdin.take().context("git log stdin unavailable")?; + let commits_owned = commits.to_vec(); + let writer = std::thread::spawn(move || -> std::io::Result<()> { + let mut buf = String::with_capacity(commits_owned.len() * 65); + for c in &commits_owned { + buf.push_str(c); + buf.push('\n'); } + stdin.write_all(buf.as_bytes()) + // stdin dropped here → git sees EOF on its revision list + }); + let out = child + .wait_with_output() + .context("git log --no-walk --format=%T --stdin failed")?; + let write_res = writer + .join() + .map_err(|_| anyhow::anyhow!("git log stdin writer thread panicked"))?; + if !out.status.success() { + // git's own error is the more informative signal than a BrokenPipe from the + // writer (which just means git exited before reading every oid), so check it + // first and fail closed. + anyhow::bail!( + "git log --no-walk --format=%T --stdin failed: {}", + String::from_utf8_lossy(&out.stderr) + ); } - Ok(allowed) + write_res.context("writing commit oids to git log stdin")?; + let mut set = HashSet::new(); + for line in String::from_utf8_lossy(&out.stdout).lines() { + let oid = line.trim(); + if !oid.is_empty() { + set.insert((oid.to_string(), "/".to_string())); + } + } + Ok(set) +} + +/// Every `(tree_oid, "/path")` pair reachable in `repo_path`: the `kind == "tree"` +/// slice of [`object_paths`] (subtree trees at their directory paths) PLUS every +/// reachable commit's root tree at "/" (see [`root_tree_pairs`]). Computes the +/// reachable-commit set ONCE and drives both the ls-tree walk and the root-tree pass +/// from it, so the two cannot diverge and neither re-enumerates. The tree analog of +/// [`blob_paths`]. +fn tree_paths(repo_path: &Path) -> Result> { + let commits = reachable_commits(repo_path)?; + let mut out: HashSet<(String, String)> = object_paths(repo_path, &commits)? + .into_iter() + .filter(|(_, _, kind)| kind == "tree") + .map(|(oid, path, _)| (oid, path)) + .collect(); + out.extend(root_tree_pairs(repo_path, &commits)?); + Ok(out) +} + +/// Reachable tree OIDs that visibility ALLOWS `caller` at some path — the tree analog +/// of [`allowed_blob_set_for_caller`]. `GET /ipfs/{cid}` gates tree objects with this +/// so the CID surface matches `get_tree`: a tree reachable only at a withheld path is +/// absent from the set and 404'd; the root tree ("/") and any tree on the path to an +/// allowed subtree are present. Fails closed on a dangling/unreachable tree (never +/// enumerated by the reachable walk, so never in the set — the #126 geometry, for +/// trees). A tree reachable at an allowed path is included even when also reachable at +/// a withheld one (its structure is visible to this caller elsewhere). +pub fn allowed_tree_set_for_caller( + repo_path: &Path, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + Ok(allowed_set_from_pairs( + &tree_paths(repo_path)?, + rules, + is_public, + owner_did, + caller, + )) } /// Objects safe to replicate, failing closed on blobs (#99). A candidate @@ -687,6 +830,293 @@ mod tests { ); } + #[test] + fn object_paths_emits_trees_and_blob_paths_is_the_blob_slice() { + let (_td, bare, secret_oid, public_oid) = fixture(); + let objs = object_paths(&bare, &reachable_commits(&bare).unwrap()).unwrap(); + + // Blob records survive the `-rzt` change, at their paths (unchanged). + assert!(objs.contains(&(secret_oid.clone(), "/secret/b.txt".into(), "blob".into()))); + assert!(objs.contains(&(public_oid.clone(), "/public/a.txt".into(), "blob".into()))); + + // The #135 addition: subtree tree objects at their directory paths. + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/secret"), + "the /secret subtree tree must be emitted at its dir path" + ); + assert!( + objs.iter().any(|(_, p, k)| k == "tree" && p == "/public"), + "the /public subtree tree must be emitted at its dir path" + ); + + // blob_paths must equal the blob slice of object_paths exactly — compared as + // SETS (both walks dedup via HashSet; the collected order is nondeterministic). + let bp: HashSet<(String, String)> = blob_paths(&bare).unwrap().into_iter().collect(); + let bp_from_obj: HashSet<(String, String)> = objs + .iter() + .filter(|(_, _, k)| k == "blob") + .map(|(o, p, _)| (o.clone(), p.clone())) + .collect(); + assert_eq!( + bp, bp_from_obj, + "blob_paths output must be byte-identical to object_paths' blob slice" + ); + } + + #[test] + fn allowed_tree_set_gates_withheld_subtree_tree() { + let (_td, bare, _s, _p) = fixture(); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse {rev}"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let secret_tree = oid("HEAD:secret"); + let public_tree = oid("HEAD:public"); + let root_tree = oid("HEAD^{tree}"); + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", &[reader])]; + + // anon: the withheld /secret tree is excluded; root ("/") and /public are in. + let anon = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, None).unwrap(); + assert!( + !anon.contains(&secret_tree), + "withheld /secret subtree tree excluded for anon" + ); + assert!(anon.contains(&root_tree), "root tree included (path /)"); + assert!(anon.contains(&public_tree), "/public subtree tree included"); + + // listed reader: sees the /secret tree (caller-aware, not a blanket deny). + let rd = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(reader)).unwrap(); + assert!( + rd.contains(&secret_tree), + "listed reader sees the /secret tree" + ); + + // owner: sees every reachable tree. + let ow = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + assert!( + ow.contains(&secret_tree) && ow.contains(&public_tree) && ow.contains(&root_tree), + "owner sees all reachable trees" + ); + } + + #[test] + fn allowed_tree_set_excludes_dangling_tree() { + use std::io::Write; + use std::process::Stdio; + let (_td, bare, secret_oid, _p) = fixture(); + // A DANGLING tree: written to the ODB but referenced by no commit. Uses a + // UNIQUE entry name so its oid is content-distinct from every reachable tree + // (a content-identical tree would dedup to a reachable oid — that is T2, not + // danglingness). The reachable-only walk never enumerates it -> fail closed. + let mut child = Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {secret_oid}\tdangling-only-unreferenced.txt" + ) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "git mktree"); + let dangling = String::from_utf8_lossy(&out.stdout).trim().to_string(); + + let rules = [rule("/secret/**", &[])]; + for caller in [None, Some(OWNER)] { + let set = allowed_tree_set_for_caller(&bare, &rules, true, OWNER, caller).unwrap(); + assert!( + !set.contains(&dangling), + "dangling tree must never be in the reachable allowed-set (caller={caller:?})" + ); + } + } + + #[test] + fn allowed_tree_set_includes_tree_shared_across_allowed_and_denied_paths() { + // T2 (content-dedup): the SAME tree oid reachable at both an allowed and a + // withheld path is INCLUDED for anon (allowed-wins) — its structure is + // visible to the caller at the allowed path. Mirrors the blob analog + // `same_blob_at_allowed_and_denied_path_is_not_withheld`. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(work.join("pub/sub")).unwrap(); + std::fs::create_dir_all(work.join("sec/sub")).unwrap(); + std::fs::write(work.join("pub/sub/f.txt"), b"same bytes\n").unwrap(); + std::fs::write(work.join("sec/sub/f.txt"), b"same bytes\n").unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + run(&["add", "."]); + run(&["commit", "-qm", "seed"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let pub_sub = oid("HEAD:pub/sub"); + let sec_sub = oid("HEAD:sec/sub"); + assert_eq!(pub_sub, sec_sub, "identical content dedups to one tree oid"); + + // Withhold /sec from anon; the shared oid is still reachable at /pub/sub. + let rules = [rule("/sec/**", &[])]; + let anon = allowed_tree_set_for_caller(&work, &rules, true, OWNER, None).unwrap(); + assert!( + anon.contains(&pub_sub), + "a tree reachable at an allowed path is included even when also at a withheld path" + ); + } + + #[test] + fn allowed_tree_set_includes_root_trees_of_all_reachable_commits() { + // The batched root-tree pass (root_tree_pairs) must return EVERY reachable + // commit's root tree, not just HEAD's — two commits with distinct root trees + // both land in the set. Guards the git-log-over-N-commits root derivation. + let td = TempDir::new().unwrap(); + let work = td.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let oid = |rev: &str| { + let out = Command::new("git") + .args(["rev-parse", rev]) + .current_dir(&work) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + std::fs::write(work.join("a.txt"), b"one\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c1"]); + let root1 = oid("HEAD^{tree}"); + std::fs::write(work.join("b.txt"), b"two\n").unwrap(); + run(&["add", "."]); + run(&["commit", "-qm", "c2"]); + let root2 = oid("HEAD^{tree}"); + assert_ne!(root1, root2, "the two commits have distinct root trees"); + + // Public repo, no rules: every reachable tree is allowed for anon. + let set = allowed_tree_set_for_caller(&work, &[], true, OWNER, None).unwrap(); + assert!( + set.contains(&root1) && set.contains(&root2), + "root trees of BOTH reachable commits are in the set (batched root pass)" + ); + } + + #[test] + fn root_tree_pairs_returns_every_root_tree_at_scale() { + // Parity + liveness at scale for root_tree_pairs (#173 P2): feed every + // reachable commit oid to `git log --format=%T --stdin` and collect each + // commit's root tree. With N commits that is ~N*41 bytes of oids in and + // ~N*41 bytes of %T out — past the ~64 KiB pipe buffer in both directions — + // so this exercises the large-bidirectional-IO path the 2-commit test above + // cannot, and asserts parity: every distinct root tree comes back. + // + // NOTE: this is NOT a deadlock guard. `git log --stdin` reads its whole + // revision list to EOF before emitting any %T, so the naive "write all of + // stdin, then drain stdout" form does not deadlock at any scale for this + // invocation — verified by reverting root_tree_pairs' writer thread and + // running at N=40000 (~1.6 MiB each way): it still returned promptly. The + // writer thread is cheap defensive isolation, not load-bearing, and this + // test does not claim otherwise. The 30s watchdog is a general liveness + // bound so a future regression that genuinely hangs fails fast here rather + // than stalling the suite. + const N: usize = 2500; + let td = TempDir::new().unwrap(); + let bare = td.path().join("many.git"); + assert!(Command::new("git") + .args(["init", "-q", "--bare", bare.to_str().unwrap()]) + .status() + .unwrap() + .success()); + + // fast-import a linear chain of N commits, each adding a distinct file so + // every root tree is distinct (dedup cannot shrink the output). One + // subprocess, ~1s — far cheaper than N `git commit` spawns. + let mut stream = String::new(); + for i in 0..N { + let (b, cm) = (2 * i + 1, 2 * i + 2); + let content = format!("v{i}"); + let msg = format!("c{i}"); + stream.push_str(&format!( + "blob\nmark :{b}\ndata {}\n{content}\n", + content.len() + )); + stream.push_str(&format!( + "commit refs/heads/main\nmark :{cm}\ncommitter t 0 +0000\ndata {}\n{msg}\n", + msg.len() + )); + if i > 0 { + stream.push_str(&format!("from :{}\n", 2 * (i - 1) + 2)); + } + stream.push_str(&format!("M 100644 :{b} f{i}\n\n")); + } + let mut fi = Command::new("git") + .args(["fast-import", "--quiet"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + { + use std::io::Write; + fi.stdin + .take() + .unwrap() + .write_all(stream.as_bytes()) + .unwrap(); + } + assert!(fi.wait().unwrap().success(), "fast-import failed"); + + let commits = reachable_commits(&bare).unwrap(); + assert_eq!(commits.len(), N, "all {N} commits reachable"); + + // Call root_tree_pairs directly (private, same module) under a liveness + // watchdog, then assert it returned every distinct root tree. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(root_tree_pairs(&bare, &commits).map(|s| s.len())); + }); + match rx.recv_timeout(std::time::Duration::from_secs(30)) { + Ok(Ok(len)) => assert_eq!(len, N, "every distinct root tree returned"), + Ok(Err(e)) => panic!("root_tree_pairs errored: {e}"), + Err(_) => panic!("root_tree_pairs did not return within 30s"), + } + } + #[test] fn fail_closed_drops_dangling_private_blob() { // #99: a private blob orphaned by a force-push/amend is unreachable but diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483d..d0a8209 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -325,6 +325,24 @@ async fn main() -> Result<()> { tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); } + // IPFS-walk flood brake: max `GET /ipfs/{cid}` full-history walks per client + // IP per hour. The route is anonymous and a valid tree CID makes every repeat + // request pay a fresh allowed-set walk (INV-10), so the walk (not the whole + // route) is braked on the resolved client IP. GITLAWB_IPFS_RATE_LIMIT + // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. + let ipfs_limit = std::env::var("GITLAWB_IPFS_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(600); + let ipfs_rate_limiter = rate_limit::RateLimiter::new_bounded( + ipfs_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if ipfs_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP IPFS walk rate limiting disabled"); + } + // Which forwarded header the edge is trusted to set. Default None (trust // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; // a node behind Caddy/NGINX sets it to x-forwarded-for. @@ -374,6 +392,7 @@ async fn main() -> Result<()> { rate_limiter, create_ip_rate_limiter, push_rate_limiter, + ipfs_rate_limiter, push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c..2c20408 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -65,6 +65,13 @@ pub struct AppState { /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. pub push_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for the `GET /ipfs/{cid}` full-history walk. + /// The route is anonymous and a valid tree CID (exposed by the public pins + /// index) makes each repeat request pay a fresh allowed-set walk (rev-list + + /// ls-tree per commit), memoized only per request — unbounded amplification + /// (INV-10). Braking the walk on the non-farmable source IP caps that cost + /// without touching cheap non-walk fetches. Keyed by `push_limiter_trust`. + pub ipfs_rate_limiter: RateLimiter, /// Which forwarded header (if any) the edge is trusted to set, for /// resolving the push limiter's client-IP key. See `GITLAWB_TRUSTED_PROXY`. /// Node-wide; also keys the two peer-sync limiters below. diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f6..66da183 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -78,6 +78,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), @@ -1809,13 +1810,19 @@ mod tests { /// Seed a SHA-256 source repo (public/a.txt + secret/b.txt), bare-clone it /// into each `/tmp//.git` path, and return guards + oids. - /// SHA-256 object format is required: `get_by_cid` resolves a CID whose - /// multihash digest IS the git object id, which only matches in sha256 repos. + /// SHA-256 object format matches production (`--object-format=sha256`) so the + /// oids are 64-hex. A real CID digests the raw object CONTENT (not the git + /// oid), so tests build the request CID with `pin_cid_for` — mirroring the pin + /// path — and `get_by_cid` maps it back to the oid via `pinned_cids` (#173). struct CidFixture { _guards: Vec, secret_oid: String, public_oid: String, secret_tree_oid: String, + public_tree_oid: String, + root_tree_oid: String, + commit_oid: String, + tag_oid: String, } impl Drop for CidFixture { fn drop(&mut self) { @@ -1849,6 +1856,8 @@ mod tests { run(&["config", "user.name", "t"], &src); run(&["add", "."], &src); run(&["commit", "-qm", "seed"], &src); + // Annotated tag of the commit — exercises the "tags stay served" guard. + run(&["tag", "-a", "-m", "annotated", "v1", "HEAD"], &src); let oid = |rev: &str| { let out = Command::new("git") .args(["rev-parse", rev]) @@ -1861,6 +1870,10 @@ mod tests { let secret_oid = oid("HEAD:secret/b.txt"); let public_oid = oid("HEAD:public/a.txt"); let secret_tree_oid = oid("HEAD:secret"); + let public_tree_oid = oid("HEAD:public"); + let root_tree_oid = oid("HEAD^{tree}"); + let commit_oid = oid("HEAD"); + let tag_oid = oid("refs/tags/v1"); let mut guards = vec![src.clone()]; for name in bare_names { let bare = std::path::PathBuf::from("/tmp") @@ -1886,16 +1899,30 @@ mod tests { secret_oid, public_oid, secret_tree_oid, + public_tree_oid, + root_tree_oid, + commit_oid, + tag_oid, } } - /// CID whose sha2-256 multihash digest equals the given 64-hex git oid, so - /// `get_by_cid` decodes it back to that oid and `git cat-file`s it. - fn cid_for_oid(oid_hex: &str) -> String { - use gitlawb_core::cid::Cid; - let bytes = hex::decode(oid_hex).expect("hex oid"); - let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); - Cid::from_sha256_bytes(&arr).to_string() + /// Record a pin exactly as the production pin path does — read the object's + /// raw bytes (`git cat-file `, no framing), CID them with + /// `Cid::from_git_object_bytes`, and store the `(oid, cid)` row — then return + /// the CID string the node advertises (`gl ipfs list`) and a client sends to + /// `GET /ipfs/{cid}`. Building the CID from the oid instead (the old + /// `cid_for_oid`) produced an identifier that never occurs in production and + /// made the gate assertions vacuous: a real pin CID digests the raw content, + /// not the git oid, so `get_by_cid` resolves it through `pinned_cids` (#173). + async fn pin_cid_for(bare_repo: &std::path::Path, oid: &str, db: &crate::db::Db) -> String { + let (_ty, raw) = crate::git::store::read_object(bare_repo, oid) + .expect("read object bytes") + .expect("object exists in repo"); + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(&raw).to_string(); + db.record_pinned_cid(oid, &cid) + .await + .expect("record pinned cid"); + cid } fn cid_router(state: &AppState) -> Router { @@ -1914,6 +1941,21 @@ mod tests { .unwrap(); (st, String::from_utf8_lossy(&b).to_string()) } + /// Raw body bytes (NOT lossy-decoded). A git tree body stores each child oid + /// as 32 RAW bytes that `from_utf8_lossy` mangles to U+FFFD, so a hex + /// `contains` check on `cid_parts`'s String is vacuous. #135 deny tests must + /// witness the leak on these raw bytes. + async fn cid_bytes(resp: axum::response::Response) -> (StatusCode, Vec) { + let st = resp.status(); + let b = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + (st, b.to_vec()) + } + /// True if `needle` appears as a contiguous byte subsequence of `haystack`. + fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) + } fn cid_anon(cid: &str) -> Request { Request::builder() .method(Method::GET) @@ -1933,6 +1975,27 @@ mod tests { .body(Body::empty()) .unwrap() } + /// Signed CID request carrying `x-forwarded-for: `. Used by the walk + /// rate-limit test to key the per-IP limiter off a chosen source under + /// `TrustedProxy::XForwardedFor` (the request goes through `oneshot`, which + /// leaves no socket peer, so the header is the only key source). + fn cid_signed_xff( + kp: &gitlawb_core::identity::Keypair, + cid: &str, + xff_ip: &str, + ) -> Request { + let path = format!("/ipfs/{cid}"); + let s = gitlawb_core::http_sig::sign_request(kp, "GET", &path, b""); + Request::builder() + .method(Method::GET) + .uri(&path) + .header("content-digest", s.content_digest) + .header("signature-input", s.signature_input) + .header("signature", s.signature) + .header("x-forwarded-for", xff_ip) + .body(Body::empty()) + .unwrap() + } /// #110: `GET /ipfs/{cid}` must gate a withheld blob by per-caller visibility. /// RED before U2 (the current handler serves the secret to anon). @@ -1951,9 +2014,18 @@ mod tests { let state = test_state(pool).await; let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let tree_cid = cid_for_oid(&fx.secret_tree_oid); - let public_cid = cid_for_oid(&fx.public_oid); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Request CIDs are the production pin CIDs (content-hash), recorded in + // pinned_cids so get_by_cid resolves each back to its oid (#173). + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + let root_tree_cid = pin_cid_for(&bare, &fx.root_tree_oid, &state.db).await; + let public_tree_cid = pin_cid_for(&bare, &fx.public_tree_oid, &state.db).await; + let commit_cid = pin_cid_for(&bare, &fx.commit_oid, &state.db).await; + let tag_cid = pin_cid_for(&bare, &fx.tag_oid, &state.db).await; state .db @@ -2033,7 +2105,10 @@ mod tests { assert_eq!(st, StatusCode::OK, "listed reader reads the blob"); assert!(body.contains("TOP SECRET")); - // KTD3: anon tree CID under /secret → 200 (trees/commits are not withheld). + // #135: anon tree CID under withheld /secret → 404. The 404 body is an opaque + // error string (never the object), so status is the load-bearing deny check; + // the real leak witness is the CONTRAST with the reader below, who DOES get a + // 200 carrying the child structure that anon is denied. let (st, _) = cid_parts( cid_router(&state) .oneshot(cid_anon(&tree_cid)) @@ -2041,7 +2116,73 @@ mod tests { .unwrap(), ) .await; - assert_eq!(st, StatusCode::OK, "tree object is served to anon (KTD3)"); + assert_eq!( + st, + StatusCode::NOT_FOUND, + "withheld subtree tree must not be served to anon (#135)" + ); + + // Over-denial guard + positive leak witness: the listed reader (signed) DOES + // read the withheld subtree's tree, and its body carries the exact child + // structure anon was denied — the child filename plus the child oid as the 32 + // RAW bytes a git tree stores (witnessed on raw bytes, since cid_parts's lossy + // decode would mangle them). This proves b.txt / secret_raw are the real leak + // markers and that the anon 404 above actually withheld them. + let secret_raw = hex::decode(&fx.secret_oid).expect("hex oid"); + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_signed(&reader, &tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "listed reader reads the withheld subtree tree" + ); + assert!( + bytes_contain(&body, b"b.txt") && bytes_contain(&body, &secret_raw), + "reader's tree body carries the child filename and raw child oid" + ); + + // Root tree (path "/") stays served to anon who passes the "/" gate. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&root_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "root tree stays served (must-serve)"); + + // /public subtree tree stays served to anon (allowed path). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&public_tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "public subtree tree stays served"); + + // Commit and annotated tag objects stay served (unchanged by #135). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&commit_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "commit object stays served"); + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&tag_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "tag object stays served"); // R3: public blob anon → 200 (non-withheld content not affected). let (st, _) = cid_parts( @@ -2053,8 +2194,11 @@ mod tests { .await; assert_eq!(st, StatusCode::OK, "public blob stays served"); - // R5: a genuine unknown CID also 404, uniform with the withheld 404. - let absent_cid = cid_for_oid(&"ab".repeat(32)); + // R5: a genuine unknown CID also 404, uniform with the withheld 404. A + // well-formed pin-style CID that was never recorded in pinned_cids, so the + // oid_for_cid resolve misses (the production not-found path). + let absent_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"never pinned to this node").to_string(); let (st, _) = cid_parts( cid_router(&state) .oneshot(cid_anon(&absent_cid)) @@ -2094,7 +2238,11 @@ mod tests { let state = test_state(pool).await; let fx = seed_cid_repos(&slug, &short, &["withhold", "pubcopy"]); - let secret_cid = cid_for_oid(&fx.secret_oid); + // Same content in both clones -> same oid/CID; read from either. + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; // Withholding repo, iterated FIRST (later updated_at; list_all_repos is DESC). let mut withhold = seed_repo(&owner_did, "withhold"); @@ -2155,7 +2303,10 @@ mod tests { let state = test_state(pool).await; let fx = seed_cid_repos(&slug, &short, &["priv"]); - let blob_cid = cid_for_oid(&fx.public_oid); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("priv.git"); + let blob_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; let mut rec = seed_repo(&owner_did, "priv"); rec.is_public = false; @@ -2211,15 +2362,17 @@ mod tests { let state = test_state(pool).await; let fx = seed_cid_repos(&slug, &short, &["withhold"]); - let secret_cid = cid_for_oid(&fx.secret_oid); - let public_cid = cid_for_oid(&fx.public_oid); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("withhold.git"); + // Recorded pins so get_by_cid resolves each CID to its oid and reaches the + // walk; the 404s below are then the fail-closed skip, not a table miss. + let secret_cid = pin_cid_for(&bare, &fx.secret_oid, &state.db).await; + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; // Force the withheld walk to fail closed: a ref pointing at a blob (not // tree-ish) makes `git ls-tree -r` error, which `withheld_blob_oids` // propagates as Err → the handler's `Ok(Err)` arm skips the repo. - let bare = std::path::PathBuf::from("/tmp") - .join(&slug) - .join("withhold.git"); std::fs::write( bare.join("refs/heads/blobref"), format!("{}\n", fx.secret_oid), @@ -2333,7 +2486,9 @@ mod tests { 64, "expected sha256 oid: {dangling_oid}" ); - let dangling_cid = cid_for_oid(&dangling_oid); + // Record the pin so oid_for_cid resolves it — the 404 must then come from + // the allowed-set gate excluding the dangling oid, not from a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_oid, &state.db).await; state .db @@ -2390,6 +2545,446 @@ mod tests { assert!(!body.contains("DANGLING SECRET")); } + /// #135: a DANGLING tree (in the ODB, referenced by no commit) 404s under + /// path-scoped rules for anon AND owner — the reachable-only allowed-tree-set + /// never enumerates it. Handler-level companion to the helper test + /// `allowed_tree_set_excludes_dangling_tree`, proving the `get_by_cid` tree arm + /// (memo insert + `!in_allowed` continue) fails closed on the dangling case. + #[sqlx::test] + async fn ipfs_cid_dangling_tree_fails_closed_under_path_rules(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["dangtree"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("dangtree.git"); + + // Dangling tree via `git mktree`: a UNIQUE entry name so its oid is + // content-distinct from every reachable tree (a content-identical tree would + // dedup to a reachable oid — that is T2, not danglingness). + let mut child = std::process::Command::new("git") + .args(["mktree"]) + .current_dir(&bare) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn git mktree"); + { + use std::io::Write; + writeln!( + child.stdin.as_mut().unwrap(), + "100644 blob {}\tdangling-only-unreferenced.txt", + fx.secret_oid + ) + .unwrap(); + } + let out = child.wait_with_output().expect("mktree output"); + assert!( + out.status.success(), + "git mktree: {}", + String::from_utf8_lossy(&out.stderr) + ); + let dangling_tree_oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + assert_eq!(dangling_tree_oid.len(), 64, "expected sha256 oid"); + // Record the pin so the 404 is the allowed-tree-set gate excluding the + // dangling tree, not a table miss. + let dangling_cid = pin_cid_for(&bare, &dangling_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "dangtree")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "dangtree") + .await + .unwrap() + .unwrap(); + state + .db + .set_visibility_rule(&rec.id, "/secret/**", VisibilityMode::B, &[], &owner_did) + .await + .expect("deny rule"); + + for req in [cid_anon(&dangling_cid), cid_signed(&owner, &dangling_cid)] { + let (st, _) = cid_parts(cid_router(&state).oneshot(req).await.unwrap()).await; + assert_eq!( + st, + StatusCode::NOT_FOUND, + "dangling tree must 404 under path-scoped rules (anon + owner)" + ); + } + } + + /// #135: with NO path-scoped rule the per-object gate is skipped, so a tree CID + /// is served (the `"/"` gate is the whole story). Guards against over-gating + /// trees — the tree analog of the blob skip-walk branch. + #[sqlx::test] + async fn ipfs_cid_tree_served_when_no_path_scoped_rule(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["nopathrule"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("nopathrule.git"); + let tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + // Public repo, no visibility rules → has_path_scoped_rule is false. + state + .db + .create_repo(&seed_repo(&owner_did, "nopathrule")) + .await + .expect("seed repo"); + + let (st, body) = cid_bytes( + cid_router(&state) + .oneshot(cid_anon(&tree_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "tree served to anon when no path-scoped rule exists" + ); + assert!( + bytes_contain(&body, b"b.txt"), + "served tree carries its child structure" + ); + } + + /// #173 (Fix 1): the pinned_cids lookup must use the canonical base32 CID, not + /// the raw request spelling. A pin is stored under `cid.to_string()` (canonical + /// base32); a request carrying the SAME CID re-encoded to a different multibase + /// (base58btc) parses and passes the sha2-256 check but, on the pre-fix handler, + /// misses the lookup key → false 404. Public repo, no path-scoped rule, so no + /// walk — this isolates the lookup-key canonicalization. + #[sqlx::test] + async fn ipfs_alt_encoding_cid_resolves(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + let fx = seed_cid_repos(&slug, &short, &["altenc"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("altenc.git"); + // Canonical base32 CID as stored by the pin path. + let public_cid = pin_cid_for(&bare, &fx.public_oid, &state.db).await; + + // Public repo, no visibility rules (no path-scoped walk). + state + .db + .create_repo(&seed_repo(&owner_did, "altenc")) + .await + .expect("seed repo"); + + // Re-encode the SAME CID to base58btc — a different, equally-valid spelling + // that is NOT the stored key. The `cid` crate re-exports `multibase`. + let alt = public_cid + .parse::>() + .unwrap() + .to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(); + assert_ne!(alt, public_cid, "alt encoding must differ from canonical"); + + let (st, body) = cid_parts(cid_router(&state).oneshot(cid_anon(&alt)).await.unwrap()).await; + assert_eq!( + st, + StatusCode::OK, + "alt-multibase spelling of a pinned CID must resolve (canonicalized lookup)" + ); + assert!( + body.contains("public bytes"), + "resolved object serves its content" + ); + } + + /// #173 (Fix 2a, db-level): `oids_for_cid` returns EVERY oid recorded under a + /// CID, not an arbitrary one. `record_pinned_cid` is unique on the git oid and + /// non-unique on cid, so two distinct oids can share one content-CID. Old + /// `oid_for_cid` did `LIMIT 1`; the new plural method must surface both. + #[sqlx::test] + async fn oids_for_cid_returns_all_duplicates(pool: PgPool) { + let state = test_state(pool).await; + let cid = gitlawb_core::cid::Cid::from_git_object_bytes(b"shared content cid").to_string(); + let oid_a = "a".repeat(64); + let oid_b = "b".repeat(64); + state.db.record_pinned_cid(&oid_a, &cid).await.unwrap(); + state.db.record_pinned_cid(&oid_b, &cid).await.unwrap(); + + let mut oids = state.db.oids_for_cid(&cid).await.unwrap(); + oids.sort(); + assert_eq!( + oids, + vec![oid_a, oid_b], + "oids_for_cid must return every oid recorded under the shared CID" + ); + } + + /// #173 (Fix 2b, handler-level): when two oids collide on one CID and the + /// first-recorded is absent from every repo while the second is a readable + /// public object, the handler must try both and serve the readable one. The + /// pre-fix handler resolved a single oid (LIMIT 1 → first-inserted for equal + /// keys) and 404'd. Ordering caveat: this relies on `oids_for_cid` returning + /// the absent oid before the readable one (heap/insert order for equal keys); + /// if that ordering ever changes, `oids_for_cid_returns_all_duplicates` remains + /// the load-bearing, deterministic driver for Fix 2. + #[sqlx::test] + async fn ipfs_cid_collision_serves_readable_duplicate(pool: PgPool) { + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + let state = test_state(pool).await; + + // The bare clone must exist for the public object to be readable, but the + // pins are recorded directly (below) to force the CID collision, so the + // `bare` path itself is not needed here — only the seeded `public_oid`. + let fx = seed_cid_repos(&slug, &short, &["collision"]); + + // A shared CID pointing first at an oid present in NO repo, then at the + // real oid of a readable public object. Recorded directly so both rows + // carry the same `cid` (the collision the fix must tolerate). + let shared_cid = + gitlawb_core::cid::Cid::from_git_object_bytes(b"collision cid seed").to_string(); + let absent_oid = "c".repeat(64); + state + .db + .record_pinned_cid(&absent_oid, &shared_cid) + .await + .expect("record absent oid first"); + state + .db + .record_pinned_cid(&fx.public_oid, &shared_cid) + .await + .expect("record readable oid second"); + + // Public repo, no rules → the readable public object is served if reached. + state + .db + .create_repo(&seed_repo(&owner_did, "collision")) + .await + .expect("seed repo"); + + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_anon(&shared_cid)) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "handler must try every oid under the CID and serve the readable duplicate" + ); + assert!( + body.contains("public bytes"), + "the readable duplicate's content is served" + ); + } + + /// #173 (Fix 3, INV-10): the full-history allowed-set WALK is rate-limited per + /// source IP. A valid tree CID makes the object-type pre-check pass, so each + /// repeat request pays a fresh walk (request-scoped memo only) — unbounded + /// amplification. The brake sits on the walk, keyed on the non-farmable source + /// IP: a second walk from the same IP is shed with 429, but a cheap non-walk + /// fetch and a walk from a different IP are unaffected. + #[sqlx::test] + async fn ipfs_walk_rate_limited_per_source(pool: PgPool) { + use crate::db::VisibilityMode; + use gitlawb_core::identity::Keypair; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + let reader_did = reader.did().to_string(); + let slug = owner_did.replace([':', '/'], "_"); + let short = owner_did.split(':').next_back().unwrap().to_string(); + + let mut state = test_state(pool).await; + // One walk per IP per hour, and key on the rightmost X-Forwarded-For hop + // so the test can choose a source IP under `oneshot` (no socket peer). + state.ipfs_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::XForwardedFor; + + let fx = seed_cid_repos(&slug, &short, &["walklimit"]); + let bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walklimit.git"); + // The tree CID drives a path-scoped walk (the load-bearing amplification + // surface). The reader is allowed under /secret so the walk returns 200. + let secret_tree_cid = pin_cid_for(&bare, &fx.secret_tree_oid, &state.db).await; + + state + .db + .create_repo(&seed_repo(&owner_did, "walklimit")) + .await + .expect("seed repo"); + let rec = state + .db + .get_repo(&owner_did, "walklimit") + .await + .unwrap() + .unwrap(); + // Mode B path rule over /secret with the reader allowed → the reader's + // secret-tree fetch runs the allowed-tree walk and returns 200. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + VisibilityMode::B, + std::slice::from_ref(&reader_did), + &owner_did, + ) + .await + .expect("path rule"); + + // The MUST-NOT object must be a genuinely CHEAP fetch: an object served + // from a repo with NO path-scoped rule takes the no-walk path, so it is + // never rate-limited. It has to live in a repo that carries no path rule + // AND whose object graph does not overlap `walklimit` (a blob shared with + // the path-scoped repo would still walk there), so we seed a second bare + // repo with UNIQUE content. `acquire(owner, "walkpublic")` resolves to + // `/tmp//walkpublic.git`. + let pub_bare = std::path::PathBuf::from("/tmp") + .join(&slug) + .join("walkpublic.git"); + { + use std::process::Command; + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let src = std::env::temp_dir().join(format!("gl-cid-pub-{short}")); + let _ = std::fs::remove_dir_all(&src); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("cheap.txt"), b"cheap public bytes\n").unwrap(); + run(&["init", "-q", "--object-format=sha256"], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + run(&["add", "."], &src); + run(&["commit", "-qm", "cheap"], &src); + let _ = std::fs::remove_dir_all(&pub_bare); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + pub_bare.to_str().unwrap(), + ], + &src, + ); + let _ = std::fs::remove_dir_all(&src); + } + let cheap_oid = { + use std::process::Command; + let out = Command::new("git") + .args(["rev-parse", "HEAD:cheap.txt"]) + .current_dir(&pub_bare) + .output() + .unwrap(); + assert!(out.status.success(), "rev-parse cheap.txt"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let public_cid = pin_cid_for(&pub_bare, &cheap_oid, &state.db).await; + // Public repo, NO visibility rules → the cheap object takes the no-walk path. + state + .db + .create_repo(&seed_repo(&owner_did, "walkpublic")) + .await + .expect("seed public repo"); + + // 1st walk from 1.2.3.4 → 200 (walk ran, reader allowed). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!(st, StatusCode::OK, "1st walk from a source IP is served"); + + // 2nd identical walk from the SAME IP → 429 (per-IP walk budget spent). + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::TOO_MANY_REQUESTS, + "2nd walk from the same source IP is shed with 429" + ); + + // MUST-NOT: a cheap public (non-walk) fetch from the SAME limited IP, even + // after the 429, is served — the brake is on the walk, not the route. + let (st, body) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &public_cid, "1.2.3.4")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "a non-walk fetch is never rate-limited, even from the exhausted IP" + ); + assert!( + body.contains("cheap public bytes"), + "the cheap fetch serves content" + ); + + // PER-SOURCE isolation: the same tree-CID walk from a DIFFERENT IP → 200. + let (st, _) = cid_parts( + cid_router(&state) + .oneshot(cid_signed_xff(&reader, &secret_tree_cid, "5.6.7.8")) + .await + .unwrap(), + ) + .await; + assert_eq!( + st, + StatusCode::OK, + "one source's exhaustion must not shed another source's walk" + ); + } + // --------------------------------------------------------------------------- // Issue #120 — repo-scoped read surfaces visibility gate // --------------------------------------------------------------------------- diff --git a/crates/gitlawb-node/src/visibility.rs b/crates/gitlawb-node/src/visibility.rs index 5661687..a8d7ddb 100644 --- a/crates/gitlawb-node/src/visibility.rs +++ b/crates/gitlawb-node/src/visibility.rs @@ -437,6 +437,31 @@ mod tests { ); } + // #135 T1: a Mode-B rule on `/secret/**` must DENY the withheld directory's + // OWN path `/secret` (the `path == prefix` arm), not just strict descendants — + // otherwise get_by_cid's tree gate would serve the /secret tree object and leak + // its children. Pins parity with get_tree, which denies the /secret path. + #[test] + fn subtree_rule_denies_the_withheld_directory_itself() { + let reader = "did:key:z6MkReader"; + let rules = [rule("/secret/**", VisibilityMode::B, &[reader])]; + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/secret"), + Decision::Deny, + "anon denied at the withheld directory's OWN path /secret" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, None, "/public"), + Decision::Allow, + "anon allowed at a sibling path outside the withheld subtree" + ); + assert_eq!( + visibility_check(&rules, true, OWNER, Some(reader), "/secret"), + Decision::Allow, + "listed reader allowed at the withheld directory (caller-aware)" + ); + } + // #153 regression: cross-method DID must still be denied even when the // trailing segment collides with a bare owner key. #[test]