From 9c5d41ea20613feef46dea128739fc55ea3abed8 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Wed, 6 Aug 2025 19:09:21 -0400 Subject: [PATCH 01/24] Make apis branch-aware --- src/fluree/db/api.cljc | 29 ++++--- src/fluree/db/connection.cljc | 91 +++++++++++++-------- src/fluree/db/nameservice/storage.cljc | 23 +++--- src/fluree/sdk/browser.cljs | 4 +- src/fluree/sdk/node.cljs | 4 +- test/fluree/db/query/misc_queries_test.clj | 20 ++--- test/fluree/db/query/stable_hashes_test.clj | 4 +- 7 files changed, 97 insertions(+), 78 deletions(-) diff --git a/src/fluree/db/api.cljc b/src/fluree/db/api.cljc index 9e827dbfdf..73555420fc 100644 --- a/src/fluree/db/api.cljc +++ b/src/fluree/db/api.cljc @@ -534,24 +534,19 @@ (transact-api/create-with-txn conn txn opts)))) (defn status - "Returns current status of a ledger branch. + "Returns current status of a ledger. Parameters: conn - Connection object - ledger-id - Ledger alias or address - branch - (optional) Branch name (defaults to current branch) + ledger-id - Ledger alias (with optional @branch) or address Returns status map with commit and index information." - ([conn ledger-id] - (status conn ledger-id nil)) - ([conn ledger-id branch] - (validate-connection conn) - (promise-wrap - (go-try - (let [ledger (Connection id state parallelism commit-catalog index-catalog primary-publisher secondary-publishers remote-systems serializer cache defaults))) +(defn normalize-ledger-alias + "Ensures ledger alias includes branch. + If no @ symbol present, appends @main as default branch." + [ledger-alias] + (if (clojure.string/includes? ledger-alias "@") + ledger-alias + (str ledger-alias "@" commit-data/default-branch))) + (defn register-ledger "Creates a promise-chan and saves it in a cache of ledgers being held in-memory on the conn. @@ -165,7 +173,9 @@ "From a connection, lookup primary address from nameservice(s) for a given ledger alias" [{:keys [primary-publisher] :as _conn} ledger-alias] - (nameservice/publishing-address primary-publisher ledger-alias)) + (->> ledger-alias + normalize-ledger-alias + (nameservice/publishing-address primary-publisher))) (defn lookup-commit* "Returns commit address from first matching nameservice on a conn @@ -216,6 +226,7 @@ (defn published-ledger? [conn ledger-alias] (go-try + (log/debug "published-ledger? checking for:" ledger-alias) (loop [[nsv & r] (publishers conn)] (if nsv (or (ledger-alias "Returns ledger alias from commit map, if present. If not present @@ -374,18 +391,22 @@ (defn load-ledger-alias [conn alias] (go-try - (let [[cached? ledger-chan] (register-ledger conn alias)] + (let [;; Normalize ledger-alias to include branch + normalized-alias (normalize-ledger-alias alias) + [cached? ledger-chan] (register-ledger conn normalized-alias)] (if cached? ( store - storage/location - (storage/build-address ledger-alias))) - (defn ns-record "Generates nameservice metadata map for JSON storage using new minimal format" [ledger-alias branch commit-address t index-address] @@ -63,14 +62,16 @@ (storage/delete store address))) (publishing-address [_ ledger-alias] - (go (publishing-address* store ledger-alias))) + ;; Just return the alias - lookup will handle branch extraction via local-filename + (go ledger-alias)) nameservice/iNameService (lookup [_ ledger-address] (go-try - (let [{:keys [alias branch]} (nameservice/resolve-address (storage/location store) ledger-address nil) - branch (or branch "main") - filename (local-filename alias branch)] + ;; ledger-address is just the alias (potentially with @branch) + (let [filename (local-filename ledger-address)] + (log/debug "StorageNameService lookup:" {:ledger-address ledger-address + :filename filename}) (when-let [record-bytes (js (fluree/status conn ledger-id))) - ([conn ledger-id branch] (clj->js (fluree/status conn ledger-id branch)))) + [conn ledger-id] + (clj->js (fluree/status conn ledger-id))) (defn ^:export db [conn ledger-id] diff --git a/src/fluree/sdk/node.cljs b/src/fluree/sdk/node.cljs index 46b665b78e..715b73553e 100644 --- a/src/fluree/sdk/node.cljs +++ b/src/fluree/sdk/node.cljs @@ -42,8 +42,8 @@ (js->clj opts :keywordize-keys true)))) (defn ^:export status - ([conn ledger-id] (clj->js (fluree/status conn ledger-id))) - ([conn ledger-id branch] (clj->js (fluree/status conn ledger-id branch)))) + [conn ledger-id] + (clj->js (fluree/status conn ledger-id))) (defn ^:export db [conn ledger-id] diff --git a/test/fluree/db/query/misc_queries_test.clj b/test/fluree/db/query/misc_queries_test.clj index 1377a9a785..66e5db18c3 100644 --- a/test/fluree/db/query/misc_queries_test.clj +++ b/test/fluree/db/query/misc_queries_test.clj @@ -177,28 +177,28 @@ ["fluree:db:sha256:btqomzs3uzs7dspzbs5ht4e7af7qrahnvomx4s4id7apr5jm7dxn" :f/flakes 11] ["fluree:db:sha256:btqomzs3uzs7dspzbs5ht4e7af7qrahnvomx4s4id7apr5jm7dxn" :f/size 1266] ["fluree:db:sha256:btqomzs3uzs7dspzbs5ht4e7af7qrahnvomx4s4id7apr5jm7dxn" :f/t 1] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" "https://www.w3.org/2018/credentials#issuer" "did:key:z6Mkf2bJEm3KiDeCzrxbQDvT8jfYiz5t2Lo3fuvwPL6E6duw"] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" :f/address - "fluree:memory://bzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew"] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + "fluree:memory://utbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3"] + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" :f/alias "query/everything"] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" :f/branch "main"] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" :f/data "fluree:db:sha256:btqomzs3uzs7dspzbs5ht4e7af7qrahnvomx4s4id7apr5jm7dxn"] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" :f/previous - "fluree:commit:sha256:bb6dtkig73qu77wvwzpumlkmy2ftq3ikv2lhltti4eqvripnpqoqz"] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + "fluree:commit:sha256:bjsatu7x6aoagsfeh5igj6hafil4nl6obmzxo4fam6qddmrtt5p3"] + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" :f/time 720000] - ["fluree:commit:sha256:bbzz6o53rstn4bwdgn54bkgvoins246lzgol6ose4g4snupoyyeew" + ["fluree:commit:sha256:butbsssgd7exnjxbs3gchv7pedh4ekrqps466euwc4w3bpqrgek3" :f/v 1] [:ex/alice :type :ex/User] diff --git a/test/fluree/db/query/stable_hashes_test.clj b/test/fluree/db/query/stable_hashes_test.clj index 1a4a79bdf7..9bc67b0d86 100644 --- a/test/fluree/db/query/stable_hashes_test.clj +++ b/test/fluree/db/query/stable_hashes_test.clj @@ -29,10 +29,10 @@ :schema/age 30}]}) db1 @(fluree/commit! conn db0)] (testing "stable commit id" - (is (= "fluree:commit:sha256:bbdainpfs7v2pg2yj76uzpybdfdldvvt5idlbasisuhwlrxbiqhii" + (is (= "fluree:commit:sha256:bbsljxqq6heekn4adhwdu67wddxzm6ghxiwmu72bopcrpih4mgbix" (get-in db1 [:commit :id])))) (testing "stable commit address" - (is (= "fluree:memory://bdainpfs7v2pg2yj76uzpybdfdldvvt5idlbasisuhwlrxbiqhii" + (is (= "fluree:memory://bsljxqq6heekn4adhwdu67wddxzm6ghxiwmu72bopcrpih4mgbix" (get-in db1 [:commit :address])))) (testing "stable db id" (is (= "fluree:db:sha256:btqomzs3uzs7dspzbs5ht4e7af7qrahnvomx4s4id7apr5jm7dxn" From 55d4d2c58e2f7b173aba2e329e96e93b8ea84935 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Sat, 9 Aug 2025 09:45:51 -0400 Subject: [PATCH 02/24] remove most of special branch handling --- src/fluree/db/async_db.cljc | 29 ++-- src/fluree/db/branch.cljc | 32 ++--- src/fluree/db/commit/storage.cljc | 4 +- src/fluree/db/connection.cljc | 51 ++----- src/fluree/db/flake/commit_data.cljc | 20 ++- src/fluree/db/flake/flake_db.cljc | 21 +-- src/fluree/db/flake/index/novelty.cljc | 4 +- src/fluree/db/flake/index/storage.cljc | 3 +- src/fluree/db/ledger.cljc | 20 +-- src/fluree/db/nameservice/storage.cljc | 11 +- src/fluree/db/serde/json.cljc | 4 +- src/fluree/db/transact.cljc | 20 ++- test/fluree/db/query/history_test.clj | 61 +++----- test/fluree/db/query/misc_queries_test.clj | 23 ++-- test/fluree/db/query/property_path_test.clj | 145 ++++++++++++-------- test/fluree/db/query/stable_hashes_test.clj | 4 +- test/fluree/db_test.cljc | 46 +++---- 17 files changed, 231 insertions(+), 267 deletions(-) diff --git a/src/fluree/db/async_db.cljc b/src/fluree/db/async_db.cljc index 2ad2b46960..43411d3821 100644 --- a/src/fluree/db/async_db.cljc +++ b/src/fluree/db/async_db.cljc @@ -21,7 +21,7 @@ (declare ->async-db ->AsyncDB deliver!) -(defrecord AsyncDB [alias branch commit t db-chan] +(defrecord AsyncDB [alias commit t db-chan] dbproto/IFlureeDb (-query [_ tracker query-map] (go-try @@ -33,7 +33,7 @@ (async-db alias branch commit* t)] + updated-db (->async-db alias commit* t)] (go-try (let [db (AsyncDB alias branch commit t db-chan-at-t)] + db-at-t (->AsyncDB alias commit t db-chan-at-t)] (go (try* (let [db (AsyncDB alias branch commit t root-ch)] + root-db (->AsyncDB alias commit t root-ch)] (go (try* (let [db (AsyncDB ledger-alias branch commit-map t (async/promise-chan))) + [ledger-alias commit-map t] + (->AsyncDB ledger-alias commit-map t (async/promise-chan))) (defn load - ([ledger-alias branch commit-catalog index-catalog commit-jsonld indexing-opts] + ([ledger-alias commit-catalog index-catalog commit-jsonld indexing-opts] (let [commit-map (commit-data/jsonld->clj commit-jsonld)] - (load ledger-alias branch commit-catalog index-catalog commit-jsonld commit-map indexing-opts))) - ([ledger-alias branch commit-catalog index-catalog commit-jsonld commit-map indexing-opts] + (load ledger-alias commit-catalog index-catalog commit-jsonld commit-map indexing-opts))) + ([ledger-alias commit-catalog index-catalog commit-jsonld commit-map indexing-opts] (let [t (-> commit-map :data :t) - ;; Ensure AsyncDB commit reflects index t when an index address exists but :t is missing - commit-map* (if (and (get-in commit-map [:index :address]) - (nil? (get-in commit-map [:index :data :t]))) - (assoc-in commit-map [:index :data :t] t) - commit-map) - async-db (->async-db ledger-alias branch commit-map* t)] + async-db (->async-db ledger-alias commit-map t)] (go - (let [db ( commit-map commit-data/->json-ld json-ld/expand)) (defn load-db - [alias branch commit-catalog index-catalog commit-map] + [combined-alias commit-catalog index-catalog commit-map] (let [commit-jsonld (commit-map->commit-jsonld commit-map)] - (async-db/load alias branch commit-catalog index-catalog + (async-db/load combined-alias commit-catalog index-catalog commit-jsonld commit-map nil))) (defn update-index-async @@ -56,11 +56,11 @@ return immediately - and for a large amount of novelty, updating the db to reflect the latest index can take some time which would lead to atom contention." - [{:keys [alias commit branch t] :as current-db} index-map] + [{:keys [alias commit t] :as current-db} index-map] (if (async-db/db? current-db) (dbproto/-index-update current-db index-map) (let [updated-commit (assoc commit :index index-map) - updated-db (async-db/->async-db alias branch updated-commit t)] + updated-db (async-db/->async-db alias updated-commit t)] (go ;; update index in the background, return updated db immediately (->> (dbproto/-index-update current-db index-map) (async-db/deliver! updated-db))) @@ -89,9 +89,9 @@ current-state)))) (defn reload-with-index - [{:keys [commit-catalog index-catalog commit] :as _db} alias branch index] + [{:keys [commit-catalog index-catalog commit alias] :as _db} index] (let [indexed-commit (assoc commit :index index)] - (load-db alias branch commit-catalog index-catalog indexed-commit))) + (load-db alias commit-catalog index-catalog indexed-commit))) (defn use-latest-db "Returns the most recent db from branch-state if it matches @@ -107,22 +107,22 @@ latest-db))) (defn use-latest-index - [{db-commit :commit, :as db} idx-commit alias branch branch-state] + [{db-commit :commit, :as db} idx-commit branch-state] (if (newer-index? idx-commit db-commit) (let [updated-db (or (use-latest-db db idx-commit branch-state) (try* (dbproto/-index-update db (:index idx-commit)) (catch* e (log/error e "Exception updating db with new index, attempting full reload. Exception:" (ex-message e)) - (reload-with-index db alias branch (:index idx-commit)))))] + (reload-with-index db (:index idx-commit)))))] updated-db) db)) (defn index-queue - [alias branch publishers branch-state] + [publishers branch-state] (let [buf (async/sliding-buffer 1) queue (async/chan buf)] (go-loop [last-index-commit nil] (when-let [{:keys [db index-files-ch complete-ch]} (clj commit-jsonld) - initial-db (async-db/load ledger-alias branch-name commit-catalog index-catalog + initial-db (async-db/load combined-alias commit-catalog index-catalog commit-jsonld commit-map indexing-opts) state (atom {:commit commit-map :current-db initial-db}) - idx-q (index-queue ledger-alias branch-name publishers state)] + idx-q (index-queue publishers state)] {:name branch-name - :alias ledger-alias + :alias combined-alias :state state :index-queue idx-q :indexing-opts indexing-opts}))) diff --git a/src/fluree/db/commit/storage.cljc b/src/fluree/db/commit/storage.cljc index 74ac59bf8b..544050bfd9 100644 --- a/src/fluree/db/commit/storage.cljc +++ b/src/fluree/db/commit/storage.cljc @@ -152,9 +152,9 @@ (storage/content-write-json storage path jsonld))) (defn write-genesis-commit - [storage ledger-alias branch publish-addresses init-time] + [storage ledger-alias publish-addresses init-time] (go-try - (let [genesis-commit (commit-data/blank-commit ledger-alias branch publish-addresses init-time) + (let [genesis-commit (commit-data/blank-commit ledger-alias publish-addresses init-time) initial-context (get genesis-commit "@context") initial-db-data (-> genesis-commit (get "data") diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index 1799d62810..5f20301a68 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -242,25 +242,6 @@ (recur r (into addrs ( conn :defaults :identity))) (defn parse-ledger-options - [conn {:keys [did branch indexing] - :or {branch commit-data/default-branch}}] + [conn {:keys [did indexing]}] (let [did* (parse-identity conn did) ledger-default (-> conn :defaults :indexing) indexing* (merge ledger-default indexing)] {:did did* - :branch branch :indexing indexing*})) (defn throw-ledger-exists @@ -302,11 +281,7 @@ [{:keys [commit-catalog index-catalog primary-publisher secondary-publishers] :as conn} ledger-alias opts] (go-try (let [;; Normalize ledger-alias to include branch - normalized-alias (normalize-ledger-alias ledger-alias) - ;; Extract the actual alias without branch for passing to ledger/create - [actual-alias _] (if (str/includes? ledger-alias "@") - (str/split ledger-alias #"@" 2) - [ledger-alias nil])] + normalized-alias (normalize-ledger-alias ledger-alias)] (if (ledger-alias conn address expanded-commit) - ;; Determine branch using helpers in priority order - branch (or (branch-from-commit expanded-commit) - (branch-from-ns ns-record) - (branch-from-alias ledger-alias)) - - {:keys [did branch indexing]} (parse-ledger-options conn {:branch branch}) - ledger (ledger/instantiate ledger-alias address branch commit-catalog index-catalog + combined-alias (commit->ledger-alias conn address expanded-commit) + + {:keys [did indexing]} (parse-ledger-options conn {}) + ledger (ledger/instantiate combined-alias address commit-catalog index-catalog primary-publisher secondary-publishers indexing did expanded-commit)] - (ns-subscribe/subscribe-ledger conn ledger-alias) + (ns-subscribe/subscribe-ledger conn combined-alias) (async/put! ledger-chan ledger) ledger) (throw (ex-info (str "Unable to load. No record of ledger at address: " address " exists.") diff --git a/src/fluree/db/flake/commit_data.cljc b/src/fluree/db/flake/commit_data.cljc index 492d889275..c773aa06b6 100644 --- a/src/fluree/db/flake/commit_data.cljc +++ b/src/fluree/db/flake/commit_data.cljc @@ -1,5 +1,6 @@ (ns fluree.db.flake.commit-data - (:require [fluree.crypto :as crypto] + (:require [clojure.string :as str] + [fluree.crypto :as crypto] [fluree.db.constants :as const] [fluree.db.datatype :as datatype] [fluree.db.flake :as flake] @@ -140,7 +141,10 @@ (let [id (get-id jsonld) v (get-first-value jsonld const/iri-v) alias (get-first-value jsonld const/iri-alias) - branch (get-first-value jsonld const/iri-branch) + ;; Normalize alias to include @main if no branch present + alias* (if (str/includes? alias "@") + alias + (str alias "@" default-branch)) address (-> jsonld (get-first-value const/iri-address) not-empty) @@ -158,8 +162,7 @@ (cond-> {:id id :v v - :alias alias - :branch branch + :alias alias* :time time :tag (mapv get-value tags) :data (parse-db-data data) @@ -223,12 +226,9 @@ (defn blank-commit "Creates a skeleton blank commit map." - [alias branch publish-addresses init-time] + [alias publish-addresses init-time] (let [commit-json (->json-ld {:alias alias :v 0 - :branch (if branch - (util/keyword->str branch) - default-branch) :data {:t 0 :flakes 0 :size 0} @@ -416,7 +416,7 @@ db-sid. Used when committing to an in-memory ledger value and when reifying a ledger from storage on load." [db t commit] - (let [{:keys [id address alias branch data time v previous author issuer message txn]} commit + (let [{:keys [id address alias data time v previous author issuer message txn]} commit {db-t :t, db-address :address, data-id :id, :keys [flakes size]} data commit-sid (iri/encode-iri db id) db-sid (iri/encode-iri db data-id)] @@ -426,8 +426,6 @@ (flake/create commit-sid const/$_address address const/$xsd:string t true nil) ;; alias (flake/create commit-sid const/$_ledger:alias alias const/$xsd:string t true nil) - ;; branch - (flake/create commit-sid const/$_ledger:branch branch const/$xsd:string t true nil) ;; v (flake/create commit-sid const/$_v v const/$xsd:int t true nil) ;; time diff --git a/src/fluree/db/flake/flake_db.cljc b/src/fluree/db/flake/flake_db.cljc index 0c2edd1169..d182c4a26a 100644 --- a/src/fluree/db/flake/flake_db.cljc +++ b/src/fluree/db/flake/flake_db.cljc @@ -290,7 +290,7 @@ (merge-flakes t-new all-flakes) (assoc :commit commit-metadata))))) -(defrecord FlakeDB [index-catalog commit-catalog alias branch commit t tt-id stats +(defrecord FlakeDB [index-catalog commit-catalog alias commit t tt-id stats spot post opst tspo vg schema comparators staged novelty policy namespaces namespace-codes max-namespace-code reindex-min-bytes reindex-max-bytes max-old-indexes] @@ -410,7 +410,7 @@ (defn display [db] - (select-keys db [:alias :branch :t :stats :policy])) + (select-keys db [:alias :t :stats :policy])) #?(:cljs (extend-type FlakeDB IPrintWithWriter @@ -535,9 +535,9 @@ ;; TODO - VG - need to reify vg from db-root!! (defn load - ([ledger-alias commit-catalog index-catalog branch commit-pair] - (load ledger-alias commit-catalog index-catalog branch commit-pair {})) - ([ledger-alias commit-catalog index-catalog branch [commit-jsonld commit-map] indexing-opts] + ([ledger-alias commit-catalog index-catalog commit-pair] + (load ledger-alias commit-catalog index-catalog commit-pair {})) + ([ledger-alias commit-catalog index-catalog [commit-jsonld commit-map] indexing-opts] (go-try (let [commit-t (-> commit-jsonld (get-first const/iri-data) @@ -546,21 +546,12 @@ ( root-map :namespace-codes iri/get-max-namespace-code) - ;; Ensure commit map reflects loaded index t when an index address exists - commit-map* (if (get-in commit-map [:index :address]) - (update commit-map :index - (fn [idx] - (let [existing-data (:data idx) - updated-data (assoc (or existing-data {}) :t (:t root-map))] - (assoc idx :data updated-data)))) - commit-map) indexed-db (-> root-map (add-reindex-thresholds indexing-opts) (assoc :index-catalog index-catalog :commit-catalog commit-catalog :alias ledger-alias - :branch branch - :commit commit-map* + :commit commit-map :tt-id nil :comparators index/comparators :staged nil diff --git a/src/fluree/db/flake/index/novelty.cljc b/src/fluree/db/flake/index/novelty.cljc index 6b4ed6db0d..20baacc8e9 100644 --- a/src/fluree/db/flake/index/novelty.cljc +++ b/src/fluree/db/flake/index/novelty.cljc @@ -391,13 +391,13 @@ refresh-ch ([{:keys [garbage], refreshed-db :db, :as _status}] - (let [{:keys [index-catalog alias branch] :as refreshed-db*} + (let [{:keys [index-catalog alias] :as refreshed-db*} (assoc-in refreshed-db [:stats :indexed] t) ;; TODO - ideally issue garbage/root writes to RAFT together ;; as a tx, currently requires waiting for both ;; through raft sync garbage-res (when (seq garbage) - (let [write-res (Ledger {:id (random-uuid) :did did @@ -160,20 +164,20 @@ context." [{:keys [alias primary-address publish-addresses commit-catalog index-catalog primary-publisher secondary-publishers]} - {:keys [did branch indexing] :as _opts}] + {:keys [did indexing] :as _opts}] (go-try - (let [ledger-alias* (normalize-alias alias) + (let [normalized-alias (normalize-alias alias) ;; internal-only opt used for migrating ledgers without genesis commits init-time (util/current-time-iso) genesis-commit (map genesis-commit nil) compact-commit (commit-data/->json-ld commit-map)] (jsonld staged-db commit-data-opts) {:keys [txn-id author annotation]} - (db-id (:hash data-write-result)) keypair {:did did, :private private} @@ -268,7 +264,7 @@ _ (log/debug "commit!: write-commit start" {:ledger ledger-alias}) {:keys [commit-map commit-jsonld write-result]} - (" {"@id" "ex:f"}}] + (let [result @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "" {"@id" "ex:f"}}] "select" {"ex:a" ["*"]}})] (is (= {:status 400, :error :db/unsupported-transitive-path} (ex-data result))) (is (= "Unsupported transitive path." (ex-message result))))))) (testing "object variable" - (let [db1 @(fluree/update db0 {"insert" + (let [db1 @(fluree/update db0 {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:a" "ex:knows" {"@id" "ex:b" "ex:knows" [{"@id" "ex:c"} @@ -40,20 +43,25 @@ "ex:knows" {"@id" "ex:e"}}]}}]})] (testing "non-transitive" (is (= ["ex:b"] - @(fluree/query db1 {"where" [{"@id" "ex:a" "ex:knows" "?who"}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "ex:knows" "?who"}] "select" "?who"})))) (testing "transitive" (testing "without cycle" (is (= ["ex:b" "ex:c" "ex:d" "ex:e"] - @(fluree/query db1 {"where" [{"@id" "ex:a" "<+>" "?who"}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "<+>" "?who"}] "select" "?who"})))) (testing "with cycle" - (let [db2 @(fluree/update db1 {"insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] + (let [db2 @(fluree/update db1 {"@context" {"ex" "http://example.org/"} + "insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] (is (= ["ex:b" "ex:c" "ex:d" "ex:e" "ex:a"] - @(fluree/query db2 {"where" [{"@id" "ex:a" "" "?who"}] + @(fluree/query db2 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "" "?who"}] "select" "?who"})))))))) (testing "subject variable" - (let [db1 @(fluree/update db0 {"insert" + (let [db1 @(fluree/update db0 {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:a" "ex:knows" {"@id" "ex:b" "ex:knows" [{"@id" "ex:c"} @@ -61,52 +69,62 @@ "ex:knows" {"@id" "ex:e"}}]}}]})] (testing "non-transitive" (is (= ["ex:d"] - @(fluree/query db1 {"where" [{"@id" "?who" "ex:knows" {"@id" "ex:e"}}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?who" "ex:knows" {"@id" "ex:e"}}] "select" "?who"})))) (testing "transitive" (testing "without cycle" (is (= ["ex:d" "ex:b" "ex:a"] - @(fluree/query db1 {"where" [{"@id" "?who" "" {"@id" "ex:e"}}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?who" "" {"@id" "ex:e"}}] "select" "?who"})))) (testing "with cycle" - (let [db2 @(fluree/update db1 {"insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] + (let [db2 @(fluree/update db1 {"@context" {"ex" "http://example.org/"} + "insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] (is (= ["ex:d" "ex:b" "ex:a" "ex:e"] - @(fluree/query db2 {"where" [{"@id" "?who" "" {"@id" "ex:e"}}] + @(fluree/query db2 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?who" "" {"@id" "ex:e"}}] "select" "?who"})))))))) (testing "subject and object variable" (let [db1 @(fluree/update db0 - {"insert" + {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:1" "ex:knows" {"@id" "ex:2" "ex:knows" {"@id" "ex:3"}}}]})] (testing "non-transitive" (is (= [["ex:1" "ex:2"] ["ex:2" "ex:3"]] - @(fluree/query db1 {"where" [{"@id" "?s" "ex:knows" "?o"}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?s" "ex:knows" "?o"}] "select" ["?s" "?o"]})))) (testing "transitive" (testing "without cycle" - (is (= [["ex:1" "ex:2"] - ["ex:2" "ex:3"] - ["ex:1" "ex:3"]] - @(fluree/query db1 {"where" [{"@id" "?x" "" "?y"}] - "select" ["?x" "?y"]})))) + (is (= #{["ex:1" "ex:2"] + ["ex:2" "ex:3"] + ["ex:1" "ex:3"]} + (set @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?x" "" "?y"}] + "select" ["?x" "?y"]}))))) (testing "with cycle" - (let [db2 @(fluree/update db1 {"insert" {"@id" "ex:3" "ex:knows" {"@id" "ex:1"}}})] - (is (= [["ex:3" "ex:2"] - ["ex:1" "ex:2"] - ["ex:2" "ex:3"] - ["ex:1" "ex:3"] - ["ex:2" "ex:2"] - ["ex:3" "ex:3"] - ["ex:3" "ex:1"] - ["ex:2" "ex:1"] - ["ex:1" "ex:1"]] - @(fluree/query db2 {"where" [{"@id" "?x" "" "?y"}] - "select" ["?x" "?y"]}))))))))) + (let [db2 @(fluree/update db1 {"@context" {"ex" "http://example.org/"} + "insert" {"@id" "ex:3" "ex:knows" {"@id" "ex:1"}}})] + (is (= #{["ex:3" "ex:2"] + ["ex:1" "ex:2"] + ["ex:2" "ex:3"] + ["ex:1" "ex:3"] + ["ex:2" "ex:2"] + ["ex:3" "ex:3"] + ["ex:3" "ex:1"] + ["ex:2" "ex:1"] + ["ex:1" "ex:1"]} + (set @(fluree/query db2 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?x" "" "?y"}] + "select" ["?x" "?y"]})))))))))) (testing "zero+" (testing "no variables" - (let [db1 @(fluree/update db0 {"insert" + (let [db1 @(fluree/update db0 {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:a" "ex:y" [{"@id" "ex:b" "ex:y" {"@id" "ex:c" @@ -120,17 +138,20 @@ "ex:y" {"@id" "ex:k"}}]}]}]})] (testing "non-transitive" (is (= [] - @(fluree/query db1 {"where" [{"@id" "ex:a" "ex:y" {"@id" "ex:f"}}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "ex:y" {"@id" "ex:f"}}] "select" {"ex:a" ["*"]}})))) (testing "transitive" - (let [result @(fluree/query db1 {"where" [{"@id" "ex:a" "" {"@id" "ex:f"}}] + (let [result @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "" {"@id" "ex:f"}}] "select" {"ex:a" ["*"]}})] (is (= {:status 400, :error :db/unsupported-transitive-path} (ex-data result))) (is (= "Unsupported transitive path." (ex-message result))))))) (testing "object variable" - (let [db1 @(fluree/update db0 {"insert" + (let [db1 @(fluree/update db0 {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:a" "ex:knows" {"@id" "ex:b" "ex:knows" [{"@id" "ex:c"} @@ -138,20 +159,25 @@ "ex:knows" {"@id" "ex:e"}}]}}]})] (testing "non-transitive" (is (= ["ex:b"] - @(fluree/query db1 {"where" [{"@id" "ex:a" "ex:knows" "?who"}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "ex:knows" "?who"}] "select" "?who"})))) (testing "transitive" (testing "without cycle" (is (= ["ex:a" "ex:b" "ex:c" "ex:d" "ex:e"] - @(fluree/query db1 {"where" [{"@id" "ex:a" "" "?who"}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "" "?who"}] "select" "?who"})))) (testing "with cycle" - (let [db2 @(fluree/update db1 {"insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] + (let [db2 @(fluree/update db1 {"@context" {"ex" "http://example.org/"} + "insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] (is (= ["ex:a" "ex:b" "ex:c" "ex:d" "ex:e"] - @(fluree/query db2 {"where" [{"@id" "ex:a" "" "?who"}] + @(fluree/query db2 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "ex:a" "" "?who"}] "select" "?who"})))))))) (testing "subject variable" - (let [db1 @(fluree/update db0 {"insert" + (let [db1 @(fluree/update db0 {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:a" "ex:knows" {"@id" "ex:b" "ex:knows" [{"@id" "ex:c"} @@ -159,28 +185,34 @@ "ex:knows" {"@id" "ex:e"}}]}}]})] (testing "non-transitive" (is (= ["ex:d"] - @(fluree/query db1 {"where" [{"@id" "?who" "ex:knows" {"@id" "ex:e"}}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?who" "ex:knows" {"@id" "ex:e"}}] "select" "?who"})))) (testing "transitive" (testing "without cycle" (is (= ["ex:e" "ex:d" "ex:b" "ex:a"] - @(fluree/query db1 {"where" [{"@id" "?who" "" {"@id" "ex:e"}}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?who" "" {"@id" "ex:e"}}] "select" "?who"})))) (testing "with cycle" - (let [db2 @(fluree/update db1 {"insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] + (let [db2 @(fluree/update db1 {"@context" {"ex" "http://example.org/"} + "insert" {"@id" "ex:e" "ex:knows" {"@id" "ex:a"}}})] (is (= ["ex:e" "ex:d" "ex:b" "ex:a"] - @(fluree/query db2 {"where" [{"@id" "?who" "" {"@id" "ex:e"}}] + @(fluree/query db2 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?who" "" {"@id" "ex:e"}}] "select" "?who"})))))))) (testing "subject and object variable" (let [db1 @(fluree/update db0 - {"insert" + {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:1" "ex:knows" {"@id" "ex:2" "ex:knows" {"@id" "ex:3"}}}]})] (testing "non-transitive" (is (= [["ex:1" "ex:2"] ["ex:2" "ex:3"]] - @(fluree/query db1 {"where" [{"@id" "?s" "ex:knows" "?o"}] + @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?s" "ex:knows" "?o"}] "select" ["?s" "?o"]})))) (testing "transitive" (testing "without cycle" @@ -190,11 +222,13 @@ ["ex:2" "ex:2"] ["ex:2" "ex:3"] ["ex:3" "ex:3"]] - (sort @(fluree/query db1 {"where" [{"@id" "?x" "" "?y"}] + (sort @(fluree/query db1 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?x" "" "?y"}] "select" ["?x" "?y"]}))))) (testing "disjoint subgraphs" (let [db2 @(fluree/update db1 - {"insert" + {"@context" {"ex" "http://example.org/"} + "insert" [{"@id" "ex:4" "ex:knows" {"@id" "ex:5" "ex:knows" {"@id" "ex:6"}}}]})] @@ -211,10 +245,12 @@ ["ex:5" "ex:5"] ["ex:5" "ex:6"] ["ex:6" "ex:6"]} - (set @(fluree/query db2 {"where" [{"@id" "?x" "" "?y"}] + (set @(fluree/query db2 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?x" "" "?y"}] "select" ["?x" "?y"]})))))) (testing "with cycle" - (let [db2 @(fluree/update db1 {"insert" {"@id" "ex:3" "ex:knows" {"@id" "ex:1"}}})] + (let [db2 @(fluree/update db1 {"@context" {"ex" "http://example.org/"} + "insert" {"@id" "ex:3" "ex:knows" {"@id" "ex:1"}}})] (is (= [["ex:1" "ex:1"] ["ex:1" "ex:2"] ["ex:1" "ex:3"] @@ -226,5 +262,6 @@ ["ex:3" "ex:1"] ["ex:3" "ex:2"] ["ex:3" "ex:3"]] - (sort @(fluree/query db2 {"where" [{"@id" "?x" "" "?y"}] + (sort @(fluree/query db2 {"@context" {"ex" "http://example.org/"} + "where" [{"@id" "?x" "" "?y"}] "select" ["?x" "?y"]})))))))))))) diff --git a/test/fluree/db/query/stable_hashes_test.clj b/test/fluree/db/query/stable_hashes_test.clj index 9bc67b0d86..e34168e358 100644 --- a/test/fluree/db/query/stable_hashes_test.clj +++ b/test/fluree/db/query/stable_hashes_test.clj @@ -29,10 +29,10 @@ :schema/age 30}]}) db1 @(fluree/commit! conn db0)] (testing "stable commit id" - (is (= "fluree:commit:sha256:bbsljxqq6heekn4adhwdu67wddxzm6ghxiwmu72bopcrpih4mgbix" + (is (= "fluree:commit:sha256:bbgrew4jalbhqdlvkvmtkcgm3zptxqjkesaintdpah2f33nrpzhfm" (get-in db1 [:commit :id])))) (testing "stable commit address" - (is (= "fluree:memory://bsljxqq6heekn4adhwdu67wddxzm6ghxiwmu72bopcrpih4mgbix" + (is (= "fluree:memory://bgrew4jalbhqdlvkvmtkcgm3zptxqjkesaintdpah2f33nrpzhfm" (get-in db1 [:commit :address])))) (testing "stable db id" (is (= "fluree:db:sha256:btqomzs3uzs7dspzbs5ht4e7af7qrahnvomx4s4id7apr5jm7dxn" diff --git a/test/fluree/db_test.cljc b/test/fluree/db_test.cljc index ef4260a4c5..dbd7c0b480 100644 --- a/test/fluree/db_test.cljc +++ b/test/fluree/db_test.cljc @@ -1085,34 +1085,34 @@ ;; wait for everything to be written (Thread/sleep 1000) (testing "before drop" - (is (= ["destined-for-drop" "ns@v1"] + (is (= ["destined-for-drop@main" "ns@v1"] (sort (async/ Date: Sat, 9 Aug 2025 21:58:49 -0400 Subject: [PATCH 03/24] remove 'branch' from most fns --- src/fluree/db/connection.cljc | 5 ++- src/fluree/db/flake/commit_data.cljc | 6 +--- src/fluree/db/ledger.cljc | 16 +++++---- src/fluree/db/migrations/nameservice_v1.cljc | 5 +-- src/fluree/db/nameservice/storage.cljc | 37 +++++++------------- 5 files changed, 31 insertions(+), 38 deletions(-) diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index 5f20301a68..2b9e8bddce 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -448,7 +448,10 @@ (try* (let [alias (if (fluree-address? alias) (nameservice/address-path alias) - alias)] + ;; Normalize alias to include branch if not present + (if (str/includes? alias "@") + alias + (str alias "@main")))] (loop [[publisher & r] (publishers conn)] (when publisher (let [ledger-addr ( jsonld (get-first-value const/iri-address) not-empty) @@ -162,7 +158,7 @@ (cond-> {:id id :v v - :alias alias* + :alias alias :time time :tag (mapv get-value tags) :data (parse-db-data data) diff --git a/src/fluree/db/ledger.cljc b/src/fluree/db/ledger.cljc index 5c6331691b..e3a1ca39e2 100644 --- a/src/fluree/db/ledger.cljc +++ b/src/fluree/db/ledger.cljc @@ -132,16 +132,16 @@ [combined-alias ledger-address commit-catalog index-catalog primary-publisher secondary-publishers indexing-opts did latest-commit] (let [;; Parse ledger name and branch from combined alias - [ledger-alias branch] (if (str/includes? combined-alias "@") - (str/split combined-alias #"@" 2) - [combined-alias "main"]) + [_ branch] (if (str/includes? combined-alias "@") + (str/split combined-alias #"@" 2) + [combined-alias "main"]) publishers (cons primary-publisher secondary-publishers) branches {branch (branch/state-map combined-alias branch commit-catalog index-catalog publishers latest-commit indexing-opts)}] (map->Ledger {:id (random-uuid) :did did :state (atom (initial-state branches branch)) - :alias ledger-alias + :alias combined-alias ;; Use the full combined alias including branch :address ledger-address :commit-catalog commit-catalog :index-catalog index-catalog @@ -167,17 +167,21 @@ {:keys [did indexing] :as _opts}] (go-try (let [normalized-alias (normalize-alias alias) + ;; Add @main if no branch is specified + ledger-alias (if (str/includes? normalized-alias "@") + normalized-alias + (str normalized-alias "@main")) ;; internal-only opt used for migrating ledgers without genesis commits init-time (util/current-time-iso) genesis-commit (map genesis-commit nil) compact-commit (commit-data/->json-ld commit-map)] ( {"@context" {"f" iri/f-ns} - "@id" (str ledger-alias "@" branch) + "@id" ledger-alias ;; Already includes @branch "@type" ["f:Database" "f:PhysicalDatabase"] - "f:ledger" {"@id" ledger-alias} + "f:ledger" {"@id" alias} ;; Just the ledger name without branch "f:branch" branch "f:commit" {"@id" commit-address} "f:t" t @@ -37,21 +33,14 @@ nameservice/Publisher (publish [_ data] (let [;; Extract data from compact JSON-LD format (both genesis and regular commits now use this) - combined-alias (get data "alias") - ;; Parse branch from combined alias if present - [ledger-alias branch] (if (str/includes? combined-alias "@") - (str/split combined-alias #"@" 2) - [combined-alias "main"]) + ledger-alias (get data "alias") ;; Already includes @branch commit-address (get data "address") t-value (get-in data ["data" "t"]) index-address (get-in data ["index" "address"]) - ns-metadata (ns-record ledger-alias branch commit-address t-value index-address) + ns-metadata (ns-record ledger-alias commit-address t-value index-address) record-bytes (json/stringify-UTF8 ns-metadata) - filename (local-filename ledger-alias branch)] - (log/debug "nameservice.storage/publish start" {:ledger ledger-alias :branch branch :filename filename}) - (let [res (storage/write-bytes store filename record-bytes)] - (log/debug "nameservice.storage/publish enqueued" {:ledger ledger-alias :branch branch :filename filename}) - res))) + filename (local-filename ledger-alias)] + (storage/write-bytes store filename record-bytes))) (retract [_ ledger-alias] (let [filename (local-filename ledger-alias) From 4d7b4c99235d1f434cd5fc6433bd9735bb7aaa11 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Sun, 10 Aug 2025 21:07:45 -0400 Subject: [PATCH 04/24] ensure branches don't use sub-directories --- src/fluree/db/commit/storage.cljc | 12 ++++--- src/fluree/db/flake/index/storage.cljc | 6 ++-- src/fluree/db/ledger.cljc | 8 ++++- src/fluree/db/transact.cljc | 18 +++++----- test/fluree/db_test.cljc | 46 +++++++++++++------------- 5 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/fluree/db/commit/storage.cljc b/src/fluree/db/commit/storage.cljc index 544050bfd9..09a91fb679 100644 --- a/src/fluree/db/commit/storage.cljc +++ b/src/fluree/db/commit/storage.cljc @@ -147,21 +147,23 @@ resp-ch)) (defn write-jsonld - [storage ledger-alias jsonld] - (let [path (str/join "/" [ledger-alias "commit"])] + [storage ledger-name jsonld] + (let [path (str/join "/" [ledger-name "commit"])] (storage/content-write-json storage path jsonld))) (defn write-genesis-commit [storage ledger-alias publish-addresses init-time] (go-try - (let [genesis-commit (commit-data/blank-commit ledger-alias publish-addresses init-time) + (let [;; Use full alias for commit data, but base name for storage paths + ledger-base-name (first (str/split ledger-alias #"@" 2)) + genesis-commit (commit-data/blank-commit ledger-alias publish-addresses init-time) initial-context (get genesis-commit "@context") initial-db-data (-> genesis-commit (get "data") (assoc "@context" initial-context)) - {db-address :address} ( genesis-commit* (assoc "address" commit-address) json-ld/expand)))) diff --git a/src/fluree/db/flake/index/storage.cljc b/src/fluree/db/flake/index/storage.cljc index 4d1385c046..55f3f164fd 100644 --- a/src/fluree/db/flake/index/storage.cljc +++ b/src/fluree/db/flake/index/storage.cljc @@ -38,8 +38,10 @@ (defn write-index-file [storage ledger-alias index-type serialized-data] - (let [index-name (name index-type) - path (str/join "/" [ledger-alias "index" index-name])] + (let [;; Extract base ledger name from alias for storage path + ledger-name (first (str/split ledger-alias #"@" 2)) + index-name (name index-type) + path (str/join "/" [ledger-name "index" index-name])] (storage/content-write-json storage path serialized-data))) (defn write-leaf diff --git a/src/fluree/db/ledger.cljc b/src/fluree/db/ledger.cljc index e3a1ca39e2..1100f168e6 100644 --- a/src/fluree/db/ledger.cljc +++ b/src/fluree/db/ledger.cljc @@ -13,6 +13,12 @@ #?(:clj (set! *warn-on-reflection* true)) +(defn ledger-base-name + "Extracts the base ledger name from a ledger alias that may include a branch. + e.g., 'my-ledger@main' -> 'my-ledger'" + [ledger-alias] + (first (str/split ledger-alias #"@" 2))) + (defn get-branch-meta "Retrieves branch metadata from ledger state" [{:keys [state] :as _ledger} requested-branch] @@ -141,7 +147,7 @@ (map->Ledger {:id (random-uuid) :did did :state (atom (initial-state branches branch)) - :alias combined-alias ;; Use the full combined alias including branch + :alias combined-alias ;; Full alias including branch :address ledger-address :commit-catalog commit-catalog :index-catalog index-catalog diff --git a/src/fluree/db/transact.cljc b/src/fluree/db/transact.cljc index 917003a43c..14b9e216b0 100644 --- a/src/fluree/db/transact.cljc +++ b/src/fluree/db/transact.cljc @@ -144,10 +144,11 @@ parse-data-helpers)) (defn save-txn! - ([{:keys [commit-catalog] ledger-alias :alias :as _ledger} txn] - (save-txn! commit-catalog ledger-alias txn)) - ([commit-catalog ledger-alias txn] - (let [path (str/join "/" [ledger-alias "txn"])] + ([{:keys [commit-catalog alias] :as _ledger} txn] + (let [ledger-name (first (str/split alias #"@" 2))] + (save-txn! commit-catalog ledger-name txn))) + ([commit-catalog ledger-name txn] + (let [path (str/join "/" [ledger-name "txn"])] (storage/content-write-json commit-catalog path txn)))) ;; TODO - as implemented the db handles 'staged' data as per below (annotation, raw txn) @@ -225,12 +226,13 @@ returns a db with an updated :commit." ([ledger db] (commit! ledger db {})) - ([{ledger-alias :alias, :as ledger} + ([{ledger-alias :alias :as ledger} {:keys [alias branch t stats commit] :as staged-db} opts] (log/debug "commit!: write-transaction start" {:ledger ledger-alias}) (go-try (let [{:keys [commit-catalog]} ledger + ledger-name (first (str/split ledger-alias #"@" 2)) {:keys [tag time message did private commit-data-opts index-files-ch] :or {time (util/current-time-iso)}} @@ -240,9 +242,9 @@ (commit-data/db->jsonld staged-db commit-data-opts) {:keys [txn-id author annotation]} - (db-id (:hash data-write-result)) keypair {:did did, :private private} @@ -264,7 +266,7 @@ _ (log/debug "commit!: write-commit start" {:ledger ledger-alias}) {:keys [commit-map commit-jsonld write-result]} - ( Date: Mon, 11 Aug 2025 13:49:48 -0400 Subject: [PATCH 05/24] update branch separator to ':' --- src/fluree/db/api.cljc | 8 ++--- src/fluree/db/commit/storage.cljc | 3 +- src/fluree/db/connection.cljc | 10 +++--- src/fluree/db/flake/index/storage.cljc | 3 +- src/fluree/db/ledger.cljc | 16 +++------ src/fluree/db/migrations/nameservice_v1.cljc | 2 +- src/fluree/db/nameservice/storage.cljc | 12 ++++--- src/fluree/db/storage.cljc | 10 ++++-- src/fluree/db/transact.cljc | 5 +-- src/fluree/db/util/ledger.cljc | 25 ++++++++++++++ src/fluree/db/virtual_graph/bm25/storage.clj | 5 ++- test/fluree/db/nameservice_query_test.clj | 6 ++-- test/fluree/db/query/history_test.clj | 34 ++++++++++---------- test/fluree/db/query/misc_queries_test.clj | 20 ++++++------ test/fluree/db/query/stable_hashes_test.clj | 4 +-- test/fluree/db/storage/s3_unit_test.clj | 6 ++-- test/fluree/db_test.cljc | 2 +- 17 files changed, 102 insertions(+), 69 deletions(-) create mode 100644 src/fluree/db/util/ledger.cljc diff --git a/src/fluree/db/api.cljc b/src/fluree/db/api.cljc index 73555420fc..4bc83f01e8 100644 --- a/src/fluree/db/api.cljc +++ b/src/fluree/db/api.cljc @@ -387,7 +387,7 @@ opts - (optional) Options map for the commit operation The ledger-id is automatically extracted from the database object's - alias and branch fields (formatted as alias@branch). + alias and branch fields (formatted as alias:branch). Creates a new commit and notifies the nameservice of the new version. Returns promise resolving to the committed database." @@ -538,7 +538,7 @@ Parameters: conn - Connection object - ledger-id - Ledger alias (with optional @branch) or address + ledger-id - Ledger alias (with optional :branch) or address Returns status map with commit and index information." [conn ledger-id] @@ -901,7 +901,7 @@ Parameters: conn - Database connection - ledger-alias - The alias/name of the ledger to index (with optional @branch) + ledger-alias - The alias/name of the ledger to index (with optional :branch) opts - (optional) Options map: :timeout - Max wait time in ms (default 300000 / 5 minutes) @@ -915,7 +915,7 @@ ) ;; Trigger indexing for a specific branch - (let [indexed-db @(trigger-index conn \"my-ledger@main\")] + (let [indexed-db @(trigger-index conn \"my-ledger:main\")] ;; Use indexed-db... )" ([conn ledger-alias] diff --git a/src/fluree/db/commit/storage.cljc b/src/fluree/db/commit/storage.cljc index 09a91fb679..3a74df2e99 100644 --- a/src/fluree/db/commit/storage.cljc +++ b/src/fluree/db/commit/storage.cljc @@ -7,6 +7,7 @@ [fluree.db.util :as util :refer [get-first get-first-id get-first-value try* catch*]] [fluree.db.util.async :refer [ genesis-commit diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index 2b9e8bddce..b10602572b 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -70,11 +70,11 @@ (defn normalize-ledger-alias "Ensures ledger alias includes branch. - If no @ symbol present, appends @main as default branch." + If no : symbol present, appends :main as default branch." [ledger-alias] - (if (clojure.string/includes? ledger-alias "@") + (if (clojure.string/includes? ledger-alias ":") ledger-alias - (str ledger-alias "@" commit-data/default-branch))) + (str ledger-alias ":" commit-data/default-branch))) (defn register-ledger "Creates a promise-chan and saves it in a cache of ledgers being held @@ -449,9 +449,9 @@ (let [alias (if (fluree-address? alias) (nameservice/address-path alias) ;; Normalize alias to include branch if not present - (if (str/includes? alias "@") + (if (str/includes? alias ":") alias - (str alias "@main")))] + (str alias ":main")))] (loop [[publisher & r] (publishers conn)] (when publisher (let [ledger-addr ( 'my-ledger'" - [ledger-alias] - (first (str/split ledger-alias #"@" 2))) - (defn get-branch-meta "Retrieves branch metadata from ledger state" [{:keys [state] :as _ledger} requested-branch] @@ -138,8 +132,8 @@ [combined-alias ledger-address commit-catalog index-catalog primary-publisher secondary-publishers indexing-opts did latest-commit] (let [;; Parse ledger name and branch from combined alias - [_ branch] (if (str/includes? combined-alias "@") - (str/split combined-alias #"@" 2) + [_ branch] (if (str/includes? combined-alias ":") + (str/split combined-alias #":" 2) [combined-alias "main"]) publishers (cons primary-publisher secondary-publishers) branches {branch (branch/state-map combined-alias branch commit-catalog index-catalog @@ -173,10 +167,10 @@ {:keys [did indexing] :as _opts}] (go-try (let [normalized-alias (normalize-alias alias) - ;; Add @main if no branch is specified - ledger-alias (if (str/includes? normalized-alias "@") + ;; Add :main if no branch is specified + ledger-alias (if (str/includes? normalized-alias ":") normalized-alias - (str normalized-alias "@main")) + (str normalized-alias ":main")) ;; internal-only opt used for migrating ledgers without genesis commits init-time (util/current-time-iso) genesis-commit ( {"@context" {"f" iri/f-ns} - "@id" ledger-alias ;; Already includes @branch + "@id" ledger-alias ;; Already includes :branch "@type" ["f:Database" "f:PhysicalDatabase"] "f:ledger" {"@id" alias} ;; Just the ledger name without branch "f:branch" branch diff --git a/src/fluree/db/storage.cljc b/src/fluree/db/storage.cljc index 1c2e6df8ee..01abac596f 100644 --- a/src/fluree/db/storage.cljc +++ b/src/fluree/db/storage.cljc @@ -54,8 +54,14 @@ (defn split-address "Splits `address` into the fully qualified storage method and local path." [address] - (let [i (str/last-index-of address ":")] - [(subs address 0 i) (subs address (inc i))])) + (let [i (str/index-of address "://")] + (if i + ;; For addresses with ://, the location is everything before :// + ;; and the path is everything after :// + [(subs address 0 i) (subs address (+ i 3))] + ;; Fallback for addresses without :// + (let [i (str/last-index-of address ":")] + [(subs address 0 i) (subs address (inc i))])))) (defn strip-extension [filename] diff --git a/src/fluree/db/transact.cljc b/src/fluree/db/transact.cljc index 14b9e216b0..99d5353946 100644 --- a/src/fluree/db/transact.cljc +++ b/src/fluree/db/transact.cljc @@ -15,6 +15,7 @@ [fluree.db.util :as util :refer [try* catch*]] [fluree.db.util.async :refer [ 'my-ledger'" + [ledger-alias] + (first (str/split ledger-alias #":" 2))) + +(defn ledger-branch + "Extracts the branch name from a ledger alias. + Returns the branch name or nil if no branch is specified. + e.g., 'my-ledger:main' -> 'main' + 'my-ledger' -> nil" + [ledger-alias] + (second (str/split ledger-alias #":" 2))) + +(defn ledger-parts + "Splits a ledger alias into [ledger-name branch-name]. + e.g., 'my-ledger:main' -> ['my-ledger' 'main'] + 'my-ledger' -> ['my-ledger' nil]" + [ledger-alias] + (let [parts (str/split ledger-alias #":" 2)] + [(first parts) (second parts)])) \ No newline at end of file diff --git a/src/fluree/db/virtual_graph/bm25/storage.clj b/src/fluree/db/virtual_graph/bm25/storage.clj index 368423abb4..62a4b1036a 100644 --- a/src/fluree/db/virtual_graph/bm25/storage.clj +++ b/src/fluree/db/virtual_graph/bm25/storage.clj @@ -4,6 +4,7 @@ [fluree.db.serde.protocol :as serde] [fluree.db.storage :as storage] [fluree.db.util.async :refer [ Date: Mon, 11 Aug 2025 15:57:03 -0400 Subject: [PATCH 06/24] ensure new ledgers don't have ':', linting --- src/fluree/db/api.cljc | 8 ++ src/fluree/db/api/transact.cljc | 8 ++ src/fluree/db/flake/commit_data.cljc | 3 +- src/fluree/db/transact.cljc | 2 +- test/fluree/db/api/create_test.clj | 140 +++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 test/fluree/db/api/create_test.clj diff --git a/src/fluree/db/api.cljc b/src/fluree/db/api.cljc index 4bc83f01e8..2c8c39dfa1 100644 --- a/src/fluree/db/api.cljc +++ b/src/fluree/db/api.cljc @@ -1,6 +1,7 @@ (ns fluree.db.api (:require [camel-snake-kebab.core :refer [->camelCaseString]] [clojure.core.async :as async :refer [go parsed-txn :opts syntax/coerce-ledger-opts) + ;; Disallow branch specification in ledger name during creation + _ (when (str/includes? ledger-id ":") + (throw (ex-info (str "Ledger name cannot contain ':' character. " + "Branches must be created separately. " + "Provided: " ledger-id) + {:error :db/invalid-ledger-name + :ledger-alias ledger-id}))) ledger ( e ex-data :error)) + "Should return correct error code"))) + + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Ledger name cannot contain ':' character" + (fluree/create conn "test:feature:v2")) + "Should reject name with multiple colons")) + + (testing "accepts valid ledger names" + (is (not (util/exception? @(fluree/create conn "valid-name"))) + "Should accept name with hyphen") + + (is (not (util/exception? @(fluree/create conn "valid_name"))) + "Should accept name with underscore") + + (is (not (util/exception? @(fluree/create conn "tenant/database"))) + "Should accept name with slash") + + (is (not (util/exception? @(fluree/create conn "my-ledger-2024"))) + "Should accept alphanumeric with special chars")) + + (testing "automatically appends ':main' branch to valid names" + (let [db @(fluree/create conn "auto-branch-test")] + (is (= "auto-branch-test:main" (get-in db [:commit :alias])) + "Should append :main to ledger name")))))) + +(deftest create-with-txn-ledger-name-validation + (testing "create-with-txn ledger name validation" + (let [conn (test-utils/create-conn)] + + (testing "rejects ledger names containing ':' character" + (let [txn-with-colon {"@context" {"ex" "http://example.org/"} + "ledger" "invalid:name" + "insert" {"@id" "ex:test" "ex:value" 1}}] + (is (util/exception? @(fluree/create-with-txn conn txn-with-colon)) + "Should reject name with colon")) + + (let [txn-with-branch {"@context" {"ex" "http://example.org/"} + "ledger" "test:branch" + "insert" {"@id" "ex:test" "ex:value" 1}} + result @(fluree/create-with-txn conn txn-with-branch)] + (is (util/exception? result) + "Should reject name with branch") + (is (= :db/invalid-ledger-name + (-> result ex-data :error)) + "Should return correct error code"))) + + (testing "accepts valid ledger names and creates with initial data" + (let [db @(fluree/create-with-txn conn + {"@context" {"ex" "http://example.org/"} + "ledger" "txn-test" + "insert" {"@id" "ex:alice" "ex:age" 42}})] + (is (= "txn-test:main" (get-in db [:commit :alias])) + "Should create ledger with :main branch") + + ;; Verify the initial data was inserted + (let [result @(fluree/query db + {"@context" {"ex" "http://example.org/"} + "select" {"ex:alice" ["*"]}})] + (is (= 42 (-> result first (get "ex:age"))) + "Should have inserted initial data"))))))) + +(deftest edge-case-validation + (testing "Edge cases for ledger name validation" + (let [conn (test-utils/create-conn)] + + (testing "empty colon cases" + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Ledger name cannot contain ':' character" + (fluree/create conn ":")) + "Should reject single colon") + + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Ledger name cannot contain ':' character" + (fluree/create conn ":branch")) + "Should reject name starting with colon") + + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Ledger name cannot contain ':' character" + (fluree/create conn "ledger:")) + "Should reject name ending with colon")) + + (testing "special characters that ARE allowed" + (is (not (util/exception? @(fluree/create conn "ledger.with.dots"))) + "Should accept dots") + + (is (not (util/exception? @(fluree/create conn "ledger-with-dashes"))) + "Should accept dashes") + + (is (not (util/exception? @(fluree/create conn "ledger_with_underscores"))) + "Should accept underscores") + + (is (not (util/exception? @(fluree/create conn "org/department/project"))) + "Should accept multiple slashes"))))) + +(deftest duplicate-ledger-creation + (testing "Cannot create duplicate ledgers" + (let [conn (test-utils/create-conn) + ledger-name "unique-test"] + + ;; First creation should succeed + (is (not (util/exception? @(fluree/create conn ledger-name))) + "First creation should succeed") + + ;; Second creation with same name should fail + (is (util/exception? @(fluree/create conn ledger-name)) + "Duplicate creation should fail") + + ;; Trying with explicit :main should be rejected by validation + (is (thrown-with-msg? + clojure.lang.ExceptionInfo + #"Ledger name cannot contain ':' character" + (fluree/create conn (str ledger-name ":main"))) + "Should reject explicit :main branch in name")))) \ No newline at end of file From 82039efd080eacd5e9f6be64dec89b37c52ce1aa Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Mon, 11 Aug 2025 16:15:07 -0400 Subject: [PATCH 07/24] put nameservice branch records in own directory --- src/fluree/db/nameservice/storage.cljc | 8 ++++---- test/fluree/db/nameservice_query_test.clj | 10 ++++++---- test/fluree/db/storage/s3_unit_test.clj | 6 +++--- test/fluree/db_test.cljc | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/fluree/db/nameservice/storage.cljc b/src/fluree/db/nameservice/storage.cljc index 62bb44c0d4..e757042a11 100644 --- a/src/fluree/db/nameservice/storage.cljc +++ b/src/fluree/db/nameservice/storage.cljc @@ -10,11 +10,11 @@ (defn local-filename "Returns the local filename for a ledger's nameservice record. - Expects ledger-alias to be in format 'ledger:branch'." + Expects ledger-alias to be in format 'ledger:branch'. + Returns path like 'ns@v1/ledger-name/branch.json'." [ledger-alias] - ;; Replace : with _ for filesystem compatibility (Windows doesn't allow : in filenames) - (let [safe-alias (str/replace ledger-alias ":" "_")] - (str "ns@v1/" safe-alias ".json"))) + (let [[ledger-name branch] (str/split ledger-alias #":" 2)] + (str "ns@v1/" ledger-name "/" branch ".json"))) (defn ns-record "Generates nameservice metadata map for JSON storage using new minimal format. diff --git a/test/fluree/db/nameservice_query_test.clj b/test/fluree/db/nameservice_query_test.clj index 03a08508bc..ffaf76011f 100644 --- a/test/fluree/db/nameservice_query_test.clj +++ b/test/fluree/db/nameservice_query_test.clj @@ -197,16 +197,18 @@ (testing "Verify file system structure" ;; Check that subdirectories were created correctly (let [ns-dir (io/file (str storage-path) "ns@v1") + ;; With ledger names like "tenant1/customers", the structure is: + ;; ns@v1/tenant1/customers/main.json tenant1-dir (io/file ns-dir "tenant1") tenant2-dir (io/file ns-dir "tenant2")] (is (.exists ns-dir) "ns@v1 directory should exist") (is (.exists tenant1-dir) "tenant1 subdirectory should exist") (is (.exists tenant2-dir) "tenant2 subdirectory should exist") - ;; Check for nameservice files - (let [customer-file (io/file ns-dir "tenant1/customers_main.json") - products-file (io/file ns-dir "tenant1/products_main.json") - orders-file (io/file ns-dir "tenant2/orders_main.json")] + ;; Check for nameservice files with new structure + (let [customer-file (io/file ns-dir "tenant1/customers/main.json") + products-file (io/file ns-dir "tenant1/products/main.json") + orders-file (io/file ns-dir "tenant2/orders/main.json")] (is (.exists customer-file) "Customer nameservice file should exist") (is (.exists products-file) "Products nameservice file should exist") (is (.exists orders-file) "Orders nameservice file should exist")))) diff --git a/test/fluree/db/storage/s3_unit_test.clj b/test/fluree/db/storage/s3_unit_test.clj index a7a66eddf0..8758a38a1f 100644 --- a/test/fluree/db/storage/s3_unit_test.clj +++ b/test/fluree/db/storage/s3_unit_test.clj @@ -84,10 +84,10 @@ (deftest s3-path-encoding-test (testing "S3 path encoding handles special characters correctly" - (let [path-with-at "ns@v1/test-ledger_main.json" + (let [path-with-at "ns@v1/test-ledger/main.json" encoded (s3-storage/encode-s3-path path-with-at)] - (is (= "ns%40v1/test-ledger_main.json" encoded) - "Should encode @ in directory name to %40 but not underscore in filename")) + (is (= "ns%40v1/test-ledger/main.json" encoded) + "Should encode @ in directory name to %40")) (let [path-normal "bucket/prefix/file.json" encoded (s3-storage/encode-s3-path path-normal)] diff --git a/test/fluree/db_test.cljc b/test/fluree/db_test.cljc index edd7ec2a45..9d5ceefd1b 100644 --- a/test/fluree/db_test.cljc +++ b/test/fluree/db_test.cljc @@ -1087,7 +1087,7 @@ (testing "before drop" (is (= ["destined-for-drop" "ns@v1"] (sort (async/ Date: Mon, 11 Aug 2025 17:37:57 -0400 Subject: [PATCH 08/24] fix test --- test/fluree/db_test.cljc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/fluree/db_test.cljc b/test/fluree/db_test.cljc index 9d5ceefd1b..05957e74a3 100644 --- a/test/fluree/db_test.cljc +++ b/test/fluree/db_test.cljc @@ -1122,8 +1122,11 @@ ;; directories are not removed (is (= ["destined-for-drop" "ns@v1"] (sort (async/ Date: Tue, 26 Aug 2025 16:08:46 -0400 Subject: [PATCH 09/24] Fix commit storage to use ledger-name instead of ledger-alias Ensures storage paths don't include branch suffix (e.g., 'destined-for-drop' instead of 'destined-for-drop:main') to maintain backward compatibility with tests and existing storage structure --- src/fluree/db/transact.cljc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fluree/db/transact.cljc b/src/fluree/db/transact.cljc index 92596ac164..c1c1e5e1b0 100644 --- a/src/fluree/db/transact.cljc +++ b/src/fluree/db/transact.cljc @@ -245,7 +245,11 @@ {:keys [txn-id author annotation]} (db-id (:hash data-write-result)) keypair {:did did, :private private} From 2f8ffd5d0824a3a948c81c206fdb6db33956620a Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 4 Sep 2025 07:14:21 -0400 Subject: [PATCH 10/24] Add ledger name validation and branch name validation functions --- src/fluree/db/api/transact.cljc | 11 +--- src/fluree/db/util/ledger.cljc | 105 +++++++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/src/fluree/db/api/transact.cljc b/src/fluree/db/api/transact.cljc index 514e66f7c1..9d2bdac01e 100644 --- a/src/fluree/db/api/transact.cljc +++ b/src/fluree/db/api/transact.cljc @@ -1,7 +1,6 @@ (ns fluree.db.api.transact (:refer-clojure :exclude [update]) (:require [clojure.core.async :as async] - [clojure.string :as str] [fluree.db.connection :as connection] [fluree.db.json-ld.credential :as cred] [fluree.db.query.fql.parse :as parse] @@ -10,6 +9,7 @@ [fluree.db.util :as util] [fluree.db.util.async :refer [ parsed-txn :opts syntax/coerce-ledger-opts) - ;; Disallow branch specification in ledger name during creation - _ (when (str/includes? ledger-id ":") - (throw (ex-info (str "Ledger name cannot contain ':' character. " - "Branches must be created separately. " - "Provided: " ledger-id) - {:error :db/invalid-ledger-name - :ledger-alias ledger-id}))) + ;; Validate ledger name + _ (util.ledger/validate-ledger-name ledger-id) ledger ( ['my-ledger' nil]" [ledger-alias] (let [parts (str/split ledger-alias #":" 2)] - [(first parts) (second parts)])) \ No newline at end of file + [(first parts) (second parts)])) + +(defn validate-ledger-name + "Validates a ledger name for creation. Throws if invalid. + Rules: + - Cannot contain ':' (reserved for branch separator) + - Cannot contain '@', '#', '?' (reserved characters) + - Cannot contain whitespace + - Cannot start with '/', '-' + - Cannot end with '/' + - Cannot be empty + - Cannot contain path traversal patterns like '../'" + [ledger-name] + (cond + (str/blank? ledger-name) + (throw (ex-info "Ledger name cannot be empty" + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (str/includes? ledger-name ":") + (throw (ex-info (str "Ledger name cannot contain ':' character. " + "Branches must be created separately. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (re-find #"[@#?]" ledger-name) + (throw (ex-info (str "Ledger name cannot contain '@', '#', or '?' characters. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (re-find #"\s" ledger-name) + (throw (ex-info (str "Ledger name cannot contain whitespace. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (str/ends-with? ledger-name "/") + (throw (ex-info (str "Ledger name cannot end with '/'. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (re-matches #"^[/\-].*" ledger-name) + (throw (ex-info (str "Ledger name cannot start with '/' or '-'. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (re-matches #"^/+$" ledger-name) + (throw (ex-info (str "Ledger name cannot consist only of '/' characters. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (str/includes? ledger-name "../") + (throw (ex-info (str "Ledger name cannot contain path traversal patterns. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + (not (re-matches #"^[a-zA-Z0-9][\w\-\./_]*$" ledger-name)) + (throw (ex-info (str "Ledger name must start with alphanumeric character and " + "contain only alphanumeric, underscore, hyphen, dot, or slash characters. " + "Provided: " ledger-name) + {:error :db/invalid-ledger-name + :ledger-name ledger-name})) + + :else ledger-name)) + +(defn validate-branch-name + "Validates a branch name. Throws if invalid. + Rules: + - Cannot contain '/' (would create subdirectories) + - Cannot contain whitespace + - Should only contain alphanumeric characters, hyphens, underscores, and dots" + [branch-name] + (cond + (str/blank? branch-name) + (throw (ex-info "Branch name cannot be empty" + {:error :db/invalid-branch-name + :branch-name branch-name})) + + (str/includes? branch-name "/") + (throw (ex-info (str "Branch name cannot contain '/' character. " + "Provided: " branch-name) + {:error :db/invalid-branch-name + :branch-name branch-name})) + + (re-find #"\s" branch-name) + (throw (ex-info (str "Branch name cannot contain whitespace. " + "Provided: " branch-name) + {:error :db/invalid-branch-name + :branch-name branch-name})) + + (not (re-matches #"^[a-zA-Z0-9][\w\-\.]*$" branch-name)) + (throw (ex-info (str "Branch name must start with alphanumeric character and " + "contain only alphanumeric, underscore, hyphen, or dot characters. " + "Provided: " branch-name) + {:error :db/invalid-branch-name + :branch-name branch-name})) + + :else branch-name)) \ No newline at end of file From 1f0f0852da908dd0e154c65ce5ec0a871dc5c668 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 4 Sep 2025 07:15:38 -0400 Subject: [PATCH 11/24] Rename combined-alias to ledger-alias for clarity in commit processing --- src/fluree/db/connection.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index b10602572b..5814b65f81 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -327,7 +327,7 @@ commit-address index-address)) expanded-commit (json-ld/expand commit) - combined-alias (commit->ledger-alias conn address expanded-commit) + ledger-alias (commit->ledger-alias conn address expanded-commit) {:keys [did indexing]} (parse-ledger-options conn {}) ledger (ledger/instantiate combined-alias address commit-catalog index-catalog From ee5821ad820c30d4374659425e0cdccfa8de6f49 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 4 Sep 2025 07:16:43 -0400 Subject: [PATCH 12/24] Update ledger alias usage in ledger instantiation and subscription --- src/fluree/db/connection.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index 5814b65f81..92a0af4d37 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -330,9 +330,9 @@ ledger-alias (commit->ledger-alias conn address expanded-commit) {:keys [did indexing]} (parse-ledger-options conn {}) - ledger (ledger/instantiate combined-alias address commit-catalog index-catalog + ledger (ledger/instantiate ledger-alias address commit-catalog index-catalog primary-publisher secondary-publishers indexing did expanded-commit)] - (ns-subscribe/subscribe-ledger conn combined-alias) + (ns-subscribe/subscribe-ledger conn ledger-alias) (async/put! ledger-chan ledger) ledger) (throw (ex-info (str "Unable to load. No record of ledger at address: " address " exists.") From c0275ef9e2ac3e28f286fd499041d8594f4dad4a Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 4 Sep 2025 07:17:08 -0400 Subject: [PATCH 13/24] Add ledger name validation during ledger creation --- src/fluree/db/api.cljc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/fluree/db/api.cljc b/src/fluree/db/api.cljc index 2c8c39dfa1..f5de70b144 100644 --- a/src/fluree/db/api.cljc +++ b/src/fluree/db/api.cljc @@ -19,6 +19,7 @@ [fluree.db.transact :as transact] [fluree.db.util :as util] [fluree.db.util.async :refer [go-try Date: Thu, 4 Sep 2025 07:17:40 -0400 Subject: [PATCH 14/24] Remove unnecessary nil argument from ledger trigger index call --- src/fluree/db/connection.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index 92a0af4d37..8c4cda6f44 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -498,7 +498,7 @@ (let [{:keys [timeout] :or {timeout 300000}} opts ledger ( Date: Thu, 4 Sep 2025 07:18:49 -0400 Subject: [PATCH 15/24] Update commit version to 2 in commit data definitions --- src/fluree/db/flake/commit_data.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/flake/commit_data.cljc b/src/fluree/db/flake/commit_data.cljc index 4a2b8f38c7..e30c9b4cb2 100644 --- a/src/fluree/db/flake/commit_data.cljc +++ b/src/fluree/db/flake/commit_data.cljc @@ -14,7 +14,7 @@ [fluree.db.util.log :as log] [fluree.db.util.reasoner :as reasoner-util])) -(def commit-version 1) +(def commit-version 2) (def data-version 0) (def default-branch @@ -223,7 +223,7 @@ "Creates a skeleton blank commit map." [alias publish-addresses init-time] (let [commit-json (->json-ld {:alias alias - :v 0 + :v commit-version :data {:t 0 :flakes 0 :size 0} From 183872f2bb6c25ea0f88f7dd2724f4f790e06d07 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 4 Sep 2025 07:23:46 -0400 Subject: [PATCH 16/24] Refactor nameservice to use constant for ns-version in filename generation --- src/fluree/db/constants.cljc | 1 + src/fluree/db/nameservice/storage.cljc | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/constants.cljc b/src/fluree/db/constants.cljc index baf0b4dff9..b9854bd3c1 100644 --- a/src/fluree/db/constants.cljc +++ b/src/fluree/db/constants.cljc @@ -5,6 +5,7 @@ ;; Version (def ^:const data_version 4) +(def ^:const ns-version "ns@v1") ;; iri constants (def ^:const iri-CommitProof (fluree-iri "CommitProof")) diff --git a/src/fluree/db/nameservice/storage.cljc b/src/fluree/db/nameservice/storage.cljc index e757042a11..f6f8d9d4da 100644 --- a/src/fluree/db/nameservice/storage.cljc +++ b/src/fluree/db/nameservice/storage.cljc @@ -1,6 +1,7 @@ (ns fluree.db.nameservice.storage (:require [clojure.core.async :refer [go]] [clojure.string :as str] + [fluree.db.constants :as const] [fluree.db.json-ld.iri :as iri] [fluree.db.nameservice :as nameservice] [fluree.db.storage :as storage] @@ -14,7 +15,7 @@ Returns path like 'ns@v1/ledger-name/branch.json'." [ledger-alias] (let [[ledger-name branch] (str/split ledger-alias #":" 2)] - (str "ns@v1/" ledger-name "/" branch ".json"))) + (str const/ns-version "/" ledger-name "/" branch ".json"))) (defn ns-record "Generates nameservice metadata map for JSON storage using new minimal format. @@ -76,7 +77,7 @@ (go-try ;; Use recursive listing to support ledger names with '/' characters (if (satisfies? storage/RecursiveListableStore store) - (if-let [list-paths-result (storage/list-paths-recursive store "ns@v1")] + (if-let [list-paths-result (storage/list-paths-recursive store const/ns-version)] (loop [remaining-paths ( Date: Thu, 4 Sep 2025 07:43:42 -0400 Subject: [PATCH 17/24] Refactor alias handling in ledger and nameservice functions for consistency --- src/fluree/db/branch.cljc | 14 ++++----- src/fluree/db/connection.cljc | 4 +-- src/fluree/db/ledger.cljc | 32 ++++++-------------- src/fluree/db/migrations/nameservice_v1.cljc | 6 ++-- src/fluree/db/nameservice/storage.cljc | 7 +++-- 5 files changed, 25 insertions(+), 38 deletions(-) diff --git a/src/fluree/db/branch.cljc b/src/fluree/db/branch.cljc index 2013553377..ef8f6fe776 100644 --- a/src/fluree/db/branch.cljc +++ b/src/fluree/db/branch.cljc @@ -44,9 +44,9 @@ (-> commit-map commit-data/->json-ld json-ld/expand)) (defn load-db - [combined-alias commit-catalog index-catalog commit-map] + [alias commit-catalog index-catalog commit-map] (let [commit-jsonld (commit-map->commit-jsonld commit-map)] - (async-db/load combined-alias commit-catalog index-catalog + (async-db/load alias commit-catalog index-catalog commit-jsonld commit-map nil))) (defn update-index-async @@ -149,17 +149,17 @@ (defn state-map "Returns a branch map for specified branch name at supplied commit" - ([combined-alias branch-name commit-catalog index-catalog publishers commit-jsonld] - (state-map combined-alias branch-name commit-catalog index-catalog publishers commit-jsonld nil)) - ([combined-alias branch-name commit-catalog index-catalog publishers commit-jsonld indexing-opts] + ([alias branch-name commit-catalog index-catalog publishers commit-jsonld] + (state-map alias branch-name commit-catalog index-catalog publishers commit-jsonld nil)) + ([alias branch-name commit-catalog index-catalog publishers commit-jsonld indexing-opts] (let [commit-map (commit-data/jsonld->clj commit-jsonld) - initial-db (async-db/load combined-alias commit-catalog index-catalog + initial-db (async-db/load alias commit-catalog index-catalog commit-jsonld commit-map indexing-opts) state (atom {:commit commit-map :current-db initial-db}) idx-q (index-queue publishers state)] {:name branch-name - :alias combined-alias + :alias alias :state state :index-queue idx-q :indexing-opts indexing-opts}))) diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index 8c4cda6f44..2ef26de79d 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -449,9 +449,7 @@ (let [alias (if (fluree-address? alias) (nameservice/address-path alias) ;; Normalize alias to include branch if not present - (if (str/includes? alias ":") - alias - (str alias ":main")))] + (normalize-ledger-alias alias))] (loop [[publisher & r] (publishers conn)] (when publisher (let [ledger-addr (Ledger {:id (random-uuid) :did did :state (atom (initial-state branches branch)) - :alias combined-alias ;; Full alias including branch + :alias alias ;; Full alias including branch :address ledger-address :commit-catalog commit-catalog :index-catalog index-catalog @@ -151,14 +150,6 @@ :reasoner #{} :indexing-opts indexing-opts}))) -(defn normalize-alias - "For a ledger alias, removes any preceding '/' or '#' if exists." - [ledger-alias] - (if (or (str/starts-with? ledger-alias "/") - (str/starts-with? ledger-alias "#")) - (subs ledger-alias 1) - ledger-alias)) - (defn create "Creates a new ledger, optionally bootstraps it as permissioned or with default context." @@ -166,22 +157,17 @@ primary-publisher secondary-publishers]} {:keys [did indexing] :as _opts}] (go-try - (let [normalized-alias (normalize-alias alias) - ;; Add :main if no branch is specified - ledger-alias (if (str/includes? normalized-alias ":") - normalized-alias - (str normalized-alias ":main")) - ;; internal-only opt used for migrating ledgers without genesis commits + (let [;; internal-only opt used for migrating ledgers without genesis commits init-time (util/current-time-iso) genesis-commit (map genesis-commit nil) compact-commit (commit-data/->json-ld commit-map)] ( {"@context" {"f" iri/f-ns} "@id" ledger-alias ;; Already includes :branch "@type" ["f:Database" "f:PhysicalDatabase"] From 273cf90168a4de6fd0b8365ef639bbad310d30a0 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Thu, 4 Sep 2025 13:24:33 -0400 Subject: [PATCH 18/24] linting --- src/fluree/db/api.cljc | 1 - src/fluree/db/ledger.cljc | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/fluree/db/api.cljc b/src/fluree/db/api.cljc index f5de70b144..6d99179101 100644 --- a/src/fluree/db/api.cljc +++ b/src/fluree/db/api.cljc @@ -1,7 +1,6 @@ (ns fluree.db.api (:require [camel-snake-kebab.core :refer [->camelCaseString]] [clojure.core.async :as async :refer [go Date: Fri, 5 Sep 2025 16:46:45 -0400 Subject: [PATCH 19/24] Update nameservice versioning from ns@v1 to ns@v2 and adjust related migrations --- src/fluree/db/constants.cljc | 2 +- src/fluree/db/migrations/nameservice_v1.cljc | 208 +++++++++++++++++-- src/fluree/db/nameservice/storage.cljc | 2 +- test/fluree/db/nameservice_query_test.clj | 7 +- test/fluree/db/query/history_test.clj | 48 ++--- test/fluree/db/query/misc_queries_test.clj | 45 ++-- test/fluree/db/query/stable_hashes_test.clj | 45 ++-- test/fluree/db_test.cljc | 11 +- 8 files changed, 268 insertions(+), 100 deletions(-) diff --git a/src/fluree/db/constants.cljc b/src/fluree/db/constants.cljc index b9854bd3c1..116bcae394 100644 --- a/src/fluree/db/constants.cljc +++ b/src/fluree/db/constants.cljc @@ -5,7 +5,7 @@ ;; Version (def ^:const data_version 4) -(def ^:const ns-version "ns@v1") +(def ^:const ns-version "ns@v2") ;; iri constants (def ^:const iri-CommitProof (fluree-iri "CommitProof")) diff --git a/src/fluree/db/migrations/nameservice_v1.cljc b/src/fluree/db/migrations/nameservice_v1.cljc index 5d8573b262..9b081a6676 100644 --- a/src/fluree/db/migrations/nameservice_v1.cljc +++ b/src/fluree/db/migrations/nameservice_v1.cljc @@ -8,12 +8,12 @@ - Detects old nameservice files stored at root level (e.g., ledger-name.json) - Extracts essential metadata from full commit JSON-LD records - Creates new minimal records with only: commit address, t value, index address - - Stores them in the new ns@v1/ directory with branch-aware naming (ledger-name@branch.json) + - Stores them in the new ns@v1/ directory with branch-aware naming (ledger-name/branch.json) - Cleans up old files after successful migration + - Additionally: migrates prior ns@v1 flat files (ledger@branch.json) to nested (ledger/branch.json) WHEN IT RUNS: - - Automatically at file storage initialization if old format is detected - - Only runs once when ledger directories exist but no ns@v1/ directory is present + - Automatically at file storage initialization (legacy and flat->nested checks are independent) SAFETY: - Read-only detection phase before any modifications @@ -25,24 +25,28 @@ ["fs" :as node-fs] ["path" :as node-path]]) [clojure.string :as str] + [fluree.db.constants :as const] [fluree.db.nameservice.storage :as ns-storage] [fluree.db.storage :as storage] [fluree.db.util.async :refer [ Nested ns@v1 (ledger/branch.json) migration --- + +(def flat-filename-regex + ;; Explicitly target ns@v1 flat files regardless of current const/ns-version + (re-pattern (str "^ns@v1/([^/]+)@([^/]+)\\.json$"))) + +(defn find-flat-nameservice-files + "Find ns@v1 files using the legacy flat naming: ns@v1/@.json" + [file-store] + (go-try + (if (satisfies? storage/RecursiveListableStore file-store) + (let [paths (" new-path) + ;; If new path already exists, just delete the old file and return + (if ( file-store storage/location (storage/build-address old-path))] + (UTF8 content) + :else content)] + (when-not bytes* + (throw (ex-info (str "Unable to read old nameservice file: " old-path) + {:status 500 :error :db/migration}))) + ( file-store storage/location (storage/build-address old-path))] + ( v2 migration (handles both flat and nested v1 layouts) + +(def v1-flat-regex (re-pattern "^ns@v1/(.+)@([^/]+)\\.json$")) +(def v1-nested-one-regex (re-pattern "^ns@v1/([^/]+)/([^/]+)\\.json$")) +(def v1-nested-two-regex (re-pattern "^ns@v1/([^/]+)/([^/]+)/([^/]+)\\.json$")) + +(defn find-v1-nameservice-files + "Find all ns@v1 nameservice files (flat and nested) and compute target ns@v2 paths. + Scopes discovery to ns@v1 only via recursive listing and avoids scanning root." + [file-store] + (go-try + (if (satisfies? storage/RecursiveListableStore file-store) + (let [paths (> paths + (keep (fn [p] + (cond + (re-matches v1-flat-regex p) + (let [[_ ledger-path branch] (re-matches v1-flat-regex p)] + {:old-path p + :alias (str ledger-path ":" branch) + :new-path (str const/ns-version "/" ledger-path "/" branch ".json")}) + + (re-matches v1-nested-two-regex p) + (let [[_ seg1 seg2 branch] (re-matches v1-nested-two-regex p) + ledger-path (str seg1 "/" seg2)] + {:old-path p + :alias (str ledger-path ":" branch) + :new-path (str const/ns-version "/" ledger-path "/" branch ".json")}) + + (re-matches v1-nested-one-regex p) + (let [[_ ledger branch] (re-matches v1-nested-one-regex p)] + {:old-path p + :alias (str ledger ":" branch) + :new-path (str const/ns-version "/" ledger "/" branch ".json")}) + + :else nil))) + vec)) + []))) + +(defn migrate-v1-file + "Migrate one ns@v1 nameservice file to ns@v2 nested layout. Idempotent if new file exists." + [file-store {:keys [old-path new-path alias]}] + (go-try + (log/info "Migrating ns@v1 record for" alias ":" old-path "->" new-path) + (if ( file-store storage/location (storage/build-address old-path))] + (UTF8 content) + :else content)] + (when-not bytes* + (throw (ex-info (str "Unable to read nameservice file: " old-path) + {:status 500 :error :db/migration}))) + ( file-store storage/location (storage/build-address old-path))] + (" const/ns-version "migration completed. Migrated" + (count (filter :migrated results)) + "files, skipped" (count (filter :skipped results)) "already updated files") + results)))) + [])))) + (defn run-migration-if-needed - "Check if migration is needed and run it if so" + "Run nameservice migrations as needed, minimizing expensive scans. + - If ns@v2 exists: do nothing + - Else if ns@v1 exists: migrate ns@v1 (flat or nested) -> ns@v2 + - Else: if legacy layout likely (has ledgers, no ns@v1/ns@v2), run legacy -> ns@v2" [file-store] (go-try - (when (" const/ns-version "...") + ( Date: Fri, 5 Sep 2025 16:58:11 -0400 Subject: [PATCH 20/24] param name change, remove unnecessary comments --- src/fluree/db/api.cljc | 1 - src/fluree/db/api/transact.cljc | 1 - src/fluree/db/flake/index/storage.cljc | 3 +-- src/fluree/db/migrations/nameservice_v1.cljc | 2 +- src/fluree/db/transact.cljc | 4 ++-- test/fluree/db_test.cljc | 1 - 6 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/fluree/db/api.cljc b/src/fluree/db/api.cljc index 6d99179101..e64bf6348a 100644 --- a/src/fluree/db/api.cljc +++ b/src/fluree/db/api.cljc @@ -249,7 +249,6 @@ ([conn ledger-alias] (create conn ledger-alias nil)) ([conn ledger-alias opts] (validate-connection conn) - ;; Validate ledger name (util.ledger/validate-ledger-name ledger-alias) (promise-wrap (go-try diff --git a/src/fluree/db/api/transact.cljc b/src/fluree/db/api/transact.cljc index 9d2bdac01e..ec925d0501 100644 --- a/src/fluree/db/api/transact.cljc +++ b/src/fluree/db/api/transact.cljc @@ -131,7 +131,6 @@ ; are no policies ; to check. ledger-opts (-> parsed-txn :opts syntax/coerce-ledger-opts) - ;; Validate ledger name _ (util.ledger/validate-ledger-name ledger-id) ledger (@.json" diff --git a/src/fluree/db/transact.cljc b/src/fluree/db/transact.cljc index c1c1e5e1b0..8c059c3405 100644 --- a/src/fluree/db/transact.cljc +++ b/src/fluree/db/transact.cljc @@ -155,12 +155,12 @@ ;; TODO - as implemented the db handles 'staged' data as per below (annotation, raw txn) ;; TODO - however this is really a concern of "commit", not staging and I don't think the db should be handling any of it (defn write-transaction! - [ledger db-alias staged] + [ledger ledger-name staged] (go-try (let [{:keys [txn author annotation]} staged {:keys [commit-catalog]} ledger] (if txn - (let [{txn-id :address} ( Date: Fri, 5 Sep 2025 17:09:18 -0400 Subject: [PATCH 21/24] linting issues --- test/fluree/db_test.cljc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/fluree/db_test.cljc b/test/fluree/db_test.cljc index 72aaf9d55b..5af3282d6c 100644 --- a/test/fluree/db_test.cljc +++ b/test/fluree/db_test.cljc @@ -1,10 +1,10 @@ (ns fluree.db-test - (:require #?@(:clj [[clojure.core.async :as async] + (:require #?@(:clj [[babashka.fs :refer [with-temp-dir]] + [clojure.core.async :as async] [clojure.test :refer [deftest is testing]] - [fluree.db.did :as did] [fluree.db.async-db :as async-db] - [fluree.db.util.filesystem :as fs] - [babashka.fs :refer [with-temp-dir]]] + [fluree.db.did :as did] + [fluree.db.util.filesystem :as fs]] :cljs [[cljs.test :refer-macros [deftest is testing async]] [clojure.core.async :refer [go Date: Fri, 5 Sep 2025 17:26:34 -0400 Subject: [PATCH 22/24] Refactor nameservice migration to use updated namespace and improve clarity --- src/fluree/db/connection/system.cljc | 2 +- .../{nameservice_v1.cljc => nameservice.cljc} | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) rename src/fluree/db/migrations/{nameservice_v1.cljc => nameservice.cljc} (98%) diff --git a/src/fluree/db/connection/system.cljc b/src/fluree/db/connection/system.cljc index 84cee553cc..a6a5a6b49b 100644 --- a/src/fluree/db/connection/system.cljc +++ b/src/fluree/db/connection/system.cljc @@ -1,7 +1,7 @@ (ns fluree.db.connection.system (:require #?(:clj [fluree.db.storage.s3 :as s3-storage] :cljs [fluree.db.storage.localstorage :as localstorage-store]) - #?(:clj [fluree.db.migrations.nameservice-v1 :as ns-migration]) + #?(:clj [fluree.db.migrations.nameservice :as ns-migration]) #?(:clj [fluree.db.storage.file :as file-storage]) [fluree.db.cache :as cache] [fluree.db.connection :as connection] diff --git a/src/fluree/db/migrations/nameservice_v1.cljc b/src/fluree/db/migrations/nameservice.cljc similarity index 98% rename from src/fluree/db/migrations/nameservice_v1.cljc rename to src/fluree/db/migrations/nameservice.cljc index 87815b57b6..68def135f7 100644 --- a/src/fluree/db/migrations/nameservice_v1.cljc +++ b/src/fluree/db/migrations/nameservice.cljc @@ -1,16 +1,17 @@ -(ns fluree.db.migrations.nameservice-v1 - "Nameservice Migration to ns@v1 Format +(ns fluree.db.migrations.nameservice + "Nameservice Migration to ns@v2 Format - This migration handles the transition from the legacy nameservice format to the new - ns@v1 directory structure with minimal storage records. + This migration handles the transition from the legacy nameservice formats (pre-ns@v1 and + ns@v1) to the new ns@v2 directory structure with minimal storage records. WHAT IT DOES: - Detects old nameservice files stored at root level (e.g., ledger-name.json) - Extracts essential metadata from full commit JSON-LD records - Creates new minimal records with only: commit address, t value, index address - - Stores them in the new ns@v1/ directory with branch-aware naming (ledger-name/branch.json) + - Stores them in the new ns@v2/ directory with branch-aware naming (ledger-name/branch.json) - Cleans up old files after successful migration - - Additionally: migrates prior ns@v1 flat files (ledger@branch.json) to nested (ledger/branch.json) + - Additionally: migrates prior ns@v1 flat files (ledger@branch.json) and nested files + (ledger/branch.json) to ns@v2 nested layout WHEN IT RUNS: - Automatically at file storage initialization (legacy and flat->nested checks are independent) From 21b4c6d2b14512d8db0ddbca6b7ef69fe25edf54 Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Sat, 6 Sep 2025 10:52:20 -0400 Subject: [PATCH 23/24] Enhance documentation for publishing-address protocol to clarify return values and usage --- src/fluree/db/nameservice.cljc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fluree/db/nameservice.cljc b/src/fluree/db/nameservice.cljc index 674fac94f1..41e01d83e1 100644 --- a/src/fluree/db/nameservice.cljc +++ b/src/fluree/db/nameservice.cljc @@ -25,8 +25,11 @@ (retract [publisher ledger-alias] "Remove the nameservice record for the ledger.") (publishing-address [publisher ledger-alias] - "Returns full publisher address/iri which will get published in commit. If - 'private', return `nil`.")) + "Returns the value to write into the commit's ns field for this nameservice. + This may be a full address/IRI (e.g., fluree:ipns://...) or a resolvable + identifier such as a ledger alias (e.g., ledger:branch), depending on the + nameservice implementation. The returned value will be used with this same + nameservice's lookup function. If publishing should be private, return nil.")) (defprotocol Publication (subscribe [publication ledger-alias] From c50e38402bac1c970e2618e9b01b4ad491908abb Mon Sep 17 00:00:00 2001 From: Brian Platz Date: Tue, 9 Sep 2025 06:41:29 -0400 Subject: [PATCH 24/24] Fix formatting issue in timeout error handling in connection.cljc --- src/fluree/db/connection.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fluree/db/connection.cljc b/src/fluree/db/connection.cljc index 2ef26de79d..bfc2b1a45f 100644 --- a/src/fluree/db/connection.cljc +++ b/src/fluree/db/connection.cljc @@ -503,4 +503,4 @@ timeout-ch (ex-info "Indexing wait timeout, but assume indexing is proceeding in the background." {:status 408 :error :db/timeout - :timeout timeout})))) ) + :timeout timeout})))))