Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
9c5d41e
Make apis branch-aware
bplatz Aug 6, 2025
55d4d2c
remove most of special branch handling
bplatz Aug 9, 2025
30938e5
remove 'branch' from most fns
bplatz Aug 10, 2025
4d7b4c9
ensure branches don't use sub-directories
bplatz Aug 11, 2025
1add842
update branch separator to ':'
bplatz Aug 11, 2025
0ebd718
ensure new ledgers don't have ':', linting
bplatz Aug 11, 2025
82039ef
put nameservice branch records in own directory
bplatz Aug 11, 2025
0f9981a
fix test
bplatz Aug 11, 2025
250e8f5
Fix commit storage to use ledger-name instead of ledger-alias
bplatz Aug 26, 2025
2f8ffd5
Add ledger name validation and branch name validation functions
bplatz Sep 4, 2025
1f0f085
Rename combined-alias to ledger-alias for clarity in commit processing
bplatz Sep 4, 2025
ee5821a
Update ledger alias usage in ledger instantiation and subscription
bplatz Sep 4, 2025
c0275ef
Add ledger name validation during ledger creation
bplatz Sep 4, 2025
e324b95
Remove unnecessary nil argument from ledger trigger index call
bplatz Sep 4, 2025
61a8426
Update commit version to 2 in commit data definitions
bplatz Sep 4, 2025
183872f
Refactor nameservice to use constant for ns-version in filename gener…
bplatz Sep 4, 2025
d5dbb27
Refactor alias handling in ledger and nameservice functions for consi…
bplatz Sep 4, 2025
273cf90
linting
bplatz Sep 4, 2025
b801ac8
Update nameservice versioning from ns@v1 to ns@v2 and adjust related …
bplatz Sep 5, 2025
bf17bcd
param name change, remove unnecessary comments
bplatz Sep 5, 2025
819213f
linting issues
bplatz Sep 5, 2025
173e90f
Refactor nameservice migration to use updated namespace and improve c…
bplatz Sep 5, 2025
21b4c6d
Enhance documentation for publishing-address protocol to clarify retu…
bplatz Sep 6, 2025
c50e384
Fix formatting issue in timeout error handling in connection.cljc
bplatz Sep 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 17 additions & 16 deletions src/fluree/db/api.cljc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
[fluree.db.transact :as transact]
[fluree.db.util :as util]
[fluree.db.util.async :refer [go-try <?]]
[fluree.db.util.ledger :as util.ledger]
[fluree.db.util.log :as log]
[fluree.json-ld :as json-ld])
(:refer-clojure :exclude [merge load range exists? update drop]))
Expand Down Expand Up @@ -248,6 +249,7 @@
([conn ledger-alias] (create conn ledger-alias nil))
([conn ledger-alias opts]
(validate-connection conn)
(util.ledger/validate-ledger-name ledger-alias)
(promise-wrap
(go-try
(log/info "Creating ledger" ledger-alias)
Expand Down Expand Up @@ -387,7 +389,7 @@
opts - (optional) Options map for the commit operation

The ledger-id is automatically extracted from the database object's
alias and branch fields (formatted as alias@branch).
alias and branch fields (formatted as alias:branch).

Creates a new commit and notifies the nameservice of the new version.
Returns promise resolving to the committed database."
Expand Down Expand Up @@ -534,24 +536,19 @@
(transact-api/create-with-txn conn txn opts))))

(defn status
"Returns current status of a ledger branch.
"Returns current status of a ledger.

Parameters:
conn - Connection object
ledger-id - Ledger alias or address
branch - (optional) Branch name (defaults to current branch)
ledger-id - Ledger alias (with optional :branch) or address

Returns status map with commit and index information."
([conn ledger-id]
(status conn ledger-id nil))
([conn ledger-id branch]
(validate-connection conn)
(promise-wrap
(go-try
(let [ledger (<? (connection/load-ledger conn ledger-id))]
(if branch
(ledger/status ledger branch)
(ledger/status ledger)))))))
[conn ledger-id]
(validate-connection conn)
(promise-wrap
(go-try
(let [ledger (<? (connection/load-ledger conn ledger-id))]
(ledger/status ledger)))))

;; db operations

Expand Down Expand Up @@ -906,9 +903,8 @@

Parameters:
conn - Database connection
ledger-alias - The alias/name of the ledger to index
ledger-alias - The alias/name of the ledger to index (with optional :branch)
opts - (optional) Options map:
:branch - Branch name (defaults to main branch)
:timeout - Max wait time in ms (default 300000 / 5 minutes)

Returns a promise that resolves to the indexed database object.
Expand All @@ -917,6 +913,11 @@
Example:
;; Trigger indexing and wait for completion
(let [indexed-db @(trigger-index conn \"my-ledger\")]
;; Use indexed-db...
)

;; Trigger indexing for a specific branch
(let [indexed-db @(trigger-index conn \"my-ledger:main\")]
;; Use indexed-db...
)"
([conn ledger-alias]
Expand Down
2 changes: 2 additions & 0 deletions src/fluree/db/api/transact.cljc
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
[fluree.db.util :as util]
[fluree.db.util.async :refer [<? go-try]]
[fluree.db.util.context :as ctx-util]
[fluree.db.util.ledger :as util.ledger]
[fluree.json-ld :as json-ld]))

(defn prep-opts
Expand Down Expand Up @@ -130,6 +131,7 @@
; are no policies
; to check.
ledger-opts (-> parsed-txn :opts syntax/coerce-ledger-opts)
_ (util.ledger/validate-ledger-name ledger-id)
ledger (<? (connection/create-ledger conn ledger-id ledger-opts))]
(<? (transact/transact-ledger! ledger parsed-txn))))))

Expand Down
29 changes: 12 additions & 17 deletions src/fluree/db/async_db.cljc
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

(declare ->async-db ->AsyncDB deliver!)

(defrecord AsyncDB [alias branch commit t db-chan]
(defrecord AsyncDB [alias commit t db-chan]
dbproto/IFlureeDb
(-query [_ tracker query-map]
(go-try
Expand All @@ -33,7 +33,7 @@
(<? (dbproto/-class-ids db tracker subject)))))
(-index-update [_ commit-index]
(let [commit* (assoc commit :index commit-index)
updated-db (->async-db alias branch commit* t)]
updated-db (->async-db alias commit* t)]
(go-try
(let [db (<? db-chan)]
(deliver! updated-db (dbproto/-index-update db commit-index))))
Expand Down Expand Up @@ -148,7 +148,7 @@

(-as-of [_ t]
(let [db-chan-at-t (async/promise-chan)
db-at-t (->AsyncDB alias branch commit t db-chan-at-t)]
db-at-t (->AsyncDB alias commit t db-chan-at-t)]
(go
(try*
(let [db (<? db-chan)]
Expand Down Expand Up @@ -189,7 +189,7 @@
(<? (policy/wrap-policy db tracker policy policy-values)))))
(root [_]
(let [root-ch (async/promise-chan)
root-db (->AsyncDB alias branch commit t root-ch)]
root-db (->AsyncDB alias commit t root-ch)]
(go
(try*
(let [db (<? db-chan)]
Expand All @@ -207,7 +207,7 @@

(defn display
[db]
(select-keys db [:alias :branch :t]))
(select-keys db [:alias :t]))

#?(:clj
(defmethod print-method AsyncDB [^AsyncDB db, ^Writer w]
Expand Down Expand Up @@ -241,23 +241,18 @@
"Creates an async-db.
This is to be used in conjunction with `deliver!` that will deliver the
loaded db to the async-db."
[ledger-alias branch commit-map t]
(->AsyncDB ledger-alias branch commit-map t (async/promise-chan)))
[ledger-alias commit-map t]
(->AsyncDB ledger-alias commit-map t (async/promise-chan)))

(defn load
([ledger-alias branch commit-catalog index-catalog commit-jsonld indexing-opts]
([ledger-alias commit-catalog index-catalog commit-jsonld indexing-opts]
(let [commit-map (commit-data/jsonld->clj commit-jsonld)]
(load ledger-alias branch commit-catalog index-catalog commit-jsonld commit-map indexing-opts)))
([ledger-alias branch commit-catalog index-catalog commit-jsonld commit-map indexing-opts]
(load ledger-alias commit-catalog index-catalog commit-jsonld commit-map indexing-opts)))
([ledger-alias commit-catalog index-catalog commit-jsonld commit-map indexing-opts]
(let [t (-> commit-map :data :t)
;; Ensure AsyncDB commit reflects index t when an index address exists but :t is missing
commit-map* (if (and (get-in commit-map [:index :address])
(nil? (get-in commit-map [:index :data :t])))
(assoc-in commit-map [:index :data :t] t)
commit-map)
async-db (->async-db ledger-alias branch commit-map* t)]
async-db (->async-db ledger-alias commit-map t)]
(go
(let [db (<! (flake-db/load ledger-alias commit-catalog index-catalog branch
(let [db (<! (flake-db/load ledger-alias commit-catalog index-catalog
[commit-jsonld commit-map] indexing-opts))]
(deliver! async-db db)))
async-db)))
32 changes: 16 additions & 16 deletions src/fluree/db/branch.cljc
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@
(-> commit-map commit-data/->json-ld json-ld/expand))

(defn load-db
[alias branch commit-catalog index-catalog commit-map]
[alias commit-catalog index-catalog commit-map]
(let [commit-jsonld (commit-map->commit-jsonld commit-map)]
(async-db/load alias branch commit-catalog index-catalog
(async-db/load alias commit-catalog index-catalog
commit-jsonld commit-map nil)))

(defn update-index-async
Expand All @@ -56,11 +56,11 @@
return immediately - and for a large amount of novelty,
updating the db to reflect the latest index can take some time
which would lead to atom contention."
[{:keys [alias commit branch t] :as current-db} index-map]
[{:keys [alias commit t] :as current-db} index-map]
(if (async-db/db? current-db)
(dbproto/-index-update current-db index-map)
(let [updated-commit (assoc commit :index index-map)
updated-db (async-db/->async-db alias branch updated-commit t)]
updated-db (async-db/->async-db alias updated-commit t)]
(go ;; update index in the background, return updated db immediately
(->> (dbproto/-index-update current-db index-map)
(async-db/deliver! updated-db)))
Expand Down Expand Up @@ -89,9 +89,9 @@
current-state))))

(defn reload-with-index
[{:keys [commit-catalog index-catalog commit] :as _db} alias branch index]
[{:keys [commit-catalog index-catalog commit alias] :as _db} index]
(let [indexed-commit (assoc commit :index index)]
(load-db alias branch commit-catalog index-catalog indexed-commit)))
(load-db alias commit-catalog index-catalog indexed-commit)))

(defn use-latest-db
"Returns the most recent db from branch-state if it matches
Expand All @@ -107,22 +107,22 @@
latest-db)))

(defn use-latest-index
[{db-commit :commit, :as db} idx-commit alias branch branch-state]
[{db-commit :commit, :as db} idx-commit branch-state]
(if (newer-index? idx-commit db-commit)
(let [updated-db (or (use-latest-db db idx-commit branch-state)
(try* (dbproto/-index-update db (:index idx-commit))
(catch* e (log/error e "Exception updating db with new index, attempting full reload. Exception:" (ex-message e))
(reload-with-index db alias branch (:index idx-commit)))))]
(reload-with-index db (:index idx-commit)))))]
updated-db)
db))

(defn index-queue
[alias branch publishers branch-state]
[publishers branch-state]
(let [buf (async/sliding-buffer 1)
queue (async/chan buf)]
(go-loop [last-index-commit nil]
(when-let [{:keys [db index-files-ch complete-ch]} (<! queue)]
(let [db* (use-latest-index db last-index-commit alias branch branch-state)
(let [db* (use-latest-index db last-index-commit branch-state)
result (try*
(let [indexed-db (<? (indexer/index db* index-files-ch)) ;; indexer/index always returns a FlakeDB (never AsyncDB)
[{prev-commit :commit} {indexed-commit :commit}]
Expand All @@ -149,17 +149,17 @@

(defn state-map
"Returns a branch map for specified branch name at supplied commit"
([ledger-alias branch-name commit-catalog index-catalog publishers commit-jsonld]
(state-map ledger-alias branch-name commit-catalog index-catalog publishers commit-jsonld nil))
([ledger-alias branch-name commit-catalog index-catalog publishers commit-jsonld indexing-opts]
([alias branch-name commit-catalog index-catalog publishers commit-jsonld]
(state-map alias branch-name commit-catalog index-catalog publishers commit-jsonld nil))
([alias branch-name commit-catalog index-catalog publishers commit-jsonld indexing-opts]
(let [commit-map (commit-data/jsonld->clj commit-jsonld)
initial-db (async-db/load ledger-alias branch-name commit-catalog index-catalog
initial-db (async-db/load alias commit-catalog index-catalog
commit-jsonld commit-map indexing-opts)
state (atom {:commit commit-map
:current-db initial-db})
idx-q (index-queue ledger-alias branch-name publishers state)]
idx-q (index-queue publishers state)]
{:name branch-name
:alias ledger-alias
:alias alias
:state state
:index-queue idx-q
:indexing-opts indexing-opts})))
Expand Down
15 changes: 9 additions & 6 deletions src/fluree/db/commit/storage.cljc
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
[fluree.db.util :as util :refer [get-first get-first-id
get-first-value try* catch*]]
[fluree.db.util.async :refer [<? go-try]]
[fluree.db.util.ledger :as util.ledger]
[fluree.db.util.log :as log]
[fluree.json-ld :as json-ld]))

Expand Down Expand Up @@ -147,21 +148,23 @@
resp-ch))

(defn write-jsonld
[storage ledger-alias jsonld]
(let [path (str/join "/" [ledger-alias "commit"])]
[storage ledger-name jsonld]
(let [path (str/join "/" [ledger-name "commit"])]
(storage/content-write-json storage path jsonld)))

(defn write-genesis-commit
[storage ledger-alias branch publish-addresses init-time]
[storage ledger-alias publish-addresses init-time]
(go-try
(let [genesis-commit (commit-data/blank-commit ledger-alias branch publish-addresses init-time)
(let [;; Use full alias for commit data, but base name for storage paths
ledger-base-name (util.ledger/ledger-base-name ledger-alias)
genesis-commit (commit-data/blank-commit ledger-alias publish-addresses init-time)
initial-context (get genesis-commit "@context")
initial-db-data (-> genesis-commit
(get "data")
(assoc "@context" initial-context))
{db-address :address} (<? (write-jsonld storage ledger-alias initial-db-data))
{db-address :address} (<? (write-jsonld storage ledger-base-name initial-db-data))
genesis-commit* (assoc-in genesis-commit ["data" "address"] db-address)
{commit-address :address} (<? (write-jsonld storage ledger-alias genesis-commit*))]
{commit-address :address} (<? (write-jsonld storage ledger-base-name genesis-commit*))]
(-> genesis-commit*
(assoc "address" commit-address)
json-ld/expand))))
Loading