Feature/index rebuild from last

deps.edn

@@ -1,8 +1,10 @@
 {:deps  {org.clojure/clojure            {:mvn/version "1.11.1"}
          org.clojure/data.xml           {:mvn/version "0.2.0-alpha8"}
          com.fluree/alphabase           {:mvn/version "3.3.0"}
-         com.fluree/db                  {:mvn/version "1.0.5"}
-         com.fluree/raft                {:mvn/version "1.0.0-beta1"}
+         com.fluree/db                  {:git/url "https://github.com/fluree/db.git"
+                                         :sha     "add56b63e276f55200f4e21d074531dbccc66765"}
+         com.fluree/raft                {:git/url "https://github.com/fluree/raft.git"
+                                         :sha     "904d915d636cdbc9eced518e5675eedf98410965"} ;; one commit after 1.0.0-beta2
          com.fluree/crypto              {:mvn/version "0.4.0"}
 
          ;; network comm
@@ -38,7 +40,7 @@
  :aliases
  {:mvn/group-id    com.fluree
   :mvn/artifact-id ledger
-  :mvn/version     "1.0.5"
+  :mvn/version     "1.0.6-RC1"
 
   :dev
   {:extra-paths ["dev", "test"]

dev/remote_debug/README.md

@@ -0,0 +1,10 @@
+# Working with remote socket REPL
+Optionally, Fluree can be started with a Remote REPL socket open for debugging purpose.
+
+The ./fluree_start.sh script has an option that automatically enables this feature.
+
+When starting with ./fluree.start.sh, simply add the option:
+```bash
+./fluree_start.sh -repl-port=5555
+```
+This will start Fluree with a remote REPL socket open on port 5555.

resources/fluree_start.sh

@@ -21,6 +21,8 @@ FLUREE_SERVER=""
 FLUREE_PROPERTIES=""
 FLUREE_LOGBACK_CONFIGURATION_FILE=""
 
+FLUREE_REMOTE_REPL_PORT=""
+
 MINIMUM_JAVA_VERSION=11
 
 function find_java() {
@@ -179,6 +181,11 @@ while [ $# -gt 0 ]; do
   -Xms*)
     XMS=$1
     ;;
+  -repl-port=*)
+    if [[ "$1" =~ ^-repl-port=([0-9]+) ]]; then
+      FLUREE_REMOTE_REPL_PORT=${BASH_REMATCH[1]}
+    fi
+    ;;
   test)
     break
     ;;
@@ -217,6 +224,15 @@ echo "Fluree ledger starting with java options: ${XMS} ${XMX} ${JAVA_OPTS}"
 echo "fluree.db log level is ${FLUREE_DB_LOG_LEVEL}"
 echo "fluree.raft log level is ${FLUREE_RAFT_LOG_LEVEL}"
 
-exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \
-  -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \
-  -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER}
+if [[ $FLUREE_REMOTE_REPL_PORT ]]
+then
+  echo "WARNING: Starting with remote REPL on port ${FLUREE_REMOTE_REPL_PORT}"
+  exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \
+    -Dclojure.server.repl="{:port ${FLUREE_REMOTE_REPL_PORT} :accept clojure.core.server/repl}" \
+    -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \
+    -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER}
+else
+  exec ${JAVA_X} -server ${XMX} ${XMS} ${JAVA_OPTS} ${FLUREE_ARGS} -Dfdb.properties.file=${FLUREE_PROPERTIES} \
+    -Dfdb.log.ansi -Dfluree.db.log.level=${FLUREE_DB_LOG_LEVEL} -Dfluree.raft.log.level=${FLUREE_RAFT_LOG_LEVEL} \
+    -Dlogback.configurationFile=${FLUREE_LOGBACK_CONFIGURATION_FILE} -jar ${FLUREE_SERVER}
+fi

src/data_readers.cljc

@@ -0,0 +1,3 @@
+{Flake fluree.db.flake/parts->Flake
+ FdbIndexConfig fluree.db.index/map->IndexConfig
+ FlureeUnresolvedNode fluree.db.storage.core/map->UnresolvedNode}

src/fluree/db/ledger/consensus/dbsync2.clj

@@ -436,10 +436,11 @@
 
 (defn consistency-full-check
   [conn ledgers-info remote-sync-servers]
-  (let [sync-chan     (async/chan)                          ;; files to sync are placed on this channel
-        res-chan      (async/chan)                          ;; results file sync (error/success) are placed on this channel
-        parallelism   8]
-    (if (empty? ledgers-info)
+  (let [sync-chan   (async/chan) ;; files to sync are placed on this channel
+        res-chan    (async/chan) ;; results file sync (error/success) are placed on this channel
+        parallelism 8]
+    (if (or (empty? ledgers-info)
+            (empty? remote-sync-servers))
       (go ::done)
       (let [remote-copy-fn (remote-copy-fn* conn remote-sync-servers 3000)]
         (check-all-ledgers-consistency conn ledgers-info sync-chan)

src/fluree/db/ledger/indexing.clj

@@ -105,6 +105,23 @@
                 :else
                 (recur (inc n) his*)))))))
 
+(defn throw-write-leaf-condition
+  "In the case there is a leaf write exception, dump out as much data as possible to help diagnose.
+
+  condition-str is overflow, underflow or standard"
+  [e condition-str network dbid block t idx-type base-id leaf-i flakes history
+   old-children new-children resolved overflow-data underflow-data]
+  (log/error e "Error creating leaf index segment from"
+             condition-str ".\n Data: "
+             (pr-str {:network        network :dbid dbid :block block :t t
+                      :idx-type       idx-type :base-id base-id :leaf-i leaf-i
+                      :flakes         flakes :history history
+                      :old-children   old-children :new-children new-children
+                      :resolved       resolved
+                      :overflow-data  overflow-data
+                      :underflow-data underflow-data}))
+  (throw e))
+
 
 (defn index-leaf
   "Given a node, idx-novelty, returns [ [nodes] skip n].
@@ -125,7 +142,8 @@
            node-bytes      (flake/size-bytes resolved-flakes)
            overflow?       (overflow? node-bytes)
            underflow?      (and (underflow? node-bytes) (not= 1 (count old-children)))
-           history         (<? history-ch)]
+           history         (or (<? history-ch) ;; with predicates with no-history turned on, this can be empty
+                               #{})]
        (cond
          overflow?
          (let [splits     (split-flakes resolved-flakes node-bytes)
@@ -139,8 +157,16 @@
                                      (first flakes))
                    base-id         (str (util/random-uuid))
                    his-split       (get-history-in-range history first-flake rhs' comparator)
-                   id              (<? (storage/write-leaf conn network dbid idx-type base-id
-                                                           flakes his-split))
+                   id              (async/<! (storage/write-leaf conn network dbid idx-type base-id
+                                                                 flakes his-split))
+                   _               (when (util/exception? id)
+                                     (throw-write-leaf-condition
+                                       id "OVERFLOW"
+                                       network dbid block t idx-type base-id leaf-i
+                                       flakes history
+                                       old-children new-children resolved
+                                       {:his-split his-split :first-flake first-flake :rhs' rhs' :splits splits}
+                                       nil))
                    child-leftmost? (and leftmost? (zero? split-i))
                    child-node      (storage/map->UnresolvedNode
                                      {:conn      conn :config config
@@ -182,8 +208,18 @@
                current-node-his (get-history-in-range history fflake rhs comparator)
                his-in-range     (into current-node-his combine-his)
                flakes           (into (:flakes resolved) (:flakes combine-leaf))
-               id               (<? (storage/write-leaf conn network dbid idx-type base-id
-                                                        flakes his-in-range))
+               id               (async/<! (storage/write-leaf conn network dbid idx-type base-id
+                                                              flakes his-in-range))
+               _                (when (util/exception? id)
+                                  (throw-write-leaf-condition
+                                    id "UNDERFLOW"
+                                    network dbid block t idx-type base-id leaf-i
+                                    flakes history
+                                    old-children new-children resolved
+                                    nil
+                                    {:skip          skip :n n :combine-leaf combine-leaf
+                                     :combine-bytes combine-bytes
+                                     :combine-his   combine-his}))
                size             (+ node-bytes combine-bytes)
                ;; current node might be empty, so we need to get first and rhs from node, NOT resolved
                [first-flake rhs] (if (= skip :previous)
@@ -204,8 +240,15 @@
          :else
          (let [base-id    (str (util/random-uuid))
                flakes     (:flakes resolved)
-               id         (<? (storage/write-leaf conn network dbid idx-type base-id
-                                                  flakes history))
+               id         (async/<! (storage/write-leaf conn network dbid idx-type base-id
+                                                        flakes history))
+               _          (when (util/exception? id)
+                            (throw-write-leaf-condition
+                              id "STANDARD"
+                              network dbid block t idx-type base-id leaf-i
+                              flakes history
+                              old-children new-children resolved
+                              nil nil))
                child-node (storage/map->UnresolvedNode
                             {:conn      conn :config config
                              :dbid      dbid :id id :leaf true
@@ -266,14 +309,14 @@
            child-n    (count children)
            at-leaf?   (:leaf (val (first children)))
            children*  (loop [child-i   (dec child-n)
-                             rhs       rhs
+                             rhs*       rhs
                              children* (empty children)]
                         (if (< child-i 0) ;; at end of result set
                           children*
                           (let [child         (val (nth children child-i))
                                 child-rhs     (:rhs child)
-                                _             (when-not (or (= (dec child-i) child-n) (= child-rhs rhs))
-                                                (throw (ex-info (str "Something went wrong. Child-rhs does not equal rhs: " {:child-rhs child-rhs :rhs rhs})
+                                _             (when-not (or (= (dec child-i) child-n) (= child-rhs rhs*))
+                                                (throw (ex-info (str "Something went wrong. Child-rhs does not equal rhs: " {:child-rhs child-rhs :rhs rhs*})
                                                                 {:status 500
                                                                  :error  :db/unexpected-error})))
                                 child-first   (:first child)
@@ -289,7 +332,21 @@
                                 dirty?        (or (seq novelty) (seq remove-preds?))
                                 [new-nodes skip n] (if dirty?
                                                      (if at-leaf?
-                                                       (<? (index-leaf conn network dbid child block t idx-novelty child-rhs children children* child-i remove-preds?))
+                                                       (let [res (async/<! (index-leaf conn network dbid child block t idx-novelty child-rhs children children* child-i remove-preds?))]
+                                                         (if (util/exception? res)
+                                                           (do
+                                                             (log/error "Error indexing leaf from index-branch. Branch Data:\n"
+                                                                        (pr-str {:base-id base-id :idx-type idx-type :child-n child-n :rhs rhs
+                                                                                  :resolved resolved})
+                                                                        "\nChild Data:\n:"
+                                                                        (pr-str {:child-i child-i :rhs* rhs*
+                                                                                 :child-first child-first :child-rhs child-rhs
+                                                                                 :leftmost? (:leftmost? child)
+                                                                                 :novelty novelty})
+                                                                        "\nIDX Novelty:\n:"
+                                                                        (pr-str idx-novelty))
+                                                             (throw res))
+                                                           res))
                                                        [(<? (index-branch conn network dbid child idx-novelty block t child-rhs progress remove-preds?)) nil nil])
                                                      [[child] nil nil])
                                 new-rhs       (:first (first new-nodes))

src/fluree/db/ledger/stats.clj

@@ -68,14 +68,29 @@
                     (assoc acc k v))
                   acc)) {} input) walk/keywordize-keys))
 
+(defn compact-group-state
+  "Report out a smaller version of the raft group state to save logging"
+  [{:keys [networks] :as group-state}]
+  (let [networks* (reduce-kv
+                    (fn [acc network network-map]
+                      (let [dbs*         (reduce-kv (fn [acc dbid db-data]
+                                                      (assoc acc dbid (dissoc db-data :indexes)))
+                                                    {} (:dbs network-map))
+                            network-map* (assoc network-map :dbs dbs*)]
+                        (assoc acc network network-map*)))
+                    {}
+                    networks)]
+    (-> group-state
+        (select-keys [:version :leases :_work])
+        (assoc :networks networks*))))
+
+
 (defn report-stats
   [system]
   (log/info "Memory: " (-> (memory-stats) (json/encode)))
-  (let [group-state  (txproto/-local-state (:group system))
-        state-report (-> group-state
-                         (select-keys [:version :leases :_work :networks]))]
-    (log/info "Group state: " (json/encode state-report))
-    (log/debug "Full group state: " (json/encode group-state))))
+  (let [group-state  (txproto/-local-state (:group system))]
+    (log/info "Group state: " (json/encode (compact-group-state group-state)))
+    (log/trace "Full group state: " (json/encode group-state))))
 
 
 

src/fluree/db/server.clj

@@ -27,6 +27,8 @@
 
 (set! *warn-on-reflection* true)
 
+(def system nil)
+
 ;; instantiates server operations
 
 (defn local-message-process
@@ -205,6 +207,7 @@
   (if-let [command (:fdb-command environ/env)]
     (execute-command command)
     (let [system (startup)]
+      (alter-var-root #'system (constantly system))
       (.addShutdownHook
         (Runtime/getRuntime)
         (Thread. ^Runnable