Iceberg virtual graph (SPARQL over Iceberg) + R2RML mapping (alpha)

.clj-kondo/ci-config.edn

@@ -1,7 +1,8 @@
 {:source-paths   #{"src"}
  :linters        {:unresolved-symbol {:exclude [(fluree.db.util/catch*)
                                                 (test-with-files.tools/with-tmp-dir)
-                                                goog.isObject]}}
+                                                goog.isObject
+                                                form#]}}
  :lint-as        {instaparse.core/defparser clojure.core/def}
  :config-in-call {cljs.core/pr-sequential-writer {:ignore [:private-call]}
                   fluree.db.query.exec.eval/parse-qualified-code {:ignore true}}

.gitignore

@@ -36,3 +36,5 @@ package-lock.json
 **report.html
 test/nodejs/store
 /dev/data/*
+/dev-resources/openflights/raw/
+/dev-resources/openflights/warehouse/

Makefile

@@ -2,7 +2,7 @@
 	cljs-browser-test cljs-node-test cljstest test graaltest ci clean			\
 	js-packages sync-package-json publish-nodejs publish-browser		\
 	publish-webworker publish-js pending-tests pt clj-kondo-lint            \
-	clj-kondo-lint-ci cljfmt-check cljfmt-fix
+	clj-kondo-lint-ci cljfmt-check cljfmt-fix iceberg-openflights iceberg-partitioned
 
 .DEFAULT_GOAL := help
 
@@ -167,3 +167,56 @@ clean: ## Remove build artifacts and caches
 	rm -rf js-packages/browser/fluree-browser-sdk.js
 	rm -rf js-packages/nodejs/fluree-node-sdk.js
 	rm -rf js-packages/webworker/fluree-webworker.js
+
+iceberg-openflights: ## Download OpenFlights CSVs and build local Iceberg warehouse (gitignored)
+	./script/fetch-openflights.sh
+	clojure -Sdeps '{:paths ["script"] :deps {org.apache.iceberg/iceberg-core {:mvn/version "1.10.0"} \
+org.apache.iceberg/iceberg-parquet {:mvn/version "1.10.0"} \
+org.apache.iceberg/iceberg-data {:mvn/version "1.10.0"} \
+org.apache.iceberg/iceberg-bundled-guava {:mvn/version "1.10.0"} \
+org.apache.parquet/parquet-hadoop {:mvn/version "1.16.0"} \
+org.apache.hadoop/hadoop-common {:mvn/version "3.3.6" :exclusions [org.slf4j/slf4j-log4j12 log4j/log4j org.slf4j/slf4j-reload4j]} \
+org.clojure/data.csv {:mvn/version "1.0.1"}}}' \
+		-M -m build-openflights-iceberg
+
+iceberg-partitioned: ## Build partitioned Iceberg table for partition pruning tests
+	clojure -Sdeps '{:paths ["script"] :deps {org.apache.iceberg/iceberg-core {:mvn/version "1.10.0"} \
+org.apache.iceberg/iceberg-parquet {:mvn/version "1.10.0"} \
+org.apache.iceberg/iceberg-data {:mvn/version "1.10.0"} \
+org.apache.iceberg/iceberg-bundled-guava {:mvn/version "1.10.0"} \
+org.apache.parquet/parquet-hadoop {:mvn/version "1.16.0"} \
+org.apache.hadoop/hadoop-common {:mvn/version "3.3.6" :exclusions [org.slf4j/slf4j-log4j12 log4j/log4j org.slf4j/slf4j-reload4j]} \
+org.clojure/data.csv {:mvn/version "1.0.1"}}}' \
+		-M -m build-partitioned-airlines
+
+iceberg-rest-up: ## Start Iceberg REST catalog with MinIO (Docker)
+	cd dev-resources/iceberg-rest && docker compose up -d
+	@echo "Waiting for REST catalog to be ready..."
+	@sleep 10
+	@echo "REST catalog available at http://localhost:8181"
+	@echo "MinIO console at http://localhost:9001 (admin/password)"
+	@echo "Run 'make iceberg-rest-load' to load OpenFlights data"
+
+iceberg-rest-load: ## Load OpenFlights data into REST catalog
+	docker exec -it iceberg-spark /scripts/load-openflights.sh
+
+iceberg-rest-down: ## Stop Iceberg REST catalog
+	cd dev-resources/iceberg-rest && docker compose down
+
+iceberg-rest-clean: ## Stop and remove all REST catalog data
+	cd dev-resources/iceberg-rest && docker compose down -v
+
+polaris-up: ## Start Apache Polaris catalog with vended credentials (Docker, port 8182)
+	cd dev-resources/polaris && docker compose up -d
+	@echo "Waiting for Polaris to be ready..."
+	@sleep 5
+	@./dev-resources/polaris/bootstrap.sh
+	@echo ""
+	@echo "Polaris REST catalog available at http://localhost:8182"
+	@echo "OAuth: POST http://localhost:8182/api/catalog/v1/oauth/tokens"
+
+polaris-down: ## Stop Apache Polaris catalog
+	cd dev-resources/polaris && docker compose down
+
+polaris-clean: ## Stop and remove Polaris (keeps MinIO data)
+	cd dev-resources/polaris && docker compose down -v

db-iceberg-arrow/deps.edn

@@ -0,0 +1,39 @@
+{:deps {com.fluree/db-iceberg {:local/root "../db-iceberg"}
+
+        ;; Arrow and Iceberg Arrow integration
+        org.apache.iceberg/iceberg-arrow {:mvn/version "1.10.0"}
+        org.apache.arrow/arrow-vector    {:mvn/version "17.0.0"}
+        org.apache.arrow/arrow-memory-netty {:mvn/version "17.0.0"}}
+
+ :paths ["src" "resources"]
+
+ ;; NOTE: Arrow requires JVM flags for direct memory access on Java 9+
+ ;; These are specified in each alias below (global :jvm-opts not supported)
+
+ :aliases
+ {:dev
+  {:extra-paths ["../db-iceberg/dev" "../dev" "../dev-resources"]
+   :jvm-opts    ["-Dlogback.configurationFile=../dev-resources/logback.xml"
+                 "--add-opens=java.base/java.nio=ALL-UNNAMED"
+                 "--add-opens=java.base/java.lang=ALL-UNNAMED"
+                 "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"]}
+
+  :test
+  {:extra-paths ["test" "../db-iceberg/test" "../test" "../test-resources" "../dev-resources"]
+   :extra-deps  {lambdaisland/kaocha {:mvn/version "1.91.1392"}
+                 org.clojure/test.check {:mvn/version "1.1.1"}
+                 babashka/fs {:mvn/version "0.5.20"}}
+   :jvm-opts    ["--add-opens=java.base/java.nio=ALL-UNNAMED"
+                 "--add-opens=java.base/java.lang=ALL-UNNAMED"
+                 "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"]
+   :exec-fn     kaocha.runner/exec-fn
+   :exec-args   {:kaocha/tests [{:kaocha.testable/id :iceberg-arrow
+                                 :kaocha.testable/type :kaocha.type/clojure.test
+                                 :kaocha/source-paths ["src" "../db-iceberg/src" "../src"]
+                                 :kaocha/test-paths ["test"]
+                                 :kaocha/ns-patterns [".*"]}]}}
+
+  ;; GraalVM-compatible alias (uses arrow-memory-unsafe instead of netty)
+  :graalvm
+  {:override-deps {org.apache.arrow/arrow-memory-unsafe {:mvn/version "17.0.0"}}
+   :exclude-deps  [org.apache.arrow/arrow-memory-netty]}}}

db-iceberg-arrow/resources/META-INF/native-image/com.fluree/db-iceberg-arrow/reflect-config.json

@@ -0,0 +1,64 @@
+[
+  {
+    "name": "org.apache.arrow.memory.RootAllocator",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true
+  },
+  {
+    "name": "org.apache.arrow.memory.BufferAllocator",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true
+  },
+  {
+    "name": "org.apache.arrow.memory.unsafe.UnsafeAllocationManager",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true
+  },
+  {
+    "name": "org.apache.arrow.memory.unsafe.DefaultAllocationManagerFactory",
+    "fields": [{"name": "FACTORY"}]
+  },
+  {
+    "name": "org.apache.arrow.vector.VectorSchemaRoot",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true
+  },
+  {
+    "name": "org.apache.arrow.vector.types.pojo.Schema",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true
+  },
+  {
+    "name": "org.apache.arrow.vector.types.pojo.Field",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true
+  },
+  {
+    "name": "org.apache.iceberg.arrow.vectorized.ArrowReader",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true
+  },
+  {
+    "name": "org.apache.iceberg.arrow.vectorized.VectorizedTableScanIterable",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true
+  }
+]