From a902fa1134a8be172f3ff64225bc113aa31f25ac Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Tue, 14 Oct 2025 11:07:03 -0500 Subject: [PATCH 01/12] add ledger-specific query route Also add support for SPARQL protocol headers for specifying query graphs. --- deps.edn | 2 +- src/fluree/server/handler.clj | 25 ++++++++++++++ src/fluree/server/handlers/ledger.clj | 8 +++-- .../fluree/server/integration/sparql_test.clj | 33 ++++++++++++++++++- 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/deps.edn b/deps.edn index 8129df76..d52a572a 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,7 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.3"} org.clojure/core.async {:mvn/version "1.6.681"} com.fluree/db {:git/url "https://github.com/fluree/db.git" - :git/sha "f2bcf88ddf90c44ef50369da4edcb82f91f023e1"} + :git/sha "f86dffe929d6f1294deb3ec22f4da58e6a989c32"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"} diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj index e0a943eb..1d4d309c 100644 --- a/src/fluree/server/handler.clj +++ b/src/fluree/server/handler.clj @@ -189,6 +189,16 @@ :coercion ^:replace query-coercer :handler #'ledger/query}) +(def ledger-query-endpoint + {:summary "Endpoint for submitting queries to a specific ledger" + :parameters {:body QueryRequestBody + :path [:map [:ledger-alias :string]]} + :responses {200 {:body QueryResponse} + 400 {:body ErrorResponse} + 500 {:body ErrorResponse}} + :coercion ^:replace query-coercer + :handler #'ledger/query}) + (def history-endpoint {:summary "Endpoint for submitting history queries" :parameters {:body HistoryQuery} @@ -299,6 +309,12 @@ (select-keys fluree-request-header-keys) (update-keys (fn [k] (keyword (subs k request-header-prefix-count))))) + ;; SPARQL protocol headers for graph specification + from (get headers "default-graph-uri") + from-named (get headers "named-graph-uri") + using (get headers "using-graph-uri") + using-named (get headers "using-named-graph-uri") + max-fuel (when max-fuel (try (Integer/parseInt max-fuel) (catch Exception e @@ -371,6 +387,10 @@ max-fuel (assoc :max-fuel max-fuel) format (assoc :format format) output (assoc :output output) + from (assoc :from from) + from-named (assoc :from from-named) + using (assoc :using using) + using-named (assoc :using using-named) ledger (assoc :ledger ledger) policy (assoc :policy policy) policy-class (assoc :policy-class policy-class) @@ -561,6 +581,10 @@ {:get query-endpoint :post query-endpoint}]) +(def fluree-ledger-query-routes + ["/query/{*ledger-alias}" + {:post ledger-query-endpoint}]) + (def fluree-history-routes ["/history" {:get history-endpoint @@ -599,6 +623,7 @@ :update fluree-update-route :transact fluree-transact-routes :query fluree-query-routes + :ledger-query fluree-ledger-query-routes :history fluree-history-routes :remote fluree-remote-routes :subscription fluree-subscription-routes}) diff --git a/src/fluree/server/handlers/ledger.clj b/src/fluree/server/handlers/ledger.clj index c6150e1a..5566a0a5 100644 --- a/src/fluree/server/handlers/ledger.clj +++ b/src/fluree/server/handlers/ledger.clj @@ -5,11 +5,13 @@ [fluree.server.handlers.shared :refer [defhandler deref!] :as shared])) (defhandler query - [{:keys [fluree/conn fluree/opts] {:keys [body]} :parameters :as _req}] + [{:keys [fluree/conn fluree/opts] {:keys [body path]} :parameters :as _req}] (let [query (or (::handler/query body) body) + ;; supply ledger-alias from path params if not overridden by a header + opts* (update opts :ledger #(or % (:ledger-alias path))) {:keys [status result] :as query-response} - (deref! (fluree/query-connection conn query opts))] - (log/debug "query handler received query:" query opts) + (deref! (fluree/query-connection conn query opts*))] + (log/debug "query handler received query:" query opts*) (shared/with-tracking-headers {:status status, :body result} query-response))) diff --git a/test/fluree/server/integration/sparql_test.clj b/test/fluree/server/integration/sparql_test.clj index 0412c1b7..554800cc 100644 --- a/test/fluree/server/integration/sparql_test.clj +++ b/test/fluree/server/integration/sparql_test.clj @@ -2,7 +2,7 @@ (:require [clojure.string :as str] [clojure.test :refer [deftest is testing use-fixtures]] [fluree.db.util.json :as json] - [fluree.server.integration.test-system + [fluree.server.integration.test-system :as test-system :refer [api-post create-rand-ledger run-test-server sparql-headers sparql-results-headers sparql-update-headers]])) @@ -63,3 +63,34 @@ "@graph" [{"@id" "ex:query-sparql-test" "foo:name" ["query-sparql-test"]}]} (-> rdf-res :body (json/parse false))))))) + +(deftest sparql-federated-query + (let [ledger-name "test-federated-query" + service (str "http://localhost:" @test-system/api-port "/fluree/query/" ledger-name) + + txn-req {"ledger" ledger-name + "@context" {"ex" "http://example.com/"} + "insert" (mapv #(do {"@id" (str "ex:" %) + "@type" (if (odd? %) "ex:Odd" "ex:Even") + "ex:name" (str "name" %) + "ex:num" [(dec %) % (inc %)] + "ex:ref" {"@id" (str "ex:" (inc %))}}) + (range 10))} + txn-res (api-post :create {:body (json/stringify txn-req) :headers test-system/json-headers}) + + service-q (str/join "\n" ["PREFIX ex: " + "SELECT ?name ?type" + (str "FROM <" ledger-name ">") + "WHERE { ?s a ex:Even ; ex:ref ?ref . " + (str "SERVICE <" service "> { ?ref ex:name ?name ; a ?type . }") + "}"]) + service-res (api-post :query {:body service-q :headers sparql-headers})] + (is (= 201 (:status txn-res))) + (is (= 200 (:status service-res))) + (testing "federated query can join across local and remote graphs" + (is (= [["name1" "ex:Odd"] + ["name3" "ex:Odd"] + ["name5" "ex:Odd"] + ["name7" "ex:Odd"] + ["name9" "ex:Odd"]] + (-> service-res :body (json/parse false))))))) From 964542a09b411a59cdff90c308db15478815d4bf Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 15 Oct 2025 02:47:11 -0500 Subject: [PATCH 02/12] add federated-query-demo task --- src/fluree/server.clj | 10 +++++----- src/fluree/server/command.clj | 1 + src/fluree/server/fedq_demo.clj | 24 ++++++++++++++++++++++++ src/fluree/server/system.clj | 19 +++++++++++++++---- 4 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 src/fluree/server/fedq_demo.clj diff --git a/src/fluree/server.clj b/src/fluree/server.clj index e5fd43e4..86bfe384 100644 --- a/src/fluree/server.clj +++ b/src/fluree/server.clj @@ -8,23 +8,23 @@ (defn start [{:keys [options] :as _cli}] - (let [{:keys [profile reindex]} options] + (let [{:keys [profile reindex setup-federated-query-demo]} options] (if-let [config-string (:string options)] (do (log/info "Starting Fluree server from command line configuration" (when profile (str "with profile: " profile))) - (system/start-config config-string :profile profile :reindex reindex)) + (system/start-config config-string :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo)) (if-let [config-path (:config options)] (do (log/info "Starting Fluree server from configuration file at path:" config-path (when profile (str "with profile: " profile))) - (system/start-file config-path :profile profile :reindex reindex)) + (system/start-file config-path :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo)) (if-let [config-resource (:resource options)] (do (log/info "Starting Fluree server from configuration resource:" config-resource (when profile (str "with profile: " profile))) - (system/start-resource config-resource :profile profile :reindex reindex)) + (system/start-resource config-resource :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo)) (do (log/info "Starting Fluree server with default configuration:" system/default-resource-name (when profile (str "with profile: " profile))) - (system/start :profile profile :reindex reindex))))))) + (system/start :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo))))))) (defn -main [& args] diff --git a/src/fluree/server/command.clj b/src/fluree/server/command.clj index feae4d31..2acd6d84 100644 --- a/src/fluree/server/command.clj +++ b/src/fluree/server/command.clj @@ -9,6 +9,7 @@ ["-c" "--config FILE" "Load configuration at a file path"] ["-s" "--string STRING" "Load stringified configuration"] ["-r" "--resource NAME" "Load pre-defined configuration resource"] + ["-fed-q" "--setup-federated-query-demo" "Prepare an in-memory database to demonstrate federated query capabilities"] ["--reindex" "--reindex LEDGER" "Reindex the specified ledger or use --all to reindex all ledgers"] ["-h" "--help" "Print this usage summary and exit"]]) diff --git a/src/fluree/server/fedq_demo.clj b/src/fluree/server/fedq_demo.clj new file mode 100644 index 00000000..1025a6e8 --- /dev/null +++ b/src/fluree/server/fedq_demo.clj @@ -0,0 +1,24 @@ +(ns fluree.server.fedq-demo + (:require [clojure.java.io :as io] + [fluree.db.api :as fluree] + [fluree.db.util.json :as json] + [fluree.db.util.log :as log])) + +(def ledger-alias "test-federated-query") + +(defn load-data + [conn] + (let [gentox (slurp (io/resource "GenTox_DB_Ontology.jsonld")) + ames (slurp (io/resource "tblincubation_ames.jsonld")) + testorg (slurp (io/resource "lst_testorganism.jsonld")) + ecoli (slurp (io/resource "eColi_expansion.jsonld")) + + db0 @(fluree/create conn ledger-alias {:indexing {:reindex-min-bytes 25000000 + :reindex-max-bytes 150000000}}) + db1 @(fluree/insert db0 (json/parse gentox false)) + db2 @(fluree/insert db1 (json/parse ames false)) + db3 @(fluree/insert db2 (json/parse testorg false)) + db4 @(fluree/insert db3 (json/parse ecoli false)) + db5 @(fluree/commit! conn db4)] + (log/info "Finished transacting demo data:" (-> db5 :stats :flakes) "flakes") + :success)) diff --git a/src/fluree/server/system.clj b/src/fluree/server/system.clj index 0e846ecb..70a77f18 100644 --- a/src/fluree/server/system.clj +++ b/src/fluree/server/system.clj @@ -18,6 +18,7 @@ [fluree.server.handler :as handler] [fluree.server.http :as-alias http] [fluree.server.reindex :as reindex] + [fluree.server.fedq-demo :as fedq-demo] [fluree.server.watcher :as watcher] [integrant.core :as ig] [ring.adapter.jetty9 :as jetty])) @@ -200,7 +201,7 @@ (log/info " Closed mode: enabled")))) (defn start-config - [config & {:keys [profile reindex]}] + [config & {:keys [profile reindex setup-federated-query-demo]}] (let [json-config (if (string? config) (json/parse config false) config) @@ -210,21 +211,31 @@ parsed-config (config/parse config-with-profile) system (conn-system/initialize parsed-config)] (log-config-summary parsed-config) - (if reindex + (cond + setup-federated-query-demo + (let [connection-key (first (filter #(isa? % :fluree.db/connection) (keys system)))] + (if-let [conn (get system connection-key)] + (do (fedq-demo/load-data conn) + system) + (do (log/error "No connection found." system) + (System/exit 1)))) + + reindex (let [connection-key (first (filter #(isa? % :fluree.db/connection) (keys system)))] (if-let [conn (get system connection-key)] (let [{:keys [status]} (async/ path config/read-file - (start-config :profile profile :reindex reindex))) + (start-config :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo))) (defn start-resource [resource-name & {:keys [profile reindex]}] From 3da3a5891d947d354056c725244d92e0d27aa7e7 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Mon, 20 Oct 2025 10:41:47 -0500 Subject: [PATCH 03/12] remove dynamically sorted middleware We do not currently support dynamic middleware, so there is no need for the ability to sort it dynamically. --- src/fluree/server/handler.clj | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj index 1d4d309c..a5cd253f 100644 --- a/src/fluree/server/handler.clj +++ b/src/fluree/server/handler.clj @@ -496,24 +496,23 @@ {::exception/default (partial exception/wrap-log-to-console fluree-exception-handler)}))] - ;; Exception middleware should always be first AND last. - ;; The last (highest sort order) one ensures that middleware that comes - ;; after it will not be skipped on response if handler code throws an - ;; exception b/c this it catches them and turns them into responses. - ;; The first (lowest sort order) one ensures that exceptions thrown by - ;; other middleware are caught and turned into appropriate responses. - ;; Seems kind of clunky. Maybe there's a better way? - WSM 2023-04-28 - (sort-middleware-by-weight [[1 exception-middleware] - [10 (partial wrap-cors cors-origins)] - [10 (partial wrap-assoc-system connection consensus - watcher subscriptions)] - [50 unwrap-credential] - [200 coercion/coerce-exceptions-middleware] - [300 coercion/coerce-response-middleware] - [400 coercion/coerce-request-middleware] - [500 wrap-request-header-opts] - [600 (wrap-closed-mode root-identities closed-mode)] - [1000 exception-middleware]]))) + ;; Exception middleware should always be first AND last. The last one ensures that + ;; middleware that comes after it will not be skipped on response if handler code + ;; throws an exception b/c this it catches them and turns them into responses. The + ;; first one ensures that exceptions thrown by other middleware are caught and + ;; turned into appropriate responses. Seems kind of clunky. Maybe there's a better + ;; way? - WSM 2023-04-28 + [exception-middleware + (partial wrap-cors cors-origins) + (partial wrap-assoc-system connection consensus + watcher subscriptions) + unwrap-credential + coercion/coerce-exceptions-middleware + coercion/coerce-response-middleware + coercion/coerce-request-middleware + wrap-request-header-opts + (wrap-closed-mode root-identities closed-mode) + exception-middleware])) (def fluree-create-routes ["/create" From 7a5ccb3a911303686a483a8826d067180e828326 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Mon, 20 Oct 2025 10:49:24 -0500 Subject: [PATCH 04/12] refactor router There was an awful lot of indirection in the construction of our router. I've consolidated all of the routes into one top-level definition so we can see all the routes at one time. --- src/fluree/server/handler.clj | 175 +++++++++++++--------------------- 1 file changed, 65 insertions(+), 110 deletions(-) diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj index a5cd253f..d45a4da2 100644 --- a/src/fluree/server/handler.clj +++ b/src/fluree/server/handler.clj @@ -406,10 +406,6 @@ (assoc :fluree/opts opts) handler)))) -(defn sort-middleware-by-weight - [weighted-middleware] - (map (fn [[_ mw]] mw) (sort-by first weighted-middleware))) - (def json-format (mf/map->Format {:name "application/json" @@ -514,127 +510,86 @@ (wrap-closed-mode root-identities closed-mode) exception-middleware])) -(def fluree-create-routes - ["/create" - {:post {:summary "Endpoint for creating new ledgers" - :parameters {:body CreateRequestBody} - :responses {201 {:body CreateResponseBody} +(def default-fluree-routes + ["/fluree" + ["/create" + {:post {:summary "Endpoint for creating new ledgers" + :parameters {:body CreateRequestBody} + :responses {201 {:body CreateResponseBody} 400 {:body ErrorResponse} 409 {:body ErrorResponse} 500 {:body ErrorResponse}} - :handler #'create/default}}]) - -(def fluree-drop-route - ["/drop" - {:post {:summary "Drop the specified ledger and delete all persisted artifacts." - :parameters {:body DropRequestBody} - :responses {200 {:body DropResponseBody} + :handler #'create/default}}] + ["/drop" + {:post {:summary "Drop the specified ledger and delete all persisted artifacts." + :parameters {:body DropRequestBody} + :responses {200 {:body DropResponseBody} 400 {:body ErrorResponse} 409 {:body ErrorResponse} 500 {:body ErrorResponse}} - :coercion (rcm/create {:transformers {:body {:default rcm/json-transformer-provider}}}) - :handler #'drop/drop-handler}}]) - -(def fluree-transact-routes - ["/transact" - {:post {:summary "Endpoint for submitting transactions" - :parameters {:body TransactRequestBody} - :responses {200 {:body TransactResponseBody} + :coercion (rcm/create {:transformers {:body {:default rcm/json-transformer-provider}}}) + :handler #'drop/drop-handler}}] + ["/transact" + {:post {:summary "Endpoint for submitting transactions" + :parameters {:body TransactRequestBody} + :responses {200 {:body TransactResponseBody} 400 {:body ErrorResponse} 409 {:body ErrorResponse} 500 {:body ErrorResponse}} - :handler #'srv-tx/update}}]) - -(def fluree-update-route - ["/update" - {:post {:summary "Endpoint for submitting transactions" - :parameters {:body TransactRequestBody} - :responses {200 {:body TransactResponseBody} + :handler #'srv-tx/update}}] + ["/update" + {:post {:summary "Endpoint for submitting transactions" + :parameters {:body TransactRequestBody} + :responses {200 {:body TransactResponseBody} 400 {:body ErrorResponse} 409 {:body ErrorResponse} 500 {:body ErrorResponse}} - :handler #'srv-tx/update}}]) - -(def fluree-insert-route - ["/insert" - {:post {:summary "Endpoint for inserting into the specified ledger." - :parameters {:body :any} - :responses {200 {:body TransactResponseBody} + :handler #'srv-tx/update}}] + ["/insert" + {:post {:summary "Endpoint for inserting into the specified ledger." + :parameters {:body :any} + :responses {200 {:body TransactResponseBody} 400 {:body ErrorResponse} 409 {:body ErrorResponse} 500 {:body ErrorResponse}} - :handler #'srv-tx/insert}}]) - -(def fluree-upsert-route - ["/upsert" - {:post {:summary "Endpoint for upserting into the specified ledger." - :parameters {:body :any} - :responses {200 {:body TransactResponseBody} - 400 {:body ErrorResponse} - 409 {:body ErrorResponse} - 500 {:body ErrorResponse}} - :handler #'srv-tx/upsert}}]) - -(def fluree-query-routes - ["/query" - {:get query-endpoint - :post query-endpoint}]) - -(def fluree-ledger-query-routes - ["/query/{*ledger-alias}" - {:post ledger-query-endpoint}]) - -(def fluree-history-routes - ["/history" - {:get history-endpoint - :post history-endpoint}]) - -(def fluree-remote-routes - ["/remote" - ["/latestCommit" - {:post {:summary "Read latest commit for a ledger" - :parameters {:body LatestCommitRequestBody} - :handler #'remote/latest-commit}}] - ["/resource" - {:post {:summary "Read resource from address" - :parameters {:body AddressRequestBody} - :handler #'remote/read-resource-address}}] - ["/hash" - {:post {:summary "Parse content hash from address" - :parameters {:body HashRequestBody} - :handler #'remote/parse-address-hash}}] - ["/addresses" - {:post {:summary "Retrieve ledger address from alias" - :parameters {:body AliasRequestBody} - :handler #'remote/published-ledger-addresses}}]]) - -(def fluree-subscription-routes - ["/subscribe" - {:get {:summary "Subscribe to ledger updates" - :parameters {:body SubscriptionRequestBody} - :handler #'subscription/default}}]) - -(def default-fluree-route-map - {:create fluree-create-routes - :drop fluree-drop-route - :insert fluree-insert-route - :upsert fluree-upsert-route - :update fluree-update-route - :transact fluree-transact-routes - :query fluree-query-routes - :ledger-query fluree-ledger-query-routes - :history fluree-history-routes - :remote fluree-remote-routes - :subscription fluree-subscription-routes}) - -(defn combine-fluree-routes - [fluree-route-map] - (->> fluree-route-map - vals - (into ["/fluree"]))) - -(def default-fluree-routes - (combine-fluree-routes default-fluree-route-map)) + :handler #'srv-tx/insert}}] + + ["/upsert" {:post {:summary "Endpoint for upserting into the specified ledger." + :parameters {:body :any} + :responses {200 {:body TransactResponseBody} + 400 {:body ErrorResponse} + 409 {:body ErrorResponse} + 500 {:body ErrorResponse}} + :handler #'srv-tx/upsert}}] + ["/query" {:get query-endpoint + :post query-endpoint}] + ["/history" {:get history-endpoint + :post history-endpoint}] + + ["/ledger/{*ledger-path}" #'ledger-specific-handler] + ["/query/{*ledger-alias}" {:post ledger-query-endpoint}] + + ["/subscribe" + {:get {:summary "Subscribe to ledger updates" + :parameters {:body SubscriptionRequestBody} + :handler #'subscription/default}}] + ["/remote" + ["/latestCommit" + {:post {:summary "Read latest commit for a ledger" + :parameters {:body LatestCommitRequestBody} + :handler #'remote/latest-commit}}] + ["/resource" + {:post {:summary "Read resource from address" + :parameters {:body AddressRequestBody} + :handler #'remote/read-resource-address}}] + ["/hash" + {:post {:summary "Parse content hash from address" + :parameters {:body HashRequestBody} + :handler #'remote/parse-address-hash}}] + ["/addresses" + {:post {:summary "Retrieve ledger address from alias" + :parameters {:body AliasRequestBody} + :handler #'remote/published-ledger-addresses}}]]]) (def fallback-handler (let [swagger-ui-handler (swagger-ui/create-swagger-ui-handler From b2bcf56213a1ead7abbf0564ed470fc477fdcdb9 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Mon, 20 Oct 2025 12:20:54 -0500 Subject: [PATCH 05/12] remove unnecessary app arity --- src/fluree/server/handler.clj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj index d45a4da2..22f35ddd 100644 --- a/src/fluree/server/handler.clj +++ b/src/fluree/server/handler.clj @@ -628,10 +628,8 @@ ([config] (app config [])) ([config custom-routes] - (app config custom-routes default-fluree-routes)) - ([config custom-routes fluree-routes] (let [app-middleware (compose-app-middleware config) - app-routes (cond-> ["" {:middleware app-middleware} fluree-routes] + app-routes (cond-> ["" {:middleware app-middleware} default-fluree-routes] (seq custom-routes) (conj custom-routes)) router (app-router app-routes)] (ring/ring-handler router fallback-handler)))) From f761f821b6d6cfae8e645a8aaeedcc1fc2511959 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Mon, 20 Oct 2025 16:48:40 -0500 Subject: [PATCH 06/12] add general support for ledger-specific operations Requests made to `/fluree/ledger/`, where op corresponds to the operation endpoint suffix (query, update, insert, upsert, transact) are rewritten and rerouted to the top-level `/fluree/` route handlers. This saves us from having to manually handle all the work the middleware does while still allowing the `/` route construction instead of the `//` form. --- src/fluree/server/handler.clj | 36 ++++++++++++++----- .../fluree/server/integration/sparql_test.clj | 2 +- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj index 22f35ddd..c6946d48 100644 --- a/src/fluree/server/handler.clj +++ b/src/fluree/server/handler.clj @@ -208,6 +208,33 @@ :coercion ^:replace history-coercer :handler #'ledger/history}) +(def fallback-handler + (let [swagger-ui-handler (swagger-ui/create-swagger-ui-handler + {:path "/" + :config {:validatorUrl nil + :operationsSorter "alpha"}}) + default-handler (ring/create-default-handler)] + (ring/routes swagger-ui-handler default-handler))) + +(defn ledger-specific-handler + "Reroutes a ledger-specific request to the appropriate endpoint, adding the + `fluree-ledger` header so that the request is directed to the appropriate ledger." + [{{ledger-path :ledger-path} :path-params + :as req}] + (let [op-delimiter-idx (str/last-index-of ledger-path "/") + + ;; deconstruct the path to obtain the ledger-alias and the endpoint + alias (subs ledger-path 0 op-delimiter-idx) + endpoint (str "/fluree" (subs ledger-path op-delimiter-idx)) + + ;; construct a new request to the correct endpoint with the fluree-ledger header + new-req (-> req + (update :headers assoc "fluree-ledger" alias) + (assoc :uri endpoint)) + ;; create a new dynamic handler and re-route the new request. + re-route (ring/ring-handler (:reitit.core/router req) fallback-handler)] + (re-route new-req))) + (defn wrap-assoc-system [conn consensus watcher subscriptions handler] (fn [req] @@ -567,7 +594,6 @@ :post history-endpoint}] ["/ledger/{*ledger-path}" #'ledger-specific-handler] - ["/query/{*ledger-alias}" {:post ledger-query-endpoint}] ["/subscribe" {:get {:summary "Subscribe to ledger updates" @@ -591,14 +617,6 @@ :parameters {:body AliasRequestBody} :handler #'remote/published-ledger-addresses}}]]]) -(def fallback-handler - (let [swagger-ui-handler (swagger-ui/create-swagger-ui-handler - {:path "/" - :config {:validatorUrl nil - :operationsSorter "alpha"}}) - default-handler (ring/create-default-handler)] - (ring/routes swagger-ui-handler default-handler))) - (def swagger-routes ["/swagger.json" {:get {:no-doc true diff --git a/test/fluree/server/integration/sparql_test.clj b/test/fluree/server/integration/sparql_test.clj index 554800cc..658a0e88 100644 --- a/test/fluree/server/integration/sparql_test.clj +++ b/test/fluree/server/integration/sparql_test.clj @@ -66,7 +66,7 @@ (deftest sparql-federated-query (let [ledger-name "test-federated-query" - service (str "http://localhost:" @test-system/api-port "/fluree/query/" ledger-name) + service (str "http://localhost:" @test-system/api-port "/fluree/ledger/" ledger-name "/query") txn-req {"ledger" ledger-name "@context" {"ex" "http://example.com/"} From 4341c0107bf51ad729dbfc4f15683da71680c4fe Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Tue, 21 Oct 2025 12:01:00 -0500 Subject: [PATCH 07/12] Revert "add federated-query-demo task" This reverts commit 964542a09b411a59cdff90c308db15478815d4bf. Now that the demo is over we no longer need this task. --- src/fluree/server.clj | 10 +++++----- src/fluree/server/command.clj | 1 - src/fluree/server/fedq_demo.clj | 24 ------------------------ src/fluree/server/system.clj | 19 ++++--------------- 4 files changed, 9 insertions(+), 45 deletions(-) delete mode 100644 src/fluree/server/fedq_demo.clj diff --git a/src/fluree/server.clj b/src/fluree/server.clj index 86bfe384..e5fd43e4 100644 --- a/src/fluree/server.clj +++ b/src/fluree/server.clj @@ -8,23 +8,23 @@ (defn start [{:keys [options] :as _cli}] - (let [{:keys [profile reindex setup-federated-query-demo]} options] + (let [{:keys [profile reindex]} options] (if-let [config-string (:string options)] (do (log/info "Starting Fluree server from command line configuration" (when profile (str "with profile: " profile))) - (system/start-config config-string :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo)) + (system/start-config config-string :profile profile :reindex reindex)) (if-let [config-path (:config options)] (do (log/info "Starting Fluree server from configuration file at path:" config-path (when profile (str "with profile: " profile))) - (system/start-file config-path :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo)) + (system/start-file config-path :profile profile :reindex reindex)) (if-let [config-resource (:resource options)] (do (log/info "Starting Fluree server from configuration resource:" config-resource (when profile (str "with profile: " profile))) - (system/start-resource config-resource :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo)) + (system/start-resource config-resource :profile profile :reindex reindex)) (do (log/info "Starting Fluree server with default configuration:" system/default-resource-name (when profile (str "with profile: " profile))) - (system/start :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo))))))) + (system/start :profile profile :reindex reindex))))))) (defn -main [& args] diff --git a/src/fluree/server/command.clj b/src/fluree/server/command.clj index 2acd6d84..feae4d31 100644 --- a/src/fluree/server/command.clj +++ b/src/fluree/server/command.clj @@ -9,7 +9,6 @@ ["-c" "--config FILE" "Load configuration at a file path"] ["-s" "--string STRING" "Load stringified configuration"] ["-r" "--resource NAME" "Load pre-defined configuration resource"] - ["-fed-q" "--setup-federated-query-demo" "Prepare an in-memory database to demonstrate federated query capabilities"] ["--reindex" "--reindex LEDGER" "Reindex the specified ledger or use --all to reindex all ledgers"] ["-h" "--help" "Print this usage summary and exit"]]) diff --git a/src/fluree/server/fedq_demo.clj b/src/fluree/server/fedq_demo.clj deleted file mode 100644 index 1025a6e8..00000000 --- a/src/fluree/server/fedq_demo.clj +++ /dev/null @@ -1,24 +0,0 @@ -(ns fluree.server.fedq-demo - (:require [clojure.java.io :as io] - [fluree.db.api :as fluree] - [fluree.db.util.json :as json] - [fluree.db.util.log :as log])) - -(def ledger-alias "test-federated-query") - -(defn load-data - [conn] - (let [gentox (slurp (io/resource "GenTox_DB_Ontology.jsonld")) - ames (slurp (io/resource "tblincubation_ames.jsonld")) - testorg (slurp (io/resource "lst_testorganism.jsonld")) - ecoli (slurp (io/resource "eColi_expansion.jsonld")) - - db0 @(fluree/create conn ledger-alias {:indexing {:reindex-min-bytes 25000000 - :reindex-max-bytes 150000000}}) - db1 @(fluree/insert db0 (json/parse gentox false)) - db2 @(fluree/insert db1 (json/parse ames false)) - db3 @(fluree/insert db2 (json/parse testorg false)) - db4 @(fluree/insert db3 (json/parse ecoli false)) - db5 @(fluree/commit! conn db4)] - (log/info "Finished transacting demo data:" (-> db5 :stats :flakes) "flakes") - :success)) diff --git a/src/fluree/server/system.clj b/src/fluree/server/system.clj index 70a77f18..0e846ecb 100644 --- a/src/fluree/server/system.clj +++ b/src/fluree/server/system.clj @@ -18,7 +18,6 @@ [fluree.server.handler :as handler] [fluree.server.http :as-alias http] [fluree.server.reindex :as reindex] - [fluree.server.fedq-demo :as fedq-demo] [fluree.server.watcher :as watcher] [integrant.core :as ig] [ring.adapter.jetty9 :as jetty])) @@ -201,7 +200,7 @@ (log/info " Closed mode: enabled")))) (defn start-config - [config & {:keys [profile reindex setup-federated-query-demo]}] + [config & {:keys [profile reindex]}] (let [json-config (if (string? config) (json/parse config false) config) @@ -211,31 +210,21 @@ parsed-config (config/parse config-with-profile) system (conn-system/initialize parsed-config)] (log-config-summary parsed-config) - (cond - setup-federated-query-demo - (let [connection-key (first (filter #(isa? % :fluree.db/connection) (keys system)))] - (if-let [conn (get system connection-key)] - (do (fedq-demo/load-data conn) - system) - (do (log/error "No connection found." system) - (System/exit 1)))) - - reindex + (if reindex (let [connection-key (first (filter #(isa? % :fluree.db/connection) (keys system)))] (if-let [conn (get system connection-key)] (let [{:keys [status]} (async/ path config/read-file - (start-config :profile profile :reindex reindex :setup-federated-query-demo setup-federated-query-demo))) + (start-config :profile profile :reindex reindex))) (defn start-resource [resource-name & {:keys [profile reindex]}] From 9f754d2cf012f49d1acc81d22c08e36cdda3dc26 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Tue, 21 Oct 2025 16:33:27 -0500 Subject: [PATCH 08/12] add tests for failed service calls Also updates the db dep to fix an issue with SERVICE SILENT. --- deps.edn | 2 +- .../fluree/server/integration/sparql_test.clj | 59 ++++++++++++++----- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/deps.edn b/deps.edn index d52a572a..91f0e8cc 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,7 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.3"} org.clojure/core.async {:mvn/version "1.6.681"} com.fluree/db {:git/url "https://github.com/fluree/db.git" - :git/sha "f86dffe929d6f1294deb3ec22f4da58e6a989c32"} + :git/sha "c99db00081aff471c734f460c0329980f4fd2ba8"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"} diff --git a/test/fluree/server/integration/sparql_test.clj b/test/fluree/server/integration/sparql_test.clj index 658a0e88..3793284f 100644 --- a/test/fluree/server/integration/sparql_test.clj +++ b/test/fluree/server/integration/sparql_test.clj @@ -76,21 +76,48 @@ "ex:num" [(dec %) % (inc %)] "ex:ref" {"@id" (str "ex:" (inc %))}}) (range 10))} - txn-res (api-post :create {:body (json/stringify txn-req) :headers test-system/json-headers}) - - service-q (str/join "\n" ["PREFIX ex: " - "SELECT ?name ?type" - (str "FROM <" ledger-name ">") - "WHERE { ?s a ex:Even ; ex:ref ?ref . " - (str "SERVICE <" service "> { ?ref ex:name ?name ; a ?type . }") - "}"]) - service-res (api-post :query {:body service-q :headers sparql-headers})] + txn-res (api-post :create {:body (json/stringify txn-req) :headers test-system/json-headers})] (is (= 201 (:status txn-res))) - (is (= 200 (:status service-res))) (testing "federated query can join across local and remote graphs" - (is (= [["name1" "ex:Odd"] - ["name3" "ex:Odd"] - ["name5" "ex:Odd"] - ["name7" "ex:Odd"] - ["name9" "ex:Odd"]] - (-> service-res :body (json/parse false))))))) + (let [service-q (str/join "\n" ["PREFIX ex: " + "SELECT ?name ?type" + (str "FROM <" ledger-name ">") + "WHERE { ?s a ex:Even ; ex:ref ?ref . " + (str "SERVICE <" service "> { ?ref ex:name ?name ; a ?type . }") + "}"]) + service-res (api-post :query {:body service-q :headers sparql-headers})] + (is (= 200 (:status service-res))) + (is (= [["name1" "ex:Odd"] + ["name3" "ex:Odd"] + ["name5" "ex:Odd"] + ["name7" "ex:Odd"] + ["name9" "ex:Odd"]] + (-> service-res :body (json/parse false)))))) + (testing "SERVICE query with unavailable service returns an error" + (let [service-q (str/join "\n" ["PREFIX ex: " + "SELECT ?name ?type" + (str "FROM <" ledger-name ">") + "WHERE { ?s a ex:Even ; ex:ref ?ref . " + "SERVICE { ?ref ex:name ?name ; a ?type . }" + "}"]) + service-res (api-post :query {:body service-q :headers sparql-headers})] + (is (= 400 (:status service-res))) + (is (= "Error executing query" + (-> service-res :body (json/parse false) (get "message")))))) + (testing "SERVICE query with unavailable service proceeds with SILENT" + (let [service-q (str/join "\n" ["PREFIX ex: " + "SELECT ?s ?type" + (str "FROM <" ledger-name ">") + "WHERE { ?s a ex:Even ; ex:ref ?ref . " + (str "SERVICE SILENT <" + (str/replace service ledger-name "foo/bar") + "> { ?ref ex:name ?name ; a ?type . }") + "}"]) + service-res (api-post :query {:body service-q :headers sparql-headers})] + (is (= 200 (:status service-res))) + (is (= [["ex:0" nil] + ["ex:2" nil] + ["ex:4" nil] + ["ex:6" nil] + ["ex:8" nil]] + (-> service-res :body (json/parse false)))))))) From f400e72c2bfdb40d0ac72a2fcb0143c69226a419 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 22 Oct 2025 12:47:32 -0500 Subject: [PATCH 09/12] add tests for request rerouting Need to dissoc the :reitit.core/match key in order to properly re-match during routing. --- src/fluree/server/handler.clj | 2 +- .../server/integration/basic_query_test.clj | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj index c6946d48..f88c25e3 100644 --- a/src/fluree/server/handler.clj +++ b/src/fluree/server/handler.clj @@ -228,7 +228,7 @@ endpoint (str "/fluree" (subs ledger-path op-delimiter-idx)) ;; construct a new request to the correct endpoint with the fluree-ledger header - new-req (-> req + new-req (-> (dissoc req :reitit.core/match) (update :headers assoc "fluree-ledger" alias) (assoc :uri endpoint)) ;; create a new dynamic handler and re-route the new request. diff --git a/test/fluree/server/integration/basic_query_test.clj b/test/fluree/server/integration/basic_query_test.clj index e072c9f1..93b9ed65 100644 --- a/test/fluree/server/integration/basic_query_test.clj +++ b/test/fluree/server/integration/basic_query_test.clj @@ -201,6 +201,90 @@ ["Leticia" 2 false]] (-> query-res :body (json/parse false))))))) +(deftest ledger-specific-request-test + (let [ledger-name "ledger-endpoint/basic-query-test"] + (testing "create" + (let [create-req1 {"ledger" ledger-name + "@context" {"ex" "http://example.com/"} + "insert" (mapv #(do {"@id" (str "ex:" %) + "@type" (if (odd? %) "ex:Odd" "ex:Even") + "ex:name" (str "name" %) + "ex:num" [(dec %) % (inc %)] + "ex:ref" {"@id" (str "ex:" (inc %))}}) + (range 10))} + create-res1 (api-post :create {:body (json/stringify create-req1) :headers json-headers}) + + ledger-name2 (str ledger-name 2) + create-req2 {"ledger" ledger-name2 + "@context" {"ex" "http://example.com/"} + "insert" (mapv #(do {"@id" (str "ex:" %) + "@type" (if (odd? %) "ex:Odd" "ex:Even") + "ex:name" (str "name" %) + "ex:num" [(dec %) % (inc %)] + "ex:ref" {"@id" (str "ex:" (inc %))}}) + (range 10))} + create-res2 (api-post (str "/ledger/" ledger-name2 "/create") {:body (json/stringify create-req2) :headers json-headers})] + (is (= 201 (:status create-res1))) + (is (= 201 (:status create-res2))))) + (testing "transact" + (let [tx-req {"ledger" ledger-name + "@context" {"ex" "http://example.com/"} + "insert" {"@id" "ex:1" "ex:op" "transact"}} + tx-res (api-post (str "/ledger/" ledger-name "/transact") {:body (json/stringify tx-req) :headers json-headers})] + (is (= 200 (:status tx-res))))) + (testing "update" + (let [tx-req {"ledger" ledger-name + "@context" {"ex" "http://example.com/"} + "insert" {"@id" "ex:1" "ex:op" "update"}} + tx-res (api-post (str "/ledger/" ledger-name "/update") {:body (json/stringify tx-req) :headers json-headers})] + (is (= 200 (:status tx-res))))) + (testing "upsert" + (let [tx-req {"@context" {"ex" "http://example.com/"} + "@id" "ex:1" "ex:op2" "upsert"} + tx-res (api-post (str "/ledger/" ledger-name "/upsert") {:body (json/stringify tx-req) :headers json-headers})] + (is (= 200 (:status tx-res))))) + (testing "insert" + (let [tx-req {"@context" {"ex" "http://example.com/"} + "@id" "ex:1" "ex:op2" "insert"} + tx-res (api-post (str "/ledger/" ledger-name "/insert") {:body (json/stringify tx-req) :headers json-headers})] + (is (= 200 (:status tx-res))))) + (testing "query" + (let [query-req {"@context" {"ex" "http://example.com/"} + "select" {"ex:1" ["ex:op" "ex:op2"]}} + query-res (api-post (str "ledger/" ledger-name "/query") + {:body (json/stringify query-req) :headers json-headers})] + (is (= 200 (:status query-res))) + (is (= [{"ex:op" ["transact" "update"] + "ex:op2" ["insert" "upsert"]}] + (-> query-res :body (json/parse false)))))) + (testing "history" + (let [query-req {"@context" {"ex" "http://example.com/"} + "from" ledger-name + "history" "ex:1" + "t" {"from" 2}} + query-res (api-post (str "ledger/" ledger-name "/history") + {:body (json/stringify query-req) :headers json-headers})] + (is (= 200 (:status query-res))) + (is (= [{"https://ns.flur.ee/ledger#t" 2, + "https://ns.flur.ee/ledger#assert" [{"@id" "ex:1", "ex:op" "transact"}], + "https://ns.flur.ee/ledger#retract" []} + {"https://ns.flur.ee/ledger#t" 3, + "https://ns.flur.ee/ledger#assert" [{"@id" "ex:1", "ex:op" "update"}], + "https://ns.flur.ee/ledger#retract" []} + {"https://ns.flur.ee/ledger#t" 4, + "https://ns.flur.ee/ledger#assert" [{"@id" "ex:1", "ex:op2" "upsert"}], + "https://ns.flur.ee/ledger#retract" []} + {"https://ns.flur.ee/ledger#t" 5, + "https://ns.flur.ee/ledger#assert" [{"@id" "ex:1", "ex:op2" "insert"}], + "https://ns.flur.ee/ledger#retract" []}] + (-> query-res :body (json/parse false)))))) + + (testing "nonmatching routes are handled uniformly" + (let [not-found1 (api-post "foo" {:body "{}" :headers json-headers}) + not-found2 (api-post (str "ledger/" ledger-name "/foo") {:body "{}" :headers json-headers})] + (is (= 404 (:status not-found1))) + (is (= 404 (:status not-found2))))))) + #_(deftest ^:integration ^:edn query-edn-test (testing "can query a basic entity w/ EDN" (let [ledger-name (create-rand-ledger "query-endpoint-basic-entity-test") From 4ff9245c268dbc6d65f4b7828ef833fea5265a64 Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 22 Oct 2025 14:49:05 -0500 Subject: [PATCH 10/12] update db dep --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 91f0e8cc..e5a37d65 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,7 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.3"} org.clojure/core.async {:mvn/version "1.6.681"} com.fluree/db {:git/url "https://github.com/fluree/db.git" - :git/sha "c99db00081aff471c734f460c0329980f4fd2ba8"} + :git/sha "f630bb658c50805100ab0c2b2012cf52da8e822f"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"} From 9539d842426eb4d8f962f253aeb68de5d7ca5e4d Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Wed, 22 Oct 2025 15:20:35 -0500 Subject: [PATCH 11/12] remove obsolete endpoint --- src/fluree/server/handler.clj | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj index f88c25e3..f389d984 100644 --- a/src/fluree/server/handler.clj +++ b/src/fluree/server/handler.clj @@ -189,16 +189,6 @@ :coercion ^:replace query-coercer :handler #'ledger/query}) -(def ledger-query-endpoint - {:summary "Endpoint for submitting queries to a specific ledger" - :parameters {:body QueryRequestBody - :path [:map [:ledger-alias :string]]} - :responses {200 {:body QueryResponse} - 400 {:body ErrorResponse} - 500 {:body ErrorResponse}} - :coercion ^:replace query-coercer - :handler #'ledger/query}) - (def history-endpoint {:summary "Endpoint for submitting history queries" :parameters {:body HistoryQuery} From af83385c053fffa3cee3e9e2de79bab69cc12adf Mon Sep 17 00:00:00 2001 From: Daniel Petranek Date: Thu, 23 Oct 2025 15:27:48 -0500 Subject: [PATCH 12/12] update db dep --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index e5a37d65..02dd0e95 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,7 @@ {:deps {org.clojure/clojure {:mvn/version "1.11.3"} org.clojure/core.async {:mvn/version "1.6.681"} com.fluree/db {:git/url "https://github.com/fluree/db.git" - :git/sha "f630bb658c50805100ab0c2b2012cf52da8e822f"} + :git/sha "d86295512569010d6bcc057bd267aa4fdc0ecfe6"} com.fluree/json-ld {:git/url "https://github.com/fluree/json-ld.git" :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"}