From 9422ace4cdd88a14d42d57c3382f9813d5f94dd4 Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Mon, 8 Dec 2025 14:28:04 -0600
Subject: [PATCH 1/9] add open telemetry tracing

---
 deps.edn                                      | 21 ++++--
 resources/logback.xml                         | 75 +++++++++++++++----
 src/fluree/server/broadcast/subscriptions.clj |  3 +-
 src/fluree/server/consensus/events.clj        | 21 ++++--
 src/fluree/server/consensus/standalone.clj    | 31 ++++----
 src/fluree/server/handler.clj                 |  6 +-
 src/fluree/server/handlers/create.clj         |  6 +-
 src/fluree/server/handlers/drop.clj           |  6 +-
 src/fluree/server/handlers/ledger.clj         |  9 ++-
 src/fluree/server/handlers/transact.clj       | 12 ++-
 10 files changed, 136 insertions(+), 54 deletions(-)

diff --git a/deps.edn b/deps.edn
index 3f870335..c3794ddd 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,7 +1,7 @@
 {:deps {org.clojure/clojure    {:mvn/version "1.11.3"}
         org.clojure/core.async {:mvn/version "1.6.681"}
         com.fluree/db          {:git/url "https://github.com/fluree/db.git"
-                                :git/sha "f5f76fb2373b712aeee96574e1bbc2cd0277fce1"}
+                                :git/sha "8925f9e677a9965aa7e1a9eb0fed0769f24ea804"}
         com.fluree/json-ld     {:git/url "https://github.com/fluree/json-ld.git"
                                 :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"}
 
@@ -23,8 +23,10 @@
         camel-snake-kebab/camel-snake-kebab {:mvn/version "0.4.3"}
 
         ;; logging
-        ch.qos.logback/logback-classic {:mvn/version "1.5.6"}
-        org.slf4j/slf4j-api            {:mvn/version "2.0.13"}
+        org.slf4j/slf4j-api                {:mvn/version "2.0.13"}
+        ch.qos.logback/logback-classic     {:mvn/version "1.5.6"}
+        net.logstash.logback/logstash-logback-encoder {:mvn/version "9.0"}
+        com.github.steffan-westcott/clj-otel-api {:mvn/version "0.2.10"}
 
         ;; http
         ;; ring-jetty9-adapter 0.30.x+ uses Jetty 12 & requires JDK 17+
@@ -50,8 +52,17 @@
    :extra-deps  {org.clojure/tools.namespace {:mvn/version "1.5.0"}
                  clj-http/clj-http           {:mvn/version "3.13.0"}
                  criterium/criterium         {:mvn/version "0.4.6"}
-                 integrant/repl              {:mvn/version "0.3.3"}}
-   :jvm-opts    ["-Djdk.attach.allowAttachSelf"]}
+                 integrant/repl              {:mvn/version "0.3.3"}
+                 com.taoensso/trove          {:mvn/version "1.1.0"}
+                 com.taoensso/telemere       {:mvn/version "1.2.0"}
+                 io.opentelemetry/opentelemetry-api {:mvn/version "1.56.0"}}
+   :jvm-opts    ["-Djdk.attach.allowAttachSelf"
+                 "-javaagent:/home/dan/scratch/learn-otel/opentelemetry-javaagent.jar"
+                 "-Dotel.exporter.otlp.endpoint=http://localhost:4318"
+                 "-Dotel.exporter.otlp.protocol=http/protobuf"
+                 "-Dotel.logs.exporter=none"
+                 "-Dotel.metrics.exporter=none"
+                 "-Dotel.service.name=fluree-server"]}
 
   :run-dev
   {:main-opts ["-m" "fluree.server" "--profile" "dev"]}
diff --git a/resources/logback.xml b/resources/logback.xml
index ca364c7f..e80d401c 100644
--- a/resources/logback.xml
+++ b/resources/logback.xml
@@ -1,19 +1,64 @@
 <configuration>
+  <property scope="context" name="LOG_LEVEL" value="${LOG_LEVEL:-INFO}"/>
+  <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>
+        %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n
+      </pattern>
+    </encoder>
+  </appender>
 
-    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>
-                %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n
-            </pattern>
-        </encoder>
-    </appender>
-
-    <logger name="fluree" level="info" additivity="false">
-        <appender-ref ref="CONSOLE"/>
-    </logger>
-
-    <root level="error">
-        <appender-ref ref="CONSOLE"/>
-    </root>
+  <appender name="JSON_CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+      <providers>
+        <!-- Timestamp in ISO8601 -->
+        <timestamp>
+          <fieldName>timestamp</fieldName>
+          <pattern>yyyy-MM-dd'T'HH:mm:ss.SSSZ</pattern>
+        </timestamp>
+
+        <!-- Log level -->
+        <logLevel>
+          <fieldName>level</fieldName>
+        </logLevel>
+
+        <!-- Thread name -->
+        <threadName>
+          <fieldName>thread</fieldName>
+        </threadName>
+
+        <!-- Logger name -->
+        <loggerName>
+          <fieldName>logger</fieldName>
+        </loggerName>
+
+        <!-- Message -->
+        <message>
+          <fieldName>message</fieldName>
+        </message>
+
+        <!-- Mapped diagnostic context (MDC) -->
+        <mdc>
+          <fieldName>mdc</fieldName>
+        </mdc>
+
+        <!-- Throwable / stacktrace -->
+        <throwable>
+          <fieldName>exception</fieldName>
+          <throwableConverter class="net.logstash.logback.stacktrace.ShortenedThrowableConverter">
+            <maxDepthPerThrowable>30</maxDepthPerThrowable>
+            <maxLength>2048</maxLength>
+            <shortenedClassNameLength>20</shortenedClassNameLength>
+          </throwableConverter>
+        </throwable>
+      </providers>
+    </encoder>
+  </appender>
+
+
+  <root level="${LOG_LEVEL}">
+    <!-- Change ref to JSON_CONSOLE for structured logs -->
+    <appender-ref ref="CONSOLE"/>
+  </root>
 
 </configuration>
diff --git a/src/fluree/server/broadcast/subscriptions.clj b/src/fluree/server/broadcast/subscriptions.clj
index 7408d7b2..a7ef4601 100644
--- a/src/fluree/server/broadcast/subscriptions.clj
+++ b/src/fluree/server/broadcast/subscriptions.clj
@@ -4,7 +4,8 @@
             [fluree.db.util.log :as log]
             [fluree.server.broadcast :refer [Broadcaster]]
             [fluree.server.consensus.events :as events]
-            [ring.adapter.jetty9.websocket :as ws])
+            [ring.adapter.jetty9.websocket :as ws]
+            [steffan-westcott.clj-otel.context :as otel-context])
   (:import (java.io Closeable IOException)
            (java.nio.channels ClosedChannelException)))
 
diff --git a/src/fluree/server/consensus/events.clj b/src/fluree/server/consensus/events.clj
index 8532b820..19dfa547 100644
--- a/src/fluree/server/consensus/events.clj
+++ b/src/fluree/server/consensus/events.clj
@@ -6,7 +6,8 @@
             [fluree.db.ledger :as ledger]
             [fluree.db.track :as-alias track]
             [fluree.db.util.async :refer [<? go-try]]
-            [fluree.db.util.log :as log]))
+            [fluree.db.util.log :as log]
+            [steffan-westcott.clj-otel.context :as otel-context]))
 
 (defn event-type
   [event]
@@ -28,6 +29,10 @@
   [x]
   (string? x))
 
+(defn with-otel-ctx
+  [evt]
+  (assoc evt :otel/context (otel-context/dyn)))
+
 (defn with-txn
   [evt txn]
   (cond (nil? txn)
@@ -65,14 +70,16 @@
              :ledger-id ledger-id
              :opts      opts
              :instant   (System/currentTimeMillis)}]
-    (with-txn evt txn)))
+    (-> evt
+        (with-txn txn)
+        (with-otel-ctx))))
 
 (defn drop-ledger
   "Create a new event message to drop an existing ledger."
   [ledger-id]
-  {:type      :ledger-drop
-   :ledger-id ledger-id
-   :instant   (System/currentTimeMillis)})
+  (with-otel-ctx {:type      :ledger-drop
+                  :ledger-id ledger-id
+                  :instant   (System/currentTimeMillis)}))
 
 (defn commit-transaction
   "Create a new event message to commit a new transaction. The `txn` argument may
@@ -85,8 +92,8 @@
              :opts      opts
              :instant   (System/currentTimeMillis)}]
     (if (= :turtle (:format opts))
-      (with-turtle-txn evt txn)
-      (with-txn evt txn))))
+      (with-otel-ctx (with-turtle-txn evt txn))
+      (with-otel-ctx (with-txn evt txn)))))
 
 (defn get-txn
   "Gets the transaction value, either a transaction document or the storage
diff --git a/src/fluree/server/consensus/standalone.clj b/src/fluree/server/consensus/standalone.clj
index 1abbb2e8..356675ba 100644
--- a/src/fluree/server/consensus/standalone.clj
+++ b/src/fluree/server/consensus/standalone.clj
@@ -7,32 +7,37 @@
             [fluree.server.consensus.events :as events]
             [fluree.server.consensus.response :as response]
             [fluree.server.consensus.shared.create :as shared-create]
-            [fluree.server.handlers.shared :refer [deref!]]))
+            [fluree.server.handlers.shared :refer [deref!]]
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (set! *warn-on-reflection* true)
 
 (defn create-ledger!
-  [conn watcher broadcaster {:keys [ledger-id tx-id txn opts] :as _params}]
+  [conn watcher broadcaster {:keys [ledger-id tx-id txn opts otel/context] :as _params}]
   (go-try
-    (let [commit-result (if txn
-                          (deref! (fluree/create-with-txn conn txn opts))
-                          (let [db (deref! (fluree/create conn ledger-id opts))]
-                            (shared-create/genesis-result db)))]
+    (let [commit-result
+          (span/with-span! {:parent context :name ::process-event}
+            (if txn
+              (deref! (fluree/create-with-txn conn txn opts))
+              (let [db (deref! (fluree/create conn ledger-id opts))]
+                (shared-create/genesis-result db))))]
       (response/announce-new-ledger watcher broadcaster ledger-id tx-id commit-result))))
 
 (defn drop-ledger!
-  [conn watcher broadcaster {:keys [ledger-id] :as _params}]
+  [conn watcher broadcaster {:keys [ledger-id otel/context] :as _params}]
   (go-try
-    (let [drop-result (deref! (fluree/drop conn ledger-id))]
+    (let [drop-result (span/with-span! {:name ::process-event :parent context}
+                        (deref! (fluree/drop conn ledger-id)))]
       (response/announce-dropped-ledger watcher broadcaster ledger-id drop-result))))
 
 (defn transact!
-  [conn watcher broadcaster {:keys [ledger-id tx-id txn opts]}]
+  [conn watcher broadcaster {:keys [ledger-id tx-id txn opts otel/context]}]
   (go-try
-    (let [commit-result (case (:op opts)
-                          :update (deref! (fluree/update! conn ledger-id txn opts))
-                          :upsert (deref! (fluree/upsert! conn ledger-id txn opts))
-                          :insert (deref! (fluree/insert! conn ledger-id txn opts)))]
+    (let [commit-result (span/with-span! {:name ::process-event :parent context}
+                          (case (:op opts)
+                            :update (deref! (fluree/update! conn ledger-id txn opts))
+                            :upsert (deref! (fluree/upsert! conn ledger-id txn opts))
+                            :insert (deref! (fluree/insert! conn ledger-id txn opts))))]
       (response/announce-commit watcher broadcaster ledger-id tx-id commit-result))))
 
 (defn process-event
diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj
index b8902b04..06e30ab0 100644
--- a/src/fluree/server/handler.clj
+++ b/src/fluree/server/handler.clj
@@ -25,7 +25,8 @@
             [reitit.swagger :as swagger]
             [reitit.swagger-ui :as swagger-ui]
             [ring.adapter.jetty9 :as http]
-            [ring.middleware.cors :as rmc])
+            [ring.middleware.cors :as rmc]
+            [steffan-westcott.clj-otel.api.trace.http :as trace-http])
   (:import (java.io InputStream)))
 
 (set! *warn-on-reflection* true)
@@ -656,4 +657,5 @@
          app-routes     (cond-> ["" {:middleware app-middleware} default-fluree-routes]
                           (seq custom-routes) (conj custom-routes))
          router         (app-router app-routes)]
-     (ring/ring-handler router fallback-handler))))
+     (trace-http/wrap-server-span
+       (ring/ring-handler router fallback-handler)))))
diff --git a/src/fluree/server/handlers/create.clj b/src/fluree/server/handlers/create.clj
index ea91f398..2743e969 100644
--- a/src/fluree/server/handlers/create.clj
+++ b/src/fluree/server/handlers/create.clj
@@ -3,7 +3,8 @@
             [fluree.server.consensus :as consensus]
             [fluree.server.handlers.shared :as shared :refer [deref! defhandler]]
             [fluree.server.handlers.transact :as srv-tx]
-            [fluree.server.watcher :as watcher]))
+            [fluree.server.watcher :as watcher]
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (set! *warn-on-reflection* true)
 
@@ -50,7 +51,8 @@
                         (throw (ex-info "Ledger ID must be provided"
                                         {:status 400 :error :db/invalid-ledger-id})))
         opts*         (prepare-create-options opts)
-        commit-event  (deref! (create-ledger! consensus watcher ledger-id txn opts*))
+        commit-event  (span/with-span! {:name ::create-handler}
+                        (deref! (create-ledger! consensus watcher ledger-id txn opts*)))
         response-body (srv-tx/commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status 201, :body response-body}
       commit-event)))
diff --git a/src/fluree/server/handlers/drop.clj b/src/fluree/server/handlers/drop.clj
index e0642ee4..b3747181 100644
--- a/src/fluree/server/handlers/drop.clj
+++ b/src/fluree/server/handlers/drop.clj
@@ -2,7 +2,8 @@
   (:require [fluree.db.util.log :as log]
             [fluree.server.consensus :as consensus]
             [fluree.server.handlers.shared :as shared :refer [deref! defhandler]]
-            [fluree.server.watcher :as watcher]))
+            [fluree.server.watcher :as watcher]
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (set! *warn-on-reflection* true)
 
@@ -24,5 +25,6 @@
     {:keys [body]} :parameters}]
   (log/debug "drop body:" body)
   (let [ledger-id (:ledger body)
-        resp-p    (drop-ledger consensus watcher ledger-id)]
+        resp-p    (span/with-span! {:name ::drop-handler}
+                    (drop-ledger consensus watcher ledger-id))]
     {:status 200, :body (deref! resp-p)}))
diff --git a/src/fluree/server/handlers/ledger.clj b/src/fluree/server/handlers/ledger.clj
index 5566a0a5..f862d196 100644
--- a/src/fluree/server/handlers/ledger.clj
+++ b/src/fluree/server/handlers/ledger.clj
@@ -2,7 +2,8 @@
   (:require [fluree.db.api :as fluree]
             [fluree.db.util.log :as log]
             [fluree.server.handler :as-alias handler]
-            [fluree.server.handlers.shared :refer [defhandler deref!] :as shared]))
+            [fluree.server.handlers.shared :refer [defhandler deref!] :as shared]
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (defhandler query
   [{:keys [fluree/conn fluree/opts] {:keys [body path]} :parameters :as _req}]
@@ -10,7 +11,8 @@
         ;; supply ledger-alias from path params if not overridden by a header
         opts* (update opts :ledger #(or % (:ledger-alias path)))
         {:keys [status result] :as query-response}
-        (deref! (fluree/query-connection conn query opts*))]
+        (span/with-span! {:name ::query-handler}
+          (deref! (fluree/query-connection conn query opts*)))]
     (log/debug "query handler received query:" query opts*)
     (shared/with-tracking-headers {:status status, :body result}
       query-response)))
@@ -18,7 +20,8 @@
 (defhandler history
   [{:keys [fluree/conn fluree/opts] {{ledger :from :as query} :body} :parameters :as _req}]
   (let [query*  (dissoc query :from)
-        result  (deref! (fluree/history conn ledger query* opts))]
+        result  (span/with-span! {:name ::history-handler}
+                  (deref! (fluree/history conn ledger query* opts)))]
     (log/debug "history handler received query:" query opts "result:" result)
     ;; fluree/history may return either raw result or wrapped in {:status :result}
     (if (and (map? result) (:status result) (:result result))
diff --git a/src/fluree/server/handlers/transact.clj b/src/fluree/server/handlers/transact.clj
index cd0ca169..374f0a4c 100644
--- a/src/fluree/server/handlers/transact.clj
+++ b/src/fluree/server/handlers/transact.clj
@@ -7,7 +7,8 @@
             [fluree.json-ld :as json-ld]
             [fluree.server.consensus :as consensus]
             [fluree.server.handlers.shared :as shared :refer [defhandler deref!]]
-            [fluree.server.watcher :as watcher]))
+            [fluree.server.watcher :as watcher]
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (set! *warn-on-reflection* true)
 
@@ -63,7 +64,8 @@
                     raw-txn (assoc :raw-txn raw-txn)
                     did     (assoc :did did))
         {:keys [status] :as commit-event}
-        (deref! (transact! consensus watcher ledger-id txn-with-ledger opts*))
+        (span/with-span! {:name ::update-handler}
+          (deref! (transact! consensus watcher ledger-id txn-with-ledger opts*)))
 
         body (commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status status, :body body}
@@ -77,7 +79,8 @@
                     raw-txn (assoc :raw-txn raw-txn)
                     did     (assoc :identity did))
         {:keys [status] :as commit-event}
-        (deref! (transact! consensus watcher ledger-id insert-txn opts*))
+        (span/with-span! {:name ::insert-handler}
+          (deref! (transact! consensus watcher ledger-id insert-txn opts*)))
 
         body (commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status status, :body body}
@@ -91,7 +94,8 @@
                     raw-txn (assoc :raw-txn raw-txn)
                     did     (assoc :identity did))
         {:keys [status] :as commit-event}
-        (deref! (transact! consensus watcher ledger-id upsert-txn opts*))
+        (span/with-span! {:name ::upsert-handler}
+          (deref! (transact! consensus watcher ledger-id upsert-txn opts*)))
 
         body (commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status status, :body body}

From 9615cb35ae5631933a317963499facc5c20ce5f0 Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Wed, 14 Jan 2026 09:29:30 -0600
Subject: [PATCH 2/9] refactor to use trace api

---
 deps.edn                                      |  2 +-
 src/fluree/server/broadcast/subscriptions.clj |  3 +-
 src/fluree/server/consensus/events.clj        |  4 +-
 src/fluree/server/consensus/standalone.clj    | 59 ++++++++++---------
 src/fluree/server/handlers/create.clj         |  6 +-
 src/fluree/server/handlers/drop.clj           |  6 +-
 src/fluree/server/handlers/ledger.clj         |  8 +--
 src/fluree/server/handlers/transact.clj       | 10 ++--
 8 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/deps.edn b/deps.edn
index c3794ddd..16bf0917 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,7 +1,7 @@
 {:deps {org.clojure/clojure    {:mvn/version "1.11.3"}
         org.clojure/core.async {:mvn/version "1.6.681"}
         com.fluree/db          {:git/url "https://github.com/fluree/db.git"
-                                :git/sha "8925f9e677a9965aa7e1a9eb0fed0769f24ea804"}
+                                :git/sha "0085590d2318e063d468081fb257863ed3f8f2c1"}
         com.fluree/json-ld     {:git/url "https://github.com/fluree/json-ld.git"
                                 :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"}
 
diff --git a/src/fluree/server/broadcast/subscriptions.clj b/src/fluree/server/broadcast/subscriptions.clj
index a7ef4601..7408d7b2 100644
--- a/src/fluree/server/broadcast/subscriptions.clj
+++ b/src/fluree/server/broadcast/subscriptions.clj
@@ -4,8 +4,7 @@
             [fluree.db.util.log :as log]
             [fluree.server.broadcast :refer [Broadcaster]]
             [fluree.server.consensus.events :as events]
-            [ring.adapter.jetty9.websocket :as ws]
-            [steffan-westcott.clj-otel.context :as otel-context])
+            [ring.adapter.jetty9.websocket :as ws])
   (:import (java.io Closeable IOException)
            (java.nio.channels ClosedChannelException)))
 
diff --git a/src/fluree/server/consensus/events.clj b/src/fluree/server/consensus/events.clj
index 19dfa547..6c5888bb 100644
--- a/src/fluree/server/consensus/events.clj
+++ b/src/fluree/server/consensus/events.clj
@@ -7,7 +7,7 @@
             [fluree.db.track :as-alias track]
             [fluree.db.util.async :refer [<? go-try]]
             [fluree.db.util.log :as log]
-            [steffan-westcott.clj-otel.context :as otel-context]))
+            [fluree.db.util.trace :as trace]))
 
 (defn event-type
   [event]
@@ -31,7 +31,7 @@
 
 (defn with-otel-ctx
   [evt]
-  (assoc evt :otel/context (otel-context/dyn)))
+  (assoc evt :otel/context (trace/get-context)))
 
 (defn with-txn
   [evt txn]
diff --git a/src/fluree/server/consensus/standalone.clj b/src/fluree/server/consensus/standalone.clj
index 356675ba..b5196770 100644
--- a/src/fluree/server/consensus/standalone.clj
+++ b/src/fluree/server/consensus/standalone.clj
@@ -3,12 +3,12 @@
             [fluree.db.api :as fluree]
             [fluree.db.util.async :refer [<? go-try]]
             [fluree.db.util.log :as log]
+            [fluree.db.util.trace :as trace]
             [fluree.server.consensus :as consensus]
             [fluree.server.consensus.events :as events]
             [fluree.server.consensus.response :as response]
             [fluree.server.consensus.shared.create :as shared-create]
-            [fluree.server.handlers.shared :refer [deref!]]
-            [steffan-westcott.clj-otel.api.trace.span :as span]))
+            [fluree.server.handlers.shared :refer [deref!]]))
 
 (set! *warn-on-reflection* true)
 
@@ -16,7 +16,7 @@
   [conn watcher broadcaster {:keys [ledger-id tx-id txn opts otel/context] :as _params}]
   (go-try
     (let [commit-result
-          (span/with-span! {:parent context :name ::process-event}
+          (trace/form ::create-ledger! {:parent context}
             (if txn
               (deref! (fluree/create-with-txn conn txn opts))
               (let [db (deref! (fluree/create conn ledger-id opts))]
@@ -26,14 +26,14 @@
 (defn drop-ledger!
   [conn watcher broadcaster {:keys [ledger-id otel/context] :as _params}]
   (go-try
-    (let [drop-result (span/with-span! {:name ::process-event :parent context}
+    (let [drop-result (trace/form ::drop-ledger! {:parent context}
                         (deref! (fluree/drop conn ledger-id)))]
       (response/announce-dropped-ledger watcher broadcaster ledger-id drop-result))))
 
 (defn transact!
   [conn watcher broadcaster {:keys [ledger-id tx-id txn opts otel/context]}]
   (go-try
-    (let [commit-result (span/with-span! {:name ::process-event :parent context}
+    (let [commit-result (trace/form ::transact! {:parent context}
                           (case (:op opts)
                             :update (deref! (fluree/update! conn ledger-id txn opts))
                             :upsert (deref! (fluree/upsert! conn ledger-id txn opts))
@@ -42,30 +42,31 @@
 
 (defn process-event
   [conn watcher broadcaster event]
-  (go
-    (try
-      (let [event*     (if (events/resolve-txn? event)
-                         (<? (events/resolve-txns conn event))
-                         event)
-            event-type (events/event-type event*)]
-        (cond
-          (= :ledger-create event-type)
-          (<? (create-ledger! conn watcher broadcaster event*))
-
-          (= :ledger-drop event-type)
-          (<? (drop-ledger! conn watcher broadcaster event*))
-
-          (= :tx-queue event-type)
-          (<? (transact! conn watcher broadcaster event*))
-
-          :else
-          (throw (ex-info (str "Unexpected event message: event type '" event-type "' not"
-                               " one of (':ledger-create', ':ledger-drop', ':tx-queue')")
-                          {:status 500, :error :consensus/unexpected-event}))))
-      (catch Exception e
-        (let [{:keys [ledger-id tx-id]} event]
-          (log/warn e "Error processing consensus event")
-          (response/announce-error watcher broadcaster ledger-id tx-id e))))))
+  (trace/with-parent-context ::process-event (:otel/context event)
+    (go
+      (try
+        (let [event*     (if (events/resolve-txn? event)
+                           (<? (events/resolve-txns conn event))
+                           event)
+              event-type (events/event-type event*)]
+          (cond
+            (= :ledger-create event-type)
+            (<? (create-ledger! conn watcher broadcaster event*))
+
+            (= :ledger-drop event-type)
+            (<? (drop-ledger! conn watcher broadcaster event*))
+
+            (= :tx-queue event-type)
+            (<? (transact! conn watcher broadcaster event*))
+
+            :else
+            (throw (ex-info (str "Unexpected event message: event type '" event-type "' not"
+                                 " one of (':ledger-create', ':ledger-drop', ':tx-queue')")
+                            {:status 500, :error :consensus/unexpected-event}))))
+        (catch Exception e
+          (let [{:keys [ledger-id tx-id]} event]
+            (log/warn e "Error processing consensus event")
+            (response/announce-error watcher broadcaster ledger-id tx-id e)))))))
 
 (defn error?
   [result]
diff --git a/src/fluree/server/handlers/create.clj b/src/fluree/server/handlers/create.clj
index 2743e969..7b329407 100644
--- a/src/fluree/server/handlers/create.clj
+++ b/src/fluree/server/handlers/create.clj
@@ -1,10 +1,10 @@
 (ns fluree.server.handlers.create
   (:require [fluree.db.api :as fluree]
+            [fluree.db.util.trace :as trace]
             [fluree.server.consensus :as consensus]
             [fluree.server.handlers.shared :as shared :refer [deref! defhandler]]
             [fluree.server.handlers.transact :as srv-tx]
-            [fluree.server.watcher :as watcher]
-            [steffan-westcott.clj-otel.api.trace.span :as span]))
+            [fluree.server.watcher :as watcher]))
 
 (set! *warn-on-reflection* true)
 
@@ -51,7 +51,7 @@
                         (throw (ex-info "Ledger ID must be provided"
                                         {:status 400 :error :db/invalid-ledger-id})))
         opts*         (prepare-create-options opts)
-        commit-event  (span/with-span! {:name ::create-handler}
+        commit-event  (trace/form ::create-handler {}
                         (deref! (create-ledger! consensus watcher ledger-id txn opts*)))
         response-body (srv-tx/commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status 201, :body response-body}
diff --git a/src/fluree/server/handlers/drop.clj b/src/fluree/server/handlers/drop.clj
index b3747181..5806c06c 100644
--- a/src/fluree/server/handlers/drop.clj
+++ b/src/fluree/server/handlers/drop.clj
@@ -1,9 +1,9 @@
 (ns fluree.server.handlers.drop
   (:require [fluree.db.util.log :as log]
+            [fluree.db.util.trace :as trace]
             [fluree.server.consensus :as consensus]
             [fluree.server.handlers.shared :as shared :refer [deref! defhandler]]
-            [fluree.server.watcher :as watcher]
-            [steffan-westcott.clj-otel.api.trace.span :as span]))
+            [fluree.server.watcher :as watcher]))
 
 (set! *warn-on-reflection* true)
 
@@ -25,6 +25,6 @@
     {:keys [body]} :parameters}]
   (log/debug "drop body:" body)
   (let [ledger-id (:ledger body)
-        resp-p    (span/with-span! {:name ::drop-handler}
+        resp-p    (trace/form ::drop-handler {}
                     (drop-ledger consensus watcher ledger-id))]
     {:status 200, :body (deref! resp-p)}))
diff --git a/src/fluree/server/handlers/ledger.clj b/src/fluree/server/handlers/ledger.clj
index f862d196..fc9cb014 100644
--- a/src/fluree/server/handlers/ledger.clj
+++ b/src/fluree/server/handlers/ledger.clj
@@ -1,9 +1,9 @@
 (ns fluree.server.handlers.ledger
   (:require [fluree.db.api :as fluree]
             [fluree.db.util.log :as log]
+            [fluree.db.util.trace :as trace]
             [fluree.server.handler :as-alias handler]
-            [fluree.server.handlers.shared :refer [defhandler deref!] :as shared]
-            [steffan-westcott.clj-otel.api.trace.span :as span]))
+            [fluree.server.handlers.shared :refer [defhandler deref!] :as shared]))
 
 (defhandler query
   [{:keys [fluree/conn fluree/opts] {:keys [body path]} :parameters :as _req}]
@@ -11,7 +11,7 @@
         ;; supply ledger-alias from path params if not overridden by a header
         opts* (update opts :ledger #(or % (:ledger-alias path)))
         {:keys [status result] :as query-response}
-        (span/with-span! {:name ::query-handler}
+        (trace/form ::query-handler {}
           (deref! (fluree/query-connection conn query opts*)))]
     (log/debug "query handler received query:" query opts*)
     (shared/with-tracking-headers {:status status, :body result}
@@ -20,7 +20,7 @@
 (defhandler history
   [{:keys [fluree/conn fluree/opts] {{ledger :from :as query} :body} :parameters :as _req}]
   (let [query*  (dissoc query :from)
-        result  (span/with-span! {:name ::history-handler}
+        result  (trace/form ::history-handler {}
                   (deref! (fluree/history conn ledger query* opts)))]
     (log/debug "history handler received query:" query opts "result:" result)
     ;; fluree/history may return either raw result or wrapped in {:status :result}
diff --git a/src/fluree/server/handlers/transact.clj b/src/fluree/server/handlers/transact.clj
index 374f0a4c..ffde9ad1 100644
--- a/src/fluree/server/handlers/transact.clj
+++ b/src/fluree/server/handlers/transact.clj
@@ -4,11 +4,11 @@
             [fluree.crypto :as crypto]
             [fluree.db.api :as fluree]
             [fluree.db.query.fql.parse :as parse]
+            [fluree.db.util.trace :as trace]
             [fluree.json-ld :as json-ld]
             [fluree.server.consensus :as consensus]
             [fluree.server.handlers.shared :as shared :refer [defhandler deref!]]
-            [fluree.server.watcher :as watcher]
-            [steffan-westcott.clj-otel.api.trace.span :as span]))
+            [fluree.server.watcher :as watcher]))
 
 (set! *warn-on-reflection* true)
 
@@ -64,7 +64,7 @@
                     raw-txn (assoc :raw-txn raw-txn)
                     did     (assoc :did did))
         {:keys [status] :as commit-event}
-        (span/with-span! {:name ::update-handler}
+        (trace/form ::update-handler {}
           (deref! (transact! consensus watcher ledger-id txn-with-ledger opts*)))
 
         body (commit-event->response-body commit-event)]
@@ -79,7 +79,7 @@
                     raw-txn (assoc :raw-txn raw-txn)
                     did     (assoc :identity did))
         {:keys [status] :as commit-event}
-        (span/with-span! {:name ::insert-handler}
+        (trace/form ::insert-handler {}
           (deref! (transact! consensus watcher ledger-id insert-txn opts*)))
 
         body (commit-event->response-body commit-event)]
@@ -94,7 +94,7 @@
                     raw-txn (assoc :raw-txn raw-txn)
                     did     (assoc :identity did))
         {:keys [status] :as commit-event}
-        (span/with-span! {:name ::upsert-handler}
+        (trace/form ::upsert-handler {}
           (deref! (transact! consensus watcher ledger-id upsert-txn opts*)))
 
         body (commit-event->response-body commit-event)]

From dfd1e7cfdaea175ceeb0992335fd7b7359ff0600 Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Wed, 14 Jan 2026 09:29:43 -0600
Subject: [PATCH 3/9] ensure x-amzn-trace-id is captured in tracing context

---
 src/fluree/server/handler.clj | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj
index 06e30ab0..2a840798 100644
--- a/src/fluree/server/handler.clj
+++ b/src/fluree/server/handler.clj
@@ -658,4 +658,6 @@
                           (seq custom-routes) (conj custom-routes))
          router         (app-router app-routes)]
      (trace-http/wrap-server-span
-       (ring/ring-handler router fallback-handler)))))
+      (ring/ring-handler router fallback-handler)
+      {:create-span? true
+       :captured-request-headers ["x-amzn-trace-id"]}))))

From 383e452a63908d739237eb984ee2b1dd577dd0c5 Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Wed, 14 Jan 2026 09:42:50 -0600
Subject: [PATCH 4/9] fix formatting and update db dep

---
 deps.edn                                   |  2 +-
 src/fluree/server/consensus/standalone.clj | 18 +++++++++---------
 src/fluree/server/handlers/create.clj      |  2 +-
 src/fluree/server/handlers/drop.clj        |  2 +-
 src/fluree/server/handlers/ledger.clj      |  4 ++--
 src/fluree/server/handlers/transact.clj    |  6 +++---
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/deps.edn b/deps.edn
index 16bf0917..4ab75487 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,7 +1,7 @@
 {:deps {org.clojure/clojure    {:mvn/version "1.11.3"}
         org.clojure/core.async {:mvn/version "1.6.681"}
         com.fluree/db          {:git/url "https://github.com/fluree/db.git"
-                                :git/sha "0085590d2318e063d468081fb257863ed3f8f2c1"}
+                                :git/sha "055676a22fbeaca7e02ff532c595bea54a931680"}
         com.fluree/json-ld     {:git/url "https://github.com/fluree/json-ld.git"
                                 :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"}
 
diff --git a/src/fluree/server/consensus/standalone.clj b/src/fluree/server/consensus/standalone.clj
index b5196770..0efbccc8 100644
--- a/src/fluree/server/consensus/standalone.clj
+++ b/src/fluree/server/consensus/standalone.clj
@@ -17,27 +17,27 @@
   (go-try
     (let [commit-result
           (trace/form ::create-ledger! {:parent context}
-            (if txn
-              (deref! (fluree/create-with-txn conn txn opts))
-              (let [db (deref! (fluree/create conn ledger-id opts))]
-                (shared-create/genesis-result db))))]
+                      (if txn
+                        (deref! (fluree/create-with-txn conn txn opts))
+                        (let [db (deref! (fluree/create conn ledger-id opts))]
+                          (shared-create/genesis-result db))))]
       (response/announce-new-ledger watcher broadcaster ledger-id tx-id commit-result))))
 
 (defn drop-ledger!
   [conn watcher broadcaster {:keys [ledger-id otel/context] :as _params}]
   (go-try
     (let [drop-result (trace/form ::drop-ledger! {:parent context}
-                        (deref! (fluree/drop conn ledger-id)))]
+                                  (deref! (fluree/drop conn ledger-id)))]
       (response/announce-dropped-ledger watcher broadcaster ledger-id drop-result))))
 
 (defn transact!
   [conn watcher broadcaster {:keys [ledger-id tx-id txn opts otel/context]}]
   (go-try
     (let [commit-result (trace/form ::transact! {:parent context}
-                          (case (:op opts)
-                            :update (deref! (fluree/update! conn ledger-id txn opts))
-                            :upsert (deref! (fluree/upsert! conn ledger-id txn opts))
-                            :insert (deref! (fluree/insert! conn ledger-id txn opts))))]
+                                    (case (:op opts)
+                                      :update (deref! (fluree/update! conn ledger-id txn opts))
+                                      :upsert (deref! (fluree/upsert! conn ledger-id txn opts))
+                                      :insert (deref! (fluree/insert! conn ledger-id txn opts))))]
       (response/announce-commit watcher broadcaster ledger-id tx-id commit-result))))
 
 (defn process-event
diff --git a/src/fluree/server/handlers/create.clj b/src/fluree/server/handlers/create.clj
index 7b329407..11425ba3 100644
--- a/src/fluree/server/handlers/create.clj
+++ b/src/fluree/server/handlers/create.clj
@@ -52,7 +52,7 @@
                                         {:status 400 :error :db/invalid-ledger-id})))
         opts*         (prepare-create-options opts)
         commit-event  (trace/form ::create-handler {}
-                        (deref! (create-ledger! consensus watcher ledger-id txn opts*)))
+                                  (deref! (create-ledger! consensus watcher ledger-id txn opts*)))
         response-body (srv-tx/commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status 201, :body response-body}
       commit-event)))
diff --git a/src/fluree/server/handlers/drop.clj b/src/fluree/server/handlers/drop.clj
index 5806c06c..87468324 100644
--- a/src/fluree/server/handlers/drop.clj
+++ b/src/fluree/server/handlers/drop.clj
@@ -26,5 +26,5 @@
   (log/debug "drop body:" body)
   (let [ledger-id (:ledger body)
         resp-p    (trace/form ::drop-handler {}
-                    (drop-ledger consensus watcher ledger-id))]
+                              (drop-ledger consensus watcher ledger-id))]
     {:status 200, :body (deref! resp-p)}))
diff --git a/src/fluree/server/handlers/ledger.clj b/src/fluree/server/handlers/ledger.clj
index fc9cb014..a72a0d4e 100644
--- a/src/fluree/server/handlers/ledger.clj
+++ b/src/fluree/server/handlers/ledger.clj
@@ -12,7 +12,7 @@
         opts* (update opts :ledger #(or % (:ledger-alias path)))
         {:keys [status result] :as query-response}
         (trace/form ::query-handler {}
-          (deref! (fluree/query-connection conn query opts*)))]
+                    (deref! (fluree/query-connection conn query opts*)))]
     (log/debug "query handler received query:" query opts*)
     (shared/with-tracking-headers {:status status, :body result}
       query-response)))
@@ -21,7 +21,7 @@
   [{:keys [fluree/conn fluree/opts] {{ledger :from :as query} :body} :parameters :as _req}]
   (let [query*  (dissoc query :from)
         result  (trace/form ::history-handler {}
-                  (deref! (fluree/history conn ledger query* opts)))]
+                            (deref! (fluree/history conn ledger query* opts)))]
     (log/debug "history handler received query:" query opts "result:" result)
     ;; fluree/history may return either raw result or wrapped in {:status :result}
     (if (and (map? result) (:status result) (:result result))
diff --git a/src/fluree/server/handlers/transact.clj b/src/fluree/server/handlers/transact.clj
index ffde9ad1..c8547a42 100644
--- a/src/fluree/server/handlers/transact.clj
+++ b/src/fluree/server/handlers/transact.clj
@@ -65,7 +65,7 @@
                     did     (assoc :did did))
         {:keys [status] :as commit-event}
         (trace/form ::update-handler {}
-          (deref! (transact! consensus watcher ledger-id txn-with-ledger opts*)))
+                    (deref! (transact! consensus watcher ledger-id txn-with-ledger opts*)))
 
         body (commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status status, :body body}
@@ -80,7 +80,7 @@
                     did     (assoc :identity did))
         {:keys [status] :as commit-event}
         (trace/form ::insert-handler {}
-          (deref! (transact! consensus watcher ledger-id insert-txn opts*)))
+                    (deref! (transact! consensus watcher ledger-id insert-txn opts*)))
 
         body (commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status status, :body body}
@@ -95,7 +95,7 @@
                     did     (assoc :identity did))
         {:keys [status] :as commit-event}
         (trace/form ::upsert-handler {}
-          (deref! (transact! consensus watcher ledger-id upsert-txn opts*)))
+                    (deref! (transact! consensus watcher ledger-id upsert-txn opts*)))
 
         body (commit-event->response-body commit-event)]
     (shared/with-tracking-headers {:status status, :body body}

From 61ed06e9cef5a2a4af2661241b65913116c3622b Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Wed, 14 Jan 2026 10:01:24 -0600
Subject: [PATCH 5/9] remove unused dev deps

---
 deps.edn | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/deps.edn b/deps.edn
index 4ab75487..b7a206f7 100644
--- a/deps.edn
+++ b/deps.edn
@@ -52,10 +52,7 @@
    :extra-deps  {org.clojure/tools.namespace {:mvn/version "1.5.0"}
                  clj-http/clj-http           {:mvn/version "3.13.0"}
                  criterium/criterium         {:mvn/version "0.4.6"}
-                 integrant/repl              {:mvn/version "0.3.3"}
-                 com.taoensso/trove          {:mvn/version "1.1.0"}
-                 com.taoensso/telemere       {:mvn/version "1.2.0"}
-                 io.opentelemetry/opentelemetry-api {:mvn/version "1.56.0"}}
+                 integrant/repl              {:mvn/version "0.3.3"}}
    :jvm-opts    ["-Djdk.attach.allowAttachSelf"
                  "-javaagent:/home/dan/scratch/learn-otel/opentelemetry-javaagent.jar"
                  "-Dotel.exporter.otlp.endpoint=http://localhost:4318"

From b3aefbfc3c65f3d3d4ee2c5fed69cf25323e210f Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Thu, 15 Jan 2026 17:03:26 -0600
Subject: [PATCH 6/9] remove new server span creation

If additional request headers are desired inside of a trace, it should
be added by the instrumentation agent.
---
 deps.edn                      | 2 +-
 src/fluree/server/handler.clj | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/deps.edn b/deps.edn
index b7a206f7..938ce30b 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,7 +1,7 @@
 {:deps {org.clojure/clojure    {:mvn/version "1.11.3"}
         org.clojure/core.async {:mvn/version "1.6.681"}
         com.fluree/db          {:git/url "https://github.com/fluree/db.git"
-                                :git/sha "055676a22fbeaca7e02ff532c595bea54a931680"}
+                                :git/sha "fa11a9a38b2935813c053a47055f622b7059bd2b"}
         com.fluree/json-ld     {:git/url "https://github.com/fluree/json-ld.git"
                                 :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"}
 
diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj
index 2a840798..5d505526 100644
--- a/src/fluree/server/handler.clj
+++ b/src/fluree/server/handler.clj
@@ -658,6 +658,4 @@
                           (seq custom-routes) (conj custom-routes))
          router         (app-router app-routes)]
      (trace-http/wrap-server-span
-      (ring/ring-handler router fallback-handler)
-      {:create-span? true
-       :captured-request-headers ["x-amzn-trace-id"]}))))
+      (ring/ring-handler router fallback-handler)))))

From c68911026005c5d8faefff8a89f949b792c7fa33 Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Fri, 16 Jan 2026 14:28:26 -0600
Subject: [PATCH 7/9] update db dep

---
 deps.edn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps.edn b/deps.edn
index 938ce30b..2f0733de 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,7 +1,7 @@
 {:deps {org.clojure/clojure    {:mvn/version "1.11.3"}
         org.clojure/core.async {:mvn/version "1.6.681"}
         com.fluree/db          {:git/url "https://github.com/fluree/db.git"
-                                :git/sha "fa11a9a38b2935813c053a47055f622b7059bd2b"}
+                                :git/sha "36656d8f47ced8be02b4a71f60aa7f638cf84bd9"}
         com.fluree/json-ld     {:git/url "https://github.com/fluree/json-ld.git"
                                 :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"}
 

From 39aef5dd7e43f5b7ce53d3c3c5afec90a6a94127 Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Fri, 16 Jan 2026 14:31:05 -0600
Subject: [PATCH 8/9] move otel settings to own alias

---
 deps.edn | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/deps.edn b/deps.edn
index 2f0733de..4d7c924d 100644
--- a/deps.edn
+++ b/deps.edn
@@ -53,13 +53,17 @@
                  clj-http/clj-http           {:mvn/version "3.13.0"}
                  criterium/criterium         {:mvn/version "0.4.6"}
                  integrant/repl              {:mvn/version "0.3.3"}}
-   :jvm-opts    ["-Djdk.attach.allowAttachSelf"
-                 "-javaagent:/home/dan/scratch/learn-otel/opentelemetry-javaagent.jar"
-                 "-Dotel.exporter.otlp.endpoint=http://localhost:4318"
-                 "-Dotel.exporter.otlp.protocol=http/protobuf"
-                 "-Dotel.logs.exporter=none"
-                 "-Dotel.metrics.exporter=none"
-                 "-Dotel.service.name=fluree-server"]}
+   :jvm-opts    ["-Djdk.attach.allowAttachSelf"]}
+
+  :otel
+  {:jvm-opts [ ;;; download jar from https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases
+              ;; "-javaagent:/path/to/opentelemetry-javaagent.jar"
+              "-Dotel.exporter.otlp.endpoint=http://localhost:4318"
+              "-Dotel.exporter.otlp.protocol=http/protobuf"
+              "-Dotel.logs.exporter=none"
+              "-Dotel.metrics.exporter=none"
+              "-Dotel.service.name=fluree-server"
+              "-Dotel.instrumentation.http.capture.headers.server.request=x-amzn-trace-id"]}
 
   :run-dev
   {:main-opts ["-m" "fluree.server" "--profile" "dev"]}

From 5d107e5dc07cf71b41f9212751ddfd2820a8a781 Mon Sep 17 00:00:00 2001
From: Daniel Petranek <dpetranek@flur.ee>
Date: Fri, 16 Jan 2026 16:24:48 -0600
Subject: [PATCH 9/9] update db dep

---
 deps.edn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps.edn b/deps.edn
index 4d7c924d..92e953be 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,7 +1,7 @@
 {:deps {org.clojure/clojure    {:mvn/version "1.11.3"}
         org.clojure/core.async {:mvn/version "1.6.681"}
         com.fluree/db          {:git/url "https://github.com/fluree/db.git"
-                                :git/sha "36656d8f47ced8be02b4a71f60aa7f638cf84bd9"}
+                                :git/sha "2843030084a3a9378b312ffaaa21f23fb45f5a26"}
         com.fluree/json-ld     {:git/url "https://github.com/fluree/json-ld.git"
                                 :git/sha "74083536c84d77f8cdd4b686b5661714010baad3"}