From 241af61bb9efdf8cc559a71d1d8b88ed126b16e7 Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kharriger@flur.ee>
Date: Thu, 6 Mar 2025 18:00:10 -0700
Subject: [PATCH 1/9] add trace-context to event

---
 .gitignore                                 |  1 +
 deps.edn                                   |  3 ++
 src/fluree/server/consensus.clj            | 35 +++++++++++++++----
 src/fluree/server/consensus/standalone.clj | 39 ++++++++++++----------
 4 files changed, 55 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4f6f5693..50769013 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@
 /.lsp/
 /dev/data/*
 /data*/**
+.calva/repl.calva-repl
diff --git a/deps.edn b/deps.edn
index 33c0dcdd..a95f66eb 100644
--- a/deps.edn
+++ b/deps.edn
@@ -24,6 +24,9 @@
         ch.qos.logback/logback-classic {:mvn/version "1.5.6"}
         org.slf4j/slf4j-api            {:mvn/version "2.0.13"}
 
+        ;; distributed tracing
+        com.github.steffan-westcott/clj-otel-api {:mvn/version "0.2.7"}
+
         ;; http
         ;; ring-jetty9-adapter 0.30.x+ uses Jetty 12 & requires JDK 17+
         ;; so we have to stay on 0.22.x b/c our minimum JDK version is 11
diff --git a/src/fluree/server/consensus.clj b/src/fluree/server/consensus.clj
index 79b1082e..555c633c 100644
--- a/src/fluree/server/consensus.clj
+++ b/src/fluree/server/consensus.clj
@@ -2,8 +2,11 @@
   "To allow for pluggable consensus, we have a Transactor protocol. In order to allow
   for a new consensus type, we need to create a record with all of the following
   methods. Currently, we support Raft and Standalone."
-  (:require [fluree.db.util.log :as log]
-            [fluree.server.consensus.events :as events]))
+  (:require [clojure.string :as string]
+            [fluree.db.util.log :as log]
+            [fluree.server.consensus.events :as events]
+            [steffan-westcott.clj-otel.api.trace.span :as span]
+            [steffan-westcott.clj-otel.context :as otel-context]))
 
 (set! *warn-on-reflection* true)
 
@@ -11,14 +14,34 @@
   (-queue-new-ledger [transactor new-ledger-params])
   (-queue-new-transaction [transactor new-tx-params]))
 
+(defn with-trace-context
+  [event]
+  (assoc event ::trace-context (otel-context/->headers)))
+
+;; need to lowercase map keys before calling otel-context/headers->merged-context
+;; https://github.com/steffan-westcott/clj-otel/issues/26
+(defn ^:private lowercase-keys [m]
+  (into {} (map (fn [[k v]] [(string/lower-case (name k)) v])) m))
+
+(defn get-trace-context [event]
+  (-> event ::trace-context lowercase-keys otel-context/headers->merged-context))
+
 (defn queue-new-ledger
   [transactor ledger-id tx-id txn opts]
   (log/trace "queue-new-ledger:" ledger-id tx-id txn)
-  (let [event-params (events/create-ledger ledger-id tx-id txn opts)]
-    (-queue-new-ledger transactor event-params)))
+  (span/with-span! {:name "Consensus Queue New Ledger"
+                    :kind :producer
+                    :attributes {"ledger.id" ledger-id
+                                 "transaction.id" tx-id}}
+    (let [event-params (events/create-ledger ledger-id tx-id txn opts)]
+      (-queue-new-ledger transactor (with-trace-context event-params)))))
 
 (defn queue-new-transaction
   [transactor ledger-id tx-id txn opts]
   (log/trace "queue-new-transaction:" txn)
-  (let [event-params (events/commit-transaction ledger-id tx-id txn opts)]
-    (-queue-new-transaction transactor event-params)))
+  (span/with-span!  {:name "Consensus Queue New Transaction"
+                     :kind :producer
+                     :attributes {"ledger.id" ledger-id
+                                  "transaction.id" tx-id}}
+    (let [event-params (events/commit-transaction ledger-id tx-id txn opts)]
+      (-queue-new-transaction transactor (with-trace-context event-params)))))
diff --git a/src/fluree/server/consensus/standalone.clj b/src/fluree/server/consensus/standalone.clj
index 3d1ce29b..fc5bf274 100644
--- a/src/fluree/server/consensus/standalone.clj
+++ b/src/fluree/server/consensus/standalone.clj
@@ -8,7 +8,9 @@
             [fluree.server.consensus :as consensus]
             [fluree.server.consensus.events :as events]
             [fluree.server.handlers.shared :refer [deref!]]
-            [fluree.server.watcher :as watcher]))
+            [fluree.server.watcher :as watcher]
+            [steffan-westcott.clj-otel.api.trace.span :as span]
+            [steffan-westcott.clj-otel.context :as otel-context]))
 
 (set! *warn-on-reflection* true)
 
@@ -52,22 +54,25 @@
 
 (defn process-event
   [conn watcher event]
-  (go
-    (try
-      (let [event-type (events/event-type event)
-            result     (<! (case event-type
-                             :ledger-create (create-ledger! conn watcher event)
-                             :tx-queue      (transact! conn watcher event)))]
-        (if (exception? result)
-          (let [{:keys [ledger-id tx-id]} event]
-            (log/debug result "Delivering tx-exception to watcher")
-            (watcher/deliver-error watcher ledger-id tx-id result))
-          result))
-      (catch Exception e
-        (log/error e
-                   "Unexpected event message - expected a map with a supported "
-                   "event type. Received:" event)
-        ::error))))
+  (otel-context/with-context! (consensus/get-trace-context event)
+    (span/with-span!  {:name "Standalone Process Event"
+                       :kind :consumer}
+      (go
+        (try
+          (let [event-type (events/event-type event)
+                result     (<! (case event-type
+                                 :ledger-create (create-ledger! conn watcher event)
+                                 :tx-queue      (transact! conn watcher event)))]
+            (if (exception? result)
+              (let [{:keys [ledger-id tx-id]} event]
+                (log/debug result "Delivering tx-exception to watcher")
+                (watcher/deliver-error watcher ledger-id tx-id result))
+              result))
+          (catch Exception e
+            (log/error e
+                       "Unexpected event message - expected a map with a supported "
+                       "event type. Received:" event)
+            ::error))))))
 
 (defn error?
   [result]

From c27388880854e74d73add547477b58a578165876 Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kharriger@flur.ee>
Date: Mon, 10 Mar 2025 17:44:41 -0600
Subject: [PATCH 2/9] readd middleware and debug data

---
 src/fluree/server/consensus.clj            | 20 ++++++-----
 src/fluree/server/consensus/standalone.clj | 39 +++++++++++-----------
 src/fluree/server/handler.clj              |  5 ++-
 src/fluree/server/handlers/create.clj      | 15 +++++----
 4 files changed, 45 insertions(+), 34 deletions(-)

diff --git a/src/fluree/server/consensus.clj b/src/fluree/server/consensus.clj
index 555c633c..d7859bcd 100644
--- a/src/fluree/server/consensus.clj
+++ b/src/fluree/server/consensus.clj
@@ -29,19 +29,23 @@
 (defn queue-new-ledger
   [transactor ledger-id tx-id txn opts]
   (log/trace "queue-new-ledger:" ledger-id tx-id txn)
-  (span/with-span! {:name "Consensus Queue New Ledger"
-                    :kind :producer
-                    :attributes {"ledger.id" ledger-id
-                                 "transaction.id" tx-id}}
+  ;; (span/set-attribute! (span/get-span) "ledger.id" ledger-id)
+  ;; (span/set-attribute! (span/get-span) "transaction.id" tx-id)
+  (span/with-span! {:name "fluree.server.consensus/queue-new-ledger"
+                    :span-kind :producer
+                    :attributes {:ledger.id ledger-id
+                                 :transaction.id tx-id
+                                 :sname (System/getenv "OTEL_SERVICE_NAME")}}
     (let [event-params (events/create-ledger ledger-id tx-id txn opts)]
       (-queue-new-ledger transactor (with-trace-context event-params)))))
 
 (defn queue-new-transaction
   [transactor ledger-id tx-id txn opts]
   (log/trace "queue-new-transaction:" txn)
-  (span/with-span!  {:name "Consensus Queue New Transaction"
-                     :kind :producer
-                     :attributes {"ledger.id" ledger-id
-                                  "transaction.id" tx-id}}
+  (span/with-span!  {:name "fluree.server.consensus/queue-new-transaction"
+                     :span-kind :producer
+                     :attributes {:ledger.id ledger-id
+                                  :transaction.id tx-id
+                                  :sname (System/getenv "OTEL_SERVICE_NAME")}}
     (let [event-params (events/commit-transaction ledger-id tx-id txn opts)]
       (-queue-new-transaction transactor (with-trace-context event-params)))))
diff --git a/src/fluree/server/consensus/standalone.clj b/src/fluree/server/consensus/standalone.clj
index fc5bf274..5e4614fb 100644
--- a/src/fluree/server/consensus/standalone.clj
+++ b/src/fluree/server/consensus/standalone.clj
@@ -54,25 +54,26 @@
 
 (defn process-event
   [conn watcher event]
-  (otel-context/with-context! (consensus/get-trace-context event)
-    (span/with-span!  {:name "Standalone Process Event"
-                       :kind :consumer}
-      (go
-        (try
-          (let [event-type (events/event-type event)
-                result     (<! (case event-type
-                                 :ledger-create (create-ledger! conn watcher event)
-                                 :tx-queue      (transact! conn watcher event)))]
-            (if (exception? result)
-              (let [{:keys [ledger-id tx-id]} event]
-                (log/debug result "Delivering tx-exception to watcher")
-                (watcher/deliver-error watcher ledger-id tx-id result))
-              result))
-          (catch Exception e
-            (log/error e
-                       "Unexpected event message - expected a map with a supported "
-                       "event type. Received:" event)
-            ::error))))))
+  (span/with-span!  {:name "fluree.server.consensus.standalone/process-event"
+                    ;; should be :consumer but only :server supported by xray  https://github.com/aws-observability/aws-otel-collector/issues/1773
+                     :span-kind :server
+                     :parent (consensus/get-trace-context event)}
+    (go
+      (try
+        (let [event-type (events/event-type event)
+              result     (<! (case event-type
+                               :ledger-create (create-ledger! conn watcher event)
+                               :tx-queue      (transact! conn watcher event)))]
+          (if (exception? result)
+            (let [{:keys [ledger-id tx-id]} event]
+              (log/debug result "Delivering tx-exception to watcher")
+              (watcher/deliver-error watcher ledger-id tx-id result))
+            result))
+        (catch Exception e
+          (log/error e
+                     "Unexpected event message - expected a map with a supported "
+                     "event type. Received:" event)
+          ::error)))))
 
 (defn error?
   [result]
diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj
index ffe1fd83..8b49b650 100644
--- a/src/fluree/server/handler.clj
+++ b/src/fluree/server/handler.clj
@@ -24,7 +24,9 @@
             [reitit.swagger :as swagger]
             [reitit.swagger-ui :as swagger-ui]
             [ring.adapter.jetty9 :as http]
-            [ring.middleware.cors :as rmc])
+            [ring.middleware.cors :as rmc]
+            [steffan-westcott.clj-otel.api.trace.span :as span]
+            [steffan-westcott.clj-otel.api.trace.http :as trace-http])
   (:import (java.io InputStream)))
 
 (set! *warn-on-reflection* true)
@@ -351,6 +353,7 @@
                                 [10 wrap-cors]
                                 [10 (partial wrap-assoc-system connection consensus
                                              watcher subscriptions broadcaster)]
+                                [20 (partial trace-http/wrap-server-span)]
                                 [50 unwrap-credential]
                                 [100 wrap-set-fuel-header]
                                 [200 coercion/coerce-exceptions-middleware]
diff --git a/src/fluree/server/handlers/create.clj b/src/fluree/server/handlers/create.clj
index 394f3db1..4f3b5cda 100644
--- a/src/fluree/server/handlers/create.clj
+++ b/src/fluree/server/handlers/create.clj
@@ -8,7 +8,8 @@
    [fluree.server.handlers.shared :refer [deref! defhandler]]
    [fluree.server.handlers.transact :refer [derive-tx-id monitor-consensus-persistence
                                             monitor-commit]]
-   [fluree.server.watcher :as watcher]))
+   [fluree.server.watcher :as watcher]
+   [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (set! *warn-on-reflection* true)
 
@@ -40,8 +41,10 @@
   (log/debug "create body:" body)
   (let [txn-context    (ctx-util/txn-context body)
         ledger-id      (transact-api/extract-ledger-id body)]
-    (if-not (deref! (fluree/exists? conn ledger-id))
-      (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
-                                  {:context txn-context})]
-        {:status 201, :body (deref! resp-p)})
-      (throw-ledger-exists ledger-id))))
+        (span/add-span-data! {:attributes {:ledger.id ledger-id :sname (System/getenv "OTEL_SERVICE_NAME")}})
+        (span/with-span! ["fluree.server.handlers.create/default" {:sname (System/getenv "OTEL_SERVICE_NAME") }]
+          (if-not (deref! (fluree/exists? conn ledger-id))
+            (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
+                                        {:context txn-context})]
+              {:status 201, :body (deref! resp-p)})
+            (throw-ledger-exists ledger-id)))))

From f6b9b80afb1446c1d66438ca564c191a392448f3 Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kharriger@flur.ee>
Date: Mon, 10 Mar 2025 19:23:47 -0600
Subject: [PATCH 3/9] add ledger to mdc

---
 src/fluree/server/handler.clj         |  3 ++-
 src/fluree/server/handlers/create.clj | 21 ++++++++++++---------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj
index 8b49b650..164f4d5f 100644
--- a/src/fluree/server/handler.clj
+++ b/src/fluree/server/handler.clj
@@ -353,7 +353,8 @@
                                 [10 wrap-cors]
                                 [10 (partial wrap-assoc-system connection consensus
                                              watcher subscriptions broadcaster)]
-                                [20 (partial trace-http/wrap-server-span)]
+                                [20 (partial trace-http/wrap-reitit-route)]
+                                [25 (partial trace-http/wrap-server-span)]
                                 [50 unwrap-credential]
                                 [100 wrap-set-fuel-header]
                                 [200 coercion/coerce-exceptions-middleware]
diff --git a/src/fluree/server/handlers/create.clj b/src/fluree/server/handlers/create.clj
index 4f3b5cda..451d6fbf 100644
--- a/src/fluree/server/handlers/create.clj
+++ b/src/fluree/server/handlers/create.clj
@@ -24,8 +24,10 @@
   (let [p         (promise)
         tx-id     (derive-tx-id txn)
         result-ch (watcher/create-watch watcher tx-id)]
-    (queue-consensus consensus watcher ledger-id tx-id txn opts)
-    (monitor-commit p ledger-id tx-id broadcaster result-ch)
+        (log/with-mdc {:tx.id tx-id}
+          (span/add-span-data! {:tx.id tx-id})
+          (queue-consensus consensus watcher ledger-id tx-id txn opts)
+          (monitor-commit p ledger-id tx-id broadcaster result-ch))
     p))
 
 (defn throw-ledger-exists
@@ -41,10 +43,11 @@
   (log/debug "create body:" body)
   (let [txn-context    (ctx-util/txn-context body)
         ledger-id      (transact-api/extract-ledger-id body)]
-        (span/add-span-data! {:attributes {:ledger.id ledger-id :sname (System/getenv "OTEL_SERVICE_NAME")}})
-        (span/with-span! ["fluree.server.handlers.create/default" {:sname (System/getenv "OTEL_SERVICE_NAME") }]
-          (if-not (deref! (fluree/exists? conn ledger-id))
-            (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
-                                        {:context txn-context})]
-              {:status 201, :body (deref! resp-p)})
-            (throw-ledger-exists ledger-id)))))
+        (log/with-mdc {:ledger.id ledger-id}
+          (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
+          (span/with-span! ["fluree.server.handlers.create/default"]
+            (if-not (deref! (fluree/exists? conn ledger-id))
+              (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
+                                          {:context txn-context})]
+                {:status 201, :body (deref! resp-p)})
+              (throw-ledger-exists ledger-id))))))

From f70d7b8515062cf05395f3e278dd9067c3912c0b Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kharriger@flur.ee>
Date: Mon, 10 Mar 2025 19:23:47 -0600
Subject: [PATCH 4/9] add tx.id to mdc

---
 src/fluree/server/consensus.clj | 32 ++++++++++++++------------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/src/fluree/server/consensus.clj b/src/fluree/server/consensus.clj
index d7859bcd..16c0d26c 100644
--- a/src/fluree/server/consensus.clj
+++ b/src/fluree/server/consensus.clj
@@ -28,24 +28,20 @@
 
 (defn queue-new-ledger
   [transactor ledger-id tx-id txn opts]
-  (log/trace "queue-new-ledger:" ledger-id tx-id txn)
-  ;; (span/set-attribute! (span/get-span) "ledger.id" ledger-id)
-  ;; (span/set-attribute! (span/get-span) "transaction.id" tx-id)
-  (span/with-span! {:name "fluree.server.consensus/queue-new-ledger"
-                    :span-kind :producer
-                    :attributes {:ledger.id ledger-id
-                                 :transaction.id tx-id
-                                 :sname (System/getenv "OTEL_SERVICE_NAME")}}
-    (let [event-params (events/create-ledger ledger-id tx-id txn opts)]
-      (-queue-new-ledger transactor (with-trace-context event-params)))))
+  (log/with-mdc {:tx.id tx-id}
+    (log/trace "queue-new-ledger:" ledger-id tx-id txn)
+    (span/with-span! {:name "fluree.server.consensus/queue-new-ledger"
+                      :span-kind :producer
+                      :attributes {:tx.id tx-id}}
+      (let [event-params (events/create-ledger ledger-id tx-id txn opts)]
+        (-queue-new-ledger transactor (with-trace-context event-params))))))
 
 (defn queue-new-transaction
   [transactor ledger-id tx-id txn opts]
-  (log/trace "queue-new-transaction:" txn)
-  (span/with-span!  {:name "fluree.server.consensus/queue-new-transaction"
-                     :span-kind :producer
-                     :attributes {:ledger.id ledger-id
-                                  :transaction.id tx-id
-                                  :sname (System/getenv "OTEL_SERVICE_NAME")}}
-    (let [event-params (events/commit-transaction ledger-id tx-id txn opts)]
-      (-queue-new-transaction transactor (with-trace-context event-params)))))
+  (log/with-mdc {:tx.id tx-id}
+    (log/trace "queue-new-transaction:" txn)
+    (span/with-span!  {:name "fluree.server.consensus/queue-new-transaction"
+                       :span-kind :producer
+                       :attributes {:tx.id tx-id}}
+      (let [event-params (events/commit-transaction ledger-id tx-id txn opts)]
+        (-queue-new-transaction transactor (with-trace-context event-params))))))

From 7f00c3711dfdb86159a69fcd2b828206ce831289 Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kurtharriger@gmail.com>
Date: Wed, 12 Mar 2025 11:24:48 -0600
Subject: [PATCH 5/9] update db to feature/kharriger/logmdc

---
 deps.edn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deps.edn b/deps.edn
index a95f66eb..421737f7 100644
--- a/deps.edn
+++ b/deps.edn
@@ -1,7 +1,7 @@
 {:deps {org.clojure/clojure    {:mvn/version "1.11.3"}
         org.clojure/core.async {:mvn/version "1.6.681"}
         com.fluree/db          {:git/url "https://github.com/fluree/db.git"
-                                :git/sha "de63ab3e47a53bce0bb4b038487f91c57cfab580"}
+                                :git/sha "66483c5c23cce163b3edf6de07d33a02a986b835"}
         com.fluree/json-ld     {:git/url "https://github.com/fluree/json-ld.git"
                                 :git/sha "73a990a4b803d0b4cfbbbe4dc16275b39a3add4e"}
 

From 67e03896fb58b180af7dcc7075e8a45873571a62 Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kurtharriger@gmail.com>
Date: Wed, 12 Mar 2025 13:21:38 -0600
Subject: [PATCH 6/9] cljfmt

---
 src/fluree/server/handler.clj         |  4 ++--
 src/fluree/server/handlers/create.clj | 24 ++++++++++++------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj
index 164f4d5f..730b39f1 100644
--- a/src/fluree/server/handler.clj
+++ b/src/fluree/server/handler.clj
@@ -25,8 +25,8 @@
             [reitit.swagger-ui :as swagger-ui]
             [ring.adapter.jetty9 :as http]
             [ring.middleware.cors :as rmc]
-            [steffan-westcott.clj-otel.api.trace.span :as span]
-            [steffan-westcott.clj-otel.api.trace.http :as trace-http])
+            [steffan-westcott.clj-otel.api.trace.http :as trace-http]
+            [steffan-westcott.clj-otel.api.trace.span :as span])
   (:import (java.io InputStream)))
 
 (set! *warn-on-reflection* true)
diff --git a/src/fluree/server/handlers/create.clj b/src/fluree/server/handlers/create.clj
index 451d6fbf..e398fada 100644
--- a/src/fluree/server/handlers/create.clj
+++ b/src/fluree/server/handlers/create.clj
@@ -24,10 +24,10 @@
   (let [p         (promise)
         tx-id     (derive-tx-id txn)
         result-ch (watcher/create-watch watcher tx-id)]
-        (log/with-mdc {:tx.id tx-id}
-          (span/add-span-data! {:tx.id tx-id})
-          (queue-consensus consensus watcher ledger-id tx-id txn opts)
-          (monitor-commit p ledger-id tx-id broadcaster result-ch))
+    (log/with-mdc {:tx.id tx-id}
+      (span/add-span-data! {:tx.id tx-id})
+      (queue-consensus consensus watcher ledger-id tx-id txn opts)
+      (monitor-commit p ledger-id tx-id broadcaster result-ch))
     p))
 
 (defn throw-ledger-exists
@@ -43,11 +43,11 @@
   (log/debug "create body:" body)
   (let [txn-context    (ctx-util/txn-context body)
         ledger-id      (transact-api/extract-ledger-id body)]
-        (log/with-mdc {:ledger.id ledger-id}
-          (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
-          (span/with-span! ["fluree.server.handlers.create/default"]
-            (if-not (deref! (fluree/exists? conn ledger-id))
-              (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
-                                          {:context txn-context})]
-                {:status 201, :body (deref! resp-p)})
-              (throw-ledger-exists ledger-id))))))
+    (log/with-mdc {:ledger.id ledger-id}
+      (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
+      (span/with-span! ["fluree.server.handlers.create/default"]
+        (if-not (deref! (fluree/exists? conn ledger-id))
+          (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
+                                      {:context txn-context})]
+            {:status 201, :body (deref! resp-p)})
+          (throw-ledger-exists ledger-id))))))

From c784f397195aefbadfa35b7f162d3f554e88c3fa Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kurtharriger@gmail.com>
Date: Wed, 12 Mar 2025 14:07:41 -0600
Subject: [PATCH 7/9] fix lint errors

---
 src/fluree/server/consensus/standalone.clj | 3 +--
 src/fluree/server/handler.clj              | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/fluree/server/consensus/standalone.clj b/src/fluree/server/consensus/standalone.clj
index 5e4614fb..37ee757e 100644
--- a/src/fluree/server/consensus/standalone.clj
+++ b/src/fluree/server/consensus/standalone.clj
@@ -9,8 +9,7 @@
             [fluree.server.consensus.events :as events]
             [fluree.server.handlers.shared :refer [deref!]]
             [fluree.server.watcher :as watcher]
-            [steffan-westcott.clj-otel.api.trace.span :as span]
-            [steffan-westcott.clj-otel.context :as otel-context]))
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (set! *warn-on-reflection* true)
 
diff --git a/src/fluree/server/handler.clj b/src/fluree/server/handler.clj
index 730b39f1..714240b9 100644
--- a/src/fluree/server/handler.clj
+++ b/src/fluree/server/handler.clj
@@ -25,8 +25,7 @@
             [reitit.swagger-ui :as swagger-ui]
             [ring.adapter.jetty9 :as http]
             [ring.middleware.cors :as rmc]
-            [steffan-westcott.clj-otel.api.trace.http :as trace-http]
-            [steffan-westcott.clj-otel.api.trace.span :as span])
+            [steffan-westcott.clj-otel.api.trace.http :as trace-http])
   (:import (java.io InputStream)))
 
 (set! *warn-on-reflection* true)

From 6ab7bd427bd00f210b9d39c8929b1e80130b4a59 Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kurtharriger@gmail.com>
Date: Wed, 12 Mar 2025 16:45:00 -0600
Subject: [PATCH 8/9] add span data on other handlers

---
 src/fluree/server/handlers/create.clj         | 11 ++++----
 src/fluree/server/handlers/ledger.clj         | 25 +++++++++++--------
 .../server/handlers/remote_resource.clj       | 15 +++++++----
 src/fluree/server/handlers/transact.clj       | 18 +++++++------
 4 files changed, 41 insertions(+), 28 deletions(-)

diff --git a/src/fluree/server/handlers/create.clj b/src/fluree/server/handlers/create.clj
index e398fada..02fcafde 100644
--- a/src/fluree/server/handlers/create.clj
+++ b/src/fluree/server/handlers/create.clj
@@ -45,9 +45,8 @@
         ledger-id      (transact-api/extract-ledger-id body)]
     (log/with-mdc {:ledger.id ledger-id}
       (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
-      (span/with-span! ["fluree.server.handlers.create/default"]
-        (if-not (deref! (fluree/exists? conn ledger-id))
-          (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
-                                      {:context txn-context})]
-            {:status 201, :body (deref! resp-p)})
-          (throw-ledger-exists ledger-id))))))
+      (if-not (deref! (fluree/exists? conn ledger-id))
+        (let [resp-p (create-ledger consensus watcher broadcaster ledger-id body
+                                    {:context txn-context})]
+          {:status 201, :body (deref! resp-p)})
+        (throw-ledger-exists ledger-id)))))
diff --git a/src/fluree/server/handlers/ledger.clj b/src/fluree/server/handlers/ledger.clj
index f5e5a248..51567978 100644
--- a/src/fluree/server/handlers/ledger.clj
+++ b/src/fluree/server/handlers/ledger.clj
@@ -3,7 +3,8 @@
    [fluree.db.api :as fluree]
    [fluree.db.util.log :as log]
    [fluree.server.handler :as-alias handler]
-   [fluree.server.handlers.shared :refer [defhandler deref!]]))
+   [fluree.server.handlers.shared :refer [defhandler deref!]]
+   [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (defn add-policy-enforcement-headers
   [override-opts {:keys [credential/did policy/identity policy/class policy/policy policy/values]}]
@@ -20,17 +21,21 @@
         format        (or (::handler/format body) :fql)
         _             (log/debug "query handler received query:" query)
         override-opts (add-policy-enforcement-headers {:format format} req)]
+    ;; todo; add any useful request data to mdc and span
     {:status 200
      :body   (deref! (fluree/query-connection conn query override-opts))}))
 
 (defhandler history
   [{:keys [fluree/conn] {{ledger :from :as query} :body} :parameters :as req}]
-  (log/debug "history handler got query:" query)
-  (let [ledger*       (->> ledger (fluree/load conn) deref!)
-        override-opts (add-policy-enforcement-headers {} req)
-        query*        (dissoc query :from)
-        _             (log/debug "history - Querying ledger" ledger "-" query*)
-        results       (deref! (fluree/history ledger* query* override-opts))]
-    (log/debug "history - query results:" results)
-    {:status 200
-     :body   results}))
+  (log/with-mdc {:ledger.id ledger}
+      (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
+   (log/debug "history handler got query:" query)
+   (let [ledger*       (->> ledger (fluree/load conn) deref!)
+         override-opts (add-policy-enforcement-headers {} req)
+         query*        (dissoc query :from)
+         _             (log/debug "history - Querying ledger" ledger "-" query*)
+         results       (deref! (fluree/history ledger* query* override-opts))]
+
+     (log/debug "history - query results:" results)
+     {:status 200
+      :body   results})))
diff --git a/src/fluree/server/handlers/remote_resource.clj b/src/fluree/server/handlers/remote_resource.clj
index 507471bc..095d96d3 100644
--- a/src/fluree/server/handlers/remote_resource.clj
+++ b/src/fluree/server/handlers/remote_resource.clj
@@ -3,11 +3,13 @@
             [fluree.db.connection :as connection]
             [fluree.db.util.async :refer [<??]]
             [fluree.db.util.log :as log]
-            [fluree.server.handlers.shared :refer [defhandler]]))
+            [fluree.server.handlers.shared :refer [defhandler]]
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (defhandler latest-commit
   [{:keys [fluree/conn]
     {{ledger-address :resource :as body} :body} :parameters}]
+    ;; todo: mdc and span properties as needed
   (log/debug "Latest commit lookup request:" body)
   (let [result (<?? (connection/read-publisher-commit conn ledger-address))]
     {:status 200
@@ -16,6 +18,7 @@
 (defhandler read-resource-address
   [{:keys [fluree/conn]
     {{resource-address :resource :as body} :body} :parameters}]
+    ;; todo: mdc and span properties as needed
   (log/debug "Remote resource read request:" body)
   (let [result (<?? (connection/read-file-address conn resource-address))]
     {:status 200
@@ -24,7 +27,9 @@
 (defhandler published-ledger-addresses
   [{:keys [fluree/conn]
     {{:keys [ledger] :as body} :body} :parameters}]
-  (log/debug "Retrieve ledger address request:" body)
-  (let [result (<?? (connection/published-addresses conn ledger))]
-    {:status 200
-     :body   {:addresses result}}))
+  (log/with-mdc {:ledger.id ledger}
+    (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
+    (log/debug "Retrieve ledger address request:" body)
+    (let [result (<?? (connection/published-addresses conn ledger))]
+      {:status 200
+       :body   {:addresses result}})))
diff --git a/src/fluree/server/handlers/transact.clj b/src/fluree/server/handlers/transact.clj
index 17b58e30..b8521492 100644
--- a/src/fluree/server/handlers/transact.clj
+++ b/src/fluree/server/handlers/transact.clj
@@ -11,7 +11,8 @@
             [fluree.server.consensus :as consensus]
             [fluree.server.consensus.events :as events]
             [fluree.server.handlers.shared :refer [defhandler deref!]]
-            [fluree.server.watcher :as watcher]))
+            [fluree.server.watcher :as watcher]
+            [steffan-westcott.clj-otel.api.trace.span :as span]))
 
 (set! *warn-on-reflection* true)
 
@@ -106,9 +107,12 @@
     {:keys [body]} :parameters}]
   (let [txn-context    (ctx-util/txn-context body)
         ledger-id      (transact-api/extract-ledger-id body)]
-    (if (deref! (fluree/exists? conn ledger-id))
-      (let [opts   (cond-> {:context txn-context :raw-txn raw-txn}
-                     did (assoc :did did))
-            resp-p (transact! consensus watcher broadcaster ledger-id body opts)]
-        {:status 200, :body (deref! resp-p)})
-      (throw-ledger-doesnt-exist ledger-id))))
+    (log/with-mdc {:ledger.id ledger-id}
+      (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
+
+      (if (deref! (fluree/exists? conn ledger-id))
+        (let [opts   (cond-> {:context txn-context :raw-txn raw-txn}
+                       did (assoc :did did))
+              resp-p (transact! consensus watcher broadcaster ledger-id body opts)]
+          {:status 200, :body (deref! resp-p)})
+        (throw-ledger-doesnt-exist ledger-id)))))

From 1c23cc73067ac04febe8e7402ef8deaae2577e80 Mon Sep 17 00:00:00 2001
From: Kurt Harriger <kurtharriger@gmail.com>
Date: Wed, 12 Mar 2025 16:49:23 -0600
Subject: [PATCH 9/9] cljfmt fix

---
 src/fluree/server/handlers/ledger.clj | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/fluree/server/handlers/ledger.clj b/src/fluree/server/handlers/ledger.clj
index 51567978..9a16464a 100644
--- a/src/fluree/server/handlers/ledger.clj
+++ b/src/fluree/server/handlers/ledger.clj
@@ -28,14 +28,14 @@
 (defhandler history
   [{:keys [fluree/conn] {{ledger :from :as query} :body} :parameters :as req}]
   (log/with-mdc {:ledger.id ledger}
-      (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
-   (log/debug "history handler got query:" query)
-   (let [ledger*       (->> ledger (fluree/load conn) deref!)
-         override-opts (add-policy-enforcement-headers {} req)
-         query*        (dissoc query :from)
-         _             (log/debug "history - Querying ledger" ledger "-" query*)
-         results       (deref! (fluree/history ledger* query* override-opts))]
+    (span/add-span-data! {:attributes (org.slf4j.MDC/getCopyOfContextMap)})
+    (log/debug "history handler got query:" query)
+    (let [ledger*       (->> ledger (fluree/load conn) deref!)
+          override-opts (add-policy-enforcement-headers {} req)
+          query*        (dissoc query :from)
+          _             (log/debug "history - Querying ledger" ledger "-" query*)
+          results       (deref! (fluree/history ledger* query* override-opts))]
 
-     (log/debug "history - query results:" results)
-     {:status 200
-      :body   results})))
+      (log/debug "history - query results:" results)
+      {:status 200
+       :body   results})))