Improvements for working with DataScript

README.md

@@ -111,7 +111,7 @@ Use `materialize` to convert a nested `XITDB` data structure to a native Clojure
 ## No query language
 
 Use `filter`, `group-by`, `reduce`, etc.
-If you want a query engine, `datascript` works out of the box, you can store the datoms as a vector in the db.
+If you want a query engine, [`datascript` works out of the box](https://gist.github.com/radarroark/663116fcd204f3f89a7e43f52fa676ef), you can store the datoms as a vector in the db.
 
 Here's a taste of how your queries could look like:
 ```clojure 
@@ -135,6 +135,14 @@ from the respective `history index`.
 The root data structure of a xitdb database is a ArrayList, called 'history'.
 Each transaction adds a new entry into this array, which points to the latest value 
 of the database (usually a map).
+
+```clojure
+(xdb/deref-at db -1) ;; the most recent value, same as @db
+(xdb/deref-at db -2) ;; the second most recent value
+(xdb/deref-at db 0)  ;; the earliest value
+(xdb/deref-at db 1)  ;; the second value
+```
+
 It is also possible to create a transaction which returns the previous and current 
 values of the database, by setting the `*return-history?*` binding to `true`.
 

deps.edn

@@ -1,6 +1,6 @@
 {:paths ["src" "test"]
  :deps  {org.clojure/clojure        {:mvn/version "1.12.0"}
-         io.github.radarroark/xitdb {:mvn/version "0.20.0"}}
+         io.github.radarroark/xitdb {:mvn/version "0.28.0"}}
 
  :aliases
  {:test  {:extra-deps {io.github.cognitect-labs/test-runner

src/xitdb/array_list.clj

@@ -18,11 +18,11 @@
   (count [_]
     (.count ral))
 
-  (cons [_ o]
-    (throw (UnsupportedOperationException. "XITDBArrayList is read-only")))
+  (cons [this o]
+    (cons o (common/-materialize-shallow this)))
 
-  (empty [_]
-    (throw (UnsupportedOperationException. "XITDBArrayList is read-only")))
+  (empty [this]
+    [])
 
   (equiv [this other]
     (and (sequential? other)
@@ -31,9 +31,20 @@
 
   clojure.lang.Sequential  ;; Add this to mark as sequential
 
+  clojure.lang.Associative
+  (assoc [this k v]
+    (assoc (common/-materialize-shallow this) k v))
+
+  (containsKey [this k]
+    (and (integer? k) (>= k 0) (< k (.count ral))))
+
+  (entryAt [this k]
+    (when (.containsKey this k)
+      (clojure.lang.MapEntry. k (.valAt this k))))
+
   clojure.lang.IPersistentVector
   (assocN [this i val]
-    (throw (UnsupportedOperationException. "XITDBArrayList is read-only")))
+    (assoc (common/-materialize-shallow this) i val))
 
   (length [this]
     (.count ral))
@@ -106,6 +117,10 @@
         (aset result len nil))
       result))
 
+  common/ISlot
+  (-slot [this]
+    (-> ral .cursor .slot))
+
   common/IUnwrap
   (-unwrap [this]
     ral)
@@ -124,6 +139,12 @@
     (reduce (fn [a v]
               (conj a (common/materialize v))) [] (seq this))))
 
+(extend-protocol common/IMaterializeShallow
+  XITDBArrayList
+  (-materialize-shallow [this]
+    (reduce (fn [a v]
+              (conj a v)) [] (seq this))))
+
 ;;-----------------------------------------------
 
 (deftype XITDBWriteArrayList [^WriteArrayList wal]

src/xitdb/common.clj

@@ -9,6 +9,9 @@
 (defprotocol IMaterialize
   (-materialize [this]))
 
+(defprotocol IMaterializeShallow
+  (-materialize-shallow [this]))
+
 (defprotocol IUnwrap
   (-unwrap [this]))
 

src/xitdb/db.clj

@@ -46,12 +46,17 @@
         nil)))
   (.count history))
 
+(defn- write-value! [^WriteCursor cursor new-value]
+  (if (satisfies? common/ISlot new-value)
+    (.write cursor (common/-slot new-value))
+    (.write cursor (conversion/v->slot! cursor new-value))))
+
 (defn xitdb-reset!
   "Sets the value of the database to `new-value`.
   Returns new history index."
   [^WriteArrayList history new-value]
   (append-context! history nil (fn [^WriteCursor cursor]
-                                 (conversion/v->slot! cursor new-value))))
+                                 (write-value! cursor new-value))))
 
 (defn v->slot!
   "Converts a value to a slot which can be written to a cursor.
@@ -79,7 +84,7 @@
         (let [cursor (conversion/keypath-cursor cursor base-keypath)
               obj (xtypes/read-from-cursor cursor true)]
           (let [retval (apply f (into [obj] args))]
-            (.write cursor (v->slot! cursor retval))))))))
+            (write-value! cursor retval)))))))
 
 (defn xitdb-swap-with-lock!
   "Performs the 'swap!' operation while locking `db.lock`.
@@ -122,6 +127,13 @@
   [xdb]
   (.count (read-history (-> xdb .tldbro .get))))
 
+(defn deref-at
+  "Returns the version of the data at the specified index."
+  [xdb index]
+  (let [history (read-history (-> xdb .tldbro .get))
+        cursor  (.getCursor history index)]
+    (xtypes/read-from-cursor cursor false)))
+
 (deftype XITDBDatabase [tldbro rwdb lock]
 
   java.io.Closeable
@@ -131,9 +143,7 @@
 
   clojure.lang.IDeref
   (deref [this]
-    (let [history (read-history (.get tldbro))
-          cursor  (.getCursor history -1)]
-      (xtypes/read-from-cursor cursor false)))
+    (deref-at this -1))
 
   clojure.lang.IAtom
 

src/xitdb/hash_map.clj

@@ -35,21 +35,21 @@
         (clojure.lang.MapEntry. key v))))
 
   (assoc [this k v]
-    (throw (UnsupportedOperationException. "XITDBHashMap is read-only")))
+    (assoc (common/-materialize-shallow this) k v))
 
   clojure.lang.IPersistentMap
-  (without [_ _]
-    (throw (UnsupportedOperationException. "XITDBHashMap is read-only")))
+  (without [this k]
+    (dissoc (common/-materialize-shallow this) k))
 
   (count [this]
     (operations/map-item-count rhm))
 
   clojure.lang.IPersistentCollection
-  (cons [_ _]
-    (throw (UnsupportedOperationException. "XITDBHashMap is read-only")))
+  (cons [this o]
+    (. clojure.lang.RT (conj (common/-materialize-shallow this) o)))
 
-  (empty [_]
-    (throw (UnsupportedOperationException. "XITDBHashMap is read-only")))
+  (empty [this]
+    {})
 
   (equiv [this other]
     (and (instance? clojure.lang.IPersistentMap other)
@@ -81,6 +81,10 @@
   (kv-reduce [this f init]
     (operations/map-kv-reduce rhm #(common/-read-from-cursor %) f init))
 
+  common/ISlot
+  (-slot [this]
+    (-> rhm .cursor .slot))
+
   common/IUnwrap
   (-unwrap [this]
     rhm)
@@ -99,6 +103,12 @@
     (reduce (fn [m [k v]]
               (assoc m k (common/materialize v))) {} (seq this))))
 
+(extend-protocol common/IMaterializeShallow
+  XITDBHashMap
+  (-materialize-shallow [this]
+    (reduce (fn [m [k v]]
+              (assoc m k v)) {} (seq this))))
+
 ;---------------------------------------------------
 
 

src/xitdb/hash_set.clj

@@ -15,8 +15,8 @@
 
 (deftype XITDBHashSet [^ReadHashSet rhs]
   clojure.lang.IPersistentSet
-  (disjoin [_ k]
-    (throw (UnsupportedOperationException. "XITDBHashSet is read-only")))
+  (disjoin [this k]
+    (disj (common/-materialize-shallow this) k))
 
   (contains [this k]
     (operations/set-contains? rhs k))
@@ -26,11 +26,11 @@
       k))
 
   clojure.lang.IPersistentCollection
-  (cons [_ o]
-    (throw (UnsupportedOperationException. "XITDBHashSet is read-only")))
+  (cons [this o]
+    (cons o (common/-materialize-shallow this)))
 
-  (empty [_]
-    (throw (UnsupportedOperationException. "XITDBHashSet is read-only")))
+  (empty [this]
+    #{})
 
   (equiv [this other]
     (and (instance? clojure.lang.IPersistentSet other)
@@ -68,6 +68,10 @@
         (remove [_]
           (throw (UnsupportedOperationException. "XITDBHashSet iterator is read-only"))))))
 
+  common/ISlot
+  (-slot [this]
+    (-> rhs .cursor .slot))
+
   common/IUnwrap
   (-unwrap [_]
     rhs)
@@ -85,6 +89,11 @@
   (-materialize [this]
     (into #{} (map common/materialize (seq this)))))
 
+(extend-protocol common/IMaterializeShallow
+  XITDBHashSet
+  (-materialize-shallow [this]
+    (into #{} (seq this))))
+
 ;; Writable version of the set
 (deftype XITDBWriteHashSet [^WriteHashSet whs]
   clojure.lang.IPersistentSet

src/xitdb/linked_list.clj

@@ -19,11 +19,11 @@
   (count [_]
     (.count rlal))
 
-  (cons [_ o]
-    (throw (UnsupportedOperationException. "XITDBLinkedArrayList is read-only")))
+  (cons [this o]
+    (cons o (common/-materialize-shallow this)))
 
-  (empty [_]
-    (throw (UnsupportedOperationException. "XITDBLinkedArrayList is read-only")))
+  (empty [this]
+    '())
 
   (equiv [this other]
     (and (sequential? other)
@@ -86,6 +86,14 @@
         (aset result len nil))
       result))
 
+  common/ISlot
+  (-slot [this]
+    (-> rlal .cursor .slot))
+
+  common/IUnwrap
+  (-unwrap [_]
+    rlal)
+
   Object
   (toString [this]
     (pr-str (into [] this))))
@@ -100,6 +108,12 @@
     (reduce (fn [a v]
               (conj a (common/materialize v))) [] (seq this))))
 
+(extend-protocol common/IMaterializeShallow
+  XITDBLinkedArrayList
+  (-materialize-shallow [this]
+    (reduce (fn [a v]
+              (conj a v)) [] (seq this))))
+
 ;; -----------------------------------------------------------------
 
 (deftype XITDBWriteLinkedArrayList [^WriteLinkedArrayList wlal]

src/xitdb/util/conversion.clj

@@ -42,7 +42,6 @@
   {:keyword     "kw"
    :boolean     "bl"
    :key-integer "ki"
-   :nil         "nl"                                        ;; TODO: Could use Tag/NONE instead
    :inst        "in"
    :date        "da"
    :coll        "co"
@@ -104,8 +103,8 @@
   [v]
   (cond
 
-    (validation/lazy-seq? v)
-    (throw (IllegalArgumentException. "Lazy sequences can be infinite and not allowed!"))
+    (or (nil? v) (instance? Slot v))
+    v
 
     (string? v)
     (database-bytes v)
@@ -122,9 +121,6 @@
     (double? v)
     (Database$Float. v)
 
-    (nil? v)
-    (database-bytes "" (fmt-tag-value :nil))
-
     (instance? java.time.Instant v)
     (database-bytes (str v) (fmt-tag-value :inst))
 
@@ -147,6 +143,9 @@
   [^WriteCursor cursor v]
   (cond
 
+    (validation/lazy-seq? v)
+    (throw (IllegalArgumentException. "Lazy sequences can be infinite and not allowed!"))
+
     (instance? WriteArrayList v)
     (-> ^WriteArrayList v .cursor .slot)
 
@@ -190,15 +189,17 @@
       (.write cursor nil)
       (.slot (list->LinkedArrayListCursor! cursor v)))
 
-    (validation/vector-or-chunked? v)
+    (set? v)
     (do
       (.write cursor nil)
-      (.slot (coll->ArrayListCursor! cursor v)))
+      (.slot (set->WriteCursor! cursor v)))
 
-    (set? v)
+    (or (validation/vector-or-chunked? v)
+        ;; any other List implementations should just be an ArrayList
+        (instance? java.util.List v))
     (do
       (.write cursor nil)
-      (.slot (set->WriteCursor! cursor v)))
+      (.slot (coll->ArrayListCursor! cursor v)))
 
     :else
     (primitive-for v)))
@@ -222,7 +223,12 @@
         (let [v-cursor (.appendCursor write-array)]
           (list->LinkedArrayListCursor! v-cursor v))
 
-        (validation/vector-or-chunked? v)
+        (set? v)
+        (let [v-cursor (.appendCursor write-array)]
+          (set->WriteCursor! v-cursor v))
+
+        (or (validation/vector-or-chunked? v)
+            (instance? java.util.List v))
         (let [v-cursor (.appendCursor write-array)]
           (coll->ArrayListCursor! v-cursor v))
 
@@ -250,6 +256,10 @@
         (let [v-cursor (.appendCursor write-list)]
           (list->LinkedArrayListCursor! v-cursor v))
 
+        (set? v)
+        (let [v-cursor (.appendCursor write-list)]
+          (set->WriteCursor! v-cursor v))
+
         (validation/vector-or-chunked? v)
         (let [v-cursor (.appendCursor write-list)]
           (coll->ArrayListCursor! v-cursor v))
@@ -311,9 +321,6 @@
       (java.util.Date/from
         (java.time.Instant/parse str))
 
-      (= fmt-tag (fmt-tag-value :nil))
-      nil
-
       :else
       str)))