From a823360903c59c840fa2dc0a35a7a13fc4e0b46d Mon Sep 17 00:00:00 2001 From: Ryan Kingsbury Date: Tue, 8 Oct 2024 13:26:44 -0400 Subject: [PATCH 1/3] JSONStore: log and and fail gracefully in read_json --- src/maggma/stores/mongolike.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 40e9a1856..dcdee8cf9 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -688,7 +688,12 @@ def connect(self, force_reset: bool = False): f.write(bytesdata.decode("utf-8")) for path in self.paths: - objects = self.read_json_file(path) + self.logger.debug(f"Reading {path}") + try: + objects = self.read_json_file(path) + except Exception as e: + self.logger.error(f"Error reading {path}: {e}. Skipping.") + try: self.update(objects) except KeyError: From 076b9610ec58c6429a378303db641aab1b576892 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 9 Oct 2024 10:04:50 -0400 Subject: [PATCH 2/3] JSONStore: open files read only on connect --- src/maggma/stores/mongolike.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index dcdee8cf9..905ea8036 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -714,7 +714,7 @@ def read_json_file(self, path) -> list: Args: path: Path to the JSON file to be read """ - with zopen(path) as f: + with zopen(path, "r") as f: data = f.read() data = data.decode() if isinstance(data, bytes) else data objects = bson.json_util.loads(data) if "$oid" in data else orjson.loads(data) From bfcb2f94532b7e8ca75a18e41f6a4b866267c6af Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 9 Oct 2024 10:05:35 -0400 Subject: [PATCH 3/3] JSONStore.read_json_file: last_updated enhancements --- src/maggma/stores/mongolike.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/maggma/stores/mongolike.py b/src/maggma/stores/mongolike.py index 905ea8036..8da294414 100644 --- a/src/maggma/stores/mongolike.py +++ b/src/maggma/stores/mongolike.py @@ -24,7 +24,7 @@ from maggma.core import Sort, Store, StoreError from maggma.stores.ssh_tunnel import SSHTunnel -from maggma.utils import confirm_field_index, to_dt +from maggma.utils import confirm_field_index try: from montydb import MontyClient, set_storage # type: ignore @@ -719,14 +719,26 @@ def read_json_file(self, path) -> list: data = data.decode() if isinstance(data, bytes) else data objects = bson.json_util.loads(data) if "$oid" in data else orjson.loads(data) objects = [objects] if not isinstance(objects, list) else objects - # datetime objects deserialize to str. Try to convert the last_updated - # field back to datetime. # # TODO - there may still be problems caused if a JSONStore is init'ed from # documents that don't contain a last_updated field # See Store.last_updated in store.py. for obj in objects: if obj.get(self.last_updated_field): - obj[self.last_updated_field] = to_dt(obj[self.last_updated_field]) + # Lists of objects that contain datetime which are serialized + # with monty dupmfn will have the last_updated field as a dict + # representation of the datetime object, but maggma expects + # just the string representation. + if ( + isinstance(obj[self.last_updated_field], dict) + and obj[self.last_updated_field].get("@class", "") == "datetime" + ): + # overwrite last_updated field with just the string + obj[self.last_updated_field] = obj[self.last_updated_field]["string"] + else: + # if there is no last_updated field, set one to the current time. + from datetime import datetime, timezone + + obj[self.last_updated_field] = datetime.now(timezone.utc) return objects