From 7cc47c5ce42dcdc05c36bb6e99048d36198530f1 Mon Sep 17 00:00:00 2001 From: mcllerena Date: Tue, 2 Dec 2025 14:59:48 -0700 Subject: [PATCH 1/6] update add property related functions to handle metadata fields --- src/plexosdb/db.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/plexosdb/db.py b/src/plexosdb/db.py index 0db38be..83d899d 100644 --- a/src/plexosdb/db.py +++ b/src/plexosdb/db.py @@ -610,7 +610,7 @@ def add_object( *, description: str | None = None, category: str | None = None, - collection_enum: CollectionEnum | None = None, + collection_enum: CollectionEnum | None | Literal[False] = None, ) -> int: """Add an object to the database and append a system membership. @@ -679,6 +679,10 @@ def add_object( assert query_result object_id = self._db.last_insert_rowid() + # Skip system membership for System class itself, or if explicitly set to False + if class_enum == ClassEnum.System or collection_enum is False: + return object_id + if not collection_enum: collection_enum = get_default_collection(class_enum) _ = self.add_membership(ClassEnum.System, class_enum, "System", name, collection_enum) @@ -852,10 +856,12 @@ def add_properties_from_records( logger.warning("No records provided for bulk property and text insertion") return - params, _ = prepare_properties_params(self, records, object_class, collection, parent_class) + params, _, metadata_map = prepare_properties_params( + self, records, object_class, collection, parent_class + ) with self._db.transaction(): - data_id_map = insert_property_data(self, params) + data_id_map = insert_property_data(self, params, metadata_map) insert_scenario_tags(self, scenario, params, chunksize) if any("datafile_text" in rec for rec in records): From 9dbaea5f29be276ce35f4d60eeb241e5bbc26e72 Mon Sep 17 00:00:00 2001 From: mcllerena Date: Tue, 2 Dec 2025 15:00:26 -0700 Subject: [PATCH 2/6] modify utils functions to handle property metadata fields --- src/plexosdb/utils.py | 67 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/src/plexosdb/utils.py b/src/plexosdb/utils.py index ef0e40a..18403c2 100644 --- a/src/plexosdb/utils.py +++ b/src/plexosdb/utils.py @@ -193,7 +193,7 @@ def prepare_properties_params( object_class: ClassEnum, collection: CollectionEnum, parent_class: ClassEnum, -) -> tuple[list[tuple[int, int, Any]], list[tuple[str, int]]]: +) -> tuple[list[tuple[int, int, Any]], list[tuple[str, int]], dict[tuple[int, int, Any], dict[str, Any]]]: """Prepare SQL parameters for property insertion. Parameters @@ -211,8 +211,9 @@ def prepare_properties_params( Returns ------- - tuple[list[tuple], list] - Tuple of (params, collection_properties) + tuple[list[tuple], list, dict] + Tuple of (params, collection_properties, metadata_map) + where metadata_map contains band/date info keyed by (membership_id, property_id, value) """ collection_id = db.get_collection_id( collection, parent_class_enum=parent_class, child_class_enum=object_class @@ -229,12 +230,47 @@ def prepare_properties_params( "Make sure you use `add_object` before adding properties." ) - params = prepare_sql_data_params(records, memberships=memberships, property_mapping=collection_properties) - return params, collection_properties + property_id_map = {prop: pid for prop, pid in collection_properties} + name_to_membership = {membership["name"]: membership["membership_id"] for membership in memberships} + + params = [] + metadata_map = {} + + for record in records: + if record["name"] not in name_to_membership: + continue + + membership_id = name_to_membership[record["name"]] + + # Extract metadata fields + band = record.get("Band") or record.get("band") + date_from = record.get("date_from") + date_to = record.get("date_to") + + for prop, value in record.items(): + if prop == "name" or prop not in property_id_map: + continue + if prop in ("Band", "band", "date_from", "date_to", "datafile_text", "timeslice"): + continue + + property_id = property_id_map[prop] + param_key = (membership_id, property_id, value) + params.append(param_key) + + # Store metadata for this property + metadata_map[param_key] = { + "band": band, + "date_from": date_from, + "date_to": date_to, + } + + return params, collection_properties, metadata_map def insert_property_data( - db: PlexosDB, params: list[tuple[int, int, Any]] + db: PlexosDB, + params: list[tuple[int, int, Any]], + metadata_map: dict[tuple[int, int, Any], dict[str, Any]] | None = None, ) -> dict[tuple[int, int, Any], tuple[int, str]]: """Insert property data and return mapping of data IDs to object names. @@ -244,6 +280,8 @@ def insert_property_data( Database instance params : list[tuple] List of (membership_id, property_id, value) tuples + metadata_map : dict | None, optional + Mapping of params to metadata (band, date_from, date_to), by default None Returns ------- @@ -268,7 +306,22 @@ def insert_property_data( for membership_id, property_id, value in params: result = db._db.fetchone(data_ids_query, (membership_id, property_id, value)) if result: - data_id_map[(membership_id, property_id, value)] = (result[0], result[1]) + data_id = result[0] + obj_name = result[1] + data_id_map[(membership_id, property_id, value)] = (data_id, obj_name) + + if metadata_map and (membership_id, property_id, value) in metadata_map: + metadata = metadata_map[(membership_id, property_id, value)] + band = metadata.get("band") + date_from = metadata.get("date_from") + date_to = metadata.get("date_to") + + if band is not None: + db.add_band(data_id, band) + + if date_from is not None or date_to is not None: + db._handle_dates(data_id, date_from, date_to) + return data_id_map From f486e457b9abe24a5e2156dc81f6ead285132229 Mon Sep 17 00:00:00 2001 From: mcllerena Date: Tue, 2 Dec 2025 15:16:43 -0700 Subject: [PATCH 3/6] update tests to match latest state of utils functions --- tests/test_utils_build_data_id_map.py | 28 ++++----- tests/test_utils_properties.py | 85 +++++++++++++++------------ 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/tests/test_utils_build_data_id_map.py b/tests/test_utils_build_data_id_map.py index 7dee8b2..72303eb 100644 --- a/tests/test_utils_build_data_id_map.py +++ b/tests/test_utils_build_data_id_map.py @@ -17,7 +17,7 @@ def test_build_data_id_map_single_record(db_with_topology: PlexosDB) -> None: records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -26,7 +26,7 @@ def test_build_data_id_map_single_record(db_with_topology: PlexosDB) -> None: ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 1 @@ -42,7 +42,7 @@ def test_build_data_id_map_returns_correct_structure(db_with_topology: PlexosDB) records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -51,7 +51,7 @@ def test_build_data_id_map_returns_correct_structure(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) for key, value in data_id_map.items(): @@ -79,7 +79,7 @@ def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: {"name": "gen-02", "Max Capacity": 200.0}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -88,7 +88,7 @@ def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 2 @@ -103,7 +103,7 @@ def test_build_data_id_map_multiple_properties(db_with_topology: PlexosDB) -> No records = [{"name": "gen-01", "Max Capacity": 100.0, "Fuel Price": 5.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -112,7 +112,7 @@ def test_build_data_id_map_multiple_properties(db_with_topology: PlexosDB) -> No ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 2 @@ -141,7 +141,7 @@ def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB {"name": "gen-02", "Max Capacity": 200.0}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -150,7 +150,7 @@ def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) # Verify all params are in the mapping @@ -175,7 +175,7 @@ def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: {"name": "gen-03", "Max Capacity": 1e15}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -184,7 +184,7 @@ def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 3 @@ -205,7 +205,7 @@ def test_build_data_id_map_data_ids_and_names_valid(db_with_topology: PlexosDB) records = [{"name": "test-generator", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -214,7 +214,7 @@ def test_build_data_id_map_data_ids_and_names_valid(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) for data_id, obj_name in data_id_map.values(): diff --git a/tests/test_utils_properties.py b/tests/test_utils_properties.py index d245d76..c5b41e1 100644 --- a/tests/test_utils_properties.py +++ b/tests/test_utils_properties.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING - +from datetime import datetime import pytest if TYPE_CHECKING: @@ -23,7 +23,7 @@ def test_prepare_properties_params_succeeds(db_with_topology: PlexosDB) -> None: {"name": "gen-02", "Max Capacity": 150.0, "Heat Rate": 9.8}, ] - params, collection_properties = prepare_properties_params( + params, collection_properties, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -34,6 +34,7 @@ def test_prepare_properties_params_succeeds(db_with_topology: PlexosDB) -> None: assert params is not None assert len(params) == 4 # 2 records with 2 properties assert collection_properties is not None + assert isinstance(metadata_map, dict) def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: PlexosDB) -> None: @@ -45,7 +46,7 @@ def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: Plexos records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -54,7 +55,7 @@ def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: Plexos ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) assert data_id_map is not None assert len(data_id_map) == 1 @@ -69,7 +70,7 @@ def test_insert_scenario_tags_creates_scenario(db_with_topology: PlexosDB) -> No records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -78,7 +79,7 @@ def test_insert_scenario_tags_creates_scenario(db_with_topology: PlexosDB) -> No ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) insert_scenario_tags(db_with_topology, "Test Scenario", params, chunksize=1000) # Verify scenario was created @@ -99,7 +100,7 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) records = [{"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/to/file.csv"}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -108,7 +109,7 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) add_texts_for_properties( db_with_topology, params, data_id_map, records, "datafile_text", ClassEnum.DataFile ) @@ -154,7 +155,7 @@ def test_prepare_properties_params_empty_collection_properties(db_with_topology: {"name": "gen-01", "NonexistentProperty": 100.0}, ] - params, collection_properties = prepare_properties_params( + params, collection_properties, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -165,6 +166,7 @@ def test_prepare_properties_params_empty_collection_properties(db_with_topology: # Should return empty params since property doesn't exist in collection assert params == [] assert collection_properties is not None + assert isinstance(metadata_map, dict) def test_prepare_properties_params_multiple_records_single_valid(db_with_topology: PlexosDB) -> None: @@ -180,7 +182,7 @@ def test_prepare_properties_params_multiple_records_single_valid(db_with_topolog {"name": "gen-02", "NonexistentProperty": 150.0}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -191,6 +193,7 @@ def test_prepare_properties_params_multiple_records_single_valid(db_with_topolog # Should only have params for gen-01 with Max Capacity assert len(params) >= 1 assert all(param[2] is not None for param in params) + assert isinstance(metadata_map, dict) def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) -> None: @@ -210,11 +213,11 @@ def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) ClassEnum.System, ) - # Check that result is a tuple with 2 elements + # Check that result is a tuple with 3 elements (updated from 2) assert isinstance(result, tuple) - assert len(result) == 2 + assert len(result) == 3 - params, collection_properties = result + params, collection_properties, metadata_map = result # Check params structure assert isinstance(params, list) @@ -225,6 +228,9 @@ def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) # Check collection_properties structure assert isinstance(collection_properties, list) + # Check metadata_map structure + assert isinstance(metadata_map, dict) + def test_insert_property_data_updates_multiple_properties(db_with_topology: PlexosDB) -> None: """Test that insert_property_data marks multiple properties as dynamic and enabled.""" @@ -239,7 +245,7 @@ def test_insert_property_data_updates_multiple_properties(db_with_topology: Plex {"name": "gen-02", "Max Capacity": 150.0, "Heat Rate": 9.8}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -248,7 +254,7 @@ def test_insert_property_data_updates_multiple_properties(db_with_topology: Plex ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Verify properties are marked as dynamic and enabled properties = db_with_topology.query( @@ -266,7 +272,7 @@ def test_insert_property_data_inserts_data_correctly(db_with_topology: PlexosDB) records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -275,7 +281,7 @@ def test_insert_property_data_inserts_data_correctly(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Verify data was inserted data_count = db_with_topology.query("SELECT COUNT(*) FROM t_data") @@ -291,7 +297,7 @@ def test_insert_property_data_builds_data_id_map(db_with_topology: PlexosDB) -> records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -300,7 +306,7 @@ def test_insert_property_data_builds_data_id_map(db_with_topology: PlexosDB) -> ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) # Verify data_id_map structure assert isinstance(data_id_map, dict) @@ -318,7 +324,7 @@ def test_insert_property_data_handles_null_values(db_with_topology: PlexosDB) -> records = [{"name": "gen-01", "Max Capacity": None}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -327,7 +333,7 @@ def test_insert_property_data_handles_null_values(db_with_topology: PlexosDB) -> ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) # Should handle NULL values without error assert isinstance(data_id_map, dict) @@ -338,7 +344,7 @@ def test_insert_property_data_empty_params_returns_empty_map(db_with_topology: P from plexosdb.utils import insert_property_data with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, []) + data_id_map = insert_property_data(db_with_topology, [], None) assert data_id_map == {} @@ -361,7 +367,7 @@ def test_insert_scenario_tags_creates_new_scenario(db_with_topology: PlexosDB) - records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -370,7 +376,7 @@ def test_insert_scenario_tags_creates_new_scenario(db_with_topology: PlexosDB) - ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) insert_scenario_tags(db_with_topology, "NewScenario", params, chunksize=1000) # Verify scenario was created @@ -390,7 +396,7 @@ def test_insert_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) records = [{"name": "gen-01", "Max Capacity": 100.0}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -399,7 +405,7 @@ def test_insert_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) insert_scenario_tags(db_with_topology, "ExistingScenario", params, chunksize=1000) # Verify scenario still exists and wasn't duplicated @@ -414,9 +420,14 @@ def test_insert_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [ + { + "name": "gen-01", "Max Capacity": 100.0, "band": 1, + "date_from": datetime(2025, 1, 1), "date_to": datetime(2025, 12, 31), + } + ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -425,7 +436,7 @@ def test_insert_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Large chunksize - should process all in one batch insert_scenario_tags(db_with_topology, "BatchTest", params, chunksize=1000) @@ -446,7 +457,7 @@ def test_insert_scenario_tags_batching_multiple_batches(db_with_topology: Plexos {"name": f"gen-{i:02d}", "Max Capacity": 100.0 + i * 10.0, "Heat Rate": 10.0 + i} for i in range(3) ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -455,7 +466,7 @@ def test_insert_scenario_tags_batching_multiple_batches(db_with_topology: Plexos ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Small chunksize to force multiple batches insert_scenario_tags(db_with_topology, "BatchTest", params, chunksize=2) @@ -489,7 +500,7 @@ def test_add_texts_for_properties_skips_records_without_field(db_with_topology: # No datafile_text field in second record ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -498,7 +509,7 @@ def test_add_texts_for_properties_skips_records_without_field(db_with_topology: ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) # Call with field that doesn't exist in all records add_texts_for_properties( @@ -517,7 +528,7 @@ def test_add_texts_for_properties_handles_data_id_none(db_with_topology: PlexosD records = [{"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/to/file.csv"}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -526,7 +537,7 @@ def test_add_texts_for_properties_handles_data_id_none(db_with_topology: PlexosD ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Create empty map - simulating missing data_ids empty_data_id_map: dict[tuple[int, int, int], tuple[int, str]] = {} @@ -550,7 +561,7 @@ def test_add_texts_for_properties_multiple_texts(db_with_topology: PlexosDB) -> {"name": "gen-02", "Max Capacity": 150.0, "datafile_text": "/path/file2.csv"}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -559,7 +570,7 @@ def test_add_texts_for_properties_multiple_texts(db_with_topology: PlexosDB) -> ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) add_texts_for_properties( db_with_topology, params, data_id_map, records, "datafile_text", ClassEnum.DataFile From 3879d94ca1d57608e4a1a5e573a12c6e127bff68 Mon Sep 17 00:00:00 2001 From: mcllerena Date: Thu, 4 Dec 2025 09:28:12 -0700 Subject: [PATCH 4/6] handle nested properties with attributes and update tests accordingly --- src/plexosdb/db.py | 4 +- src/plexosdb/utils.py | 36 +++++---- tests/test_plexosdb_from_records.py | 29 ++++++- tests/test_utils_build_data_id_map.py | 27 ++++--- tests/test_utils_properties.py | 109 ++++++++++++++++++-------- 5 files changed, 140 insertions(+), 65 deletions(-) diff --git a/src/plexosdb/db.py b/src/plexosdb/db.py index 83d899d..ad0d4b0 100644 --- a/src/plexosdb/db.py +++ b/src/plexosdb/db.py @@ -627,7 +627,7 @@ def add_object( Category of the object, by default "-" description : str | None, optional Description of the object, by default None - collection_enum : CollectionEnum | None, optional + collection_enum : CollectionEnum | None | Literal[False] = None, optional Collection for the system membership. If None, a default collection is determined based on the class, by default None @@ -680,7 +680,7 @@ def add_object( object_id = self._db.last_insert_rowid() # Skip system membership for System class itself, or if explicitly set to False - if class_enum == ClassEnum.System or collection_enum is False: + if collection_enum is False: return object_id if not collection_enum: diff --git a/src/plexosdb/utils.py b/src/plexosdb/utils.py index 18403c2..08ec39f 100644 --- a/src/plexosdb/utils.py +++ b/src/plexosdb/utils.py @@ -201,7 +201,14 @@ def prepare_properties_params( db : PlexosDB Database instance records : list[dict] - List of property records + List of property records with property-specific format: + { + "name": "obj1", + "properties": { + "Property1": {"value": value1, "band": 1, "date_from": date1}, + "Property2": {"value": value2, "band": 2, "date_from": date2} + } + } object_class : ClassEnum Class enumeration of the objects collection : CollectionEnum @@ -213,7 +220,6 @@ def prepare_properties_params( ------- tuple[list[tuple], list, dict] Tuple of (params, collection_properties, metadata_map) - where metadata_map contains band/date info keyed by (membership_id, property_id, value) """ collection_id = db.get_collection_id( collection, parent_class_enum=parent_class, child_class_enum=object_class @@ -237,27 +243,25 @@ def prepare_properties_params( metadata_map = {} for record in records: - if record["name"] not in name_to_membership: + membership_id = name_to_membership.get(record["name"]) + if not membership_id: continue - membership_id = name_to_membership[record["name"]] - - # Extract metadata fields - band = record.get("Band") or record.get("band") - date_from = record.get("date_from") - date_to = record.get("date_to") + properties = record.get("properties", {}) - for prop, value in record.items(): - if prop == "name" or prop not in property_id_map: - continue - if prop in ("Band", "band", "date_from", "date_to", "datafile_text", "timeslice"): + for prop_name, prop_data in properties.items(): + property_id = property_id_map.get(prop_name) + if not property_id: continue - property_id = property_id_map[prop] + # Extract value and metadata - handle both dict and simple value + value = prop_data.get("value") if isinstance(prop_data, dict) else prop_data + band = prop_data.get("band") or prop_data.get("Band") if isinstance(prop_data, dict) else None + date_from = prop_data.get("date_from") if isinstance(prop_data, dict) else None + date_to = prop_data.get("date_to") if isinstance(prop_data, dict) else None + param_key = (membership_id, property_id, value) params.append(param_key) - - # Store metadata for this property metadata_map[param_key] = { "band": band, "date_from": date_from, diff --git a/tests/test_plexosdb_from_records.py b/tests/test_plexosdb_from_records.py index 62e3127..eba81e6 100644 --- a/tests/test_plexosdb_from_records.py +++ b/tests/test_plexosdb_from_records.py @@ -11,16 +11,37 @@ def test_bulk_insert_properties_from_records(db_base: PlexosDB): from plexosdb import ClassEnum, CollectionEnum - db: PlexosDB() = db_base + db: PlexosDB = db_base db.add_object(ClassEnum.Generator, "Generator1") db.add_object(ClassEnum.Generator, "Generator2") db.add_object(ClassEnum.Generator, "Generator3") records = [ - {"name": "Generator1", "Max Capacity": 100.0, "Min Stable Level": 20.0, "Heat Rate": 10.5}, - {"name": "Generator2", "Max Capacity": 150.0, "Min Stable Level": 30.0, "Heat Rate": 9.8}, - {"name": "Generator3", "Max Capacity": 200.0, "Min Stable Level": 40.0, "Heat Rate": 8.7}, + { + "name": "Generator1", + "properties": { + "Max Capacity": {"value": 100.0}, + "Min Stable Level": {"value": 20.0}, + "Heat Rate": {"value": 10.5}, + }, + }, + { + "name": "Generator2", + "properties": { + "Max Capacity": {"value": 150.0}, + "Min Stable Level": {"value": 30.0}, + "Heat Rate": {"value": 9.8}, + }, + }, + { + "name": "Generator3", + "properties": { + "Max Capacity": {"value": 200.0}, + "Min Stable Level": {"value": 40.0}, + "Heat Rate": {"value": 8.7}, + }, + }, ] db.add_properties_from_records( diff --git a/tests/test_utils_build_data_id_map.py b/tests/test_utils_build_data_id_map.py index 72303eb..59caa32 100644 --- a/tests/test_utils_build_data_id_map.py +++ b/tests/test_utils_build_data_id_map.py @@ -15,7 +15,7 @@ def test_build_data_id_map_single_record(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -40,7 +40,7 @@ def test_build_data_id_map_returns_correct_structure(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -75,8 +75,8 @@ def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - {"name": "gen-02", "Max Capacity": 200.0}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "gen-02", "properties": {"Max Capacity": {"value": 200.0}}}, ] params, _, metadata_map = prepare_properties_params( @@ -101,7 +101,12 @@ def test_build_data_id_map_multiple_properties(db_with_topology: PlexosDB) -> No db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0, "Fuel Price": 5.0}] + records = [ + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}, "Fuel Price": {"value": 5.0}}, + } + ] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -137,8 +142,8 @@ def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - {"name": "gen-02", "Max Capacity": 200.0}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "gen-02", "properties": {"Max Capacity": {"value": 200.0}}}, ] params, _, metadata_map = prepare_properties_params( @@ -170,9 +175,9 @@ def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-03") records = [ - {"name": "gen-01", "Max Capacity": 0.0}, - {"name": "gen-02", "Max Capacity": -100.0}, - {"name": "gen-03", "Max Capacity": 1e15}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 0.0}}}, + {"name": "gen-02", "properties": {"Max Capacity": {"value": -100.0}}}, + {"name": "gen-03", "properties": {"Max Capacity": {"value": 1e15}}}, ] params, _, metadata_map = prepare_properties_params( @@ -203,7 +208,7 @@ def test_build_data_id_map_data_ids_and_names_valid(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "test-generator") - records = [{"name": "test-generator", "Max Capacity": 100.0}] + records = [{"name": "test-generator", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, diff --git a/tests/test_utils_properties.py b/tests/test_utils_properties.py index c5b41e1..f8648a0 100644 --- a/tests/test_utils_properties.py +++ b/tests/test_utils_properties.py @@ -2,8 +2,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING from datetime import datetime +from typing import TYPE_CHECKING + import pytest if TYPE_CHECKING: @@ -19,8 +20,14 @@ def test_prepare_properties_params_succeeds(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0, "Heat Rate": 10.5}, - {"name": "gen-02", "Max Capacity": 150.0, "Heat Rate": 9.8}, + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}, "Heat Rate": {"value": 10.5}}, + }, + { + "name": "gen-02", + "properties": {"Max Capacity": {"value": 150.0}, "Heat Rate": {"value": 9.8}}, + }, ] params, collection_properties, metadata_map = prepare_properties_params( @@ -44,7 +51,7 @@ def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: Plexos db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -68,7 +75,7 @@ def test_insert_scenario_tags_creates_scenario(db_with_topology: PlexosDB) -> No db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -98,7 +105,13 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/to/file.csv"}] + records = [ + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}}, + "datafile_text": "/path/to/file.csv", + } + ] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -116,7 +129,6 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) # Verify text was added by checking database text_records = db_with_topology.query("SELECT COUNT(*) as count FROM t_text") - # query returns tuples, so access by index assert text_records[0][0] > 0 @@ -128,8 +140,8 @@ def test_prepare_properties_params_raises_error_when_no_memberships(db_with_topo # Don't add objects to database - they should not exist records = [ - {"name": "nonexistent-gen-01", "Max Capacity": 100.0}, - {"name": "nonexistent-gen-02", "Max Capacity": 150.0}, + {"name": "nonexistent-gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "nonexistent-gen-02", "properties": {"Max Capacity": {"value": 150.0}}}, ] # Raises NotFoundError when objects don't exist in database @@ -152,7 +164,7 @@ def test_prepare_properties_params_empty_collection_properties(db_with_topology: # Records with properties that don't exist in the collection records = [ - {"name": "gen-01", "NonexistentProperty": 100.0}, + {"name": "gen-01", "properties": {"NonexistentProperty": {"value": 100.0}}}, ] params, collection_properties, metadata_map = prepare_properties_params( @@ -178,8 +190,8 @@ def test_prepare_properties_params_multiple_records_single_valid(db_with_topolog db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - {"name": "gen-02", "NonexistentProperty": 150.0}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "gen-02", "properties": {"NonexistentProperty": {"value": 150.0}}}, ] params, _, metadata_map = prepare_properties_params( @@ -203,7 +215,7 @@ def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] result = prepare_properties_params( db_with_topology, @@ -213,7 +225,7 @@ def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) ClassEnum.System, ) - # Check that result is a tuple with 3 elements (updated from 2) + # Check that result is a tuple with 3 elements assert isinstance(result, tuple) assert len(result) == 3 @@ -241,8 +253,14 @@ def test_insert_property_data_updates_multiple_properties(db_with_topology: Plex db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0, "Heat Rate": 10.5}, - {"name": "gen-02", "Max Capacity": 150.0, "Heat Rate": 9.8}, + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}, "Heat Rate": {"value": 10.5}}, + }, + { + "name": "gen-02", + "properties": {"Max Capacity": {"value": 150.0}, "Heat Rate": {"value": 9.8}}, + }, ] params, _, metadata_map = prepare_properties_params( @@ -270,7 +288,7 @@ def test_insert_property_data_inserts_data_correctly(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -295,7 +313,7 @@ def test_insert_property_data_builds_data_id_map(db_with_topology: PlexosDB) -> db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -322,7 +340,7 @@ def test_insert_property_data_handles_null_values(db_with_topology: PlexosDB) -> db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": None}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": None}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -354,7 +372,6 @@ def test_insert_scenario_tags_early_return_when_scenario_none(db_with_topology: from plexosdb.utils import insert_scenario_tags # Should not raise error and should return early when scenario is None - # Using type: ignore because we're testing the None case insert_scenario_tags(db_with_topology, None, [], chunksize=1000) # type: ignore[arg-type] @@ -365,7 +382,7 @@ def test_insert_scenario_tags_creates_new_scenario(db_with_topology: PlexosDB) - db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -394,7 +411,7 @@ def test_insert_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -422,8 +439,15 @@ def test_insert_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) records = [ { - "name": "gen-01", "Max Capacity": 100.0, "band": 1, - "date_from": datetime(2025, 1, 1), "date_to": datetime(2025, 12, 31), + "name": "gen-01", + "properties": { + "Max Capacity": { + "value": 100.0, + "band": 1, + "date_from": datetime(2025, 1, 1), + "date_to": datetime(2025, 12, 31), + } + }, } ] @@ -454,7 +478,14 @@ def test_insert_scenario_tags_batching_multiple_batches(db_with_topology: Plexos db_with_topology.add_object(ClassEnum.Generator, f"gen-{i:02d}") records = [ - {"name": f"gen-{i:02d}", "Max Capacity": 100.0 + i * 10.0, "Heat Rate": 10.0 + i} for i in range(3) + { + "name": f"gen-{i:02d}", + "properties": { + "Max Capacity": {"value": 100.0 + i * 10.0}, + "Heat Rate": {"value": 10.0 + i}, + }, + } + for i in range(3) ] params, _, metadata_map = prepare_properties_params( @@ -496,8 +527,8 @@ def test_add_texts_for_properties_skips_records_without_field(db_with_topology: db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - # No datafile_text field in second record + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + # No datafile_text field in record ] params, _, metadata_map = prepare_properties_params( @@ -526,7 +557,13 @@ def test_add_texts_for_properties_handles_data_id_none(db_with_topology: PlexosD db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/to/file.csv"}] + records = [ + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}}, + "datafile_text": "/path/to/file.csv", + } + ] params, _, metadata_map = prepare_properties_params( db_with_topology, @@ -557,8 +594,16 @@ def test_add_texts_for_properties_multiple_texts(db_with_topology: PlexosDB) -> db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/file1.csv"}, - {"name": "gen-02", "Max Capacity": 150.0, "datafile_text": "/path/file2.csv"}, + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}}, + "datafile_text": "/path/file1.csv", + }, + { + "name": "gen-02", + "properties": {"Max Capacity": {"value": 150.0}}, + "datafile_text": "/path/file2.csv", + }, ] params, _, metadata_map = prepare_properties_params( @@ -595,7 +640,7 @@ def test_prepare_properties_params_raises_on_no_memberships(db_with_topology: Pl from plexosdb.utils import prepare_properties_params # Try to prepare params for object that doesn't exist - records = [{"name": "NonExistentObject", "property": 100}] + records = [{"name": "NonExistentObject", "properties": {"property": {"value": 100}}}] with pytest.raises(NotFoundError, match="Object = NonExistentObject not found"): prepare_properties_params( @@ -604,4 +649,4 @@ def test_prepare_properties_params_raises_on_no_memberships(db_with_topology: Pl ClassEnum.Generator, CollectionEnum.Generators, ClassEnum.System, - ) + ) \ No newline at end of file From 72d4efd4eb888fd78242ffd5e6baf5f607c8d592 Mon Sep 17 00:00:00 2001 From: mcllerena Date: Thu, 4 Dec 2025 09:29:44 -0700 Subject: [PATCH 5/6] run pre-commit --- tests/test_utils_properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_properties.py b/tests/test_utils_properties.py index f8648a0..5768ec0 100644 --- a/tests/test_utils_properties.py +++ b/tests/test_utils_properties.py @@ -649,4 +649,4 @@ def test_prepare_properties_params_raises_on_no_memberships(db_with_topology: Pl ClassEnum.Generator, CollectionEnum.Generators, ClassEnum.System, - ) \ No newline at end of file + ) From e1efaccd4ee1be4a928c1122b956131398f89ef1 Mon Sep 17 00:00:00 2001 From: mcllerena Date: Thu, 4 Dec 2025 10:13:13 -0700 Subject: [PATCH 6/6] fix: remove comment to test commit message name --- src/plexosdb/db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plexosdb/db.py b/src/plexosdb/db.py index ad0d4b0..7052793 100644 --- a/src/plexosdb/db.py +++ b/src/plexosdb/db.py @@ -743,7 +743,6 @@ def add_objects( query_result = self._db.executemany(query, params) assert query_result - # Add system memberships in bulk collection_enum = get_default_collection(class_enum) object_ids = self.get_objects_id(names, class_enum=class_enum) parent_class_id = self.get_class_id(ClassEnum.System)