diff --git a/src/plexosdb/db.py b/src/plexosdb/db.py index 0db38be..7052793 100644 --- a/src/plexosdb/db.py +++ b/src/plexosdb/db.py @@ -610,7 +610,7 @@ def add_object( *, description: str | None = None, category: str | None = None, - collection_enum: CollectionEnum | None = None, + collection_enum: CollectionEnum | None | Literal[False] = None, ) -> int: """Add an object to the database and append a system membership. @@ -627,7 +627,7 @@ def add_object( Category of the object, by default "-" description : str | None, optional Description of the object, by default None - collection_enum : CollectionEnum | None, optional + collection_enum : CollectionEnum | None | Literal[False] = None, optional Collection for the system membership. If None, a default collection is determined based on the class, by default None @@ -679,6 +679,10 @@ def add_object( assert query_result object_id = self._db.last_insert_rowid() + # Skip system membership for System class itself, or if explicitly set to False + if collection_enum is False: + return object_id + if not collection_enum: collection_enum = get_default_collection(class_enum) _ = self.add_membership(ClassEnum.System, class_enum, "System", name, collection_enum) @@ -739,7 +743,6 @@ def add_objects( query_result = self._db.executemany(query, params) assert query_result - # Add system memberships in bulk collection_enum = get_default_collection(class_enum) object_ids = self.get_objects_id(names, class_enum=class_enum) parent_class_id = self.get_class_id(ClassEnum.System) @@ -852,10 +855,12 @@ def add_properties_from_records( logger.warning("No records provided for bulk property and text insertion") return - params, _ = prepare_properties_params(self, records, object_class, collection, parent_class) + params, _, metadata_map = prepare_properties_params( + self, records, object_class, collection, parent_class + ) with self._db.transaction(): - data_id_map = insert_property_data(self, params) + data_id_map = insert_property_data(self, params, metadata_map) insert_scenario_tags(self, scenario, params, chunksize) if any("datafile_text" in rec for rec in records): diff --git a/src/plexosdb/utils.py b/src/plexosdb/utils.py index ef0e40a..08ec39f 100644 --- a/src/plexosdb/utils.py +++ b/src/plexosdb/utils.py @@ -193,7 +193,7 @@ def prepare_properties_params( object_class: ClassEnum, collection: CollectionEnum, parent_class: ClassEnum, -) -> tuple[list[tuple[int, int, Any]], list[tuple[str, int]]]: +) -> tuple[list[tuple[int, int, Any]], list[tuple[str, int]], dict[tuple[int, int, Any], dict[str, Any]]]: """Prepare SQL parameters for property insertion. Parameters @@ -201,7 +201,14 @@ def prepare_properties_params( db : PlexosDB Database instance records : list[dict] - List of property records + List of property records with property-specific format: + { + "name": "obj1", + "properties": { + "Property1": {"value": value1, "band": 1, "date_from": date1}, + "Property2": {"value": value2, "band": 2, "date_from": date2} + } + } object_class : ClassEnum Class enumeration of the objects collection : CollectionEnum @@ -211,8 +218,8 @@ def prepare_properties_params( Returns ------- - tuple[list[tuple], list] - Tuple of (params, collection_properties) + tuple[list[tuple], list, dict] + Tuple of (params, collection_properties, metadata_map) """ collection_id = db.get_collection_id( collection, parent_class_enum=parent_class, child_class_enum=object_class @@ -229,12 +236,45 @@ def prepare_properties_params( "Make sure you use `add_object` before adding properties." ) - params = prepare_sql_data_params(records, memberships=memberships, property_mapping=collection_properties) - return params, collection_properties + property_id_map = {prop: pid for prop, pid in collection_properties} + name_to_membership = {membership["name"]: membership["membership_id"] for membership in memberships} + + params = [] + metadata_map = {} + + for record in records: + membership_id = name_to_membership.get(record["name"]) + if not membership_id: + continue + + properties = record.get("properties", {}) + + for prop_name, prop_data in properties.items(): + property_id = property_id_map.get(prop_name) + if not property_id: + continue + + # Extract value and metadata - handle both dict and simple value + value = prop_data.get("value") if isinstance(prop_data, dict) else prop_data + band = prop_data.get("band") or prop_data.get("Band") if isinstance(prop_data, dict) else None + date_from = prop_data.get("date_from") if isinstance(prop_data, dict) else None + date_to = prop_data.get("date_to") if isinstance(prop_data, dict) else None + + param_key = (membership_id, property_id, value) + params.append(param_key) + metadata_map[param_key] = { + "band": band, + "date_from": date_from, + "date_to": date_to, + } + + return params, collection_properties, metadata_map def insert_property_data( - db: PlexosDB, params: list[tuple[int, int, Any]] + db: PlexosDB, + params: list[tuple[int, int, Any]], + metadata_map: dict[tuple[int, int, Any], dict[str, Any]] | None = None, ) -> dict[tuple[int, int, Any], tuple[int, str]]: """Insert property data and return mapping of data IDs to object names. @@ -244,6 +284,8 @@ def insert_property_data( Database instance params : list[tuple] List of (membership_id, property_id, value) tuples + metadata_map : dict | None, optional + Mapping of params to metadata (band, date_from, date_to), by default None Returns ------- @@ -268,7 +310,22 @@ def insert_property_data( for membership_id, property_id, value in params: result = db._db.fetchone(data_ids_query, (membership_id, property_id, value)) if result: - data_id_map[(membership_id, property_id, value)] = (result[0], result[1]) + data_id = result[0] + obj_name = result[1] + data_id_map[(membership_id, property_id, value)] = (data_id, obj_name) + + if metadata_map and (membership_id, property_id, value) in metadata_map: + metadata = metadata_map[(membership_id, property_id, value)] + band = metadata.get("band") + date_from = metadata.get("date_from") + date_to = metadata.get("date_to") + + if band is not None: + db.add_band(data_id, band) + + if date_from is not None or date_to is not None: + db._handle_dates(data_id, date_from, date_to) + return data_id_map diff --git a/tests/test_plexosdb_from_records.py b/tests/test_plexosdb_from_records.py index 62e3127..eba81e6 100644 --- a/tests/test_plexosdb_from_records.py +++ b/tests/test_plexosdb_from_records.py @@ -11,16 +11,37 @@ def test_bulk_insert_properties_from_records(db_base: PlexosDB): from plexosdb import ClassEnum, CollectionEnum - db: PlexosDB() = db_base + db: PlexosDB = db_base db.add_object(ClassEnum.Generator, "Generator1") db.add_object(ClassEnum.Generator, "Generator2") db.add_object(ClassEnum.Generator, "Generator3") records = [ - {"name": "Generator1", "Max Capacity": 100.0, "Min Stable Level": 20.0, "Heat Rate": 10.5}, - {"name": "Generator2", "Max Capacity": 150.0, "Min Stable Level": 30.0, "Heat Rate": 9.8}, - {"name": "Generator3", "Max Capacity": 200.0, "Min Stable Level": 40.0, "Heat Rate": 8.7}, + { + "name": "Generator1", + "properties": { + "Max Capacity": {"value": 100.0}, + "Min Stable Level": {"value": 20.0}, + "Heat Rate": {"value": 10.5}, + }, + }, + { + "name": "Generator2", + "properties": { + "Max Capacity": {"value": 150.0}, + "Min Stable Level": {"value": 30.0}, + "Heat Rate": {"value": 9.8}, + }, + }, + { + "name": "Generator3", + "properties": { + "Max Capacity": {"value": 200.0}, + "Min Stable Level": {"value": 40.0}, + "Heat Rate": {"value": 8.7}, + }, + }, ] db.add_properties_from_records( diff --git a/tests/test_utils_build_data_id_map.py b/tests/test_utils_build_data_id_map.py index 7dee8b2..59caa32 100644 --- a/tests/test_utils_build_data_id_map.py +++ b/tests/test_utils_build_data_id_map.py @@ -15,9 +15,9 @@ def test_build_data_id_map_single_record(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -26,7 +26,7 @@ def test_build_data_id_map_single_record(db_with_topology: PlexosDB) -> None: ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 1 @@ -40,9 +40,9 @@ def test_build_data_id_map_returns_correct_structure(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -51,7 +51,7 @@ def test_build_data_id_map_returns_correct_structure(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) for key, value in data_id_map.items(): @@ -75,11 +75,11 @@ def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - {"name": "gen-02", "Max Capacity": 200.0}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "gen-02", "properties": {"Max Capacity": {"value": 200.0}}}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -88,7 +88,7 @@ def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 2 @@ -101,9 +101,14 @@ def test_build_data_id_map_multiple_properties(db_with_topology: PlexosDB) -> No db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0, "Fuel Price": 5.0}] + records = [ + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}, "Fuel Price": {"value": 5.0}}, + } + ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -112,7 +117,7 @@ def test_build_data_id_map_multiple_properties(db_with_topology: PlexosDB) -> No ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 2 @@ -137,11 +142,11 @@ def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - {"name": "gen-02", "Max Capacity": 200.0}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "gen-02", "properties": {"Max Capacity": {"value": 200.0}}}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -150,7 +155,7 @@ def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) # Verify all params are in the mapping @@ -170,12 +175,12 @@ def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-03") records = [ - {"name": "gen-01", "Max Capacity": 0.0}, - {"name": "gen-02", "Max Capacity": -100.0}, - {"name": "gen-03", "Max Capacity": 1e15}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 0.0}}}, + {"name": "gen-02", "properties": {"Max Capacity": {"value": -100.0}}}, + {"name": "gen-03", "properties": {"Max Capacity": {"value": 1e15}}}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -184,7 +189,7 @@ def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 3 @@ -203,9 +208,9 @@ def test_build_data_id_map_data_ids_and_names_valid(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "test-generator") - records = [{"name": "test-generator", "Max Capacity": 100.0}] + records = [{"name": "test-generator", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -214,7 +219,7 @@ def test_build_data_id_map_data_ids_and_names_valid(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) for data_id, obj_name in data_id_map.values(): diff --git a/tests/test_utils_properties.py b/tests/test_utils_properties.py index d245d76..5768ec0 100644 --- a/tests/test_utils_properties.py +++ b/tests/test_utils_properties.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import datetime from typing import TYPE_CHECKING import pytest @@ -19,11 +20,17 @@ def test_prepare_properties_params_succeeds(db_with_topology: PlexosDB) -> None: db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0, "Heat Rate": 10.5}, - {"name": "gen-02", "Max Capacity": 150.0, "Heat Rate": 9.8}, + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}, "Heat Rate": {"value": 10.5}}, + }, + { + "name": "gen-02", + "properties": {"Max Capacity": {"value": 150.0}, "Heat Rate": {"value": 9.8}}, + }, ] - params, collection_properties = prepare_properties_params( + params, collection_properties, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -34,6 +41,7 @@ def test_prepare_properties_params_succeeds(db_with_topology: PlexosDB) -> None: assert params is not None assert len(params) == 4 # 2 records with 2 properties assert collection_properties is not None + assert isinstance(metadata_map, dict) def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: PlexosDB) -> None: @@ -43,9 +51,9 @@ def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: Plexos db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -54,7 +62,7 @@ def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: Plexos ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) assert data_id_map is not None assert len(data_id_map) == 1 @@ -67,9 +75,9 @@ def test_insert_scenario_tags_creates_scenario(db_with_topology: PlexosDB) -> No db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -78,7 +86,7 @@ def test_insert_scenario_tags_creates_scenario(db_with_topology: PlexosDB) -> No ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) insert_scenario_tags(db_with_topology, "Test Scenario", params, chunksize=1000) # Verify scenario was created @@ -97,9 +105,15 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/to/file.csv"}] + records = [ + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}}, + "datafile_text": "/path/to/file.csv", + } + ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -108,14 +122,13 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) add_texts_for_properties( db_with_topology, params, data_id_map, records, "datafile_text", ClassEnum.DataFile ) # Verify text was added by checking database text_records = db_with_topology.query("SELECT COUNT(*) as count FROM t_text") - # query returns tuples, so access by index assert text_records[0][0] > 0 @@ -127,8 +140,8 @@ def test_prepare_properties_params_raises_error_when_no_memberships(db_with_topo # Don't add objects to database - they should not exist records = [ - {"name": "nonexistent-gen-01", "Max Capacity": 100.0}, - {"name": "nonexistent-gen-02", "Max Capacity": 150.0}, + {"name": "nonexistent-gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "nonexistent-gen-02", "properties": {"Max Capacity": {"value": 150.0}}}, ] # Raises NotFoundError when objects don't exist in database @@ -151,10 +164,10 @@ def test_prepare_properties_params_empty_collection_properties(db_with_topology: # Records with properties that don't exist in the collection records = [ - {"name": "gen-01", "NonexistentProperty": 100.0}, + {"name": "gen-01", "properties": {"NonexistentProperty": {"value": 100.0}}}, ] - params, collection_properties = prepare_properties_params( + params, collection_properties, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -165,6 +178,7 @@ def test_prepare_properties_params_empty_collection_properties(db_with_topology: # Should return empty params since property doesn't exist in collection assert params == [] assert collection_properties is not None + assert isinstance(metadata_map, dict) def test_prepare_properties_params_multiple_records_single_valid(db_with_topology: PlexosDB) -> None: @@ -176,11 +190,11 @@ def test_prepare_properties_params_multiple_records_single_valid(db_with_topolog db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - {"name": "gen-02", "NonexistentProperty": 150.0}, + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + {"name": "gen-02", "properties": {"NonexistentProperty": {"value": 150.0}}}, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -191,6 +205,7 @@ def test_prepare_properties_params_multiple_records_single_valid(db_with_topolog # Should only have params for gen-01 with Max Capacity assert len(params) >= 1 assert all(param[2] is not None for param in params) + assert isinstance(metadata_map, dict) def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) -> None: @@ -200,7 +215,7 @@ def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] result = prepare_properties_params( db_with_topology, @@ -210,11 +225,11 @@ def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) ClassEnum.System, ) - # Check that result is a tuple with 2 elements + # Check that result is a tuple with 3 elements assert isinstance(result, tuple) - assert len(result) == 2 + assert len(result) == 3 - params, collection_properties = result + params, collection_properties, metadata_map = result # Check params structure assert isinstance(params, list) @@ -225,6 +240,9 @@ def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) # Check collection_properties structure assert isinstance(collection_properties, list) + # Check metadata_map structure + assert isinstance(metadata_map, dict) + def test_insert_property_data_updates_multiple_properties(db_with_topology: PlexosDB) -> None: """Test that insert_property_data marks multiple properties as dynamic and enabled.""" @@ -235,11 +253,17 @@ def test_insert_property_data_updates_multiple_properties(db_with_topology: Plex db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0, "Heat Rate": 10.5}, - {"name": "gen-02", "Max Capacity": 150.0, "Heat Rate": 9.8}, + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}, "Heat Rate": {"value": 10.5}}, + }, + { + "name": "gen-02", + "properties": {"Max Capacity": {"value": 150.0}, "Heat Rate": {"value": 9.8}}, + }, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -248,7 +272,7 @@ def test_insert_property_data_updates_multiple_properties(db_with_topology: Plex ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Verify properties are marked as dynamic and enabled properties = db_with_topology.query( @@ -264,9 +288,9 @@ def test_insert_property_data_inserts_data_correctly(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -275,7 +299,7 @@ def test_insert_property_data_inserts_data_correctly(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Verify data was inserted data_count = db_with_topology.query("SELECT COUNT(*) FROM t_data") @@ -289,9 +313,9 @@ def test_insert_property_data_builds_data_id_map(db_with_topology: PlexosDB) -> db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -300,7 +324,7 @@ def test_insert_property_data_builds_data_id_map(db_with_topology: PlexosDB) -> ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) # Verify data_id_map structure assert isinstance(data_id_map, dict) @@ -316,9 +340,9 @@ def test_insert_property_data_handles_null_values(db_with_topology: PlexosDB) -> db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": None}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": None}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -327,7 +351,7 @@ def test_insert_property_data_handles_null_values(db_with_topology: PlexosDB) -> ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) # Should handle NULL values without error assert isinstance(data_id_map, dict) @@ -338,7 +362,7 @@ def test_insert_property_data_empty_params_returns_empty_map(db_with_topology: P from plexosdb.utils import insert_property_data with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, []) + data_id_map = insert_property_data(db_with_topology, [], None) assert data_id_map == {} @@ -348,7 +372,6 @@ def test_insert_scenario_tags_early_return_when_scenario_none(db_with_topology: from plexosdb.utils import insert_scenario_tags # Should not raise error and should return early when scenario is None - # Using type: ignore because we're testing the None case insert_scenario_tags(db_with_topology, None, [], chunksize=1000) # type: ignore[arg-type] @@ -359,9 +382,9 @@ def test_insert_scenario_tags_creates_new_scenario(db_with_topology: PlexosDB) - db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -370,7 +393,7 @@ def test_insert_scenario_tags_creates_new_scenario(db_with_topology: PlexosDB) - ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) insert_scenario_tags(db_with_topology, "NewScenario", params, chunksize=1000) # Verify scenario was created @@ -388,9 +411,9 @@ def test_insert_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -399,7 +422,7 @@ def test_insert_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) insert_scenario_tags(db_with_topology, "ExistingScenario", params, chunksize=1000) # Verify scenario still exists and wasn't duplicated @@ -414,9 +437,21 @@ def test_insert_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0}] + records = [ + { + "name": "gen-01", + "properties": { + "Max Capacity": { + "value": 100.0, + "band": 1, + "date_from": datetime(2025, 1, 1), + "date_to": datetime(2025, 12, 31), + } + }, + } + ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -425,7 +460,7 @@ def test_insert_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Large chunksize - should process all in one batch insert_scenario_tags(db_with_topology, "BatchTest", params, chunksize=1000) @@ -443,10 +478,17 @@ def test_insert_scenario_tags_batching_multiple_batches(db_with_topology: Plexos db_with_topology.add_object(ClassEnum.Generator, f"gen-{i:02d}") records = [ - {"name": f"gen-{i:02d}", "Max Capacity": 100.0 + i * 10.0, "Heat Rate": 10.0 + i} for i in range(3) + { + "name": f"gen-{i:02d}", + "properties": { + "Max Capacity": {"value": 100.0 + i * 10.0}, + "Heat Rate": {"value": 10.0 + i}, + }, + } + for i in range(3) ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -455,7 +497,7 @@ def test_insert_scenario_tags_batching_multiple_batches(db_with_topology: Plexos ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Small chunksize to force multiple batches insert_scenario_tags(db_with_topology, "BatchTest", params, chunksize=2) @@ -485,11 +527,11 @@ def test_add_texts_for_properties_skips_records_without_field(db_with_topology: db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [ - {"name": "gen-01", "Max Capacity": 100.0}, - # No datafile_text field in second record + {"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}, + # No datafile_text field in record ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -498,7 +540,7 @@ def test_add_texts_for_properties_skips_records_without_field(db_with_topology: ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) # Call with field that doesn't exist in all records add_texts_for_properties( @@ -515,9 +557,15 @@ def test_add_texts_for_properties_handles_data_id_none(db_with_topology: PlexosD db_with_topology.add_object(ClassEnum.Generator, "gen-01") - records = [{"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/to/file.csv"}] + records = [ + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}}, + "datafile_text": "/path/to/file.csv", + } + ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -526,7 +574,7 @@ def test_add_texts_for_properties_handles_data_id_none(db_with_topology: PlexosD ) with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params) + insert_property_data(db_with_topology, params, metadata_map) # Create empty map - simulating missing data_ids empty_data_id_map: dict[tuple[int, int, int], tuple[int, str]] = {} @@ -546,11 +594,19 @@ def test_add_texts_for_properties_multiple_texts(db_with_topology: PlexosDB) -> db_with_topology.add_object(ClassEnum.Generator, "gen-02") records = [ - {"name": "gen-01", "Max Capacity": 100.0, "datafile_text": "/path/file1.csv"}, - {"name": "gen-02", "Max Capacity": 150.0, "datafile_text": "/path/file2.csv"}, + { + "name": "gen-01", + "properties": {"Max Capacity": {"value": 100.0}}, + "datafile_text": "/path/file1.csv", + }, + { + "name": "gen-02", + "properties": {"Max Capacity": {"value": 150.0}}, + "datafile_text": "/path/file2.csv", + }, ] - params, _ = prepare_properties_params( + params, _, metadata_map = prepare_properties_params( db_with_topology, records, ClassEnum.Generator, @@ -559,7 +615,7 @@ def test_add_texts_for_properties_multiple_texts(db_with_topology: PlexosDB) -> ) with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params) + data_id_map = insert_property_data(db_with_topology, params, metadata_map) add_texts_for_properties( db_with_topology, params, data_id_map, records, "datafile_text", ClassEnum.DataFile @@ -584,7 +640,7 @@ def test_prepare_properties_params_raises_on_no_memberships(db_with_topology: Pl from plexosdb.utils import prepare_properties_params # Try to prepare params for object that doesn't exist - records = [{"name": "NonExistentObject", "property": 100}] + records = [{"name": "NonExistentObject", "properties": {"property": {"value": 100}}}] with pytest.raises(NotFoundError, match="Object = NonExistentObject not found"): prepare_properties_params(