diff --git a/docs/source/howtos/add_properties.md b/docs/source/howtos/add_properties.md index 37589d1..b7bd253 100644 --- a/docs/source/howtos/add_properties.md +++ b/docs/source/howtos/add_properties.md @@ -82,19 +82,23 @@ db.add_property( ## Bulk Adding Properties -For efficiency when adding many properties at once: +For efficiency when adding many properties at once (use the flat format; the nested format is accepted but deprecated and will emit a warning): ```python -# Prepare property records -property_records = [ - {"name": "Generator1", "Max Capacity": 100.0, "Min Stable Level": 20.0}, - {"name": "Generator2", "Max Capacity": 150.0, "Min Stable Level": 30.0}, - {"name": "Generator3", "Max Capacity": 250.0, "Min Stable Level": 10.0}, +# Flat format (recommended) +flat_records = [ + {"name": "Generator1", "property": "Max Capacity", "value": 100, "band": 1}, + {"name": "Generator1", "property": "Max Capacity", "value": 200, "band": 2}, + {"name": "Generator2", "property": "Heat Rate", "value": 9.9, "datafile_text": "gen2.csv"}, +] + +# Nested format (legacy; will be removed in the future) +nested_records = [ + {"name": "Generator3", "properties": {"Max Capacity": {"value": 150, "band": 1}}}, ] -# Bulk add properties db.add_properties_from_records( - property_records, + flat_records + nested_records, object_class=ClassEnum.Generator, parent_class=ClassEnum.System, collection=CollectionEnum.Generators, diff --git a/src/plexosdb/db.py b/src/plexosdb/db.py index aa363d7..19461b8 100644 --- a/src/plexosdb/db.py +++ b/src/plexosdb/db.py @@ -8,6 +8,7 @@ from pathlib import Path from string import Template from typing import Any, Literal, TypedDict, cast +import warnings from loguru import logger @@ -20,13 +21,13 @@ NotFoundError, ) from .utils import ( - add_texts_for_properties, + apply_scenario_tags, create_membership_record, - insert_property_data, - insert_scenario_tags, + insert_property_texts, + insert_property_values, no_space, normalize_names, - prepare_properties_params, + plan_property_inserts, ) from .xml_handler import XMLHandler @@ -855,20 +856,57 @@ def add_properties_from_records( logger.warning("No records provided for bulk property and text insertion") return - params, _, metadata_map = prepare_properties_params( - self, records, object_class, collection, parent_class + prepared = plan_property_inserts( + self, + records, + object_class=object_class, + collection=collection, + parent_class=parent_class, ) + if prepared.deprecated_format_used: + warnings.warn( + "The nested 'properties' payload is deprecated; prefer flat property records with " + "keys 'name', 'property', and 'value'.", + DeprecationWarning, + stacklevel=2, + ) - with self._db.transaction(): - data_id_map = insert_property_data(self, params, metadata_map) - insert_scenario_tags(self, scenario, params, chunksize) + params = prepared.params + metadata_map = prepared.metadata_map + + if not params: + msg = f"Failed to parse the properties for the given {collection=} and {object_class=}. " + msg += "Check the function plan_property_inserts" + return + # raise PropertyError(msg) - if any("datafile_text" in rec for rec in records): - add_texts_for_properties( - self, params, data_id_map, records, "datafile_text", ClassEnum.DataFile + has_datafile_text = any(meta.get("datafile_text") for meta in metadata_map.values()) + has_timeslice_text = any(meta.get("timeslice") for meta in metadata_map.values()) + + with self._db.transaction(): + data_id_map = insert_property_values(self, params, metadata_map=metadata_map) + apply_scenario_tags(self, params, scenario=scenario, chunksize=chunksize) + + if has_datafile_text: + insert_property_texts( + self, + params, + data_id_map=data_id_map, + records=records, + field_name="datafile_text", + text_class=ClassEnum.DataFile, + metadata_map=metadata_map, + ) + if has_timeslice_text: + insert_property_texts( + self, + params, + data_id_map=data_id_map, + records=records, + field_name="timeslice", + text_class=ClassEnum.Timeslice, + metadata_map=metadata_map, ) - if any("timeslice" in rec for rec in records): - add_texts_for_properties(self, params, data_id_map, records, "timeslice", ClassEnum.Timeslice) logger.debug(f"Successfully processed {len(records)} property and text records in batches") return @@ -2355,7 +2393,8 @@ def get_category_max_id(self, class_enum: ClassEnum) -> int: """ result = self._db.fetchone(query, (class_enum,)) assert result - return cast(int, result[0]) + rank = result[0] + return 0 if rank is None else cast(int, rank) def get_class_id(self, class_enum: ClassEnum) -> int: """Return the ID for a given class. diff --git a/src/plexosdb/exceptions.py b/src/plexosdb/exceptions.py index 058f192..4ea81c6 100644 --- a/src/plexosdb/exceptions.py +++ b/src/plexosdb/exceptions.py @@ -23,3 +23,7 @@ class NameError(ValueError): class NoPropertiesError(Exception): """Raised when a lookup finds no properties for a given object.""" + + +class PropertyError(Exception): + """Raised when we have a problem with a property.""" diff --git a/src/plexosdb/utils.py b/src/plexosdb/utils.py index 08ec39f..d6b2e9f 100644 --- a/src/plexosdb/utils.py +++ b/src/plexosdb/utils.py @@ -4,17 +4,32 @@ import ast from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from datetime import datetime from importlib.resources import files from itertools import islice from typing import TYPE_CHECKING, Any from loguru import logger +from .exceptions import NotFoundError + if TYPE_CHECKING: from plexosdb import ClassEnum, CollectionEnum, PlexosDB from plexosdb.db_manager import SQLiteManager +@dataclass +class PreparedPropertiesResult: + """Prepared inputs for bulk property insertion.""" + + params: list[tuple[int, int, Any]] + collection_properties: list[tuple[str, int]] + metadata_map: dict[tuple[int, int, Any], dict[str, Any]] + normalized_records: list[dict[str, Any]] + deprecated_format_used: bool + + def batched(iterable: Iterable[Any], n: int) -> Iterator[tuple[Any, ...]]: """Implement batched iterator. @@ -187,93 +202,168 @@ def create_membership_record( ] -def prepare_properties_params( +def _flatten_property_records(records: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], bool]: + """Normalize incoming property records into a flat list.""" + normalized_records: list[dict[str, Any]] = [] + deprecated_format_used = False + + for record in records: + object_name = record.get("name") + if "properties" in record: + deprecated_format_used = True + record_text = record.get("datafile_text") + record_timeslice = record.get("timeslice") + for prop_name, prop_value in record.get("properties", {}).items(): + if isinstance(prop_value, dict): + value = prop_value.get("value") + band = prop_value.get("band") or prop_value.get("Band") + date_from = prop_value.get("date_from") + date_to = prop_value.get("date_to") + datafile_text = prop_value.get("datafile_text", record_text) + timeslice = prop_value.get("timeslice", record_timeslice) + else: + value = prop_value + band = None + date_from = None + date_to = None + datafile_text = record_text + timeslice = record_timeslice + + normalized_records.append( + { + "name": object_name, + "property": prop_name, + "value": value, + "band": band, + "date_from": date_from, + "date_to": date_to, + "datafile_text": datafile_text, + "timeslice": timeslice, + "source_format": "nested", + } + ) + elif "property" in record and "value" in record: + normalized_records.append( + { + "name": object_name, + "property": record.get("property"), + "value": record.get("value"), + "band": record.get("band"), + "date_from": record.get("date_from"), + "date_to": record.get("date_to"), + "datafile_text": record.get("datafile_text"), + "timeslice": record.get("timeslice"), + "source_format": "flat", + } + ) + else: + raise ValueError( + "Each record must include either a 'properties' dict or 'property'/'value' keys. " + "Reshape the input to one of those forms before adding properties." + ) + + return normalized_records, deprecated_format_used + + +def plan_property_inserts( db: PlexosDB, records: list[dict[str, Any]], + *, object_class: ClassEnum, collection: CollectionEnum, parent_class: ClassEnum, -) -> tuple[list[tuple[int, int, Any]], list[tuple[str, int]], dict[tuple[int, int, Any], dict[str, Any]]]: - """Prepare SQL parameters for property insertion. - - Parameters - ---------- - db : PlexosDB - Database instance - records : list[dict] - List of property records with property-specific format: - { - "name": "obj1", - "properties": { - "Property1": {"value": value1, "band": 1, "date_from": date1}, - "Property2": {"value": value2, "band": 2, "date_from": date2} - } - } - object_class : ClassEnum - Class enumeration of the objects - collection : CollectionEnum - Collection enumeration for the properties - parent_class : ClassEnum - Parent class enumeration +) -> PreparedPropertiesResult: + """Prepare SQL parameters for property insertion.""" + normalized_records, deprecated_format_used = _flatten_property_records(records) + if not normalized_records: + return PreparedPropertiesResult([], [], {}, [], deprecated_format_used) - Returns - ------- - tuple[list[tuple], list, dict] - Tuple of (params, collection_properties, metadata_map) - """ collection_id = db.get_collection_id( collection, parent_class_enum=parent_class, child_class_enum=object_class ) - collection_properties = db.query( - f"select name, property_id from t_property where collection_id={collection_id}" + collection_properties = _fetch_collection_properties(db, collection_id=collection_id) + name_to_membership = _resolve_membership_map(db, normalized_records, object_class=object_class) + property_id_map = {prop: pid for prop, pid in collection_properties} + + params, metadata_map = _build_property_rows( + normalized_records, name_to_membership=name_to_membership, property_id_map=property_id_map + ) + + return PreparedPropertiesResult( + params, collection_properties, metadata_map, normalized_records, deprecated_format_used ) - component_names = tuple(d["name"] for d in records) - memberships = db.get_memberships_system(component_names, object_class=object_class) + + +def _fetch_collection_properties(db: PlexosDB, *, collection_id: int) -> list[tuple[str, int]]: + """Fetch property rows for a collection as (name, id) tuples.""" + return db.query(f"select name, property_id from t_property where collection_id={collection_id}") + + +def _resolve_membership_map( + db: PlexosDB, + normalized_records: list[dict[str, Any]], + *, + object_class: ClassEnum, +) -> dict[str, int]: + """Resolve membership ids for each object name.""" + component_names = tuple({d["name"] for d in normalized_records if d.get("name") is not None}) + try: + memberships = db.get_memberships_system(component_names, object_class=object_class) + except Exception as exc: + missing = ", ".join(sorted(name for name in component_names if name)) + raise NotFoundError( + f"Objects not found: {missing}. Add them with `add_object` or `add_objects` before " + "adding properties." + ) from exc if not memberships: - raise KeyError( - "Object do not exists on the database yet. " - "Make sure you use `add_object` before adding properties." + missing = ", ".join(sorted(name for name in component_names if name)) + raise NotFoundError( + f"Objects not found: {missing}. Add them with `add_object` or `add_objects` before " + "adding properties." ) - property_id_map = {prop: pid for prop, pid in collection_properties} - name_to_membership = {membership["name"]: membership["membership_id"] for membership in memberships} + return {membership["name"]: membership["membership_id"] for membership in memberships} - params = [] - metadata_map = {} - for record in records: +def _build_property_rows( + normalized_records: list[dict[str, Any]], + *, + name_to_membership: dict[str, int], + property_id_map: dict[str, int], +) -> tuple[list[tuple[int, int, Any]], dict[tuple[int, int, Any], dict[str, Any]]]: + """Build parameter tuples and metadata for normalized records.""" + params: list[tuple[int, int, Any]] = [] + metadata_map: dict[tuple[int, int, Any], dict[str, Any]] = {} + + for record in normalized_records: membership_id = name_to_membership.get(record["name"]) if not membership_id: continue - properties = record.get("properties", {}) - - for prop_name, prop_data in properties.items(): - property_id = property_id_map.get(prop_name) - if not property_id: - continue - - # Extract value and metadata - handle both dict and simple value - value = prop_data.get("value") if isinstance(prop_data, dict) else prop_data - band = prop_data.get("band") or prop_data.get("Band") if isinstance(prop_data, dict) else None - date_from = prop_data.get("date_from") if isinstance(prop_data, dict) else None - date_to = prop_data.get("date_to") if isinstance(prop_data, dict) else None + property_id = property_id_map.get(record["property"]) + if not property_id: + continue - param_key = (membership_id, property_id, value) - params.append(param_key) - metadata_map[param_key] = { - "band": band, - "date_from": date_from, - "date_to": date_to, - } + param_key = (membership_id, property_id, record["value"]) + params.append(param_key) + metadata_map[param_key] = { + "band": record.get("band"), + "date_from": record.get("date_from"), + "date_to": record.get("date_to"), + "datafile_text": record.get("datafile_text"), + "timeslice": record.get("timeslice"), + "property_name": record.get("property"), + "object_name": record.get("name"), + } - return params, collection_properties, metadata_map + return params, metadata_map -def insert_property_data( +def insert_property_values( db: PlexosDB, params: list[tuple[int, int, Any]], + *, metadata_map: dict[tuple[int, int, Any], dict[str, Any]] | None = None, ) -> dict[tuple[int, int, Any], tuple[int, str]]: """Insert property data and return mapping of data IDs to object names. @@ -292,10 +382,13 @@ def insert_property_data( dict Mapping of (membership_id, property_id, value) to (data_id, obj_name) """ - filter_property_ids = [d[1] for d in params] - for property_id in filter_property_ids: - db._db.execute("UPDATE t_property set is_dynamic=1 where property_id = ?", (property_id,)) - db._db.execute("UPDATE t_property set is_enabled=1 where property_id = ?", (property_id,)) + if not params: + return {} + + unique_property_ids = {property_id for _, property_id, _ in params} + property_params = [(property_id,) for property_id in unique_property_ids] + db._db.executemany("UPDATE t_property set is_dynamic=1 where property_id = ?", property_params) + db._db.executemany("UPDATE t_property set is_enabled=1 where property_id = ?", property_params) db._db.executemany("INSERT into t_data(membership_id, property_id, value) values (?,?,?)", params) @@ -314,23 +407,19 @@ def insert_property_data( obj_name = result[1] data_id_map[(membership_id, property_id, value)] = (data_id, obj_name) - if metadata_map and (membership_id, property_id, value) in metadata_map: - metadata = metadata_map[(membership_id, property_id, value)] - band = metadata.get("band") - date_from = metadata.get("date_from") - date_to = metadata.get("date_to") - - if band is not None: - db.add_band(data_id, band) - - if date_from is not None or date_to is not None: - db._handle_dates(data_id, date_from, date_to) + if metadata_map: + _persist_metadata_for_data(db, metadata_map=metadata_map, data_id_map=data_id_map) return data_id_map -def insert_scenario_tags( - db: PlexosDB, scenario: str, params: list[tuple[int, int, Any]], chunksize: int +def apply_scenario_tags( + db: PlexosDB, + params: list[tuple[int, int, Any]], + /, + *, + scenario: str, + chunksize: int, ) -> None: """Insert scenario tags for property data. @@ -338,10 +427,10 @@ def insert_scenario_tags( ---------- db : PlexosDB Database instance - scenario : str - Scenario name params : list[tuple] List of (membership_id, property_id, value) tuples + scenario : str + Scenario name chunksize : int Number of records to process in each batch """ @@ -367,13 +456,16 @@ def insert_scenario_tags( db._db.executemany(scenario_query, batched_list) -def add_texts_for_properties( +def insert_property_texts( db: PlexosDB, params: list[tuple[int, int, Any]], + /, + *, data_id_map: dict[tuple[int, int, Any], tuple[int, str]], records: list[dict[str, Any]], field_name: str, text_class: ClassEnum, + metadata_map: dict[tuple[int, int, Any], dict[str, Any]] | None = None, ) -> None: """Add text data for properties from specified field. @@ -391,12 +483,119 @@ def add_texts_for_properties( Name of the field in records containing text data text_class : ClassEnum ClassEnum for the text data + metadata_map : dict | None, optional + Metadata map keyed by param tuple to drive property-specific text mapping """ - text_map = {rec["name"]: rec[field_name] for rec in records if field_name in rec} + text_map = _build_text_lookup(records, field_name=field_name) + class_id = db.get_class_id(text_class) + texts_to_insert = _collect_text_rows( + params, data_id_map, metadata_map=metadata_map, text_map=text_map, class_id=class_id + ) + + if texts_to_insert: + db._db.executemany( + "INSERT INTO t_text(data_id, class_id, value) VALUES (?,?,?)", + texts_to_insert, + ) + + +def _persist_metadata_for_data( + db: PlexosDB, + *, + metadata_map: dict[tuple[int, int, Any], dict[str, Any]], + data_id_map: dict[tuple[int, int, Any], tuple[int, str]], +) -> None: + """Attach band and date metadata for inserted data rows.""" + bands_to_insert: list[tuple[int, int]] = [] + dates_from_to_insert: list[tuple[int, str]] = [] + dates_to_to_insert: list[tuple[int, str]] = [] + + for key, metadata in metadata_map.items(): + data_entry = data_id_map.get(key) + if not data_entry: + continue + + data_id = data_entry[0] + band = metadata.get("band") + date_from = metadata.get("date_from") + date_to = metadata.get("date_to") + + if band is not None: + bands_to_insert.append((data_id, band)) + + _append_date_if_present(dates_from_to_insert, data_id, date_value=date_from, label="date_from") + _append_date_if_present(dates_to_to_insert, data_id, date_value=date_to, label="date_to") + + if bands_to_insert: + db._db.executemany("INSERT INTO t_band(data_id, band_id) VALUES (?, ?)", bands_to_insert) + if dates_from_to_insert: + db._db.executemany("INSERT INTO t_date_from(data_id, date) VALUES (?, ?)", dates_from_to_insert) + if dates_to_to_insert: + db._db.executemany("INSERT INTO t_date_to(data_id, date) VALUES (?, ?)", dates_to_to_insert) + + +def _append_date_if_present( + target: list[tuple[int, str]], data_id: int, *, date_value: datetime | None, label: str +) -> None: + """Validate and append date metadata when provided.""" + if date_value is None: + return + if not isinstance(date_value, datetime): + raise TypeError(f"{label} must be a datetime object") + target.append((data_id, date_value.isoformat())) + + +def _build_text_lookup( + records: list[dict[str, Any]], *, field_name: str +) -> dict[tuple[str, str | None], Any]: + """Create a lookup of object/property combinations to text values.""" + text_map: dict[tuple[str, str | None], Any] = {} + for rec in records: + obj_name = rec.get("name") + if obj_name is None: + continue + + if field_name in rec: + text_map[(obj_name, rec.get("property"))] = rec[field_name] + + if "properties" in rec and field_name in rec: + text_map[(obj_name, None)] = rec[field_name] + + for prop_name, prop_value in rec.get("properties", {}).items(): + if isinstance(prop_value, dict) and field_name in prop_value: + text_map[(obj_name, prop_name)] = prop_value[field_name] + + return text_map + + +def _collect_text_rows( + params: list[tuple[int, int, Any]], + data_id_map: dict[tuple[int, int, Any], tuple[int, str]], + *, + metadata_map: dict[tuple[int, int, Any], dict[str, Any]] | None, + text_map: dict[tuple[str, str | None], Any], + class_id: int, +) -> list[tuple[int, int, Any]]: + """Convert params and metadata into t_text insert rows.""" + texts_to_insert: list[tuple[int, int, Any]] = [] + for membership_id, property_id, value in params: data_id, obj_name = data_id_map.get((membership_id, property_id, value), (None, None)) - if data_id and obj_name and obj_name in text_map: - db.add_text(text_class, text_map[obj_name], data_id) + if not data_id or not obj_name: + continue + + property_name = ( + metadata_map.get((membership_id, property_id, value), {}).get("property_name") + if metadata_map + else None + ) + lookup_keys = [(obj_name, property_name), (obj_name, None)] + for lookup in lookup_keys: + if lookup in text_map: + texts_to_insert.append((data_id, class_id, text_map[lookup])) + break + + return texts_to_insert def build_data_id_map( diff --git a/tests/test_plexosdb_from_records.py b/tests/test_plexosdb_from_records.py index eba81e6..3e4c702 100644 --- a/tests/test_plexosdb_from_records.py +++ b/tests/test_plexosdb_from_records.py @@ -57,6 +57,83 @@ def test_bulk_insert_properties_from_records(db_base: PlexosDB): assert properties[0]["scenario_name"] == "Base Case" +def test_add_properties_supports_flat_records_with_metadata(db_instance_with_schema: PlexosDB): + from datetime import datetime + from plexosdb import ClassEnum, CollectionEnum + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "FlatGen") + + records = [ + { + "name": "FlatGen", + "property": "Max Capacity", + "value": 120.5, + "band": 1, + "date_from": datetime(2025, 1, 1), + "date_to": datetime(2025, 2, 1), + }, + { + "name": "FlatGen", + "property": "Max Energy", + "value": 350.0, + "datafile_text": "profile.csv", + }, + ] + + db.add_properties_from_records( + records, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + scenario="Planning", + ) + + data_rows = db._db.fetchall("SELECT membership_id, property_id, value FROM t_data") + assert len(data_rows) == 2 + + band_rows = db._db.fetchall("SELECT data_id, band_id FROM t_band") + assert len(band_rows) == 1 + assert band_rows[0][1] == 1 + + date_from_rows = db._db.fetchall("SELECT date FROM t_date_from") + date_to_rows = db._db.fetchall("SELECT date FROM t_date_to") + assert date_from_rows[0][0].startswith("2025-01-01") + assert date_to_rows[0][0].startswith("2025-02-01") + + text_rows = db._db.fetchall("SELECT class_id, value FROM t_text") + assert len(text_rows) == 1 + assert text_rows[0][1] == "profile.csv" + + +def test_add_properties_nested_records_emit_deprecation(db_instance_with_schema: PlexosDB): + from plexosdb import ClassEnum, CollectionEnum + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "LegacyGen") + + records = [ + { + "name": "LegacyGen", + "properties": { + "Max Capacity": {"value": 75.0}, + }, + } + ] + + with pytest.warns(DeprecationWarning): + db.add_properties_from_records( + records, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + scenario="Legacy", + ) + + values = db._db.fetchall("SELECT value FROM t_data") + assert values == [(75.0,)] + + def test_bulk_insert_memberships_from_records(db_base: PlexosDB): from plexosdb import ClassEnum, CollectionEnum @@ -110,3 +187,40 @@ def test_bulk_insert_memberships_from_records(db_base: PlexosDB): ] with pytest.raises(KeyError): _ = db.add_memberships_from_records(memberships) + + +def test_add_properties_from_records_no_records(db_instance_with_schema: PlexosDB, caplog): + """Gracefully handle empty payload.""" + from plexosdb import ClassEnum, CollectionEnum + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "EmptyGen") + + db.add_properties_from_records( + [], + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + scenario="None", + ) + + assert "No records provided" in caplog.text + assert db._db.fetchone("SELECT COUNT(*) FROM t_data")[0] == 0 + + +def test_add_properties_from_records_unknown_property(db_instance_with_schema: PlexosDB): + """Return early when properties are not recognized for the collection.""" + from plexosdb import ClassEnum, CollectionEnum + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "BadPropGen") + + db.add_properties_from_records( + [{"name": "BadPropGen", "property": "Unknown", "value": 1}], + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + scenario="None", + ) + + assert db._db.fetchone("SELECT COUNT(*) FROM t_data")[0] == 0 diff --git a/tests/test_utils_build_data_id_map.py b/tests/test_utils_build_data_id_map.py index 59caa32..52fa6f7 100644 --- a/tests/test_utils_build_data_id_map.py +++ b/tests/test_utils_build_data_id_map.py @@ -11,22 +11,24 @@ def test_build_data_id_map_single_record(db_with_topology: PlexosDB) -> None: """Test build_data_id_map with single record returns correct mapping.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import build_data_id_map, insert_property_data, prepare_properties_params + from plexosdb.utils import build_data_id_map, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 1 @@ -36,22 +38,24 @@ def test_build_data_id_map_single_record(db_with_topology: PlexosDB) -> None: def test_build_data_id_map_returns_correct_structure(db_with_topology: PlexosDB) -> None: """Test build_data_id_map returns (data_id, obj_name) tuples with correct key format.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import build_data_id_map, insert_property_data, prepare_properties_params + from plexosdb.utils import build_data_id_map, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) for key, value in data_id_map.items(): @@ -69,7 +73,7 @@ def test_build_data_id_map_returns_correct_structure(db_with_topology: PlexosDB) def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: """Test build_data_id_map with multiple records.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import build_data_id_map, insert_property_data, prepare_properties_params + from plexosdb.utils import build_data_id_map, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") db_with_topology.add_object(ClassEnum.Generator, "gen-02") @@ -79,16 +83,18 @@ def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: {"name": "gen-02", "properties": {"Max Capacity": {"value": 200.0}}}, ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 2 @@ -97,7 +103,7 @@ def test_build_data_id_map_multiple_records(db_with_topology: PlexosDB) -> None: def test_build_data_id_map_multiple_properties(db_with_topology: PlexosDB) -> None: """Test build_data_id_map with multiple properties per record.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import build_data_id_map, insert_property_data, prepare_properties_params + from plexosdb.utils import build_data_id_map, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") @@ -108,16 +114,18 @@ def test_build_data_id_map_multiple_properties(db_with_topology: PlexosDB) -> No } ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 2 @@ -136,7 +144,7 @@ def test_build_data_id_map_empty_params(db_with_topology: PlexosDB) -> None: def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB) -> None: """Test build_data_id_map maintains accurate mapping between params and results.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import build_data_id_map, insert_property_data, prepare_properties_params + from plexosdb.utils import build_data_id_map, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") db_with_topology.add_object(ClassEnum.Generator, "gen-02") @@ -146,16 +154,18 @@ def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB {"name": "gen-02", "properties": {"Max Capacity": {"value": 200.0}}}, ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) # Verify all params are in the mapping @@ -168,7 +178,7 @@ def test_build_data_id_map_preserves_mapping_accuracy(db_with_topology: PlexosDB def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: """Test build_data_id_map handles zero, negative, and large values.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import build_data_id_map, insert_property_data, prepare_properties_params + from plexosdb.utils import build_data_id_map, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") db_with_topology.add_object(ClassEnum.Generator, "gen-02") @@ -180,16 +190,18 @@ def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: {"name": "gen-03", "properties": {"Max Capacity": {"value": 1e15}}}, ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) assert len(data_id_map) == 3 @@ -204,22 +216,24 @@ def test_build_data_id_map_edge_case_values(db_with_topology: PlexosDB) -> None: def test_build_data_id_map_data_ids_and_names_valid(db_with_topology: PlexosDB) -> None: """Test build_data_id_map returns valid data_ids and non-empty object names.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import build_data_id_map, insert_property_data, prepare_properties_params + from plexosdb.utils import build_data_id_map, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "test-generator") records = [{"name": "test-generator", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) data_id_map = build_data_id_map(db_with_topology._db, params) for data_id, obj_name in data_id_map.values(): diff --git a/tests/test_utils_properties.py b/tests/test_utils_properties.py index 5768ec0..1bf73c2 100644 --- a/tests/test_utils_properties.py +++ b/tests/test_utils_properties.py @@ -11,10 +11,10 @@ from plexosdb import PlexosDB -def test_prepare_properties_params_succeeds(db_with_topology: PlexosDB) -> None: - """Test that prepare_properties_params correctly prepares SQL parameters.""" +def test_plan_property_inserts_succeeds(db_with_topology: PlexosDB) -> None: + """Test that plan_property_inserts correctly prepares SQL parameters.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import prepare_properties_params + from plexosdb.utils import plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") db_with_topology.add_object(ClassEnum.Generator, "gen-02") @@ -30,77 +30,110 @@ def test_prepare_properties_params_succeeds(db_with_topology: PlexosDB) -> None: }, ] - params, collection_properties, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) - assert params is not None - assert len(params) == 4 # 2 records with 2 properties - assert collection_properties is not None - assert isinstance(metadata_map, dict) + assert prepared.params is not None + assert len(prepared.params) == 4 # 2 records with 2 properties + assert prepared.collection_properties is not None + assert isinstance(prepared.metadata_map, dict) -def test_insert_property_data_marks_dynamic_and_enabled(db_with_topology: PlexosDB) -> None: - """Test that insert_property_data marks properties as dynamic and enabled.""" +def test_insert_property_values_marks_dynamic_and_enabled(db_with_topology: PlexosDB) -> None: + """Test that insert_property_values marks properties as dynamic and enabled.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, prepare_properties_params + from plexosdb.utils import insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params, metadata_map) + data_id_map = insert_property_values(db_with_topology, params, metadata_map=metadata_map) assert data_id_map is not None assert len(data_id_map) == 1 -def test_insert_scenario_tags_creates_scenario(db_with_topology: PlexosDB) -> None: - """Test that insert_scenario_tags creates scenario if it doesn't exist.""" +def test_insert_property_values_raises_for_invalid_date_type(db_instance_with_schema: PlexosDB) -> None: + """Invalid date types should trigger TypeError.""" + from plexosdb import ClassEnum, CollectionEnum + from plexosdb.utils import insert_property_values, plan_property_inserts + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "DateGen") + + records = [ + { + "name": "DateGen", + "properties": {"Max Capacity": {"value": 10.0, "date_from": "2025-01-01"}}, + } + ] + + prepared = plan_property_inserts( + db, + records, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + ) + params = prepared.params + metadata_map = prepared.metadata_map + + with db._db.transaction(), pytest.raises(TypeError): + insert_property_values(db, params, metadata_map=metadata_map) + + +def test_apply_scenario_tags_creates_scenario(db_with_topology: PlexosDB) -> None: + """Test that apply_scenario_tags creates scenario if it doesn't exist.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, insert_scenario_tags, prepare_properties_params + from plexosdb.utils import insert_property_values, apply_scenario_tags, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) - insert_scenario_tags(db_with_topology, "Test Scenario", params, chunksize=1000) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) + apply_scenario_tags(db_with_topology, params, scenario="Test Scenario", chunksize=1000) # Verify scenario was created scenario_exists = db_with_topology.check_scenario_exists("Test Scenario") assert scenario_exists is True -def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) -> None: - """Test that add_texts_for_properties correctly adds datafile_text field.""" +def test_insert_property_texts_with_datafile_text(db_with_topology: PlexosDB) -> None: + """Test that insert_property_texts correctly adds datafile_text field.""" from plexosdb import ClassEnum, CollectionEnum from plexosdb.utils import ( - add_texts_for_properties, - insert_property_data, - prepare_properties_params, + insert_property_texts, + insert_property_values, + plan_property_inserts, ) db_with_topology.add_object(ClassEnum.Generator, "gen-01") @@ -113,18 +146,25 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) } ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params, metadata_map) - add_texts_for_properties( - db_with_topology, params, data_id_map, records, "datafile_text", ClassEnum.DataFile + data_id_map = insert_property_values(db_with_topology, params, metadata_map=metadata_map) + insert_property_texts( + db_with_topology, + params, + data_id_map=data_id_map, + records=records, + field_name="datafile_text", + text_class=ClassEnum.DataFile, ) # Verify text was added by checking database @@ -132,11 +172,59 @@ def test_add_texts_for_properties_with_datafile_text(db_with_topology: PlexosDB) assert text_records[0][0] > 0 -def test_prepare_properties_params_raises_error_when_no_memberships(db_with_topology: PlexosDB) -> None: - """Test that prepare_properties_params raises error when objects don't exist.""" +def test_insert_property_texts_handles_missing_names_and_prop_level( + db_instance_with_schema: PlexosDB, +) -> None: + """Text mapping should ignore nameless records and accept property-level text.""" + from plexosdb import ClassEnum, CollectionEnum + from plexosdb.utils import ( + insert_property_texts, + insert_property_values, + plan_property_inserts, + ) + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "TextGen") + + records = [ + { + "name": "TextGen", + "properties": {"Max Capacity": {"value": 5.0, "datafile_text": "prop-level.txt"}}, + } + ] + + prepared = plan_property_inserts( + db, + records, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + ) + params = prepared.params + metadata_map = prepared.metadata_map + records_with_extra = [*records, {"name": None, "datafile_text": "skip-me"}] + + with db._db.transaction(): + data_id_map = insert_property_values(db, params, metadata_map=metadata_map) + insert_property_texts( + db, + params, + data_id_map=data_id_map, + records=records_with_extra, + field_name="datafile_text", + text_class=ClassEnum.DataFile, + metadata_map=metadata_map, + ) + + text_rows = db._db.fetchall("SELECT value FROM t_text") + assert text_rows == [("prop-level.txt",)] + + +def test_plan_property_inserts_raises_error_when_no_memberships(db_with_topology: PlexosDB) -> None: + """Test that plan_property_inserts raises error when objects don't exist.""" from plexosdb import ClassEnum, CollectionEnum from plexosdb.exceptions import NotFoundError - from plexosdb.utils import prepare_properties_params + from plexosdb.utils import plan_property_inserts # Don't add objects to database - they should not exist records = [ @@ -146,19 +234,19 @@ def test_prepare_properties_params_raises_error_when_no_memberships(db_with_topo # Raises NotFoundError when objects don't exist in database with pytest.raises(NotFoundError): - prepare_properties_params( + plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) -def test_prepare_properties_params_empty_collection_properties(db_with_topology: PlexosDB) -> None: - """Test prepare_properties_params with valid objects but no properties in collection.""" +def test_plan_property_inserts_empty_collection_properties(db_with_topology: PlexosDB) -> None: + """Test plan_property_inserts with valid objects but no properties in collection.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import prepare_properties_params + from plexosdb.utils import plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") @@ -167,13 +255,16 @@ def test_prepare_properties_params_empty_collection_properties(db_with_topology: {"name": "gen-01", "properties": {"NonexistentProperty": {"value": 100.0}}}, ] - params, collection_properties, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + collection_properties = prepared.collection_properties + metadata_map = prepared.metadata_map # Should return empty params since property doesn't exist in collection assert params == [] @@ -181,10 +272,10 @@ def test_prepare_properties_params_empty_collection_properties(db_with_topology: assert isinstance(metadata_map, dict) -def test_prepare_properties_params_multiple_records_single_valid(db_with_topology: PlexosDB) -> None: - """Test prepare_properties_params with multiple records but only some have valid properties.""" +def test_plan_property_inserts_multiple_records_single_valid(db_with_topology: PlexosDB) -> None: + """Test plan_property_inserts with multiple records but only some have valid properties.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import prepare_properties_params + from plexosdb.utils import plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") db_with_topology.add_object(ClassEnum.Generator, "gen-02") @@ -194,13 +285,15 @@ def test_prepare_properties_params_multiple_records_single_valid(db_with_topolog {"name": "gen-02", "properties": {"NonexistentProperty": {"value": 150.0}}}, ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map # Should only have params for gen-01 with Max Capacity assert len(params) >= 1 @@ -208,46 +301,145 @@ def test_prepare_properties_params_multiple_records_single_valid(db_with_topolog assert isinstance(metadata_map, dict) -def test_prepare_properties_params_return_structure(db_with_topology: PlexosDB) -> None: - """Test that prepare_properties_params returns correct tuple structure.""" +def test_plan_property_inserts_return_structure(db_with_topology: PlexosDB) -> None: + """Test that plan_property_inserts returns structured result.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import prepare_properties_params + from plexosdb.utils import plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - result = prepare_properties_params( + result = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) - # Check that result is a tuple with 3 elements - assert isinstance(result, tuple) - assert len(result) == 3 - - params, collection_properties, metadata_map = result + assert hasattr(result, "params") + assert hasattr(result, "collection_properties") + assert hasattr(result, "metadata_map") + assert hasattr(result, "normalized_records") # Check params structure - assert isinstance(params, list) - if len(params) > 0: - assert isinstance(params[0], tuple) - assert len(params[0]) == 3 # (membership_id, property_id, value) + assert isinstance(result.params, list) + if len(result.params) > 0: + assert isinstance(result.params[0], tuple) + assert len(result.params[0]) == 3 # (membership_id, property_id, value) # Check collection_properties structure - assert isinstance(collection_properties, list) + assert isinstance(result.collection_properties, list) # Check metadata_map structure - assert isinstance(metadata_map, dict) + assert isinstance(result.metadata_map, dict) -def test_insert_property_data_updates_multiple_properties(db_with_topology: PlexosDB) -> None: - """Test that insert_property_data marks multiple properties as dynamic and enabled.""" +def test_plan_property_inserts_handles_simple_values(db_instance_with_schema: PlexosDB) -> None: + """Ensure simple (non-dict) property values are normalized.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, prepare_properties_params + from plexosdb.utils import plan_property_inserts + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "SimpleGen") + + records = [{"name": "SimpleGen", "properties": {"Max Capacity": 99.0}}] + + prepared = plan_property_inserts( + db, + records, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + ) + + assert len(prepared.params) == 1 + meta = next(iter(prepared.metadata_map.values())) + assert meta["band"] is None and meta["date_from"] is None and meta["date_to"] is None + + +def test_plan_property_inserts_invalid_record_raises(db_instance_with_schema: PlexosDB) -> None: + """Invalid payload lacking property info should raise ValueError.""" + from plexosdb import ClassEnum, CollectionEnum + from plexosdb.utils import plan_property_inserts + + db = db_instance_with_schema + with pytest.raises(ValueError): + plan_property_inserts( + db, + [{"name": "NoProperty"}], + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + ) + + +def test_plan_property_inserts_empty_records(db_instance_with_schema: PlexosDB) -> None: + """Empty input returns empty parameter collections.""" + from plexosdb import ClassEnum, CollectionEnum + from plexosdb.utils import plan_property_inserts + + db = db_instance_with_schema + prepared = plan_property_inserts( + db, + [], + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + ) + assert prepared.params == [] and prepared.collection_properties == [] and prepared.metadata_map == {} + + +def test_plan_property_inserts_membership_missing(monkeypatch, db_instance_with_schema: PlexosDB) -> None: + """Raises NotFoundError when memberships lookup returns empty.""" + from plexosdb import ClassEnum, CollectionEnum + from plexosdb.exceptions import NotFoundError + from plexosdb.utils import plan_property_inserts + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "GhostGen") + monkeypatch.setattr(db, "get_memberships_system", lambda *args, **kwargs: []) + + with pytest.raises(NotFoundError, match="Objects not found: GhostGen"): + plan_property_inserts( + db, + [{"name": "GhostGen", "properties": {"Max Capacity": {"value": 10}}}], + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + ) + + +def test_plan_property_inserts_skips_missing_membership( + monkeypatch, db_instance_with_schema: PlexosDB +) -> None: + """Records without matching membership are skipped gracefully.""" + from plexosdb import ClassEnum, CollectionEnum + from plexosdb.utils import plan_property_inserts + + db = db_instance_with_schema + db.add_object(ClassEnum.Generator, "HasMembership") + + fake_memberships = [{"name": "OtherObject", "membership_id": 999}] + monkeypatch.setattr(db, "get_memberships_system", lambda *args, **kwargs: fake_memberships) + + prepared = plan_property_inserts( + db, + [{"name": "HasMembership", "properties": {"Max Capacity": {"value": 42}}}], + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, + ) + + assert prepared.params == [] + assert prepared.metadata_map == {} + + +def test_insert_property_values_updates_multiple_properties(db_with_topology: PlexosDB) -> None: + """Test that insert_property_values marks multiple properties as dynamic and enabled.""" + from plexosdb import ClassEnum, CollectionEnum + from plexosdb.utils import insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") db_with_topology.add_object(ClassEnum.Generator, "gen-02") @@ -263,16 +455,18 @@ def test_insert_property_data_updates_multiple_properties(db_with_topology: Plex }, ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Verify properties are marked as dynamic and enabled properties = db_with_topology.query( @@ -281,50 +475,54 @@ def test_insert_property_data_updates_multiple_properties(db_with_topology: Plex assert len(properties) >= 2 -def test_insert_property_data_inserts_data_correctly(db_with_topology: PlexosDB) -> None: - """Test that insert_property_data inserts data rows into t_data table.""" +def test_insert_property_values_inserts_data_correctly(db_with_topology: PlexosDB) -> None: + """Test that insert_property_values inserts data rows into t_data table.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, prepare_properties_params + from plexosdb.utils import insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Verify data was inserted data_count = db_with_topology.query("SELECT COUNT(*) FROM t_data") assert data_count[0][0] > 0 -def test_insert_property_data_builds_data_id_map(db_with_topology: PlexosDB) -> None: - """Test that insert_property_data builds correct data_id_map.""" +def test_insert_property_values_builds_data_id_map(db_with_topology: PlexosDB) -> None: + """Test that insert_property_values builds correct data_id_map.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, prepare_properties_params + from plexosdb.utils import insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params, metadata_map) + data_id_map = insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Verify data_id_map structure assert isinstance(data_id_map, dict) @@ -333,78 +531,82 @@ def test_insert_property_data_builds_data_id_map(db_with_topology: PlexosDB) -> assert len(value) == 2 # (data_id, obj_name) -def test_insert_property_data_handles_null_values(db_with_topology: PlexosDB) -> None: - """Test that insert_property_data handles None/NULL values correctly.""" +def test_insert_property_values_handles_null_values(db_with_topology: PlexosDB) -> None: + """Test that insert_property_values handles None/NULL values correctly.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, prepare_properties_params + from plexosdb.utils import insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": None}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params, metadata_map) + data_id_map = insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Should handle NULL values without error assert isinstance(data_id_map, dict) -def test_insert_property_data_empty_params_returns_empty_map(db_with_topology: PlexosDB) -> None: - """Test that insert_property_data returns empty map when params is empty.""" - from plexosdb.utils import insert_property_data +def test_insert_property_values_empty_params_returns_empty_map(db_with_topology: PlexosDB) -> None: + """Test that insert_property_values returns empty map when params is empty.""" + from plexosdb.utils import insert_property_values with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, [], None) + data_id_map = insert_property_values(db_with_topology, [], metadata_map=None) assert data_id_map == {} -def test_insert_scenario_tags_early_return_when_scenario_none(db_with_topology: PlexosDB) -> None: - """Test that insert_scenario_tags early returns when scenario is None.""" - from plexosdb.utils import insert_scenario_tags +def test_apply_scenario_tags_early_return_when_scenario_none(db_with_topology: PlexosDB) -> None: + """Test that apply_scenario_tags early returns when scenario is None.""" + from plexosdb.utils import apply_scenario_tags # Should not raise error and should return early when scenario is None - insert_scenario_tags(db_with_topology, None, [], chunksize=1000) # type: ignore[arg-type] + apply_scenario_tags(db_with_topology, [], scenario=None, chunksize=1000) # type: ignore[arg-type] -def test_insert_scenario_tags_creates_new_scenario(db_with_topology: PlexosDB) -> None: - """Test that insert_scenario_tags creates scenario if it doesn't exist.""" +def test_apply_scenario_tags_creates_new_scenario(db_with_topology: PlexosDB) -> None: + """Test that apply_scenario_tags creates scenario if it doesn't exist.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, insert_scenario_tags, prepare_properties_params + from plexosdb.utils import insert_property_values, apply_scenario_tags, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) - insert_scenario_tags(db_with_topology, "NewScenario", params, chunksize=1000) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) + apply_scenario_tags(db_with_topology, params, scenario="NewScenario", chunksize=1000) # Verify scenario was created scenario_exists = db_with_topology.check_scenario_exists("NewScenario") assert scenario_exists is True -def test_insert_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) -> None: - """Test that insert_scenario_tags uses existing scenario instead of creating new one.""" +def test_apply_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) -> None: + """Test that apply_scenario_tags uses existing scenario instead of creating new one.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, insert_scenario_tags, prepare_properties_params + from plexosdb.utils import insert_property_values, apply_scenario_tags, plan_property_inserts # Pre-create scenario scenario_id_before = db_with_topology.add_scenario("ExistingScenario") @@ -413,27 +615,29 @@ def test_insert_scenario_tags_uses_existing_scenario(db_with_topology: PlexosDB) records = [{"name": "gen-01", "properties": {"Max Capacity": {"value": 100.0}}}] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) - insert_scenario_tags(db_with_topology, "ExistingScenario", params, chunksize=1000) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) + apply_scenario_tags(db_with_topology, params, scenario="ExistingScenario", chunksize=1000) # Verify scenario still exists and wasn't duplicated scenario_id_after = db_with_topology.get_scenario_id("ExistingScenario") assert scenario_id_after == scenario_id_before -def test_insert_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) -> None: - """Test insert_scenario_tags with params less than chunksize.""" +def test_apply_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) -> None: + """Test apply_scenario_tags with params less than chunksize.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, insert_scenario_tags, prepare_properties_params + from plexosdb.utils import insert_property_values, apply_scenario_tags, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") @@ -451,28 +655,30 @@ def test_insert_scenario_tags_batching_single_batch(db_with_topology: PlexosDB) } ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Large chunksize - should process all in one batch - insert_scenario_tags(db_with_topology, "BatchTest", params, chunksize=1000) + apply_scenario_tags(db_with_topology, params, scenario="BatchTest", chunksize=1000) # Verify tags were inserted tag_count = db_with_topology.query("SELECT COUNT(*) FROM t_tag") assert tag_count[0][0] > 0 -def test_insert_scenario_tags_batching_multiple_batches(db_with_topology: PlexosDB) -> None: - """Test insert_scenario_tags with params split into multiple batches.""" +def test_apply_scenario_tags_batching_multiple_batches(db_with_topology: PlexosDB) -> None: + """Test apply_scenario_tags with params split into multiple batches.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import insert_property_data, insert_scenario_tags, prepare_properties_params + from plexosdb.utils import insert_property_values, apply_scenario_tags, plan_property_inserts for i in range(3): db_with_topology.add_object(ClassEnum.Generator, f"gen-{i:02d}") @@ -488,41 +694,43 @@ def test_insert_scenario_tags_batching_multiple_batches(db_with_topology: Plexos for i in range(3) ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Small chunksize to force multiple batches - insert_scenario_tags(db_with_topology, "BatchTest", params, chunksize=2) + apply_scenario_tags(db_with_topology, params, scenario="BatchTest", chunksize=2) # Verify tags were inserted tag_count = db_with_topology.query("SELECT COUNT(*) FROM t_tag") assert tag_count[0][0] > 0 -def test_insert_scenario_tags_empty_params(db_with_topology: PlexosDB) -> None: - """Test insert_scenario_tags with empty params list.""" - from plexosdb.utils import insert_scenario_tags +def test_apply_scenario_tags_empty_params(db_with_topology: PlexosDB) -> None: + """Test apply_scenario_tags with empty params list.""" + from plexosdb.utils import apply_scenario_tags with db_with_topology._db.transaction(): # Should not raise error with empty params - insert_scenario_tags(db_with_topology, "EmptyTest", [], chunksize=1000) + apply_scenario_tags(db_with_topology, [], scenario="EmptyTest", chunksize=1000) # Verify scenario was still created scenario_exists = db_with_topology.check_scenario_exists("EmptyTest") assert scenario_exists is True -def test_add_texts_for_properties_skips_records_without_field(db_with_topology: PlexosDB) -> None: - """Test that add_texts_for_properties skips records without specified field.""" +def test_insert_property_texts_skips_records_without_field(db_with_topology: PlexosDB) -> None: + """Test that insert_property_texts skips records without specified field.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import add_texts_for_properties, insert_property_data, prepare_properties_params + from plexosdb.utils import insert_property_texts, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") @@ -531,29 +739,36 @@ def test_add_texts_for_properties_skips_records_without_field(db_with_topology: # No datafile_text field in record ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params, metadata_map) + data_id_map = insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Call with field that doesn't exist in all records - add_texts_for_properties( - db_with_topology, params, data_id_map, records, "datafile_text", ClassEnum.DataFile + insert_property_texts( + db_with_topology, + params, + data_id_map=data_id_map, + records=records, + field_name="datafile_text", + text_class=ClassEnum.DataFile, ) # Should not raise error -def test_add_texts_for_properties_handles_data_id_none(db_with_topology: PlexosDB) -> None: - """Test that add_texts_for_properties handles missing data_id in map.""" +def test_insert_property_texts_handles_data_id_none(db_with_topology: PlexosDB) -> None: + """Test that insert_property_texts handles missing data_id in map.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import add_texts_for_properties, insert_property_data, prepare_properties_params + from plexosdb.utils import insert_property_texts, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") @@ -565,30 +780,37 @@ def test_add_texts_for_properties_handles_data_id_none(db_with_topology: PlexosD } ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - insert_property_data(db_with_topology, params, metadata_map) + insert_property_values(db_with_topology, params, metadata_map=metadata_map) # Create empty map - simulating missing data_ids empty_data_id_map: dict[tuple[int, int, int], tuple[int, str]] = {} # Should not raise error when data_id is missing - add_texts_for_properties( - db_with_topology, params, empty_data_id_map, records, "datafile_text", ClassEnum.DataFile + insert_property_texts( + db_with_topology, + params, + data_id_map=empty_data_id_map, + records=records, + field_name="datafile_text", + text_class=ClassEnum.DataFile, ) -def test_add_texts_for_properties_multiple_texts(db_with_topology: PlexosDB) -> None: - """Test that add_texts_for_properties handles multiple text records.""" +def test_insert_property_texts_multiple_texts(db_with_topology: PlexosDB) -> None: + """Test that insert_property_texts handles multiple text records.""" from plexosdb import ClassEnum, CollectionEnum - from plexosdb.utils import add_texts_for_properties, insert_property_data, prepare_properties_params + from plexosdb.utils import insert_property_texts, insert_property_values, plan_property_inserts db_with_topology.add_object(ClassEnum.Generator, "gen-01") db_with_topology.add_object(ClassEnum.Generator, "gen-02") @@ -606,47 +828,61 @@ def test_add_texts_for_properties_multiple_texts(db_with_topology: PlexosDB) -> }, ] - params, _, metadata_map = prepare_properties_params( + prepared = plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, ) + params = prepared.params + metadata_map = prepared.metadata_map with db_with_topology._db.transaction(): - data_id_map = insert_property_data(db_with_topology, params, metadata_map) + data_id_map = insert_property_values(db_with_topology, params, metadata_map=metadata_map) - add_texts_for_properties( - db_with_topology, params, data_id_map, records, "datafile_text", ClassEnum.DataFile + insert_property_texts( + db_with_topology, + params, + data_id_map=data_id_map, + records=records, + field_name="datafile_text", + text_class=ClassEnum.DataFile, ) text_count = db_with_topology.query("SELECT COUNT(*) FROM t_text") assert text_count[0][0] >= 2 -def test_add_texts_for_properties_empty_params(db_with_topology: PlexosDB) -> None: - """Test add_texts_for_properties with empty inputs.""" +def test_insert_property_texts_empty_params(db_with_topology: PlexosDB) -> None: + """Test insert_property_texts with empty inputs.""" from plexosdb import ClassEnum - from plexosdb.utils import add_texts_for_properties + from plexosdb.utils import insert_property_texts - add_texts_for_properties(db_with_topology, [], {}, [], "datafile_text", ClassEnum.DataFile) + insert_property_texts( + db_with_topology, + [], + data_id_map={}, + records=[], + field_name="datafile_text", + text_class=ClassEnum.DataFile, + ) -def test_prepare_properties_params_raises_on_no_memberships(db_with_topology: PlexosDB) -> None: - """Test prepare_properties_params raises NotFoundError when no memberships exist.""" +def test_plan_property_inserts_raises_on_no_memberships(db_with_topology: PlexosDB) -> None: + """Test plan_property_inserts raises NotFoundError when no memberships exist.""" from plexosdb import ClassEnum, CollectionEnum from plexosdb.exceptions import NotFoundError - from plexosdb.utils import prepare_properties_params + from plexosdb.utils import plan_property_inserts # Try to prepare params for object that doesn't exist records = [{"name": "NonExistentObject", "properties": {"property": {"value": 100}}}] - with pytest.raises(NotFoundError, match="Object = NonExistentObject not found"): - prepare_properties_params( + with pytest.raises(NotFoundError, match="Objects not found: NonExistentObject"): + plan_property_inserts( db_with_topology, records, - ClassEnum.Generator, - CollectionEnum.Generators, - ClassEnum.System, + object_class=ClassEnum.Generator, + collection=CollectionEnum.Generators, + parent_class=ClassEnum.System, )