diff --git a/simvue/api/objects/filter.py b/simvue/api/objects/filter.py new file mode 100644 index 00000000..214a9bb9 --- /dev/null +++ b/simvue/api/objects/filter.py @@ -0,0 +1,298 @@ +import abc +from collections.abc import Generator +import enum +import json +import sys + +import pydantic + +if sys.version_info < (3, 11): + from typing_extensions import Self, TYPE_CHECKING +else: + from typing import Self, TYPE_CHECKING + + +if TYPE_CHECKING: + from .base import SimvueObject + + +class Status(str, enum.Enum): + Created = "created" + Running = "running" + Completed = "completed" + Lost = "lost" + Terminated = "terminated" + Failed = "failed" + + +class Time(str, enum.Enum): + Created = "created" + Started = "started" + Modified = "modified" + Ended = "ended" + + +class System(str, enum.Enum): + Working_Directory = "cwd" + Hostname = "hostname" + Python_Version = "pythonversion" + Platform_System = "platform.system" + Platform_Release = "platform.release" + Platform_Version = "platform.version" + CPU_Architecture = "cpu.arch" + CPU_Processor = "cpu.processor" + GPU_Name = "gpu.name" + GPU_Driver = "gpu.driver" + + +class RestAPIFilter(abc.ABC): + """RestAPI query filter object.""" + + def __init__(self, simvue_object: "type[SimvueObject] | None" = None) -> None: + """Initialise a query object using a Simvue object class.""" + self._sv_object: "type[SimvueObject] | None" = simvue_object + self._filters: list[str] = [] + self._generate_members() + + def _time_within( + self, time_type: Time, *, hours: int = 0, days: int = 0, years: int = 0 + ) -> Self: + """Define filter using time range.""" + if len(_non_zero := list(i for i in (hours, days, years) if i != 0)) > 1: + raise AssertionError( + "Only one duration type may be provided: hours, days or years" + ) + if len(_non_zero) < 1: + raise AssertionError( + f"No duration provided for filter '{time_type.value}_within'" + ) + + if hours: + self._filters.append(f"{time_type.value} < {hours}h") + elif days: + self._filters.append(f"{time_type.value} < {days}d") + else: + self._filters.append(f"{time_type.value} < {years}y") + return self + + @abc.abstractmethod + def _generate_members(self) -> None: + """Generate filters using specified definitions.""" + + def has_name(self, name: str) -> Self: + """Filter based on absolute object name.""" + self._filters.append(f"name == {name}") + return self + + def has_name_containing(self, name: str) -> Self: + """Filter base on object name containing a term.""" + self._filters.append(f"name contains {name}") + return self + + def created_within(self, *, hours: int = 0, days: int = 0, years: int = 0) -> Self: + """Find objects created within the last specified time period.""" + return self._time_within(Time.Created, hours=hours, days=days, years=years) + + def has_description_containing(self, search_str: str) -> Self: + """Return objects containing the specified term within the description.""" + self._filters.append(f"description contains {search_str}") + return self + + def exclude_description_containing(self, search_str: str) -> Self: + """Find objects not containing the specified term in their description.""" + self._filters.append(f"description not contains {search_str}") + return self + + def has_tag(self, tag: str) -> Self: + """Find objects with the given tag.""" + self._filters.append(f"has tag.{tag}") + return self + + def starred(self) -> Self: + self._filters.append("starred") + return self + + def as_list(self) -> list[str]: + """Returns the filters as a list.""" + return self._filters + + def clear(self) -> None: + """Clear all current filters.""" + self._filters = [] + + def get( + self, + count: pydantic.PositiveInt | None = None, + offset: pydantic.NonNegativeInt | None = None, + **kwargs, + ) -> Generator[tuple[str, "SimvueObject | None"], None, None]: + """Call the get method from the simvue object class.""" + if not self._sv_object: + raise RuntimeError("No object type associated with filter.") + _filters: str = json.dumps(self._filters) + return self._sv_object.get( + count=count, offset=offset, filters=_filters, **kwargs + ) + + def count(self, **kwargs) -> int: + if not self._sv_object: + raise RuntimeError("No object type associated with filter.") + _ = kwargs.pop("count", None) + _filters: str = json.dumps(self._filters) + return self._sv_object.count(filters=_filters, **kwargs) + + +class FoldersFilter(RestAPIFilter): + def has_path(self, name: str) -> "FoldersFilter": + self._filters.append(f"path == {name}") + return self + + def has_path_containing(self, name: str) -> "FoldersFilter": + self._filters.append(f"path contains {name}") + return self + + def _generate_members(self) -> None: + return super()._generate_members() + + +class RunsFilter(RestAPIFilter): + def _generate_members(self) -> None: + _global_comparators = [self._value_contains, self._value_eq, self._value_neq] + + _numeric_comparators = [ + self._value_geq, + self._value_leq, + self._value_lt, + self._value_gt, + ] + + for label, system_spec in System.__members__.items(): + for function in _global_comparators: + _label: str = label.lower() + _func_name: str = function.__name__.replace("_value", _label) + + def _out_func(value: str | int | float, func=function) -> Self: + return func("system", system_spec.value, value) + + _out_func.__name__ = _func_name + setattr(self, _func_name, _out_func) + + for function in _global_comparators + _numeric_comparators: + _func_name = function.__name__.replace("_value", "metadata") + + def _out_func( + attribute: str, value: str | int | float, func=function + ) -> Self: + return func("metadata", attribute, value) + + _out_func.__name__ = _func_name + setattr(self, _func_name, _out_func) + + def owner(self, username: str = "self") -> "RunsFilter": + self._filters.append(f"user == {username}") + return self + + def exclude_owner(self, username: str = "self") -> "RunsFilter": + self._filters.append(f"user != {username}") + return self + + def has_status(self, status: Status) -> "RunsFilter": + self._filters.append(f"status == {status.value}") + return self + + def is_running(self) -> "RunsFilter": + return self.has_status(Status.Running) + + def is_lost(self) -> "RunsFilter": + return self.has_status(Status.Lost) + + def has_completed(self) -> "RunsFilter": + return self.has_status(Status.Completed) + + def has_failed(self) -> "RunsFilter": + return self.has_status(Status.Failed) + + def has_alert( + self, alert_name: str, is_critical: bool | None = None + ) -> "RunsFilter": + self._filters.append(f"alert.name == {alert_name}") + if is_critical is True: + self._filters.append("alert.status == critical") + elif is_critical is False: + self._filters.append("alert.status == ok") + return self + + def started_within( + self, *, hours: int = 0, days: int = 0, years: int = 0 + ) -> "RunsFilter": + return self._time_within(Time.Started, hours=hours, days=days, years=years) + + def modified_within( + self, *, hours: int = 0, days: int = 0, years: int = 0 + ) -> "RunsFilter": + return self._time_within(Time.Modified, hours=hours, days=days, years=years) + + def ended_within( + self, *, hours: int = 0, days: int = 0, years: int = 0 + ) -> "RunsFilter": + return self._time_within(Time.Ended, hours=hours, days=days, years=years) + + def in_folder(self, folder_name: str) -> "RunsFilter": + self._filters.append(f"folder.path == {folder_name}") + return self + + def has_metadata_attribute(self, attribute: str) -> "RunsFilter": + self._filters.append(f"metadata.{attribute} exists") + return self + + def exclude_metadata_attribute(self, attribute: str) -> "RunsFilter": + self._filters.append(f"metadata.{attribute} not exists") + return self + + def _value_eq( + self, category: str, attribute: str, value: str | int | float + ) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} == {value}") + return self + + def _value_neq( + self, category: str, attribute: str, value: str | int | float + ) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} != {value}") + return self + + def _value_contains( + self, category: str, attribute: str, value: str | int | float + ) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} contains {value}") + return self + + def _value_leq( + self, category: str, attribute: str, value: int | float + ) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} <= {value}") + return self + + def _value_geq( + self, category: str, attribute: str, value: int | float + ) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} >= {value}") + return self + + def _value_lt( + self, category: str, attribute: str, value: int | float + ) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} < {value}") + return self + + def _value_gt( + self, category: str, attribute: str, value: int | float + ) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} > {value}") + return self + + def __str__(self) -> str: + return " && ".join(self._filters) if self._filters else "None" + + def __repr__(self) -> str: + return f"{super().__repr__()[:-1]}, filters={self._filters}>" diff --git a/simvue/api/objects/folder.py b/simvue/api/objects/folder.py index 845368e3..165254e0 100644 --- a/simvue/api/objects/folder.py +++ b/simvue/api/objects/folder.py @@ -7,22 +7,25 @@ """ +import http import typing import datetime import json import pydantic +from simvue.api.objects.filter import FoldersFilter from simvue.exception import ObjectNotFoundError +from simvue.api.request import put as sv_put, get_json_from_response from .base import SimvueObject, staging_check, write_only, Sort from simvue.models import FOLDER_REGEX, DATETIME_FORMAT # Need to use this inside of Generator typing to fix bug present in Python 3.10 - see issue #745 try: - from typing import Self + from typing import Self, override except ImportError: - from typing_extensions import Self + from typing_extensions import Self, override __all__ = ["Folder"] @@ -94,6 +97,23 @@ def get( sorting: list[FolderSort] | None = None, **kwargs, ) -> typing.Generator[tuple[str, T | None], None, None]: + """Get folders from the server. + + Parameters + ---------- + count : int, optional + limit the number of objects returned, default no limit. + offset : int, optional + start index for results, default is 0. + sorting : list[dict] | None, optional + list of sorting definitions in the form {'column': str, 'descending': bool} + + Yields + ------ + tuple[str, Folder] + id of run + Folder object representing object on server + """ _params: dict[str, str] = kwargs if sorting: @@ -101,6 +121,19 @@ def get( return super().get(count=count, offset=offset, **_params) + @classmethod + def filter(cls) -> FoldersFilter: + _filter_instance: FoldersFilter = FoldersFilter(cls) + _filter_instance.get.__func__.__doc__ = cls.get.__func__.__doc__ + return _filter_instance + + @override + def commit(self) -> dict | list[dict] | None: + if "starred" in self._staging: + _star_run: bool = self._staging.pop("starred") + self._set_favourite(starred=_star_run) + return super().commit() + @property def tree(self) -> dict[str, object]: """Return hierarchy for this folder. @@ -233,6 +266,18 @@ def created(self) -> datetime.datetime | None: else None ) + def _set_favourite(self, *, starred: bool) -> dict: + """Set starred status.""" + _url = self.url / "starred" + _response = sv_put( + f"{_url}", headers=self._user_config.headers, data={"starred": starred} + ) + return get_json_from_response( + expected_status=[http.HTTPStatus.OK], + response=_response, + scenario=f"Applying favourite preference to folder '{self.id}'", + ) + @pydantic.validate_call def get_folder_from_path( diff --git a/simvue/api/objects/run.py b/simvue/api/objects/run.py index a8c6d9e2..5a9119eb 100644 --- a/simvue/api/objects/run.py +++ b/simvue/api/objects/run.py @@ -16,9 +16,9 @@ import json try: - from typing import Self + from typing import Self, override except ImportError: - from typing_extensions import Self + from typing_extensions import Self, override from .base import ( ObjectBatchArgs, @@ -29,6 +29,7 @@ Visibility, write_only, ) +from .filter import RunsFilter from simvue.api.request import ( get as sv_get, put as sv_put, @@ -95,6 +96,19 @@ def __init__(self, identifier: str | None = None, **kwargs) -> None: self.visibility = Visibility(self) super().__init__(identifier, **kwargs) + @classmethod + def filter(cls) -> RunsFilter: + _run_filter = RunsFilter(cls) + _run_filter.get.__func__.__doc__ = cls.get.__func__.__doc__ + return _run_filter + + @override + def commit(self) -> dict | list[dict] | None: + if "starred" in self._staging: + _star_run: bool = self._staging.pop("starred") + self._set_favourite(starred=_star_run) + return super().commit() + @classmethod @pydantic.validate_call def new( @@ -491,6 +505,31 @@ def started(self) -> datetime.datetime | None: def started(self, started: datetime.datetime) -> None: self._staging["started"] = simvue_timestamp(started) + @property + @staging_check + def star(self) -> bool: + """Return if this folder is starred""" + return self._get().get("starred", False) + + @star.setter + @write_only + @pydantic.validate_call + def star(self, is_true: bool = True) -> None: + """Star this folder as a favourite""" + self._staging["starred"] = is_true + + def _set_favourite(self, *, starred: bool) -> dict: + """Set starred status.""" + _url = self.url / "starred" + _response = sv_put( + f"{_url}", headers=self._user_config.headers, data={"starred": starred} + ) + return get_json_from_response( + expected_status=[http.HTTPStatus.OK], + response=_response, + scenario=f"Applying favourite preference to run '{self.id}'", + ) + @property @staging_check def endtime(self) -> datetime.datetime | None: diff --git a/tests/unit/test_folder.py b/tests/unit/test_folder.py index 8f0dd83a..2404369f 100644 --- a/tests/unit/test_folder.py +++ b/tests/unit/test_folder.py @@ -56,13 +56,28 @@ def test_folder_creation_offline(offline_cache_setup) -> None: @pytest.mark.api @pytest.mark.online -def test_get_folder_count() -> None: +@pytest.mark.parametrize( + "subset", (True, False), + ids=("subset", "all") +) +def test_get_folder_count(subset: bool) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder_1 = Folder.new(path=f"{_folder_name}/dir_1") + _folder_1.tags = [f"simvue_{_uuid}"] _folder_2 = Folder.new(path=f"{_folder_name}/dir_2") - assert len(list(Folder.get(count=2, offset=None))) == 2 - + _folder_1.commit() + _folder_2.commit() + _folder_1.star = True + _folder_1.commit() + if not subset: + assert len(list(Folder.get(count=2, offset=None))) == 2 + else: + _generator = Folder.filter().has_tag(f"simvue_{_uuid}").starred() + assert len(list(_generator.get())) == 1 + assert _generator.count() == 1 + _folder_1.delete() + _folder_2.delete() @pytest.mark.api @pytest.mark.online diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index 1ee2e6f9..d22046d3 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -154,13 +154,31 @@ def test_run_modification_offline(offline_cache_setup) -> None: @pytest.mark.api @pytest.mark.online -def test_get_run_count() -> None: +@pytest.mark.parametrize( + "subset", (True, False), + ids=("subset", "all") +) +def test_get_run_count(subset: bool) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name) + _folder.commit() _run_1 = Run.new(folder=_folder_name) _run_2 = Run.new(folder=_folder_name) - assert len(list(Run.get(count=2, offset=None))) == 2 + _run_1.tags = [f"run_1_{_uuid}"] + _run_1.metadata = {"uuid": _uuid} + _run_1.commit() + _run_1.star = True + _run_1.commit() + _run_2.commit() + if not subset: + assert len(list(Run.get(count=2, offset=None))) == 2 + else: + _generator = Run.filter().has_tag(f"run_1_{_uuid}").in_folder(_folder_name) + _generator = _generator.has_metadata_attribute("uuid").metadata_eq("uuid", _uuid).starred() + assert len(list(_generator.get())) == 1 + assert _generator.count() == 1 + _folder.delete(recursive=True, delete_runs=True, runs_only=False) @pytest.mark.api