From 852b0805a346bc64623554fb0e23303d1957ef37 Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Sat, 13 Dec 2025 11:40:07 +0100 Subject: [PATCH 01/16] Experiment with `TypedDict` in CWL types This commit checks that the new `TypedDict` type aliases specified in cwl-utils with PR common-workflow-language/cwl-utils#393 do not break anything in the cwltool type system. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6dcb90bb9..0ccb07436 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ coloredlogs pydot>=1.4.1 argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 -cwl-utils>=0.32 +cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head spython>=0.3.0 rich-argparse typing-extensions>=4.1.0 From 0f8280510f92998006f734421de7d7ec8bdc56bb Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Sat, 13 Dec 2025 13:03:46 +0100 Subject: [PATCH 02/16] Solving some typing problems --- cwltool/builder.py | 3 +-- cwltool/checker.py | 3 ++- cwltool/command_line_tool.py | 37 +++++++++++++++------------ cwltool/context.py | 3 ++- cwltool/cuda.py | 3 ++- cwltool/cwlprov/provenance_profile.py | 3 ++- cwltool/cwlprov/ro.py | 11 ++++---- cwltool/cwlprov/writablebagfile.py | 3 ++- cwltool/docker.py | 3 ++- cwltool/executors.py | 3 ++- cwltool/factory.py | 3 ++- cwltool/job.py | 4 +-- cwltool/load_tool.py | 3 ++- cwltool/main.py | 3 +-- cwltool/mutation.py | 3 ++- cwltool/pack.py | 2 +- cwltool/pathmapper.py | 3 ++- cwltool/process.py | 3 +-- cwltool/procgenerator.py | 3 ++- cwltool/secrets.py | 2 +- cwltool/singularity.py | 3 ++- cwltool/subgraph.py | 3 ++- cwltool/update.py | 3 ++- cwltool/utils.py | 17 ++---------- cwltool/workflow.py | 2 +- cwltool/workflow_job.py | 4 +-- tests/test_context.py | 3 ++- tests/test_cuda.py | 2 +- tests/test_examples.py | 3 ++- tests/test_http_input.py | 2 +- tests/test_load_tool.py | 2 +- tests/test_path_checks.py | 3 ++- tests/test_pathmapper.py | 3 ++- tests/test_secrets.py | 2 +- tests/test_streaming.py | 2 +- 35 files changed, 78 insertions(+), 77 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index cf6d6d5ac..6f32a4197 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -9,6 +9,7 @@ from cwl_utils import expression from cwl_utils.file_formats import check_format +from cwl_utils.types import CWLObjectType, CWLOutputType from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap @@ -26,8 +27,6 @@ from .stdfsaccess import StdFsAccess from .utils import ( CONTENT_LIMIT, - CWLObjectType, - CWLOutputType, HasReqsHints, LoadListingType, aslist, diff --git a/cwltool/checker.py b/cwltool/checker.py index d81642e1d..4e6541f9c 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -3,6 +3,7 @@ from collections.abc import Iterator, MutableMapping, MutableSequence, Sized from typing import Any, Literal, NamedTuple, Optional, Union, cast +from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno from schema_salad.utils import json_dumps @@ -10,7 +11,7 @@ from .errors import WorkflowException from .loghandler import _logger from .process import shortname -from .utils import CWLObjectType, CWLOutputType, SinkType, aslist +from .utils import aslist def _get_type(tp: Any) -> Any: diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index a9d5e57e2..20e799cda 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -24,6 +24,14 @@ from re import Pattern from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast +from cwl_utils.types import ( + CWLObjectType, + CWLOutputType, + DirectoryType, + is_directory, + is_file, + is_file_or_directory, +) from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import RecordSchema @@ -60,9 +68,6 @@ from .udocker import UDockerCommandLineJob from .update import ORDERED_VERSIONS, ORIGINAL_CWLVERSION from .utils import ( - CWLObjectType, - CWLOutputType, - DirectoryType, JobsGeneratorType, OutputCallbackType, adjustDirObjs, @@ -619,11 +624,8 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: et: CWLObjectType = {} writable = t.get("writable", False) et["writable"] = writable - if isinstance(entry, Mapping) and entry.get("class") in ( - "File", - "Directory", - ): - if writable and "secondaryFiles" in entry: + if is_file_or_directory(entry): + if writable and is_file(entry) and "secondaryFiles" in entry: secFiles = cast(MutableSequence[CWLObjectType], entry["secondaryFiles"]) for sf in secFiles: sf["writable"] = writable @@ -765,22 +767,23 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: for entry in ls: if "basename" in entry: basename = cast(str, entry["basename"]) - dirname = os.path.join(builder.outdir, os.path.dirname(basename)) - entry["dirname"] = dirname entry["basename"] = os.path.basename(basename) - if "secondaryFiles" in entry: - for sec_file in cast( - MutableSequence[CWLObjectType], entry["secondaryFiles"] - ): - sec_file["dirname"] = dirname + if is_file(entry): + dirname = os.path.join(builder.outdir, os.path.dirname(basename)) + entry["dirname"] = dirname + if "secondaryFiles" in entry: + for sec_file in cast( + MutableSequence[CWLObjectType], entry["secondaryFiles"] + ): + sec_file["dirname"] = dirname normalizeFilesDirs(entry) self.updatePathmap( - cast(Optional[str], entry.get("dirname")) or builder.outdir, + (entry.get("dirname") if is_file(entry) else None) or builder.outdir, cast(PathMapper, builder.pathmapper), entry, ) - if "listing" in entry: + if is_directory(entry) and "listing" in entry: def remove_dirname(d: CWLObjectType) -> None: if "dirname" in d: diff --git a/cwltool/context.py b/cwltool/context.py index 4e106ac48..44129b40f 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -8,6 +8,7 @@ from collections.abc import Callable, Iterable from typing import IO, TYPE_CHECKING, Any, Literal, Optional, TextIO, Union +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names from schema_salad.ref_resolver import Loader @@ -16,7 +17,7 @@ from .mpi import MpiConfig from .pathmapper import PathMapper from .stdfsaccess import StdFsAccess -from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType +from .utils import DEFAULT_TMP_PREFIX, HasReqsHints, ResolverType if TYPE_CHECKING: from _typeshed import SupportsWrite diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 86dcb3cdd..ec3635373 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -3,8 +3,9 @@ import subprocess # nosec import xml.dom.minidom # nosec +from cwl_utils.types import CWLObjectType + from .loghandler import _logger -from .utils import CWLObjectType def cuda_version_and_device_count() -> tuple[str, int]: diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index c0aa5ec5b..0bd2b220c 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -8,6 +8,7 @@ from pathlib import PurePath, PurePosixPath from typing import TYPE_CHECKING, Any, cast +from cwl_utils.types import CWLObjectType from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity from schema_salad.sourceline import SourceLine @@ -17,7 +18,7 @@ from ..loghandler import _logger from ..process import Process, shortname from ..stdfsaccess import StdFsAccess -from ..utils import CWLObjectType, JobsType, get_listing, posix_path, versionstring +from ..utils import JobsType, get_listing, posix_path, versionstring from ..workflow_job import WorkflowJob from .provenance_constants import ( ACCOUNT_UUID, diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 28b7c86df..783d3cf1f 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -13,13 +13,12 @@ from typing import IO, TYPE_CHECKING, Any, Optional, cast import prov.model as provM +from cwl_utils.types import CWLObjectType, CWLOutputType, is_directory, is_file from prov.model import ProvDocument from ..loghandler import _logger from ..stdfsaccess import StdFsAccess from ..utils import ( - CWLObjectType, - CWLOutputType, create_tmp_dir, local_path, posix_path, @@ -642,10 +641,10 @@ def _relativise_files( _logger.debug("[provenance] Relativising: %s", structure) if isinstance(structure, MutableMapping): - if structure.get("class") == "File": + if is_file(structure): relative_path: str | PurePosixPath | None = None if "checksum" in structure: - raw_checksum = cast(str, structure["checksum"]) + raw_checksum = structure["checksum"] alg, checksum = raw_checksum.split("$") if alg != SHA1: raise TypeError( @@ -659,7 +658,7 @@ def _relativise_files( # Register in RO; but why was this not picked # up by used_artefacts? _logger.info("[provenance] Adding to RO %s", structure["location"]) - with self.fsaccess.open(cast(str, structure["location"]), "rb") as fp: + with self.fsaccess.open(structure["location"], "rb") as fp: relative_path = self.add_data_file(fp) checksum = PurePosixPath(relative_path).name structure["checksum"] = f"{SHA1}${checksum}" @@ -668,7 +667,7 @@ def _relativise_files( if "path" in structure: del structure["path"] - if structure.get("class") == "Directory": + if is_directory(structure): # TODO: Generate anonymous Directory with a "listing" # pointing to the hashed files del structure["location"] diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index ecd6463d6..f0f7fb323 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -14,10 +14,11 @@ from pathlib import Path, PurePosixPath from typing import Any, BinaryIO, cast +from cwl_utils.types import CWLObjectType from schema_salad.utils import json_dumps from ..loghandler import _logger -from ..utils import CWLObjectType, local_path, posix_path +from ..utils import local_path, posix_path from .provenance_constants import ( CWLPROV, CWLPROV_VERSION, diff --git a/cwltool/docker.py b/cwltool/docker.py index 69fecc830..7dd15d7d9 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -14,6 +14,7 @@ from typing import Optional, cast import requests +from cwl_utils.types import CWLObjectType from .builder import Builder from .context import RuntimeContext @@ -22,7 +23,7 @@ from .job import ContainerCommandLineJob from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper -from .utils import CWLObjectType, create_tmp_dir, ensure_writable +from .utils import create_tmp_dir, ensure_writable _IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() diff --git a/cwltool/executors.py b/cwltool/executors.py index 491078e83..8bcdfa519 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -13,6 +13,7 @@ from typing import Optional, cast import psutil +from cwl_utils.types import CWLObjectType from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine @@ -27,7 +28,7 @@ from .process import Process, cleanIntermediate, relocateOutputs from .task_queue import TaskQueue from .update import ORIGINAL_CWLVERSION -from .utils import CWLObjectType, JobsType +from .utils import JobsType from .workflow import Workflow from .workflow_job import WorkflowJob, WorkflowJobStep diff --git a/cwltool/factory.py b/cwltool/factory.py index fc00eb061..aee4abb6f 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,12 +1,13 @@ import os from typing import Any +from cwl_utils.types import CWLObjectType + from . import load_tool from .context import LoadingContext, RuntimeContext from .errors import WorkflowException from .executors import JobExecutor, SingleJobExecutor from .process import Process -from .utils import CWLObjectType class WorkflowStatus(Exception): diff --git a/cwltool/job.py b/cwltool/job.py index 4f49c95fe..091a0dbe7 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -21,6 +21,7 @@ from typing import IO, TYPE_CHECKING, Optional, TextIO, Union, cast import psutil +from cwl_utils.types import CWLObjectType, CWLOutputType, DirectoryType from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps @@ -35,9 +36,6 @@ from .process import stage_files from .secrets import SecretStore from .utils import ( - CWLObjectType, - CWLOutputType, - DirectoryType, HasReqsHints, OutputCallbackType, bytes2str_in_dicts, diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 853c203be..4c34094c6 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -12,6 +12,7 @@ from typing import Any, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.fetcher import Fetcher @@ -32,7 +33,7 @@ from .loghandler import _logger from .process import Process, get_schema, shortname from .update import ALLUPDATES -from .utils import CWLObjectType, ResolverType, visit_class +from .utils import ResolverType, visit_class docloaderctx: ContextType = { "cwl": "https://w3id.org/cwl/cwl#", diff --git a/cwltool/main.py b/cwltool/main.py index b8d74ded0..3716cd141 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -23,6 +23,7 @@ import coloredlogs import requests import ruamel.yaml +from cwl_utils.types import CWLObjectType, CWLOutputType from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML @@ -89,8 +90,6 @@ from .update import ALLUPDATES, UPDATES from .utils import ( DEFAULT_TMP_PREFIX, - CWLObjectType, - CWLOutputType, HasReqsHints, adjustDirObjs, normalizeFilesDirs, diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 622807ec6..b3da0a004 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -2,8 +2,9 @@ from typing import NamedTuple, cast +from cwl_utils.types import CWLObjectType + from .errors import WorkflowException -from .utils import CWLObjectType class _MutationState(NamedTuple): diff --git a/cwltool/pack.py b/cwltool/pack.py index 8c250c51b..686a463c6 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -5,6 +5,7 @@ from collections.abc import Callable, MutableMapping, MutableSequence from typing import Any, Optional, Union, cast +from cwl_utils.types import CWLObjectType, CWLOutputType from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader from schema_salad.utils import ResolveType @@ -13,7 +14,6 @@ from .load_tool import fetch_document, resolve_and_validate_document from .process import shortname, uniquename from .update import ORDERED_VERSIONS, ORIGINAL_CWLVERSION, update -from .utils import CWLObjectType, CWLOutputType LoadRefType = Callable[[Optional[str], str], ResolveType] diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 0cf5f7086..3225235d0 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -6,6 +6,7 @@ from collections.abc import ItemsView, Iterable, Iterator, KeysView from typing import NamedTuple, Optional, cast +from cwl_utils.types import CWLObjectType from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import uri_file_path @@ -13,7 +14,7 @@ from .loghandler import _logger from .stdfsaccess import abspath -from .utils import CWLObjectType, dedup, downloadHttpFile +from .utils import dedup, downloadHttpFile class MapperEnt(NamedTuple): diff --git a/cwltool/process.py b/cwltool/process.py index 75069b859..9b3cee2f8 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union, cast from cwl_utils import expression +from cwl_utils.types import CWLObjectType, CWLOutputType from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -52,8 +53,6 @@ from .stdfsaccess import StdFsAccess from .update import INTERNAL_VERSION, ORDERED_VERSIONS, ORIGINAL_CWLVERSION from .utils import ( - CWLObjectType, - CWLOutputType, HasReqsHints, JobsGeneratorType, LoadListingType, diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index eb7eca076..1dd963b06 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,6 +1,7 @@ import copy from typing import cast +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException from schema_salad.sourceline import indent @@ -10,7 +11,7 @@ from .load_tool import load_tool from .loghandler import _logger from .process import Process, shortname -from .utils import CWLObjectType, JobsGeneratorType, OutputCallbackType +from .utils import JobsGeneratorType, OutputCallbackType class ProcessGeneratorJob: diff --git a/cwltool/secrets.py b/cwltool/secrets.py index 6a39231e4..4fea75994 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -3,7 +3,7 @@ import uuid from collections.abc import MutableMapping, MutableSequence -from .utils import CWLObjectType, CWLOutputType +from cwl_utils.types import CWLObjectType, CWLOutputType class SecretStore: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 5551882b8..dd050f732 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -13,6 +13,7 @@ from subprocess import check_call, check_output # nosec from typing import cast +from cwl_utils.types import CWLObjectType from packaging.version import Version from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dumps @@ -28,7 +29,7 @@ from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper from .singularity_utils import singularity_supports_userns -from .utils import CWLObjectType, create_tmp_dir, ensure_non_writable, ensure_writable +from .utils import create_tmp_dir, ensure_non_writable, ensure_writable # Cached version number of singularity # This is a list containing major and minor versions as integer. diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 8bad72a47..5378ec3ad 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -2,11 +2,12 @@ from collections.abc import Mapping, MutableMapping, MutableSequence from typing import Any, NamedTuple, Union, cast +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap, CommentedSeq from .context import LoadingContext from .load_tool import load_tool, make_tool -from .utils import CWLObjectType, aslist +from .utils import aslist from .workflow import Workflow, WorkflowStep diff --git a/cwltool/update.py b/cwltool/update.py index 91a63f496..de92b3bdf 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -3,13 +3,14 @@ from functools import partial from typing import cast +from cwl_utils.types import CWLObjectType, CWLOutputType from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader from schema_salad.sourceline import SourceLine from .loghandler import _logger -from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field +from .utils import aslist, visit_class, visit_field def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> tuple[CommentedMap, str]: diff --git a/cwltool/utils.py b/cwltool/utils.py index 2975f0199..feea65039 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -2,6 +2,8 @@ import collections +from cwl_utils.types import CWLObjectType, CWLOutputType, DirectoryType + try: import fcntl except ImportError: @@ -41,7 +43,6 @@ NamedTuple, Optional, TypeAlias, - TypedDict, Union, cast, ) @@ -68,16 +69,6 @@ processes_to_kill: Deque["subprocess.Popen[str]"] = collections.deque() -CWLOutputType: TypeAlias = Union[ - None, - bool, - str, - int, - float, - MutableSequence["CWLOutputType"], - MutableMapping[str, "CWLOutputType"], -] -CWLObjectType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] """Typical raw dictionary found in lightly parsed CWL.""" JobsType: TypeAlias = Union[ @@ -89,10 +80,6 @@ DestinationsType: TypeAlias = MutableMapping[str, Optional[CWLOutputType]] ScatterDestinationsType: TypeAlias = MutableMapping[str, list[Optional[CWLOutputType]]] ScatterOutputCallbackType: TypeAlias = Callable[[Optional[ScatterDestinationsType], str], None] -SinkType: TypeAlias = Union[CWLOutputType, CWLObjectType] -DirectoryType = TypedDict( - "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} -) JSONType: TypeAlias = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] diff --git a/cwltool/workflow.py b/cwltool/workflow.py index fe34c75ce..78078905a 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -7,6 +7,7 @@ from typing import cast from uuid import UUID +from cwl_utils.types import CWLObjectType from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -22,7 +23,6 @@ from .loghandler import _logger from .process import Process, get_overrides, shortname from .utils import ( - CWLObjectType, JobsGeneratorType, OutputCallbackType, StepType, diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 6e1527c20..c1a3f7d2d 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression +from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dumps @@ -18,14 +19,11 @@ from .process import shortname, uniquename from .stdfsaccess import StdFsAccess from .utils import ( - CWLObjectType, - CWLOutputType, JobsGeneratorType, OutputCallbackType, ParametersType, ScatterDestinationsType, ScatterOutputCallbackType, - SinkType, WorkflowStateItem, adjustDirObjs, aslist, diff --git a/tests/test_context.py b/tests/test_context.py index 505dfd635..b417180c4 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -6,9 +6,10 @@ from pathlib import Path from typing import cast +from cwl_utils.types import CWLObjectType + from cwltool.context import RuntimeContext from cwltool.factory import Factory -from cwltool.utils import CWLObjectType from cwltool.workflow_job import WorkflowJobStep from .util import get_data, needs_docker diff --git a/tests/test_cuda.py b/tests/test_cuda.py index 93b1c7a47..391d3fa65 100644 --- a/tests/test_cuda.py +++ b/tests/test_cuda.py @@ -2,6 +2,7 @@ from unittest.mock import MagicMock import pytest +from cwl_utils.types import CWLObjectType from schema_salad.avro import schema from cwltool.builder import Builder @@ -15,7 +16,6 @@ from cwltool.process import use_custom_schema from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CWLObjectType from .util import get_data, needs_docker, needs_singularity_3_or_newer diff --git a/tests/test_examples.py b/tests/test_examples.py index f371f45ae..a8b2fa072 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -16,6 +16,7 @@ import pytest from cwl_utils.errors import JavascriptException from cwl_utils.sandboxjs import param_re +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -29,7 +30,7 @@ from cwltool.errors import WorkflowException from cwltool.main import main from cwltool.process import CWL_IANA -from cwltool.utils import CWLObjectType, dedup +from cwltool.utils import dedup from .util import get_data, get_main_output, needs_docker, working_directory diff --git a/tests/test_http_input.py b/tests/test_http_input.py index e80260ff9..16d40a292 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -2,10 +2,10 @@ from datetime import datetime from pathlib import Path +from cwl_utils.types import CWLObjectType from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper -from cwltool.utils import CWLObjectType def test_http_path_mapping(tmp_path: Path) -> None: diff --git a/tests/test_load_tool.py b/tests/test_load_tool.py index 2b9e9b6ed..c1c1ce017 100644 --- a/tests/test_load_tool.py +++ b/tests/test_load_tool.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest +from cwl_utils.types import CWLObjectType from schema_salad.exceptions import ValidationException from cwltool.context import LoadingContext, RuntimeContext @@ -13,7 +14,6 @@ from cwltool.loghandler import _logger, configure_logging from cwltool.process import use_custom_schema, use_standard_schema from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CWLObjectType from .util import get_data diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 096de9942..c01d84944 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -4,6 +4,7 @@ from typing import IO, Any, cast import pytest +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap @@ -14,7 +15,7 @@ from cwltool.main import main from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CONTENT_LIMIT, CWLObjectType, bytes2str_in_dicts +from cwltool.utils import CONTENT_LIMIT, bytes2str_in_dicts from .util import needs_docker diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index 4ffac24bd..8850e7a8d 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,7 +1,8 @@ import pytest +from cwl_utils.types import CWLObjectType from cwltool.pathmapper import PathMapper -from cwltool.utils import CWLObjectType, normalizeFilesDirs +from cwltool.utils import normalizeFilesDirs def test_subclass() -> None: diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 206b001bd..8bc7b8654 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -4,10 +4,10 @@ from io import StringIO import pytest +from cwl_utils.types import CWLObjectType from cwltool.main import main from cwltool.secrets import SecretStore -from cwltool.utils import CWLObjectType from .util import get_data, needs_docker, needs_singularity diff --git a/tests/test_streaming.py b/tests/test_streaming.py index aef4a7efe..c68454409 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -5,6 +5,7 @@ from typing import cast import pytest +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap @@ -13,7 +14,6 @@ from cwltool.errors import WorkflowException from cwltool.job import JobBase from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION -from cwltool.utils import CWLObjectType from .util import get_data From f99751acc54d3f6e494cb0d9c7cb60481217950c Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Sun, 14 Dec 2025 12:59:17 +0100 Subject: [PATCH 03/16] Rewrite tests for CWLParameterContext --- tests/test_examples.py | 291 +++++++++++++++++++++-------------------- tests/test_secrets.py | 7 +- 2 files changed, 156 insertions(+), 142 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index a8b2fa072..c6ffd72d4 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -7,6 +7,7 @@ import subprocess import sys import urllib.parse +from collections.abc import MutableMapping from io import StringIO from pathlib import Path from typing import Any, cast @@ -16,7 +17,7 @@ import pytest from cwl_utils.errors import JavascriptException from cwl_utils.sandboxjs import param_re -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLObjectType, CWLOutputType, CWLParameterContext from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -37,30 +38,30 @@ sys.argv = [""] expression_match = [ - ("(foo)", True), - ("(foo.bar)", True), - ("(foo['bar'])", True), - ('(foo["bar"])', True), - ("(foo.bar.baz)", True), - ("(foo['bar'].baz)", True), - ("(foo['bar']['baz'])", True), - ("(foo['b\\'ar']['baz'])", True), - ("(foo['b ar']['baz'])", True), - ("(foo_bar)", True), - ('(foo.["bar"])', False), - ('(.foo["bar"])', False), - ('(foo ["bar"])', False), - ('( foo["bar"])', False), - ("(foo[bar].baz)", False), - ("(foo['bar\"].baz)", False), - ("(foo['bar].baz)", False), - ("{foo}", False), - ("(foo.bar", False), - ("foo.bar)", False), - ("foo.b ar)", False), - ("foo.b'ar)", False), - ("(foo+bar", False), - ("(foo bar", False), + ("(inputs)", True), + ("(inputs.bar)", True), + ("(inputs['bar'])", True), + ('(inputs["bar"])', True), + ("(inputs.bar.baz)", True), + ("(inputs['bar'].baz)", True), + ("(inputs['bar']['baz'])", True), + ("(inputs['b\\'ar']['baz'])", True), + ("(inputs['b ar']['baz'])", True), + ("(inputs_bar)", True), + ('(inputs.["bar"])', False), + ('(.inputs["bar"])', False), + ('(inputs ["bar"])', False), + ('( inputs["bar"])', False), + ("(inputs[bar].baz)", False), + ("(inputs['bar\"].baz)", False), + ("(inputs['bar].baz)", False), + ("{inputs}", False), + ("(inputs.bar", False), + ("inputs.bar)", False), + ("inputs.b ar)", False), + ("inputs.b'ar)", False), + ("(inputs+bar", False), + ("(inputs bar", False), ] @@ -70,54 +71,66 @@ def test_expression_match(expression: str, expected: bool) -> None: assert (match is not None) == expected -interpolate_input: dict[str, Any] = { - "foo": { +interpolate_input = CWLParameterContext( + inputs={ "bar": {"baz": "zab1"}, "b ar": {"baz": 2}, "b'ar": {"baz": True}, 'b"ar': {"baz": None}, }, - "lst": ["A", "B"], -} + self=["A", "B"], +) interpolate_parameters = [ - ("$(foo)", interpolate_input["foo"]), - ("$(foo.bar)", interpolate_input["foo"]["bar"]), - ("$(foo['bar'])", interpolate_input["foo"]["bar"]), - ('$(foo["bar"])', interpolate_input["foo"]["bar"]), - ("$(foo.bar.baz)", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo['bar'].baz)", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo['bar'][\"baz\"])", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo.bar['baz'])", interpolate_input["foo"]["bar"]["baz"]), - ("$(foo['b\\'ar'].baz)", True), - ('$(foo["b\'ar"].baz)', True), - ("$(foo['b\\\"ar'].baz)", None), - ("$(lst[0])", "A"), - ("$(lst[1])", "B"), - ("$(lst.length)", 2), - ("$(lst['length'])", 2), - ("-$(foo.bar)", """-{"baz": "zab1"}"""), - ("-$(foo['bar'])", """-{"baz": "zab1"}"""), - ('-$(foo["bar"])', """-{"baz": "zab1"}"""), - ("-$(foo.bar.baz)", "-zab1"), - ("-$(foo['bar'].baz)", "-zab1"), - ("-$(foo['bar'][\"baz\"])", "-zab1"), - ("-$(foo.bar['baz'])", "-zab1"), - ("-$(foo['b ar'].baz)", "-2"), - ("-$(foo['b\\'ar'].baz)", "-true"), - ('-$(foo["b\\\'ar"].baz)', "-true"), - ("-$(foo['b\\\"ar'].baz)", "-null"), - ("$(foo.bar) $(foo.bar)", """{"baz": "zab1"} {"baz": "zab1"}"""), - ("$(foo['bar']) $(foo['bar'])", """{"baz": "zab1"} {"baz": "zab1"}"""), - ('$(foo["bar"]) $(foo["bar"])', """{"baz": "zab1"} {"baz": "zab1"}"""), - ("$(foo.bar.baz) $(foo.bar.baz)", "zab1 zab1"), - ("$(foo['bar'].baz) $(foo['bar'].baz)", "zab1 zab1"), - ("$(foo['bar'][\"baz\"]) $(foo['bar'][\"baz\"])", "zab1 zab1"), - ("$(foo.bar['baz']) $(foo.bar['baz'])", "zab1 zab1"), - ("$(foo['b ar'].baz) $(foo['b ar'].baz)", "2 2"), - ("$(foo['b\\'ar'].baz) $(foo['b\\'ar'].baz)", "true true"), - ('$(foo["b\\\'ar"].baz) $(foo["b\\\'ar"].baz)', "true true"), - ("$(foo['b\\\"ar'].baz) $(foo['b\\\"ar'].baz)", "null null"), + ("$(inputs)", interpolate_input["inputs"]), + ("$(inputs.bar)", interpolate_input["inputs"]["bar"]), + ("$(inputs['bar'])", interpolate_input["inputs"]["bar"]), + ('$(inputs["bar"])', interpolate_input["inputs"]["bar"]), + ( + "$(inputs.bar.baz)", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ( + "$(inputs['bar'].baz)", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ( + "$(inputs['bar'][\"baz\"])", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ( + "$(inputs.bar['baz'])", + cast(MutableMapping[str, CWLOutputType], interpolate_input["inputs"]["bar"])["baz"], + ), + ("$(inputs['b\\'ar'].baz)", True), + ('$(inputs["b\'ar"].baz)', True), + ("$(inputs['b\\\"ar'].baz)", None), + ("$(self[0])", "A"), + ("$(self[1])", "B"), + ("$(self.length)", 2), + ("$(self['length'])", 2), + ("-$(inputs.bar)", """-{"baz": "zab1"}"""), + ("-$(inputs['bar'])", """-{"baz": "zab1"}"""), + ('-$(inputs["bar"])', """-{"baz": "zab1"}"""), + ("-$(inputs.bar.baz)", "-zab1"), + ("-$(inputs['bar'].baz)", "-zab1"), + ("-$(inputs['bar'][\"baz\"])", "-zab1"), + ("-$(inputs.bar['baz'])", "-zab1"), + ("-$(inputs['b ar'].baz)", "-2"), + ("-$(inputs['b\\'ar'].baz)", "-true"), + ('-$(inputs["b\\\'ar"].baz)', "-true"), + ("-$(inputs['b\\\"ar'].baz)", "-null"), + ("$(inputs.bar) $(inputs.bar)", """{"baz": "zab1"} {"baz": "zab1"}"""), + ("$(inputs['bar']) $(inputs['bar'])", """{"baz": "zab1"} {"baz": "zab1"}"""), + ('$(inputs["bar"]) $(inputs["bar"])', """{"baz": "zab1"} {"baz": "zab1"}"""), + ("$(inputs.bar.baz) $(inputs.bar.baz)", "zab1 zab1"), + ("$(inputs['bar'].baz) $(inputs['bar'].baz)", "zab1 zab1"), + ("$(inputs['bar'][\"baz\"]) $(inputs['bar'][\"baz\"])", "zab1 zab1"), + ("$(inputs.bar['baz']) $(inputs.bar['baz'])", "zab1 zab1"), + ("$(inputs['b ar'].baz) $(inputs['b ar'].baz)", "2 2"), + ("$(inputs['b\\'ar'].baz) $(inputs['b\\'ar'].baz)", "true true"), + ('$(inputs["b\\\'ar"].baz) $(inputs["b\\\'ar"].baz)', "true true"), + ("$(inputs['b\\\"ar'].baz) $(inputs['b\\\"ar'].baz)", "null null"), ] @@ -128,31 +141,31 @@ def test_expression_interpolate(pattern: str, expected: Any) -> None: parameter_to_expressions = [ ( - "-$(foo)", + "-$(inputs)", r"""-{"bar":{"baz":"zab1"},"b ar":{"baz":2},"b'ar":{"baz":true},"b\"ar":{"baz":null}}""", ), - ("-$(foo.bar)", """-{"baz":"zab1"}"""), - ("-$(foo['bar'])", """-{"baz":"zab1"}"""), - ('-$(foo["bar"])', """-{"baz":"zab1"}"""), - ("-$(foo.bar.baz)", "-zab1"), - ("-$(foo['bar'].baz)", "-zab1"), - ("-$(foo['bar'][\"baz\"])", "-zab1"), - ("-$(foo.bar['baz'])", "-zab1"), - ("-$(foo['b ar'].baz)", "-2"), - ("-$(foo['b\\'ar'].baz)", "-true"), - ('-$(foo["b\\\'ar"].baz)', "-true"), - ("-$(foo['b\\\"ar'].baz)", "-null"), - ("$(foo.bar) $(foo.bar)", """{"baz":"zab1"} {"baz":"zab1"}"""), - ("$(foo['bar']) $(foo['bar'])", """{"baz":"zab1"} {"baz":"zab1"}"""), - ('$(foo["bar"]) $(foo["bar"])', """{"baz":"zab1"} {"baz":"zab1"}"""), - ("$(foo.bar.baz) $(foo.bar.baz)", "zab1 zab1"), - ("$(foo['bar'].baz) $(foo['bar'].baz)", "zab1 zab1"), - ("$(foo['bar'][\"baz\"]) $(foo['bar'][\"baz\"])", "zab1 zab1"), - ("$(foo.bar['baz']) $(foo.bar['baz'])", "zab1 zab1"), - ("$(foo['b ar'].baz) $(foo['b ar'].baz)", "2 2"), - ("$(foo['b\\'ar'].baz) $(foo['b\\'ar'].baz)", "true true"), - ('$(foo["b\\\'ar"].baz) $(foo["b\\\'ar"].baz)', "true true"), - ("$(foo['b\\\"ar'].baz) $(foo['b\\\"ar'].baz)", "null null"), + ("-$(inputs.bar)", """-{"baz":"zab1"}"""), + ("-$(inputs['bar'])", """-{"baz":"zab1"}"""), + ('-$(inputs["bar"])', """-{"baz":"zab1"}"""), + ("-$(inputs.bar.baz)", "-zab1"), + ("-$(inputs['bar'].baz)", "-zab1"), + ("-$(inputs['bar'][\"baz\"])", "-zab1"), + ("-$(inputs.bar['baz'])", "-zab1"), + ("-$(inputs['b ar'].baz)", "-2"), + ("-$(inputs['b\\'ar'].baz)", "-true"), + ('-$(inputs["b\\\'ar"].baz)', "-true"), + ("-$(inputs['b\\\"ar'].baz)", "-null"), + ("$(inputs.bar) $(inputs.bar)", """{"baz":"zab1"} {"baz":"zab1"}"""), + ("$(inputs['bar']) $(inputs['bar'])", """{"baz":"zab1"} {"baz":"zab1"}"""), + ('$(inputs["bar"]) $(inputs["bar"])', """{"baz":"zab1"} {"baz":"zab1"}"""), + ("$(inputs.bar.baz) $(inputs.bar.baz)", "zab1 zab1"), + ("$(inputs['bar'].baz) $(inputs['bar'].baz)", "zab1 zab1"), + ("$(inputs['bar'][\"baz\"]) $(inputs['bar'][\"baz\"])", "zab1 zab1"), + ("$(inputs.bar['baz']) $(inputs.bar['baz'])", "zab1 zab1"), + ("$(inputs['b ar'].baz) $(inputs['b ar'].baz)", "2 2"), + ("$(inputs['b\\'ar'].baz) $(inputs['b\\'ar'].baz)", "true true"), + ('$(inputs["b\\\'ar"].baz) $(inputs["b\\\'ar"].baz)', "true true"), + ("$(inputs['b\\\"ar'].baz) $(inputs['b\\\"ar'].baz)", "null null"), ] @@ -174,22 +187,22 @@ def test_parameter_to_expression(pattern: str, expected: Any) -> None: param_to_expr_interpolate_escapebehavior = ( - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 1), - ("\\\\$(foo.bar.baz)", "\\zab1", 1), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 1), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 1), - ("\\$foo", "$foo", 1), - ("\\foo", "foo", 1), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 1), + ("\\\\$(inputs.bar.baz)", "\\zab1", 1), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 1), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 1), + ("\\$inputs", "$inputs", 1), + ("\\inputs", "inputs", 1), ("\\x", "x", 1), ("\\\\x", "\\x", 1), ("\\\\\\x", "\\x", 1), ("\\\\\\\\x", "\\\\x", 1), - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 2), - ("\\\\$(foo.bar.baz)", "\\zab1", 2), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 2), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 2), - ("\\$foo", "\\$foo", 2), - ("\\foo", "\\foo", 2), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 2), + ("\\\\$(inputs.bar.baz)", "\\zab1", 2), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 2), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 2), + ("\\$inputs", "\\$inputs", 2), + ("\\inputs", "\\inputs", 2), ("\\x", "\\x", 2), ("\\\\x", "\\x", 2), ("\\\\\\x", "\\\\x", 2), @@ -219,32 +232,32 @@ def test_parameter_to_expression_interpolate_escapebehavior( interpolate_bad_parameters = [ - ("$(fooz)"), - ("$(foo.barz)"), - ("$(foo['barz'])"), - ('$(foo["barz"])'), - ("$(foo.bar.bazz)"), - ("$(foo['bar'].bazz)"), - ("$(foo['bar'][\"bazz\"])"), - ("$(foo.bar['bazz'])"), - ("$(foo['b\\'ar'].bazz)"), - ('$(foo["b\'ar"].bazz)'), - ("$(foo['b\\\"ar'].bazz)"), - ("$(lst[O])"), # not "0" the number, but the letter O - ("$(lst[2])"), - ("$(lst.lengthz)"), - ("$(lst['lengthz'])"), - ("-$(foo.barz)"), - ("-$(foo['barz'])"), - ('-$(foo["barz"])'), - ("-$(foo.bar.bazz)"), - ("-$(foo['bar'].bazz)"), - ("-$(foo['bar'][\"bazz\"])"), - ("-$(foo.bar['bazz'])"), - ("-$(foo['b ar'].bazz)"), - ("-$(foo['b\\'ar'].bazz)"), - ('-$(foo["b\\\'ar"].bazz)'), - ("-$(foo['b\\\"ar'].bazz)"), + ("$(inputsz)"), + ("$(inputs.barz)"), + ("$(inputs['barz'])"), + ('$(inputs["barz"])'), + ("$(inputs.bar.bazz)"), + ("$(inputs['bar'].bazz)"), + ("$(inputs['bar'][\"bazz\"])"), + ("$(inputs.bar['bazz'])"), + ("$(inputs['b\\'ar'].bazz)"), + ('$(inputs["b\'ar"].bazz)'), + ("$(inputs['b\\\"ar'].bazz)"), + ("$(self[O])"), # not "0" the number, but the letter O + ("$(self[2])"), + ("$(self.lengthz)"), + ("$(self['lengthz'])"), + ("-$(inputs.barz)"), + ("-$(inputs['barz'])"), + ('-$(inputs["barz"])'), + ("-$(inputs.bar.bazz)"), + ("-$(inputs['bar'].bazz)"), + ("-$(inputs['bar'][\"bazz\"])"), + ("-$(inputs.bar['bazz'])"), + ("-$(inputs['b ar'].bazz)"), + ("-$(inputs['b\\'ar'].bazz)"), + ('-$(inputs["b\\\'ar"].bazz)'), + ("-$(inputs['b\\\"ar'].bazz)"), ] @@ -255,22 +268,22 @@ def test_expression_interpolate_failures(pattern: str) -> None: interpolate_escapebehavior = ( - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 1), - ("\\\\$(foo.bar.baz)", "\\zab1", 1), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 1), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 1), - ("\\$foo", "$foo", 1), - ("\\foo", "foo", 1), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 1), + ("\\\\$(inputs.bar.baz)", "\\zab1", 1), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 1), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 1), + ("\\$inputs", "$inputs", 1), + ("\\inputs", "inputs", 1), ("\\x", "x", 1), ("\\\\x", "\\x", 1), ("\\\\\\x", "\\x", 1), ("\\\\\\\\x", "\\\\x", 1), - ("\\$(foo.bar.baz)", "$(foo.bar.baz)", 2), - ("\\\\$(foo.bar.baz)", "\\zab1", 2), - ("\\\\\\$(foo.bar.baz)", "\\$(foo.bar.baz)", 2), - ("\\\\\\\\$(foo.bar.baz)", "\\\\zab1", 2), - ("\\$foo", "\\$foo", 2), - ("\\foo", "\\foo", 2), + ("\\$(inputs.bar.baz)", "$(inputs.bar.baz)", 2), + ("\\\\$(inputs.bar.baz)", "\\zab1", 2), + ("\\\\\\$(inputs.bar.baz)", "\\$(inputs.bar.baz)", 2), + ("\\\\\\\\$(inputs.bar.baz)", "\\\\zab1", 2), + ("\\$inputs", "\\$inputs", 2), + ("\\inputs", "\\inputs", 2), ("\\x", "\\x", 2), ("\\\\x", "\\x", 2), ("\\\\\\x", "\\\\x", 2), diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 8bc7b8654..b6c316ce1 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,10 +1,11 @@ import shutil import tempfile -from collections.abc import Callable +from collections.abc import Callable, MutableMapping from io import StringIO +from typing import cast import pytest -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLObjectType, CWLOutputType from cwltool.main import main from cwltool.secrets import SecretStore @@ -27,7 +28,7 @@ def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: storage, obscured = secrets assert obscured["foo"] != "bar" assert obscured["baz"] == "quux" - result = storage.retrieve(obscured) + result = cast(MutableMapping[str, CWLOutputType], storage.retrieve(obscured)) assert isinstance(result, dict) and result["foo"] == "bar" From bc0cae7c221955108d665fa6617b5f0072c4708c Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Sun, 14 Dec 2025 13:58:23 +0100 Subject: [PATCH 04/16] Reworking type system --- cwltool/builder.py | 82 +++++------ cwltool/checker.py | 14 +- cwltool/command_line_tool.py | 70 ++++----- cwltool/context.py | 2 +- cwltool/cwlprov/provenance_profile.py | 62 ++++---- cwltool/cwlprov/ro.py | 20 ++- cwltool/cwlviewer.py | 2 +- cwltool/docker.py | 12 +- cwltool/job.py | 24 ++-- cwltool/main.py | 34 ++--- cwltool/pack.py | 2 +- cwltool/pathmapper.py | 30 ++-- cwltool/process.py | 200 ++++++++++++++------------ cwltool/singularity.py | 8 +- cwltool/task_queue.py | 5 +- cwltool/update.py | 2 +- cwltool/utils.py | 61 ++++---- cwltool/workflow_job.py | 4 +- mypy-requirements.txt | 1 - pyproject.toml | 2 +- setup.py | 2 +- tests/test_examples.py | 44 +++--- tests/test_http_input.py | 33 +++-- tests/test_js_sandbox.py | 7 +- tests/test_parallel.py | 4 +- tests/test_pathmapper.py | 8 +- 26 files changed, 399 insertions(+), 336 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 6f32a4197..86e1a4b77 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -9,7 +9,7 @@ from cwl_utils import expression from cwl_utils.file_formats import check_format -from cwl_utils.types import CWLObjectType, CWLOutputType +from cwl_utils.types import CWLObjectType, CWLOutputType, CWLDirectoryType, CWLFileType from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap @@ -94,7 +94,7 @@ class Builder(HasReqsHints): def __init__( self, job: CWLObjectType, - files: list[CWLObjectType], + files: MutableSequence[CWLFileType | CWLDirectoryType], bindings: list[CWLObjectType], schemaDefs: MutableMapping[str, CWLObjectType], names: Names, @@ -165,7 +165,7 @@ def build_job_script(self, commands: list[str]) -> str | None: return self.job_script_provider.build_job_script(self, commands) return None - def _capture_files(self, f: CWLObjectType) -> CWLObjectType: + def _capture_files(self, f: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType: self.files.append(f) return f @@ -355,13 +355,13 @@ def bind_input( ) binding = {} - def _capture_files(f: CWLObjectType) -> CWLObjectType: + def _capture_files(f: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType: self.files.append(f) return f if schema["type"] == "org.w3id.cwl.cwl.File": - datum = cast(CWLObjectType, datum) - self.files.append(datum) + file_datum = cast(CWLFileType, datum) + self.files.append(file_datum) loadContents_sourceline: ( None | MutableMapping[str, str | list[int]] | CWLObjectType @@ -379,14 +379,16 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: debug, ): try: - with self.fs_access.open(cast(str, datum["location"]), "rb") as f2: - datum["contents"] = content_limit_respected_read(f2) + with self.fs_access.open(file_datum["location"], "rb") as f2: + file_datum["contents"] = content_limit_respected_read(f2) except Exception as e: - raise Exception("Reading {}\n{}".format(datum["location"], e)) from e + raise Exception( + "Reading {}\n{}".format(file_datum["location"], e) + ) from e if "secondaryFiles" in schema: - if "secondaryFiles" not in datum: - datum["secondaryFiles"] = [] + if "secondaryFiles" not in file_datum: + file_datum["secondaryFiles"] = [] sf_schema = aslist(schema["secondaryFiles"]) elif not discover_secondaryFiles: sf_schema = [] # trust the inputs @@ -395,7 +397,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: for num, sf_entry in enumerate(sf_schema): if "required" in sf_entry and sf_entry["required"] is not None: - required_result = self.do_eval(sf_entry["required"], context=datum) + required_result = self.do_eval(sf_entry["required"], context=file_datum) if not (isinstance(required_result, bool) or required_result is None): if sf_schema == schema["secondaryFiles"]: sf_item: Any = sf_schema[num] @@ -417,7 +419,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: if "$(" in sf_entry["pattern"] or "${" in sf_entry["pattern"]: sfpath = self.do_eval(sf_entry["pattern"], context=datum) else: - sfpath = substitute(cast(str, datum["basename"]), sf_entry["pattern"]) + sfpath = substitute(file_datum["basename"], sf_entry["pattern"]) for sfname in aslist(sfpath): if not sfname: @@ -425,7 +427,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: found = False if isinstance(sfname, str): - d_location = cast(str, datum["location"]) + d_location = file_datum["location"] if "/" in d_location: sf_location = ( d_location[0 : d_location.rindex("/") + 1] + sfname @@ -434,6 +436,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: sf_location = d_location + sfname sfbasename = sfname elif isinstance(sfname, MutableMapping): + sfname = cast(CWLFileType | CWLDirectoryType, sfname) sf_location = sfname["location"] sfbasename = sfname["basename"] else: @@ -446,10 +449,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: f"{type(sfname)!r} from {sf_entry['pattern']!r}." ) - for d in cast( - MutableSequence[MutableMapping[str, str]], - datum["secondaryFiles"], - ): + for d in file_datum["secondaryFiles"]: if not d.get("basename"): d["basename"] = d["location"][d["location"].rindex("/") + 1 :] if d["basename"] == sfbasename: @@ -458,8 +458,8 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: if not found: def addsf( - files: MutableSequence[CWLObjectType], - newsf: CWLObjectType, + files: MutableSequence[CWLFileType | CWLDirectoryType], + newsf: CWLFileType | CWLDirectoryType, ) -> None: for f in files: if f["location"] == newsf["location"]: @@ -469,23 +469,19 @@ def addsf( if isinstance(sfname, MutableMapping): addsf( - cast( - MutableSequence[CWLObjectType], - datum["secondaryFiles"], - ), - sfname, + file_datum["secondaryFiles"], + cast(CWLFileType | CWLDirectoryType, sfname), ) elif discover_secondaryFiles and self.fs_access.exists(sf_location): addsf( - cast( - MutableSequence[CWLObjectType], - datum["secondaryFiles"], + file_datum["secondaryFiles"], + CWLFileType( + **{ + "location": sf_location, + "basename": sfname, + "class": "File", + } ), - { - "location": sf_location, - "basename": sfname, - "class": "File", - }, ) elif sf_required: raise SourceLine( @@ -495,12 +491,10 @@ def addsf( debug, ).makeError( "Missing required secondary file '%s' from file object: %s" - % (sfname, json_dumps(datum, indent=4)) + % (sfname, json_dumps(file_datum, indent=4)) ) - normalizeFilesDirs( - cast(MutableSequence[CWLObjectType], datum["secondaryFiles"]) - ) + normalizeFilesDirs(file_datum["secondaryFiles"]) if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) @@ -545,7 +539,7 @@ def addsf( ) try: check_format( - datum, + file_datum, evaluated_format, self.formatgraph, ) @@ -556,21 +550,21 @@ def addsf( ) from ve visit_class( - datum.get("secondaryFiles", []), + file_datum.get("secondaryFiles", []), ("File", "Directory"), self._capture_files, ) if schema["type"] == "org.w3id.cwl.cwl.Directory": - datum = cast(CWLObjectType, datum) + dir_datum = cast(CWLDirectoryType, datum) ll = schema.get("loadListing") or self.loadListing if ll and ll != "no_listing": get_listing( self.fs_access, - datum, + dir_datum, (ll == "deep_listing"), ) - self.files.append(datum) + self.files.append(dir_datum) if schema["type"] == "Any": visit_class(datum, ("File", "Directory"), self._capture_files) @@ -595,9 +589,7 @@ def tostr(self, value: MutableMapping[str, str] | Any) -> str: match value: case {"class": "File" | "Directory" as class_name, **rest}: if "path" not in rest: - raise WorkflowException( - '{} object missing "path": {}'.format(class_name, value) - ) + raise WorkflowException(f'{class_name} object missing "path": {value}') return str(rest["path"]) case ScalarFloat(): rep = RoundTripRepresenter() diff --git a/cwltool/checker.py b/cwltool/checker.py index 4e6541f9c..c30a0d0b5 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -22,8 +22,8 @@ def _get_type(tp: Any) -> Any: def check_types( - srctype: SinkType, - sinktype: SinkType, + srctype: SinkType | None, + sinktype: SinkType | None, linkMerge: str | None, valueFrom: str | None, ) -> Literal["pass"] | Literal["warning"] | Literal["exception"]: @@ -56,7 +56,7 @@ def check_types( raise WorkflowException(f"Unrecognized linkMerge enum {linkMerge!r}") -def merge_flatten_type(src: SinkType) -> CWLOutputType: +def merge_flatten_type(src: SinkType | None) -> CWLOutputType | None: """Return the merge flattened type of the source type.""" match src: case MutableSequence(): @@ -67,7 +67,9 @@ def merge_flatten_type(src: SinkType) -> CWLOutputType: return {"items": src, "type": "array"} -def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = False) -> bool: +def can_assign_src_to_sink( + src: SinkType | None, sink: SinkType | None, strict: bool = False +) -> bool: """ Check for identical type specifications, ignoring extra keys like inputBinding. @@ -85,8 +87,8 @@ def can_assign_src_to_sink(src: SinkType, sink: SinkType | None, strict: bool = return False if src["type"] == "array" and sink["type"] == "array": return can_assign_src_to_sink( - cast(MutableSequence[CWLOutputType], src["items"]), - cast(MutableSequence[CWLOutputType], sink["items"]), + cast(MutableSequence[CWLOutputType | None], src["items"]), + cast(MutableSequence[CWLOutputType | None], sink["items"]), strict, ) if src["type"] == "record" and sink["type"] == "record": diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 20e799cda..8b3dbe2d7 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -27,7 +27,8 @@ from cwl_utils.types import ( CWLObjectType, CWLOutputType, - DirectoryType, + CWLDirectoryType, + CWLFileType, is_directory, is_file, is_file_or_directory, @@ -178,23 +179,12 @@ def __init__( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Union[threading.Lock, None] = None, + tmpdir_lock: threading.Lock | None = None, ) -> None: try: normalizeFilesDirs(self.builder.job) ev = self.builder.do_eval(self.script) - normalizeFilesDirs( - cast( - Optional[ - Union[ - MutableSequence[MutableMapping[str, Any]], - MutableMapping[str, Any], - DirectoryType, - ] - ], - ev, - ) - ) + normalizeFilesDirs(ev) if self.output_callback: self.output_callback(cast(Optional[CWLObjectType], ev), "success") except WorkflowException as err: @@ -244,7 +234,9 @@ def remove_path(f: CWLObjectType) -> None: del f["path"] -def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectType | None: +def revmap_file( + builder: Builder, outdir: str, f: CWLFileType | CWLDirectoryType +) -> CWLFileType | CWLDirectoryType: """ Remap a file from internal path to external path. @@ -263,18 +255,18 @@ def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> CWLObjectTyp # quoted any further. if "location" in f and "path" not in f: - location = cast(str, f["location"]) + location = f["location"] if location.startswith("file://"): f["path"] = uri_file_path(location) else: - f["location"] = builder.fs_access.join(outdir, cast(str, f["location"])) + f["location"] = builder.fs_access.join(outdir, f["location"]) return f - if "dirname" in f: + if is_file(f) and "dirname" in f: del f["dirname"] if "path" in f: - path = builder.fs_access.join(builder.outdir, cast(str, f["path"])) + path = builder.fs_access.join(builder.outdir, f["path"]) uripath = file_uri(path) del f["path"] @@ -337,7 +329,7 @@ def __init__( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Union[threading.Lock, None] = None, + tmpdir_lock: threading.Lock | None = None, ) -> None: if self.output_callback: self.output_callback( @@ -384,9 +376,9 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: location = cast(str, ob["location"]) if location.startswith("_:"): pass - if ob["class"] == "File" and not fs_access.isfile(location): + if is_file(ob) and not fs_access.isfile(location): raise ValidationException("Does not exist or is not a File: '%s'" % location) - if ob["class"] == "Directory" and not fs_access.isdir(location): + if is_directory(ob) and not fs_access.isdir(location): raise ValidationException("Does not exist or is not a Directory: '%s'" % location) @@ -454,7 +446,7 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: @staticmethod def make_path_mapper( - reffiles: list[CWLObjectType], + reffiles: MutableSequence[CWLFileType | CWLDirectoryType], stagedir: str, runtimeContext: RuntimeContext, separateDirs: bool, @@ -601,10 +593,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: filelist = True for e in entry: - if not isinstance(e, MutableMapping) or e.get("class") not in ( - "File", - "Directory", - ): + if not is_file_or_directory(e): filelist = False break @@ -706,7 +695,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: ) ) - if t2["entry"].get("class") not in ("File", "Directory"): + if not is_file_or_directory(t2["entry"]): raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( "Entry at index %s of listing is not a File or Directory object, was %s" % (i, t2) @@ -720,10 +709,10 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: t2entry["writable"] = t2copy.get("writable") t2["entry"] = t2entry - ls[i] = t2["entry"] + ls[i] = cast(CWLObjectType, t2["entry"]) for i, t3 in enumerate(ls): - if t3.get("class") not in ("File", "Directory"): + if not is_file_or_directory(t3): # Check that every item is a File or Directory object now raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( f"Entry at index {i} of listing is not a Dirent, File or " @@ -731,7 +720,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: ) if "basename" not in t3: continue - basename = os.path.normpath(cast(str, t3["basename"])) + basename = os.path.normpath(t3["basename"]) t3["basename"] = basename if basename.startswith("../"): raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( @@ -763,7 +752,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: return # Only testing with SourceLine(initialWorkdir, "listing", WorkflowException, debug): - j.generatefiles["listing"] = ls + j.generatefiles["listing"] = cast(MutableSequence[CWLFileType | CWLDirectoryType], ls) for entry in ls: if "basename" in entry: basename = cast(str, entry["basename"]) @@ -860,10 +849,11 @@ def calc_checksum(location: str) -> str | None: if ( "location" in e and e["location"] == location + and is_file(e) and "checksum" in e and e["checksum"] != "sha1$hash" ): - return cast(str, e["checksum"]) + return e["checksum"] return None def remove_prefix(s: str, prefix: str) -> str: @@ -1022,7 +1012,7 @@ def update_status_output_callback( ) j.stdin = stdin_eval if j.stdin: - reffiles.append({"class": "File", "path": j.stdin}) + reffiles.append(CWLFileType(**{"class": "File", "path": j.stdin})) if self.tool.get("stderr"): with SourceLine(self.tool, "stderr", ValidationException, debug): @@ -1380,28 +1370,28 @@ def collect_output( _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(list[dict[str, Optional[CWLOutputType]]], r): + for files in cast(MutableSequence[CWLFileType | CWLDirectoryType], r): rfile = files.copy() revmap(rfile) - if files["class"] == "Directory": + if is_directory(files): ll = binding.get("loadListing") or builder.loadListing if ll and ll != "no_listing": get_listing(fs_access, files, (ll == "deep_listing")) - else: + elif is_file(files): if binding.get("loadContents"): - with fs_access.open(cast(str, rfile["location"]), "rb") as f: + with fs_access.open(rfile["location"], "rb") as f: files["contents"] = str( content_limit_respected_read_bytes(f), "utf-8" ) if compute_checksum: - with fs_access.open(cast(str, rfile["location"]), "rb") as f: + with fs_access.open(rfile["location"], "rb") as f: checksum = hashlib.sha1() # nosec contents = f.read(1024 * 1024) while contents != b"": checksum.update(contents) contents = f.read(1024 * 1024) files["checksum"] = "sha1$%s" % checksum.hexdigest() - files["size"] = fs_access.size(cast(str, rfile["location"])) + files["size"] = fs_access.size(rfile["location"]) optional = False single = False diff --git a/cwltool/context.py b/cwltool/context.py index 44129b40f..d9c8772f1 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -180,7 +180,7 @@ def __init__(self, kwargs: dict[str, Any] | None = None) -> None: self.cidfile_dir: str | None = None self.cidfile_prefix: str | None = None - self.workflow_eval_lock: Union[threading.Condition, None] = None + self.workflow_eval_lock: threading.Condition | None = None self.research_obj: ResearchObject | None = None self.orcid: str = "" self.cwl_full_name: str = "" diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index 0bd2b220c..f2168a224 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -6,20 +6,13 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from typing import TYPE_CHECKING, Any, cast +from typing import Any, TYPE_CHECKING, TypedDict, cast -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, is_directory, is_file from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity from schema_salad.sourceline import SourceLine -from ..errors import WorkflowException -from ..job import CommandLineJob, JobBase -from ..loghandler import _logger -from ..process import Process, shortname -from ..stdfsaccess import StdFsAccess -from ..utils import JobsType, get_listing, posix_path, versionstring -from ..workflow_job import WorkflowJob from .provenance_constants import ( ACCOUNT_UUID, CWLPROV, @@ -37,10 +30,27 @@ WFPROV, ) from .writablebagfile import create_job, write_bag_file # change this later +from ..errors import WorkflowException +from ..job import CommandLineJob, JobBase +from ..loghandler import _logger +from ..process import Process, shortname +from ..stdfsaccess import StdFsAccess +from ..utils import JobsType, get_listing, posix_path, versionstring +from ..workflow_job import WorkflowJob if TYPE_CHECKING: from .ro import ResearchObject +CWLArtifact = TypedDict("CWLArtifact", {"@id": str}) + + +class CWLDirectoryArtifact(CWLArtifact, CWLDirectoryType): + pass + + +class CWLFileArtifact(CWLArtifact, CWLFileType): + pass + def copy_job_order(job: Process | JobsType, job_order_object: CWLObjectType) -> CWLObjectType: """Create copy of job object for provenance.""" @@ -244,15 +254,15 @@ def record_process_end( self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, str]: """Construct a FileEntity for the given CWL File object.""" - if value["class"] != "File": + if not is_file(value): raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename entity: ProvEntity | None = None checksum = None if "checksum" in value: - csum = cast(str, value["checksum"]) + csum = value["checksum"] (method, checksum) = csum.split("$", 1) if method == SHA1 and self.research_object.has_data_file(checksum): entity = self.document.entity("data:" + checksum) @@ -270,7 +280,7 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st if not entity and "contents" in value: # Anonymous file, add content as string - entity, checksum = self.declare_string(cast(str, value["contents"])) + entity, checksum = self.declare_string(value["contents"]) # By here one of them should have worked! if not entity or not checksum: @@ -280,7 +290,7 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st # secondaryFiles. Note that multiple uses of a file might thus record # different names for the same entity, so we'll # make/track a specialized entity by UUID - file_id = cast(str, value.setdefault("@id", uuid.uuid4().urn)) + file_id = value.setdefault("@id", uuid.uuid4().urn) # A specialized entity that has just these names file_entity = self.document.entity( file_id, @@ -288,20 +298,20 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st ) if "basename" in value: - file_entity.add_attributes({CWLPROV["basename"]: cast(str, value["basename"])}) + file_entity.add_attributes({CWLPROV["basename"]: value["basename"]}) if "nameroot" in value: - file_entity.add_attributes({CWLPROV["nameroot"]: cast(str, value["nameroot"])}) + file_entity.add_attributes({CWLPROV["nameroot"]: value["nameroot"]}) if "nameext" in value: - file_entity.add_attributes({CWLPROV["nameext"]: cast(str, value["nameext"])}) + file_entity.add_attributes({CWLPROV["nameext"]: value["nameext"]}) self.document.specializationOf(file_entity, entity) # Check for secondaries for sec in cast(MutableSequence[CWLObjectType], value.get("secondaryFiles", [])): # TODO: Record these in a specializationOf entity with UUID? - if sec["class"] == "File": - (sec_entity, _, _) = self.declare_file(sec) - elif sec["class"] == "Directory": - sec_entity = self.declare_directory(sec) + if is_file(sec): + (sec_entity, _, _) = self.declare_file(cast(CWLFileArtifact, sec)) + elif is_directory(sec): + sec_entity = self.declare_directory(cast(CWLDirectoryArtifact, sec)) else: raise ValueError(f"Got unexpected secondaryFiles value: {sec}") # We don't know how/when/where the secondary file was generated, @@ -316,14 +326,14 @@ def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, st return file_entity, entity, checksum - def declare_directory(self, value: CWLObjectType) -> ProvEntity: + def declare_directory(self, value: CWLDirectoryArtifact) -> ProvEntity: """Register any nested files/directories.""" # FIXME: Calculate a hash-like identifier for directory # so we get same value if it's the same filenames/hashes # in a different location. # For now, mint a new UUID to identify this directory, but # attempt to keep it inside the value dictionary - dir_id = cast(str, value.setdefault("@id", uuid.uuid4().urn)) + dir_id = value.setdefault("@id", uuid.uuid4().urn) # New annotation file to keep the ORE Folder listing ore_doc_fn = dir_id.replace("urn:uuid:", "directory-") + ".ttl" @@ -340,7 +350,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: ) if "basename" in value: - coll.add_attributes({CWLPROV["basename"]: cast(str, value["basename"])}) + coll.add_attributes({CWLPROV["basename"]: value["basename"]}) # ORE description of ro:Folder, saved separately coll_b = dir_bundle.entity( @@ -476,11 +486,11 @@ def declare_artefact(self, value: Any) -> ProvEntity: # Base case - we found a File we need to update case {"class": "File"}: - entity = self.declare_file(value)[0] + entity = self.declare_file(cast(CWLFileArtifact, value))[0] value["@id"] = entity.identifier.uri return entity case {"class": "Directory"}: - entity = self.declare_directory(value) + entity = self.declare_directory(cast(CWLDirectoryArtifact, value)) value["@id"] = entity.identifier.uri return entity case {**rest}: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 783d3cf1f..98683b1d7 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -13,7 +13,15 @@ from typing import IO, TYPE_CHECKING, Any, Optional, cast import prov.model as provM -from cwl_utils.types import CWLObjectType, CWLOutputType, is_directory, is_file +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_directory, + is_file, + is_file_or_directory, +) from prov.model import ProvDocument from ..loghandler import _logger @@ -494,7 +502,7 @@ def _authored_by(self) -> AuthoredBy | None: return authored_by return None - def generate_snapshot(self, prov_dep: CWLObjectType) -> None: + def generate_snapshot(self, prov_dep: CWLFileType | CWLDirectoryType) -> None: """Copy all of the CWL files to the snapshot/ directory.""" self.self_check() for key, value in prov_dep.items(): @@ -520,8 +528,8 @@ def generate_snapshot(self, prov_dep: CWLObjectType) -> None: except PermissionError: pass # FIXME: avoids duplicate snapshotting; need better solution elif key in ("secondaryFiles", "listing"): - for files in cast(MutableSequence[CWLObjectType], value): - if isinstance(files, MutableMapping): + for files in cast(MutableSequence[CWLFileType | CWLDirectoryType], value): + if is_file_or_directory(files): self.generate_snapshot(files) else: pass @@ -634,7 +642,7 @@ def _add_to_bagit(self, rel_path: str, **checksums: str) -> None: def _relativise_files( self, - structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType], + structure: CWLObjectType | CWLOutputType | MutableSequence[CWLObjectType] | None, ) -> None: """Save any file objects into the RO and update the local paths.""" # Base case - we found a File we need to update @@ -674,7 +682,7 @@ def _relativise_files( for val in structure.values(): try: - self._relativise_files(val) + self._relativise_files(cast(CWLOutputType, val)) except OSError: pass return diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index db6b50358..d84cc9dbd 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -12,7 +12,7 @@ if Version(pydot.__version__) > Version("3.0"): quote_id_if_necessary = pydot.quote_id_if_necessary else: - quote_id_if_necessary = pydot.quote_if_necessary # type: ignore[attr-defined] + quote_id_if_necessary = pydot.quote_if_necessary def _get_inner_edges_query() -> str: diff --git a/cwltool/docker.py b/cwltool/docker.py index 7dd15d7d9..5620def24 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,12 +9,12 @@ import subprocess # nosec import sys import threading -from collections.abc import Callable, MutableMapping +from collections.abc import Callable, MutableMapping, MutableSequence from io import StringIO # pylint: disable=redefined-builtin from typing import Optional, cast import requests -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLObjectType, CWLDirectoryType, CWLFileType from .builder import Builder from .context import RuntimeContext @@ -85,7 +85,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, @@ -447,7 +449,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, diff --git a/cwltool/job.py b/cwltool/job.py index 091a0dbe7..7dda6d026 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -21,7 +21,7 @@ from typing import IO, TYPE_CHECKING, Optional, TextIO, Union, cast import psutil -from cwl_utils.types import CWLObjectType, CWLOutputType, DirectoryType +from cwl_utils.types import CWLObjectType, CWLOutputType, CWLDirectoryType, CWLFileType from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps @@ -106,7 +106,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, @@ -136,11 +138,13 @@ def __init__( self.tmpdir = "" self.environment: MutableMapping[str, str] = {} - self.generatefiles: DirectoryType = { - "class": "Directory", - "listing": [], - "basename": "", - } + self.generatefiles = CWLDirectoryType( + **{ + "class": "Directory", + "listing": [], + "basename": "", + } + ) self.stagedir: str | None = None self.inplace_update = False self.prov_obj: ProvenanceProfile | None = None @@ -157,7 +161,7 @@ def __repr__(self) -> str: def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Union[threading.Lock, None] = None, + tmpdir_lock: threading.Lock | None = None, ) -> None: pass @@ -562,7 +566,7 @@ class CommandLineJob(JobBase): def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Union[threading.Lock, None] = None, + tmpdir_lock: threading.Lock | None = None, ) -> None: if tmpdir_lock: with tmpdir_lock: @@ -748,7 +752,7 @@ def add_volumes( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Union[threading.Lock, None] = None, + tmpdir_lock: threading.Lock | None = None, ) -> None: debug = runtimeContext.debug if tmpdir_lock: diff --git a/cwltool/main.py b/cwltool/main.py index 3716cd141..89256afa5 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -23,7 +23,7 @@ import coloredlogs import requests import ruamel.yaml -from cwl_utils.types import CWLObjectType, CWLOutputType +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType, is_file from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML @@ -244,8 +244,8 @@ def generate_example_input( for field in cast(list[CWLObjectType], fields): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) - case {"type": str(inp_type), "default": default}: - example = default + case {"type": str(inp_type), "default": default_value}: + example = default_value comment = f"default value of type {inp_type!r}" case {"type": str(inp_type)}: example = defaults.get(inp_type, str(inptype)) @@ -297,7 +297,7 @@ def realize_input_schema( if isinstance(entry["type"], Mapping): entry["type"] = cast( CWLOutputType, - realize_input_schema([entry["type"]], schema_defs), + realize_input_schema([cast(CWLObjectType, entry["type"])], schema_defs), ) if entry["type"] == "array": items = entry["items"] if not isinstance(entry["items"], str) else [entry["items"]] @@ -564,14 +564,14 @@ def prov_deps( document_loader: Loader, uri: str, basedir: str | None = None, -) -> CWLObjectType: +) -> CWLFileType: deps = find_deps(obj, document_loader, uri, basedir=basedir) - def remove_non_cwl(deps: CWLObjectType) -> None: - if "secondaryFiles" in deps: - sec_files = cast(list[CWLObjectType], deps["secondaryFiles"]) + def remove_non_cwl(deps: CWLFileType | CWLDirectoryType) -> None: + if is_file(deps) and "secondaryFiles" in deps: + sec_files = deps["secondaryFiles"] for index, entry in enumerate(sec_files): - if not ("format" in entry and entry["format"] == CWL_IANA): + if not (is_file(entry) and "format" in entry and entry["format"] == CWL_IANA): del sec_files[index] else: remove_non_cwl(entry) @@ -586,13 +586,15 @@ def find_deps( uri: str, basedir: str | None = None, nestdirs: bool = True, -) -> CWLObjectType: +) -> CWLFileType: """Find the dependencies of the CWL document.""" - deps: CWLObjectType = { - "class": "File", - "location": uri, - "format": CWL_IANA, - } + deps = CWLFileType( + **{ + "class": "File", + "location": uri, + "format": CWL_IANA, + } + ) def loadref(base: str, uri: str) -> CommentedMap | CommentedSeq | str | None: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) @@ -606,7 +608,7 @@ def loadref(base: str, uri: str) -> CommentedMap | CommentedSeq | str | None: nestdirs=nestdirs, ) if sfs is not None: - deps["secondaryFiles"] = cast(MutableSequence[CWLOutputType], mergedirs(sfs)) + deps["secondaryFiles"] = mergedirs(sfs) return deps diff --git a/cwltool/pack.py b/cwltool/pack.py index 686a463c6..968e8364c 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -79,7 +79,7 @@ def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> No def import_embed( - d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType, + d: MutableSequence[CWLObjectType] | CWLObjectType | CWLOutputType | None, seen: set[str], ) -> None: if isinstance(d, MutableSequence): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 3225235d0..5c80295a4 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -3,10 +3,10 @@ import stat import urllib import uuid -from collections.abc import ItemsView, Iterable, Iterator, KeysView +from collections.abc import ItemsView, Iterable, Iterator, KeysView, MutableSequence from typing import NamedTuple, Optional, cast -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLDirectoryType, CWLFileType, is_directory, is_file from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import uri_file_path @@ -76,7 +76,7 @@ class PathMapper: def __init__( self, - referenced_files: list[CWLObjectType], + referenced_files: MutableSequence[CWLFileType | CWLDirectoryType], basedir: str, stagedir: str, separateDirs: bool = True, @@ -89,7 +89,7 @@ def __init__( def visitlisting( self, - listing: list[CWLObjectType], + listing: MutableSequence[CWLFileType | CWLDirectoryType], stagedir: str, basedir: str, copy: bool = False, @@ -106,7 +106,7 @@ def visitlisting( def visit( self, - obj: CWLObjectType, + obj: CWLFileType | CWLDirectoryType, stagedir: str, basedir: str, copy: bool = False, @@ -117,10 +117,10 @@ def visit( stagedir = cast(Optional[str], obj.get("dirname")) or stagedir tgt = os.path.join( stagedir, - cast(str, obj["basename"]), + obj["basename"], ) - if obj["class"] == "Directory": - location = cast(str, obj["location"]) + if is_directory(obj): + location = obj["location"] if location.startswith("file://"): resolved = uri_file_path(location) else: @@ -131,18 +131,18 @@ def visit( if location.startswith("file://"): staged = False self.visitlisting( - cast(list[CWLObjectType], obj.get("listing", [])), + obj.get("listing", []), tgt, basedir, copy=copy, staged=staged, ) - elif obj["class"] == "File": - path = cast(str, obj["location"]) + elif is_file(obj): + path = obj["location"] ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): self._pathmap[path] = MapperEnt( - cast(str, obj["contents"]), + obj["contents"], tgt, "CreateWritableFile" if copy else "CreateFile", staged, @@ -173,14 +173,16 @@ def visit( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( - cast(list[CWLObjectType], obj.get("secondaryFiles", [])), + obj.get("secondaryFiles", []), stagedir, basedir, copy=copy, staged=staged, ) - def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: + def setup( + self, referenced_files: MutableSequence[CWLFileType | CWLDirectoryType], basedir: str + ) -> None: """ For each file, set the target to its own directory. diff --git a/cwltool/process.py b/cwltool/process.py index 9b3cee2f8..8d9c842ea 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -19,6 +19,7 @@ Iterator, MutableMapping, MutableSequence, + Sequence, Sized, ) from importlib.resources import files @@ -26,7 +27,15 @@ from typing import TYPE_CHECKING, Any, Optional, Union, cast from cwl_utils import expression -from cwl_utils.types import CWLObjectType, CWLOutputType +from cwl_utils.types import ( + CWLObjectType, + CWLOutputType, + CWLDirectoryType, + CWLFileType, + is_directory, + is_file, + is_file_or_directory, +) from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -308,9 +317,9 @@ def relocateOutputs( def _collectDirEntries( obj: CWLObjectType | MutableSequence[CWLObjectType] | None, - ) -> Iterator[CWLObjectType]: + ) -> Iterator[CWLFileType | CWLDirectoryType]: if isinstance(obj, dict): - if obj.get("class") in ("File", "Directory"): + if is_file_or_directory(obj): yield obj else: for sub_obj in obj.values(): @@ -446,7 +455,9 @@ def avroize_type( cast(MutableSequence[CWLOutputType], items), name_prefix ) case {"type": f_type}: - cast(CWLObjectType, field_type)["type"] = avroize_type(f_type, name_prefix) + cast(CWLObjectType, field_type)["type"] = avroize_type( + cast(CWLObjectType, f_type), name_prefix + ) case "File": return "org.w3id.cwl.cwl.File" case "Directory": @@ -476,7 +487,7 @@ def get_overrides(overrides: MutableSequence[CWLObjectType], toolid: str) -> CWL def var_spool_cwl_detector( - obj: CWLOutputType, + obj: CWLOutputType | None, item: Any | None = None, obj_key: Any | None = None, ) -> bool: @@ -810,7 +821,7 @@ def inc(d: list[int]) -> None: except (ValidationException, WorkflowException) as err: raise WorkflowException("Invalid job input record:\n" + str(err)) from err - files: list[CWLObjectType] = [] + files: MutableSequence[CWLFileType | CWLDirectoryType] = [] bindings = CommentedSeq() outdir = "" tmpdir = "" @@ -1126,9 +1137,9 @@ def uniquename(stem: str, names: set[str] | None = None) -> str: return u -def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: +def nestdir(base: str, deps: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType: dirname = os.path.dirname(base) + "/" - subid = cast(str, deps["location"]) + subid = deps["location"] if subid.startswith(dirname): s2 = subid[len(dirname) :] sp = s2.split("/") @@ -1136,22 +1147,24 @@ def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: while sp: loc = dirname + "/".join(sp) nx = sp.pop() - deps = { - "class": "Directory", - "basename": nx, - "listing": [deps], - "location": loc, - } + deps = CWLDirectoryType( + **{ + "class": "Directory", + "basename": nx, + "listing": [deps], + "location": loc, + } + ) return deps def mergedirs( - listing: MutableSequence[CWLObjectType], -) -> MutableSequence[CWLObjectType]: - r: list[CWLObjectType] = [] - ents: dict[str, CWLObjectType] = {} + listing: MutableSequence[CWLFileType | CWLDirectoryType], +) -> MutableSequence[CWLFileType | CWLDirectoryType]: + r: list[CWLFileType | CWLDirectoryType] = [] + ents: dict[str, CWLFileType | CWLDirectoryType] = {} for e in listing: - basename = cast(str, e["basename"]) + basename = e["basename"] if basename not in ents: ents[basename] = e elif e["location"] != ents[basename]["location"]: @@ -1159,19 +1172,16 @@ def mergedirs( "Conflicting basename in listing or secondaryFiles, '%s' used by both '%s' and '%s'" % (basename, e["location"], ents[basename]["location"]) ) - elif e["class"] == "Directory": + elif is_directory(e): if e.get("listing"): # name already in entries # merge it into the existing listing - cast(list[CWLObjectType], ents[basename].setdefault("listing", [])).extend( - cast(list[CWLObjectType], e["listing"]) + cast(CWLDirectoryType, ents[basename]).setdefault("listing", []).extend( + e["listing"] ) for e in ents.values(): - if e["class"] == "Directory" and "listing" in e: - e["listing"] = cast( - MutableSequence[CWLOutputType], - mergedirs(cast(list[CWLObjectType], e["listing"])), - ) + if is_directory(e) and "listing" in e: + e["listing"] = mergedirs(e["listing"]) r.extend(ents.values()) return r @@ -1181,13 +1191,18 @@ def mergedirs( def scandeps( base: str, - doc: CWLObjectType | MutableSequence[CWLObjectType], + doc: ( + CWLFileType + | CWLDirectoryType + | CWLObjectType + | Sequence[CWLFileType | CWLDirectoryType | CWLObjectType] + ), reffields: set[str], urlfields: set[str], loadref: Callable[[str, str], CommentedMap | CommentedSeq | str | None], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, -) -> MutableSequence[CWLObjectType]: +) -> MutableSequence[CWLFileType | CWLDirectoryType]: """ Search for external files references in a CWL document or input object. @@ -1207,72 +1222,74 @@ def scandeps( produce the same relative file system locations. :returns: A list of File or Directory dependencies """ - r: MutableSequence[CWLObjectType] = [] + r: MutableSequence[CWLFileType | CWLDirectoryType] = [] if isinstance(doc, MutableMapping): if "id" in doc: if cast(str, doc["id"]).startswith("file://"): df, _ = urllib.parse.urldefrag(cast(str, doc["id"])) if base != df: - r.append({"class": "File", "location": df, "format": CWL_IANA}) + r.append(CWLFileType(**{"class": "File", "location": df, "format": CWL_IANA})) base = df - if doc.get("class") in ("File", "Directory") and "location" in urlfields: - u = cast(Optional[str], doc.get("location", doc.get("path"))) + if is_file_or_directory(doc) and "location" in urlfields: + u = doc.get("location", doc.get("path")) if u and not u.startswith("_:"): - deps: CWLObjectType = { - "class": doc["class"], - "location": urljoin(base, u), - } + deps: CWLFileType | CWLDirectoryType + if is_file(doc): + deps = CWLFileType( + **{ + "class": "File", + } + ) + if "secondaryFiles" in doc: + deps["secondaryFiles"] = scandeps( + base, + doc["secondaryFiles"], + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, + ) + else: + deps = CWLDirectoryType( + **{ + "class": "Directory", + } + ) + if "listing" in doc: + deps["listing"] = cast(CWLDirectoryType, doc)["listing"] + deps["location"] = urljoin(base, u) if "basename" in doc: deps["basename"] = doc["basename"] - match doc: - case {"class": "Directory", "listing": listing}: - deps["listing"] = listing - case {"class": "File", "secondaryFiles": sec_files}: - deps["secondaryFiles"] = cast( - CWLOutputType, - scandeps( - base, - cast( - Union[CWLObjectType, MutableSequence[CWLObjectType]], - sec_files, - ), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ), - ) if nestdirs: deps = nestdir(base, deps) r.append(deps) else: - match doc: - case {"class": "Directory", "listing": listing}: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], listing), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ) + if is_directory(doc) and "listing" in doc: + r.extend( + scandeps( + base, + doc["listing"], + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, ) - case {"class": "File", "secondaryFiles": sec_files}: - r.extend( - scandeps( - base, - cast(MutableSequence[CWLObjectType], sec_files), - reffields, - urlfields, - loadref, - urljoin=urljoin, - nestdirs=nestdirs, - ) + ) + elif is_file(doc) and "secondaryFiles" in doc: + r.extend( + scandeps( + base, + doc["secondaryFiles"], + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, ) + ) for k, v in doc.items(): if k in reffields: @@ -1299,11 +1316,13 @@ def scandeps( Union[MutableSequence[CWLObjectType], CWLObjectType], loadref(base, u2), ) - deps2: CWLObjectType = { - "class": "File", - "location": subid, - "format": CWL_IANA, - } + deps2 = CWLFileType( + **{ + "class": "File", + "location": subid, + "format": CWL_IANA, + } + ) sf = scandeps( subid, sub, @@ -1314,19 +1333,18 @@ def scandeps( nestdirs=nestdirs, ) if sf: - deps2["secondaryFiles"] = cast( - MutableSequence[CWLOutputType], mergedirs(sf) - ) + deps2["secondaryFiles"] = mergedirs(sf) if nestdirs: - deps2 = nestdir(base, deps2) - r.append(deps2) + r.append(nestdir(base, deps2)) + else: + r.append(deps2) elif k in urlfields and k != "location": for u3 in aslist(v): - deps = {"class": "File", "location": urljoin(base, u3)} + deps = CWLFileType(**{"class": "File", "location": urljoin(base, u3)}) if nestdirs: deps = nestdir(base, deps) r.append(deps) - elif doc.get("class") in ("File", "Directory") and k in ( + elif is_file_or_directory(doc) and k in ( "listing", "secondaryFiles", ): diff --git a/cwltool/singularity.py b/cwltool/singularity.py index dd050f732..0cf20899e 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -9,11 +9,11 @@ import shutil import sys import threading -from collections.abc import Callable, MutableMapping +from collections.abc import Callable, MutableMapping, MutableSequence from subprocess import check_call, check_output # nosec from typing import cast -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLObjectType, CWLDirectoryType, CWLFileType from packaging.version import Version from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dumps @@ -170,7 +170,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + make_path_mapper: Callable[ + [MutableSequence[CWLFileType | CWLDirectoryType], str, RuntimeContext, bool], PathMapper + ], requirements: list[CWLObjectType], hints: list[CWLObjectType], name: str, diff --git a/cwltool/task_queue.py b/cwltool/task_queue.py index e86172a9a..18d6cb096 100644 --- a/cwltool/task_queue.py +++ b/cwltool/task_queue.py @@ -6,7 +6,6 @@ import queue import threading from collections.abc import Callable -from typing import Union from .loghandler import _logger @@ -66,8 +65,8 @@ def _task_queue_func(self) -> None: def add( self, task: Callable[[], None], - unlock: Union[threading.Condition, None] = None, - check_done: Union[threading.Event, None] = None, + unlock: threading.Condition | None = None, + check_done: threading.Event | None = None, ) -> None: """ Add your task to the queue. diff --git a/cwltool/update.py b/cwltool/update.py index de92b3bdf..48b9484c9 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -125,7 +125,7 @@ def rewrite_requirements(t: CWLObjectType) -> None: rewrite_requirements(s) def update_secondaryFiles( - t: CWLOutputType, top: bool = False + t: CWLOutputType | None, top: bool = False ) -> MutableSequence[MutableMapping[str, str]] | MutableMapping[str, str]: if isinstance(t, CommentedSeq): new_seq = copy.deepcopy(t) diff --git a/cwltool/utils.py b/cwltool/utils.py index feea65039..0de5626ac 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -2,7 +2,14 @@ import collections -from cwl_utils.types import CWLObjectType, CWLOutputType, DirectoryType +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_directory, + is_file, +) try: import fcntl @@ -27,6 +34,7 @@ Iterable, MutableMapping, MutableSequence, + Sequence, ) from datetime import datetime from email.utils import parsedate_to_datetime @@ -44,7 +52,6 @@ Optional, TypeAlias, Union, - cast, ) import requests @@ -241,15 +248,17 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No visit_class(rec, ("Directory",), op) -def dedup(listing: list[CWLObjectType]) -> list[CWLObjectType]: +def dedup( + listing: MutableSequence[CWLFileType | CWLDirectoryType], +) -> MutableSequence[CWLFileType | CWLDirectoryType]: marksub = set() def mark(d: dict[str, str]) -> None: marksub.add(d["location"]) for entry in listing: - if entry["class"] == "Directory": - for e in cast(list[CWLObjectType], entry.get("listing", [])): + if is_directory(entry): + for e in entry.get("listing", []): adjustFileObjs(e, mark) adjustDirObjs(e, mark) @@ -258,37 +267,41 @@ def mark(d: dict[str, str]) -> None: for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) - markdup.add(cast(str, r["location"])) + markdup.add(r["location"]) return dd -def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = True) -> None: +def get_listing( + fs_access: "StdFsAccess", rec: CWLObjectType | CWLDirectoryType, recursive: bool = True +) -> None: """Expand, recursively, any 'listing' fields in a Directory.""" - if rec.get("class") != "Directory": - finddirs: list[CWLObjectType] = [] + if not is_directory(rec): + finddirs: list[CWLDirectoryType] = [] visit_class(rec, ("Directory",), finddirs.append) for f in finddirs: get_listing(fs_access, f, recursive=recursive) return if "listing" in rec: return - listing: list[CWLOutputType] = [] - loc = cast(str, rec["location"]) + listing: MutableSequence[CWLFileType | CWLDirectoryType] = [] + loc = rec["location"] for ld in fs_access.listdir(loc): parse = urllib.parse.urlparse(ld) bn = os.path.basename(urllib.request.url2pathname(parse.path)) if fs_access.isdir(ld): - ent: MutableMapping[str, Any] = { - "class": "Directory", - "location": ld, - "basename": bn, - } + ent = CWLDirectoryType( + **{ + "class": "Directory", + "location": ld, + "basename": bn, + } + ) if recursive: get_listing(fs_access, ent, recursive) listing.append(ent) else: - listing.append({"class": "File", "location": ld, "basename": bn}) + listing.append(CWLFileType(**{"class": "File", "location": ld, "basename": bn})) rec["listing"] = listing @@ -388,17 +401,15 @@ def ensure_non_writable(path: str) -> None: def normalizeFilesDirs( - job: None | ( - MutableSequence[MutableMapping[str, Any]] | MutableMapping[str, Any] | DirectoryType - ), + job: Sequence[CWLObjectType | CWLOutputType | None] | CWLObjectType | CWLOutputType | None, ) -> None: - def addLocation(d: dict[str, Any]) -> None: + def addLocation(d: CWLFileType | CWLDirectoryType) -> None: if "location" not in d: - if d["class"] == "File" and ("contents" not in d): + if is_file(d) and ("contents" not in d): raise ValidationException( "Anonymous file object must have 'contents' and 'basename' fields." ) - if d["class"] == "Directory" and ("listing" not in d or "basename" not in d): + if is_directory(d) and ("listing" not in d or "basename" not in d): raise ValidationException( "Anonymous directory object must have 'listing' and 'basename' fields." ) @@ -410,7 +421,7 @@ def addLocation(d: dict[str, Any]) -> None: path = parse.path # strip trailing slash if path.endswith("/"): - if d["class"] != "Directory": + if not is_directory(d): raise ValidationException( "location '%s' ends with '/' but is not a Directory" % d["location"] ) @@ -432,7 +443,7 @@ def addLocation(d: dict[str, Any]) -> None: else: d["basename"] = str(os.path.basename(urllib.request.url2pathname(path))) - if d["class"] == "File": + if is_file(d): nr, ne = os.path.splitext(d["basename"]) if d.get("nameroot") != nr: d["nameroot"] = str(nr) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index c1a3f7d2d..1096aff08 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -4,7 +4,7 @@ import logging import threading from collections.abc import MutableMapping, MutableSequence, Sized -from typing import TYPE_CHECKING, Optional, Union, cast +from typing import Optional, TYPE_CHECKING, cast from cwl_utils import expression from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType @@ -764,7 +764,7 @@ def valueFromFunc(k: str, v: CWLOutputType | None) -> CWLOutputType | None: def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: Union[threading.Lock, None] = None, + tmpdir_lock: threading.Lock | None = None, ) -> None: """Log the start of each workflow.""" _logger.info("[%s] start", self.name) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index e2dd3606d..c6c3bb105 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,6 +1,5 @@ mypy==1.19.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 -cwl-utils>=0.32 cwltest types-requests types-setuptools diff --git a/pyproject.toml b/pyproject.toml index 57b89aa93..81d17192c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "types-psutil>=7.1.3.20251210", "ruamel.yaml>=0.16.0,<0.19", "schema-salad>=8.9,<9", - "cwl-utils>=0.32", + "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head", "toml", "argcomplete>=1.12.0", "rich-argparse", diff --git a/setup.py b/setup.py index e473c36fe..ba3c2c8d0 100644 --- a/setup.py +++ b/setup.py @@ -159,7 +159,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "pydot >= 1.4.1", "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 - "cwl-utils >= 0.32", + "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head", "spython >= 0.3.0", "rich-argparse", "typing-extensions >= 4.1.0", diff --git a/tests/test_examples.py b/tests/test_examples.py index c6ffd72d4..bc4301959 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -7,7 +7,7 @@ import subprocess import sys import urllib.parse -from collections.abc import MutableMapping +from collections.abc import MutableMapping, MutableSequence from io import StringIO from pathlib import Path from typing import Any, cast @@ -17,7 +17,13 @@ import pytest from cwl_utils.errors import JavascriptException from cwl_utils.sandboxjs import param_re -from cwl_utils.types import CWLObjectType, CWLOutputType, CWLParameterContext +from cwl_utils.types import ( + CWLObjectType, + CWLOutputType, + CWLParameterContext, + CWLDirectoryType, + CWLFileType, +) from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -590,24 +596,28 @@ def test_scandeps_defaults_with_secondaryfiles() -> None: def test_dedupe() -> None: - not_deduped: list[CWLObjectType] = [ - {"class": "File", "location": "file:///example/a"}, - {"class": "File", "location": "file:///example/a"}, - {"class": "File", "location": "file:///example/d"}, - { - "class": "Directory", - "location": "file:///example/c", - "listing": [{"class": "File", "location": "file:///example/d"}], - }, + not_deduped: MutableSequence[CWLFileType | CWLDirectoryType] = [ + CWLFileType(**{"class": "File", "location": "file:///example/a"}), + CWLFileType(**{"class": "File", "location": "file:///example/a"}), + CWLFileType(**{"class": "File", "location": "file:///example/d"}), + CWLDirectoryType( + **{ + "class": "Directory", + "location": "file:///example/c", + "listing": [{"class": "File", "location": "file:///example/d"}], + } + ), ] expected = [ - {"class": "File", "location": "file:///example/a"}, - { - "class": "Directory", - "location": "file:///example/c", - "listing": [{"class": "File", "location": "file:///example/d"}], - }, + CWLFileType(**{"class": "File", "location": "file:///example/a"}), + CWLDirectoryType( + **{ + "class": "Directory", + "location": "file:///example/c", + "listing": [{"class": "File", "location": "file:///example/d"}], + } + ), ] assert dedup(not_deduped) == expected diff --git a/tests/test_http_input.py b/tests/test_http_input.py index 16d40a292..d98e8a368 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,8 +1,9 @@ import os +from collections.abc import MutableSequence from datetime import datetime from pathlib import Path -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLDirectoryType, CWLFileType from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper @@ -12,13 +13,15 @@ def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = ( "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" ) - base_file: list[CWLObjectType] = [ - { - "class": "File", - "location": "https://raw.githubusercontent.com/common-workflow-language/" - "cwltool/main/tests/2.fasta", - "basename": "chr20.fa", - } + base_file: MutableSequence[CWLFileType | CWLDirectoryType] = [ + CWLFileType( + **{ + "class": "File", + "location": "https://raw.githubusercontent.com/common-workflow-language/" + "cwltool/main/tests/2.fasta", + "basename": "chr20.fa", + } + ) ] pathmap = PathMapper(base_file, os.getcwd(), str(tmp_path))._pathmap @@ -54,12 +57,14 @@ def test_modification_date(tmp_path: Path) -> None: ) location = httpserver.url_for(f"/{remote_file_name}") - base_file: list[CWLObjectType] = [ - { - "class": "File", - "location": location, - "basename": remote_file_name, - } + base_file: MutableSequence[CWLFileType | CWLDirectoryType] = [ + CWLFileType( + **{ + "class": "File", + "location": location, + "basename": remote_file_name, + } + ) ] date_now = datetime.now() diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index 2c5df6339..a7144a8f0 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -9,6 +9,7 @@ import pytest from cwl_utils import sandboxjs +from cwl_utils.types import CWLFileType from cwltool.factory import Factory from cwltool.loghandler import _logger, configure_logging @@ -41,7 +42,7 @@ def test_value_from_two_concatenated_expressions() -> None: js_engine.localdata = threading.local() # type: ignore[attr-defined] factory = Factory() echo = factory.make(get_data("tests/wf/vf-concat.cwl")) - file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file) == {"out": "a string\n"} @@ -87,7 +88,7 @@ def test_value_from_two_concatenated_expressions_podman( with monkeypatch.context() as m: m.setenv("PATH", new_paths) echo = factory.make(get_data("tests/wf/vf-concat.cwl")) - file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file) == {"out": "a string\n"} @@ -112,7 +113,7 @@ def test_value_from_two_concatenated_expressions_singularity( m.setenv("CWL_SINGULARITY_CACHE", str(singularity_cache)) m.setenv("PATH", new_paths) echo = factory.make(get_data("tests/wf/vf-concat.cwl")) - file = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file) == {"out": "a string\n"} diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 2d76e07a7..367714f27 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,6 +1,8 @@ import json from pathlib import Path +from cwl_utils.types import CWLFileType + from cwltool.context import RuntimeContext from cwltool.executors import MultithreadedJobExecutor from cwltool.factory import Factory @@ -17,7 +19,7 @@ def test_sequential_workflow(tmp_path: Path) -> None: runtime_context.select_resources = executor.select_resources factory = Factory(executor, None, runtime_context) echo = factory.make(get_data(test_file)) - file_contents = {"class": "File", "location": get_data("tests/wf/whale.txt")} + file_contents = CWLFileType(**{"class": "File", "location": get_data("tests/wf/whale.txt")}) assert echo(file1=file_contents) == {"count_output": 16} diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index 8850e7a8d..289661fed 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,5 +1,7 @@ +from collections.abc import MutableSequence + import pytest -from cwl_utils.types import CWLObjectType +from cwl_utils.types import CWLObjectType, CWLDirectoryType, CWLFileType from cwltool.pathmapper import PathMapper from cwltool.utils import normalizeFilesDirs @@ -9,7 +11,7 @@ def test_subclass() -> None: class SubPathMapper(PathMapper): def __init__( self, - referenced_files: list[CWLObjectType], + referenced_files: MutableSequence[CWLFileType | CWLDirectoryType], basedir: str, stagedir: str, new: str, @@ -90,7 +92,7 @@ def test_basename_field_generation(filename: str, expected: tuple[str, str]) -> "nameext": nameext, } - my_file = {"class": "File", "location": "/foo/" + filename} + my_file = CWLFileType(**{"class": "File", "location": "/foo/" + filename}) normalizeFilesDirs(my_file) assert my_file == expected2 From e9582d4061ead98e86f8dbe3f4850e3f20ffaadd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Mon, 15 Dec 2025 14:48:53 +0100 Subject: [PATCH 05/16] Update cwltool/cwlviewer.py --- cwltool/cwlviewer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index d84cc9dbd..db6b50358 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -12,7 +12,7 @@ if Version(pydot.__version__) > Version("3.0"): quote_id_if_necessary = pydot.quote_id_if_necessary else: - quote_id_if_necessary = pydot.quote_if_necessary + quote_id_if_necessary = pydot.quote_if_necessary # type: ignore[attr-defined] def _get_inner_edges_query() -> str: From 99aec496052dbad19643f102d64bb7a85bee4f51 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 16 Dec 2025 09:33:55 +0100 Subject: [PATCH 06/16] creating a type union with the pipe operator is not support by threading.Lock until Python 3.13 https://github.com/python/cpython/issues/114315 fixed in 3.13 https://github.com/python/cpython/commit/d96358ff9de646dbf64dfdfed46d510da7ec4803 --- cwltool/command_line_tool.py | 4 ++-- cwltool/context.py | 2 +- cwltool/job.py | 7 ++++--- cwltool/task_queue.py | 5 +++-- cwltool/workflow_job.py | 4 ++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 8b3dbe2d7..5e7605c8c 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -179,7 +179,7 @@ def __init__( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: threading.Lock | None = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: try: normalizeFilesDirs(self.builder.job) @@ -329,7 +329,7 @@ def __init__( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: threading.Lock | None = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: if self.output_callback: self.output_callback( diff --git a/cwltool/context.py b/cwltool/context.py index d9c8772f1..44129b40f 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -180,7 +180,7 @@ def __init__(self, kwargs: dict[str, Any] | None = None) -> None: self.cidfile_dir: str | None = None self.cidfile_prefix: str | None = None - self.workflow_eval_lock: threading.Condition | None = None + self.workflow_eval_lock: Union[threading.Condition, None] = None self.research_obj: ResearchObject | None = None self.orcid: str = "" self.cwl_full_name: str = "" diff --git a/cwltool/job.py b/cwltool/job.py index 7dda6d026..6dabf8632 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -161,7 +161,8 @@ def __repr__(self) -> str: def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: threading.Lock | None = None, + tmpdir_lock: Union[threading.Lock, None] = None, + # use `threading.Lock | None` when we drop support for Python 3.12 ) -> None: pass @@ -566,7 +567,7 @@ class CommandLineJob(JobBase): def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: threading.Lock | None = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: if tmpdir_lock: with tmpdir_lock: @@ -752,7 +753,7 @@ def add_volumes( def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: threading.Lock | None = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: debug = runtimeContext.debug if tmpdir_lock: diff --git a/cwltool/task_queue.py b/cwltool/task_queue.py index 18d6cb096..e86172a9a 100644 --- a/cwltool/task_queue.py +++ b/cwltool/task_queue.py @@ -6,6 +6,7 @@ import queue import threading from collections.abc import Callable +from typing import Union from .loghandler import _logger @@ -65,8 +66,8 @@ def _task_queue_func(self) -> None: def add( self, task: Callable[[], None], - unlock: threading.Condition | None = None, - check_done: threading.Event | None = None, + unlock: Union[threading.Condition, None] = None, + check_done: Union[threading.Event, None] = None, ) -> None: """ Add your task to the queue. diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 1096aff08..c1a3f7d2d 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -4,7 +4,7 @@ import logging import threading from collections.abc import MutableMapping, MutableSequence, Sized -from typing import Optional, TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression from cwl_utils.types import CWLObjectType, CWLOutputType, SinkType @@ -764,7 +764,7 @@ def valueFromFunc(k: str, v: CWLOutputType | None) -> CWLOutputType | None: def run( self, runtimeContext: RuntimeContext, - tmpdir_lock: threading.Lock | None = None, + tmpdir_lock: Union[threading.Lock, None] = None, ) -> None: """Log the start of each workflow.""" _logger.info("[%s] start", self.name) From ec7537f54185eed4cc3e932ab37977fd07293d8d Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 16 Dec 2025 09:35:23 +0100 Subject: [PATCH 07/16] sort imports --- cwltool/builder.py | 2 +- cwltool/command_line_tool.py | 4 ++-- cwltool/cwlprov/provenance_profile.py | 26 ++++++++++++++++---------- cwltool/cwlprov/ro.py | 7 +------ cwltool/docker.py | 2 +- cwltool/job.py | 2 +- cwltool/main.py | 8 +++++++- cwltool/process.py | 4 ++-- cwltool/singularity.py | 2 +- cwltool/workflow.py | 7 +------ tests/test_examples.py | 4 ++-- tests/test_pathmapper.py | 2 +- 12 files changed, 36 insertions(+), 34 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 86e1a4b77..5c615a3ec 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -9,7 +9,7 @@ from cwl_utils import expression from cwl_utils.file_formats import check_format -from cwl_utils.types import CWLObjectType, CWLOutputType, CWLDirectoryType, CWLFileType +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType from mypy_extensions import mypyc_attr from rdflib import Graph from ruamel.yaml.comments import CommentedMap diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 5e7605c8c..ef7c6fe88 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -25,10 +25,10 @@ from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast from cwl_utils.types import ( - CWLObjectType, - CWLOutputType, CWLDirectoryType, CWLFileType, + CWLObjectType, + CWLOutputType, is_directory, is_file, is_file_or_directory, diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index f2168a224..ffafe3dc3 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -6,13 +6,26 @@ from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from typing import Any, TYPE_CHECKING, TypedDict, cast - -from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, is_directory, is_file +from typing import TYPE_CHECKING, Any, TypedDict, cast + +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + is_directory, + is_file, +) from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity from schema_salad.sourceline import SourceLine +from ..errors import WorkflowException +from ..job import CommandLineJob, JobBase +from ..loghandler import _logger +from ..process import Process, shortname +from ..stdfsaccess import StdFsAccess +from ..utils import JobsType, get_listing, posix_path, versionstring +from ..workflow_job import WorkflowJob from .provenance_constants import ( ACCOUNT_UUID, CWLPROV, @@ -30,13 +43,6 @@ WFPROV, ) from .writablebagfile import create_job, write_bag_file # change this later -from ..errors import WorkflowException -from ..job import CommandLineJob, JobBase -from ..loghandler import _logger -from ..process import Process, shortname -from ..stdfsaccess import StdFsAccess -from ..utils import JobsType, get_listing, posix_path, versionstring -from ..workflow_job import WorkflowJob if TYPE_CHECKING: from .ro import ResearchObject diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 98683b1d7..392b183f5 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -26,12 +26,7 @@ from ..loghandler import _logger from ..stdfsaccess import StdFsAccess -from ..utils import ( - create_tmp_dir, - local_path, - posix_path, - versionstring, -) +from ..utils import create_tmp_dir, local_path, posix_path, versionstring from . import Aggregate, Annotation, AuthoredBy, _valid_orcid, _whoami, checksum_copy from .provenance_constants import ( ACCOUNT_UUID, diff --git a/cwltool/docker.py b/cwltool/docker.py index 5620def24..b26ac78c2 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -14,7 +14,7 @@ from typing import Optional, cast import requests -from cwl_utils.types import CWLObjectType, CWLDirectoryType, CWLFileType +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType from .builder import Builder from .context import RuntimeContext diff --git a/cwltool/job.py b/cwltool/job.py index 6dabf8632..9def5fddd 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -21,7 +21,7 @@ from typing import IO, TYPE_CHECKING, Optional, TextIO, Union, cast import psutil -from cwl_utils.types import CWLObjectType, CWLOutputType, CWLDirectoryType, CWLFileType +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps diff --git a/cwltool/main.py b/cwltool/main.py index 89256afa5..582066b05 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -23,7 +23,13 @@ import coloredlogs import requests import ruamel.yaml -from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType, CWLOutputType, is_file +from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, + CWLObjectType, + CWLOutputType, + is_file, +) from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML diff --git a/cwltool/process.py b/cwltool/process.py index 8d9c842ea..f49193a65 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -28,10 +28,10 @@ from cwl_utils import expression from cwl_utils.types import ( - CWLObjectType, - CWLOutputType, CWLDirectoryType, CWLFileType, + CWLObjectType, + CWLOutputType, is_directory, is_file, is_file_or_directory, diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 0cf20899e..75ebb769d 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -13,7 +13,7 @@ from subprocess import check_call, check_output # nosec from typing import cast -from cwl_utils.types import CWLObjectType, CWLDirectoryType, CWLFileType +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType from packaging.version import Version from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dumps diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 78078905a..6a66b3ef2 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -22,12 +22,7 @@ from .load_tool import load_tool from .loghandler import _logger from .process import Process, get_overrides, shortname -from .utils import ( - JobsGeneratorType, - OutputCallbackType, - StepType, - aslist, -) +from .utils import JobsGeneratorType, OutputCallbackType, StepType, aslist from .workflow_job import WorkflowJob diff --git a/tests/test_examples.py b/tests/test_examples.py index bc4301959..329bb6d47 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -18,11 +18,11 @@ from cwl_utils.errors import JavascriptException from cwl_utils.sandboxjs import param_re from cwl_utils.types import ( + CWLDirectoryType, + CWLFileType, CWLObjectType, CWLOutputType, CWLParameterContext, - CWLDirectoryType, - CWLFileType, ) from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index 289661fed..c0005cc77 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,7 +1,7 @@ from collections.abc import MutableSequence import pytest -from cwl_utils.types import CWLObjectType, CWLDirectoryType, CWLFileType +from cwl_utils.types import CWLDirectoryType, CWLFileType, CWLObjectType from cwltool.pathmapper import PathMapper from cwltool.utils import normalizeFilesDirs From f55ce3e5f56067afe870ab06cd995f0d3f74354f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 16 Dec 2025 10:28:15 +0100 Subject: [PATCH 08/16] restore previous logic Yes, a bit odd that `class: Directory` objects in `listing` had a `dirname`, but we are temporarily using that (and fixing them) for setting the outdir when updating the Pathmapper. Then the `dirname` fields are removed immediately after. --- cwltool/command_line_tool.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index ef7c6fe88..7d4302b82 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -757,9 +757,9 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: if "basename" in entry: basename = cast(str, entry["basename"]) entry["basename"] = os.path.basename(basename) + dirname = os.path.join(builder.outdir, os.path.dirname(basename)) + entry["dirname"] = dirname if is_file(entry): - dirname = os.path.join(builder.outdir, os.path.dirname(basename)) - entry["dirname"] = dirname if "secondaryFiles" in entry: for sec_file in cast( MutableSequence[CWLObjectType], entry["secondaryFiles"] @@ -767,7 +767,7 @@ def _initialworkdir(self, j: JobBase | None, builder: Builder) -> None: sec_file["dirname"] = dirname normalizeFilesDirs(entry) self.updatePathmap( - (entry.get("dirname") if is_file(entry) else None) or builder.outdir, + cast(Optional[str], entry.get("dirname")) or builder.outdir, cast(PathMapper, builder.pathmapper), entry, ) From 8d95cb6a585faefeffddbd8ea15420029b41152d Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 16 Dec 2025 10:42:45 +0100 Subject: [PATCH 09/16] tox: match {shellcheck,pydocstyle,lintreadme} py version to CI --- Makefile | 2 +- tox.ini | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 28d9b8219..9b864ed19 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py -DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \ +DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox>4' auto-walrus \ isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\ -rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \ diff --git a/tox.ini b/tox.ini index 305603343..69d0d1437 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,9 @@ envlist = py3{10,11,12,13,14}-unit py3{10,11,12,13,14}-bandit py3{10,11,12,13,14}-mypy - py312-lintreadme - py312-shellcheck - py312-pydocstyle + py313-lintreadme + py313-shellcheck + py313-pydocstyle skip_missing_interpreters = True @@ -31,9 +31,9 @@ description = py3{10,11,12,13,14}-lint: Lint the Python code py3{10,11,12,13,14}-bandit: Search for common security issues py3{10,11,12,13,14}-mypy: Check for type safety - py312-pydocstyle: docstring style checker - py312-shellcheck: syntax check for shell scripts - py312-lintreadme: Lint the README.rst→.md conversion + py313-pydocstyle: docstring style checker + py313-shellcheck: syntax check for shell scripts + py313-lintreadme: Lint the README.rst→.md conversion passenv = CI @@ -52,27 +52,27 @@ deps = py3{10,11,12,13,14}-lint: -rlint-requirements.txt py3{10,11,12,13,14}-bandit: bandit py3{10,11,12,13,14}-mypy: -rmypy-requirements.txt - py312-pydocstyle: pydocstyle - py312-pydocstyle: diff-cover - py312-lintreadme: twine - py312-lintreadme: build - py312-lintreadme: readme_renderer[rst] + py313-pydocstyle: pydocstyle + py313-pydocstyle: diff-cover + py313-lintreadme: twine + py313-lintreadme: build + py313-lintreadme: readme_renderer[rst] setenv = LC_ALL = C.UTF-8 HOME = {envtmpdir} commands_pre = - py312-lintreadme: python -m build --outdir {pkg_dir} + py313-lintreadme: python -m build --outdir {pkg_dir} commands = py3{10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA="{posargs}" py3{10,11,12,13,14}-bandit: bandit -r cwltool py3{10,11,12,13,14}-lint: make flake8 format-check codespell-check py3{10,11,12,13,14}-mypy: make mypy mypyc PYTEST_EXTRA="{posargs}" - py312-shellcheck: make shellcheck - py312-pydocstyle: make diff_pydocstyle_report - py312-lintreadme: twine check {pkg_dir}/* + py313-shellcheck: make shellcheck + py313-pydocstyle: make diff_pydocstyle_report + py313-lintreadme: twine check {pkg_dir}/* skip_install = py3{10,11,12,13,14}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true From 8920b7d1993b6573950dcedbdf39911382b32b3a Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 16 Dec 2025 10:58:30 +0100 Subject: [PATCH 10/16] add missing docstrings to modified functions --- cwltool/cwlprov/provenance_profile.py | 16 ++++++++-------- cwltool/process.py | 1 + cwltool/utils.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index ffafe3dc3..b90f442f8 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -50,11 +50,11 @@ CWLArtifact = TypedDict("CWLArtifact", {"@id": str}) -class CWLDirectoryArtifact(CWLArtifact, CWLDirectoryType): +class _CWLDirectoryArtifact(CWLArtifact, CWLDirectoryType): pass -class CWLFileArtifact(CWLArtifact, CWLFileType): +class _CWLFileArtifact(CWLArtifact, CWLFileType): pass @@ -260,7 +260,7 @@ def record_process_end( self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: _CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, str]: """Construct a FileEntity for the given CWL File object.""" if not is_file(value): raise ValueError("Must have class:File: %s" % value) @@ -315,9 +315,9 @@ def declare_file(self, value: CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, for sec in cast(MutableSequence[CWLObjectType], value.get("secondaryFiles", [])): # TODO: Record these in a specializationOf entity with UUID? if is_file(sec): - (sec_entity, _, _) = self.declare_file(cast(CWLFileArtifact, sec)) + (sec_entity, _, _) = self.declare_file(cast(_CWLFileArtifact, sec)) elif is_directory(sec): - sec_entity = self.declare_directory(cast(CWLDirectoryArtifact, sec)) + sec_entity = self.declare_directory(cast(_CWLDirectoryArtifact, sec)) else: raise ValueError(f"Got unexpected secondaryFiles value: {sec}") # We don't know how/when/where the secondary file was generated, @@ -332,7 +332,7 @@ def declare_file(self, value: CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, return file_entity, entity, checksum - def declare_directory(self, value: CWLDirectoryArtifact) -> ProvEntity: + def declare_directory(self, value: _CWLDirectoryArtifact) -> ProvEntity: """Register any nested files/directories.""" # FIXME: Calculate a hash-like identifier for directory # so we get same value if it's the same filenames/hashes @@ -492,11 +492,11 @@ def declare_artefact(self, value: Any) -> ProvEntity: # Base case - we found a File we need to update case {"class": "File"}: - entity = self.declare_file(cast(CWLFileArtifact, value))[0] + entity = self.declare_file(cast(_CWLFileArtifact, value))[0] value["@id"] = entity.identifier.uri return entity case {"class": "Directory"}: - entity = self.declare_directory(cast(CWLDirectoryArtifact, value)) + entity = self.declare_directory(cast(_CWLDirectoryArtifact, value)) value["@id"] = entity.identifier.uri return entity case {**rest}: diff --git a/cwltool/process.py b/cwltool/process.py index f49193a65..92f6a6903 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1138,6 +1138,7 @@ def uniquename(stem: str, names: set[str] | None = None) -> str: def nestdir(base: str, deps: CWLFileType | CWLDirectoryType) -> CWLFileType | CWLDirectoryType: + """Add intermediate directory objects to preserve the relative layout.""" dirname = os.path.dirname(base) + "/" subid = deps["location"] if subid.startswith(dirname): diff --git a/cwltool/utils.py b/cwltool/utils.py index 0de5626ac..f3702192c 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -251,6 +251,7 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No def dedup( listing: MutableSequence[CWLFileType | CWLDirectoryType], ) -> MutableSequence[CWLFileType | CWLDirectoryType]: + """Remove duplicate entries from a CWL Directory 'listing'.""" marksub = set() def mark(d: dict[str, str]) -> None: @@ -403,6 +404,15 @@ def ensure_non_writable(path: str) -> None: def normalizeFilesDirs( job: Sequence[CWLObjectType | CWLOutputType | None] | CWLObjectType | CWLOutputType | None, ) -> None: + """ + Add missing `location`s and `basename`s to CWL File and Directory objects. + + :raises ValidationException: if anonymous objects are missing required fields, + or if the location ends in '/' but the object isn't + a directory + + """ + def addLocation(d: CWLFileType | CWLDirectoryType) -> None: if "location" not in d: if is_file(d) and ("contents" not in d): From 168678d15a87e510c8b45f500a661381ef56ea89 Mon Sep 17 00:00:00 2001 From: GlassOfWhiskey Date: Tue, 16 Dec 2025 17:59:40 +0100 Subject: [PATCH 11/16] Apply IsType optimizations --- cwltool/command_line_tool.py | 40 ++++++++++++--------------- cwltool/cwlprov/provenance_profile.py | 2 +- cwltool/pathmapper.py | 4 +-- cwltool/process.py | 2 +- 4 files changed, 21 insertions(+), 27 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 7d4302b82..499601979 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -14,7 +14,6 @@ import urllib.parse from collections.abc import ( Generator, - Iterable, Mapping, MutableMapping, MutableSequence, @@ -22,7 +21,7 @@ from enum import Enum from functools import cmp_to_key, partial from re import Pattern -from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast +from typing import Any, Optional, TYPE_CHECKING, TextIO, Union, cast from cwl_utils.types import ( CWLDirectoryType, @@ -42,12 +41,7 @@ from schema_salad.utils import json_dumps from schema_salad.validate import validate_ex -from .builder import ( - INPUT_OBJ_VOCAB, - Builder, - content_limit_respected_read_bytes, - substitute, -) +from .builder import Builder, INPUT_OBJ_VOCAB, content_limit_respected_read_bytes, substitute from .context import LoadingContext, RuntimeContext, getdefault from .docker import DockerCommandLineJob, PodmanCommandLineJob from .errors import UnsupportedRequirement, WorkflowException @@ -1292,7 +1286,7 @@ def collect_output( fs_access: StdFsAccess, compute_checksum: bool = True, ) -> CWLOutputType | None: - r: list[CWLOutputType] = [] + r: MutableSequence[CWLFileType | CWLDirectoryType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) result: CWLOutputType | None = None @@ -1340,9 +1334,9 @@ def collect_output( key=cmp_to_key(locale.strcoll), ) r.extend( - cast( - Iterable[CWLOutputType], - [ + [ + cast( + CWLFileType | CWLDirectoryType, { "location": g, "path": fs_access.join( @@ -1353,16 +1347,16 @@ def collect_output( "nameroot": os.path.splitext(decoded_basename)[0], "nameext": os.path.splitext(decoded_basename)[1], "class": "File" if fs_access.isfile(g) else "Directory", - } - for g, decoded_basename in zip( + }, + ) + for g, decoded_basename in zip( + sorted_glob_result, + map( + lambda x: os.path.basename(urllib.parse.unquote(x)), sorted_glob_result, - map( - lambda x: os.path.basename(urllib.parse.unquote(x)), - sorted_glob_result, - ), - ) - ], - ) + ), + ) + ] ) except OSError as e: _logger.warning(str(e), exc_info=builder.debug) @@ -1370,14 +1364,14 @@ def collect_output( _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(MutableSequence[CWLFileType | CWLDirectoryType], r): + for files in r: rfile = files.copy() revmap(rfile) if is_directory(files): ll = binding.get("loadListing") or builder.loadListing if ll and ll != "no_listing": get_listing(fs_access, files, (ll == "deep_listing")) - elif is_file(files): + else: if binding.get("loadContents"): with fs_access.open(rfile["location"], "rb") as f: files["contents"] = str( diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index b90f442f8..6658bd54c 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -312,7 +312,7 @@ def declare_file(self, value: _CWLFileArtifact) -> tuple[ProvEntity, ProvEntity, self.document.specializationOf(file_entity, entity) # Check for secondaries - for sec in cast(MutableSequence[CWLObjectType], value.get("secondaryFiles", [])): + for sec in value.get("secondaryFiles", []): # TODO: Record these in a specializationOf entity with UUID? if is_file(sec): (sec_entity, _, _) = self.declare_file(cast(_CWLFileArtifact, sec)) diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 5c80295a4..9e25d6c0f 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -6,7 +6,7 @@ from collections.abc import ItemsView, Iterable, Iterator, KeysView, MutableSequence from typing import NamedTuple, Optional, cast -from cwl_utils.types import CWLDirectoryType, CWLFileType, is_directory, is_file +from cwl_utils.types import CWLDirectoryType, CWLFileType, is_directory from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import uri_file_path @@ -137,7 +137,7 @@ def visit( copy=copy, staged=staged, ) - elif is_file(obj): + else: path = obj["location"] ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): diff --git a/cwltool/process.py b/cwltool/process.py index 92f6a6903..4f11b8a24 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1259,7 +1259,7 @@ def scandeps( } ) if "listing" in doc: - deps["listing"] = cast(CWLDirectoryType, doc)["listing"] + deps["listing"] = doc["listing"] deps["location"] = urljoin(base, u) if "basename" in doc: deps["basename"] = doc["basename"] From 611253345bbafb921e74b2e081848b3ade65d4bd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 21 Dec 2025 12:14:04 +0100 Subject: [PATCH 12/16] Makefile: PEP-517 build already installs the dependencies --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 9b864ed19..4d025133c 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,6 @@ install-dep: install-dependencies install-dependencies: FORCE pip install --upgrade $(DEVPKGS) - pip install -r requirements.txt install-doc-dep: pip install -r docs/requirements.txt From ab6bd9ce3c14549823d3db44b4466570b0f1f04a Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 21 Dec 2025 12:15:47 +0100 Subject: [PATCH 13/16] import fix and reformat --- cwltool/command_line_tool.py | 16 ++++++++-------- tests/test_tmpdir.py | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 499601979..02b88182c 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -12,16 +12,11 @@ import threading import urllib import urllib.parse -from collections.abc import ( - Generator, - Mapping, - MutableMapping, - MutableSequence, -) +from collections.abc import Generator, Mapping, MutableMapping, MutableSequence from enum import Enum from functools import cmp_to_key, partial from re import Pattern -from typing import Any, Optional, TYPE_CHECKING, TextIO, Union, cast +from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast from cwl_utils.types import ( CWLDirectoryType, @@ -41,7 +36,12 @@ from schema_salad.utils import json_dumps from schema_salad.validate import validate_ex -from .builder import Builder, INPUT_OBJ_VOCAB, content_limit_respected_read_bytes, substitute +from .builder import ( + INPUT_OBJ_VOCAB, + Builder, + content_limit_respected_read_bytes, + substitute, +) from .context import LoadingContext, RuntimeContext, getdefault from .docker import DockerCommandLineJob, PodmanCommandLineJob from .errors import UnsupportedRequirement, WorkflowException diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 540c368ff..367c8fbd2 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -11,6 +11,7 @@ from typing import cast import pytest +from cwl_utils.types import CWLObjectType from ruamel.yaml.comments import CommentedMap from schema_salad.avro import schema from schema_salad.sourceline import cmap @@ -26,7 +27,7 @@ from cwltool.singularity import _IMAGES, _IMAGES_LOCK, SingularityCommandLineJob from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION -from cwltool.utils import CWLObjectType, create_tmp_dir +from cwltool.utils import create_tmp_dir from .util import get_data, get_main_output, needs_docker, needs_singularity From adc3286db344d64b27ba428b3daa86504f0e750e Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 21 Dec 2025 12:28:39 +0100 Subject: [PATCH 14/16] no leftovers test: cwl-utils uses hatchling now --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9619af49f..6f37adb44 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -151,7 +151,7 @@ jobs: - name: install with test dependencies run: | - pip install -U pip setuptools wheel + pip install -U pip setuptools wheel "hatchling>=1.27.0" pip install --no-build-isolation -rtest-requirements.txt .[deps] - name: make working directory read-only From 201ce1a57082b440c6e7cf2b2c3496bb92da09ff Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 21 Dec 2025 13:26:13 +0100 Subject: [PATCH 15/16] test with newer cwl-utils PR --- pyproject.toml | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 81d17192c..2ad522f4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "types-psutil>=7.1.3.20251210", "ruamel.yaml>=0.16.0,<0.19", "schema-salad>=8.9,<9", - "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head", + "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/396/head", "toml", "argcomplete>=1.12.0", "rich-argparse", diff --git a/requirements.txt b/requirements.txt index 0ccb07436..7b47b52db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ coloredlogs pydot>=1.4.1 argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 -cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head +cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/396/head spython>=0.3.0 rich-argparse typing-extensions>=4.1.0 diff --git a/setup.py b/setup.py index 6c66e1e05..4ef1d0617 100644 --- a/setup.py +++ b/setup.py @@ -159,7 +159,7 @@ def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> li "pydot >= 1.4.1", "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 - "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/393/head", + "cwl-utils @ git+https://github.com/common-workflow-language/cwl-utils.git@refs/pull/396/head", "spython >= 0.3.0", "rich-argparse", "typing-extensions >= 4.1.0", From acc0411bd6deaf1b1d215d26526d409010b1d8ce Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 21 Dec 2025 13:36:58 +0100 Subject: [PATCH 16/16] more type adjustments --- cwltool/load_tool.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 4c34094c6..dcc47ca59 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -13,6 +13,7 @@ from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from cwl_utils.types import CWLObjectType +from mypy_extensions import i32, i64 from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.fetcher import Fetcher @@ -255,10 +256,12 @@ def _fast_parser_convert_stdstreams_to_files( cwl_v1_2_utils.convert_stdstreams_to_files(processobj) case cwl_v1_2.Workflow(steps=steps): for st in steps: - _fast_parser_convert_stdstreams_to_files(st.run) + if not isinstance(st.run, str): + _fast_parser_convert_stdstreams_to_files(st.run) case MutableSequence(): for p in processobj: - _fast_parser_convert_stdstreams_to_files(p) + if not isinstance(p, str): + _fast_parser_convert_stdstreams_to_files(p) def _fast_parser_expand_hint_class( @@ -282,7 +285,8 @@ def _fast_parser_handle_hints( case cwl_v1_2.Workflow(steps=steps): for st in steps: _fast_parser_expand_hint_class(st.hints, loadingOptions) - _fast_parser_handle_hints(st.run, loadingOptions) + if not isinstance(st.run, str): + _fast_parser_handle_hints(st.run, loadingOptions) case MutableSequence(): for p in processobj: _fast_parser_handle_hints(p, loadingOptions) @@ -315,7 +319,7 @@ def fast_parser( _fast_parser_convert_stdstreams_to_files(objects) _fast_parser_handle_hints(objects, loadopt) - processobj: MutableMapping[str, Any] | MutableSequence[Any] | float | str | None + processobj: MutableMapping[str, Any] | MutableSequence[Any] | i32 | i64 | float | str | None processobj = cwl_v1_2.save(objects, relative_uris=False)