From ab62a149ef3d97d1e75582b4b77d30502fba82a0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 8 Jun 2021 21:49:14 -0500 Subject: [PATCH 01/14] Add super targets to satisfy the globs used in some tests --- .../tests/unit/test_actions_registrar.py | 4 +- .../controllers/v1/test_pack_config_schema.py | 6 +-- .../unit/controllers/v1/test_pack_configs.py | 7 +-- .../test_register_content_script.py | 4 +- .../tests/unit/test_policies_registrar.py | 5 +- .../tests/unit/test_triggers_registrar.py | 5 +- st2tests/st2tests/fixtures/BUILD | 16 +++++++ .../st2tests/fixtures/child_packs_glob/BUILD | 3 ++ .../fixtures/child_packs_glob/__init__.py | 3 ++ st2tests/st2tests/fixtures/packs/BUILD | 47 +++++++++++++++++++ .../fixtures/packs/all_packs_glob/BUILD | 3 ++ .../fixtures/packs/all_packs_glob/__init__.py | 3 ++ 12 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 st2tests/st2tests/fixtures/child_packs_glob/BUILD create mode 100644 st2tests/st2tests/fixtures/child_packs_glob/__init__.py create mode 100644 st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD create mode 100644 st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py diff --git a/st2actions/tests/unit/test_actions_registrar.py b/st2actions/tests/unit/test_actions_registrar.py index 0f3fe76e32..349ec7fd1b 100644 --- a/st2actions/tests/unit/test_actions_registrar.py +++ b/st2actions/tests/unit/test_actions_registrar.py @@ -30,6 +30,7 @@ PACK_NAME as GENERIC_PACK, PACK_PATH as GENERIC_PACK_PATH, ) +from st2tests.fixtures.child_packs_glob import PACKS_PATH import st2tests.fixturesloader as fixtures_loader MOCK_RUNNER_TYPE_DB = RunnerTypeDB(name="run-local", runner_module="st2.runners.local") @@ -52,9 +53,8 @@ class ActionsRegistrarTest(tests_base.DbTestCase): ) def test_register_all_actions(self): try: - packs_base_path = fixtures_loader.get_fixtures_base_path() all_actions_in_db = Action.get_all() - actions_registrar.register_actions(packs_base_paths=[packs_base_path]) + actions_registrar.register_actions(packs_base_paths=[PACKS_PATH]) except Exception as e: print(six.text_type(e)) self.fail("All actions must be registered without exceptions.") diff --git a/st2api/tests/unit/controllers/v1/test_pack_config_schema.py b/st2api/tests/unit/controllers/v1/test_pack_config_schema.py index a38c278f07..9ca48aea98 100644 --- a/st2api/tests/unit/controllers/v1/test_pack_config_schema.py +++ b/st2api/tests/unit/controllers/v1/test_pack_config_schema.py @@ -17,12 +17,12 @@ from st2tests.api import FunctionalTest -from st2tests.fixturesloader import get_fixtures_packs_base_path +# import this so that pants can infer dependencies for the glob below +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH __all__ = ["PackConfigSchemasControllerTestCase"] -PACKS_PATH = get_fixtures_packs_base_path() -CONFIG_SCHEMA_COUNT = len(glob.glob("%s/*/config.schema.yaml" % (PACKS_PATH))) +CONFIG_SCHEMA_COUNT = len(glob.glob(f"{PACKS_PATH}/*/config.schema.yaml")) assert CONFIG_SCHEMA_COUNT > 1 diff --git a/st2api/tests/unit/controllers/v1/test_pack_configs.py b/st2api/tests/unit/controllers/v1/test_pack_configs.py index 5a87719eaa..91fd712826 100644 --- a/st2api/tests/unit/controllers/v1/test_pack_configs.py +++ b/st2api/tests/unit/controllers/v1/test_pack_configs.py @@ -19,12 +19,13 @@ from st2tests.api import FunctionalTest from st2api.controllers.v1.pack_configs import PackConfigsController -from st2tests.fixturesloader import get_fixtures_packs_base_path + +# import this so that pants can infer dependencies for the glob below +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH __all__ = ["PackConfigsControllerTestCase"] -PACKS_PATH = get_fixtures_packs_base_path() -CONFIGS_COUNT = len(glob.glob("%s/configs/*.yaml" % (PACKS_PATH))) +CONFIGS_COUNT = len(glob.glob(f"{PACKS_PATH}/configs/*.yaml")) assert CONFIGS_COUNT > 1 diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index d416ae9af1..0c5f20676f 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -25,6 +25,7 @@ from st2tests.fixturesloader import get_fixtures_packs_base_path # import this so that pants can infer dependencies for the glob below +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH from st2tests.fixtures.packs.dummy_pack_1.fixture import PACK_PATH as DUMMY_PACK_1_PATH from st2tests.fixtures.packs.dummy_pack_4.fixture import PACK_PATH as DUMMY_PACK_4_PATH from st2tests.fixtures.packs.runners.fixture import FIXTURE_PATH as RUNNER_DIRS @@ -38,8 +39,7 @@ BASE_CMD_ARGS = [sys.executable, SCRIPT_PATH, "--config-file=conf/st2.tests.conf", "-v"] BASE_REGISTER_ACTIONS_CMD_ARGS = BASE_CMD_ARGS + ["--register-actions"] -PACKS_PATH = get_fixtures_packs_base_path() -PACKS_COUNT = len(glob.glob("%s/*/pack.yaml" % (PACKS_PATH))) +PACKS_COUNT = len(glob.glob(f"{PACKS_PATH}/*/pack.yaml")) assert PACKS_COUNT >= 2 diff --git a/st2common/tests/unit/test_policies_registrar.py b/st2common/tests/unit/test_policies_registrar.py index 231c477446..3a435ab0f3 100644 --- a/st2common/tests/unit/test_policies_registrar.py +++ b/st2common/tests/unit/test_policies_registrar.py @@ -27,7 +27,7 @@ from st2common.persistence.policy import Policy from st2common.persistence.policy import PolicyType from st2tests.base import CleanDbTestCase -from st2tests.fixturesloader import get_fixtures_packs_base_path +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH from st2tests.fixtures.packs.dummy_pack_1.fixture import ( PACK_NAME as DUMMY_PACK_1, PACK_PATH as DUMMY_PACK_1_PATH, @@ -63,8 +63,7 @@ def test_register_all_policies(self): policies_dbs = Policy.get_all() self.assertEqual(len(policies_dbs), 0) - packs_base_path = get_fixtures_packs_base_path() - count = policies_registrar.register_policies(packs_base_paths=[packs_base_path]) + count = policies_registrar.register_policies(packs_base_paths=[PACKS_PATH]) # Verify PolicyDB objects have been created policies_dbs = Policy.get_all() diff --git a/st2common/tests/unit/test_triggers_registrar.py b/st2common/tests/unit/test_triggers_registrar.py index ef06ec5d8a..9ad3526b18 100644 --- a/st2common/tests/unit/test_triggers_registrar.py +++ b/st2common/tests/unit/test_triggers_registrar.py @@ -19,7 +19,7 @@ from st2common.persistence.trigger import Trigger from st2common.persistence.trigger import TriggerType from st2tests.base import CleanDbTestCase -from st2tests.fixturesloader import get_fixtures_packs_base_path +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH from st2tests.fixtures.packs.dummy_pack_1.fixture import ( PACK_NAME as DUMMY_PACK_1, PACK_PATH as DUMMY_PACK_1_PATH, @@ -33,8 +33,7 @@ def test_register_all_triggers(self): trigger_type_dbs = TriggerType.get_all() self.assertEqual(len(trigger_type_dbs), 0) - packs_base_path = get_fixtures_packs_base_path() - count = triggers_registrar.register_triggers(packs_base_paths=[packs_base_path]) + count = triggers_registrar.register_triggers(packs_base_paths=[PACKS_PATH]) self.assertEqual(count, 2) # Verify TriggerTypeDB and corresponding TriggerDB objects have been created diff --git a/st2tests/st2tests/fixtures/BUILD b/st2tests/st2tests/fixtures/BUILD index 010e795e35..f35dd8853c 100644 --- a/st2tests/st2tests/fixtures/BUILD +++ b/st2tests/st2tests/fixtures/BUILD @@ -6,3 +6,19 @@ resources( "./rbac_invalid/**/*.yaml", ], ) + +# Please make sure to register all of the packs in this dir. +# For each pack, depend on the python_sources target (which depends on pack_metadata). +target( + name="child_packs", + dependencies=[ + "./aliases", + "./backstop", + "./descendants", + "./generic", + "./localrunner_pack", + "./rule_enforcements", + "./timers", + "./traces", + ], +) diff --git a/st2tests/st2tests/fixtures/child_packs_glob/BUILD b/st2tests/st2tests/fixtures/child_packs_glob/BUILD new file mode 100644 index 0000000000..9cbfb9b54f --- /dev/null +++ b/st2tests/st2tests/fixtures/child_packs_glob/BUILD @@ -0,0 +1,3 @@ +python_sources( + dependencies=["st2tests/st2tests/fixtures:child_packs"], +) diff --git a/st2tests/st2tests/fixtures/child_packs_glob/__init__.py b/st2tests/st2tests/fixtures/child_packs_glob/__init__.py new file mode 100644 index 0000000000..9ea5aab79b --- /dev/null +++ b/st2tests/st2tests/fixtures/child_packs_glob/__init__.py @@ -0,0 +1,3 @@ +from st2tests.fixturesloader import get_fixtures_base_path + +PACKS_PATH = get_fixtures_base_path() diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index c53022a3ca..665501dd96 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -34,3 +34,50 @@ python_sources( "test_content_version/**/*.py", ], ) + +# Please make sure to register all of the packs in this dir. +# For each pack, depend on the python_sources target (which depends on pack_metadata). +target( + name="all_packs", + dependencies=[ + "./action_chain_tests", + "./core", + "./dummy_pack_1", + "./dummy_pack_2", + "./dummy_pack_3", + "./dummy_pack_4", + "./dummy_pack_5", + "./dummy_pack_6", + "./dummy_pack_7", + "./dummy_pack_8", + "./dummy_pack_9", + "./dummy_pack_10", + "./dummy_pack_11", + "./dummy_pack_12", + "./dummy_pack_13", + "./dummy_pack_14", + "./dummy_pack_15", + "./dummy_pack_16", + "./dummy_pack_17", + "./dummy_pack_18", + "./dummy_pack_19", + "./dummy_pack_20", + "./dummy_pack_21", + "./dummy_pack_22", + "./dummy_pack_23", + "./dummy_pack_schema_with_additional_items_1", + "./dummy_pack_schema_with_additional_properties_1", + "./dummy_pack_schema_with_nested_object_1", + "./dummy_pack_schema_with_nested_object_2", + "./dummy_pack_schema_with_nested_object_3", + "./dummy_pack_schema_with_nested_object_4", + "./dummy_pack_schema_with_nested_object_5", + "./dummy_pack_schema_with_pattern_and_additional_properties_1", + "./dummy_pack_schema_with_pattern_properties_1", + "./orquesta_tests", + "./pack_dir_name_doesnt_match_ref", + "./pack_invalid_requirements", + ":test_content_version", + "./test_library_dependencies", + ], +) diff --git a/st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD b/st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD new file mode 100644 index 0000000000..13726c3e4b --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD @@ -0,0 +1,3 @@ +python_sources( + dependencies=["st2tests/st2tests/fixtures/packs:all_packs"], +) diff --git a/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py b/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py new file mode 100644 index 0000000000..571f0d6111 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py @@ -0,0 +1,3 @@ +from st2tests.fixturesloader import get_fixtures_packs_base_path + +PACKS_PATH = get_fixtures_packs_base_path() From dbaa91a2ccd2592a8761f9ea59926774ae5b0d44 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 14 Jun 2021 12:37:20 -0500 Subject: [PATCH 02/14] add license header --- .../st2tests/fixtures/child_packs_glob/__init__.py | 13 +++++++++++++ .../fixtures/packs/all_packs_glob/__init__.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/st2tests/st2tests/fixtures/child_packs_glob/__init__.py b/st2tests/st2tests/fixtures/child_packs_glob/__init__.py index 9ea5aab79b..76d1e6e14b 100644 --- a/st2tests/st2tests/fixtures/child_packs_glob/__init__.py +++ b/st2tests/st2tests/fixtures/child_packs_glob/__init__.py @@ -1,3 +1,16 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from st2tests.fixturesloader import get_fixtures_base_path PACKS_PATH = get_fixtures_base_path() diff --git a/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py b/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py index 571f0d6111..d41e2dad06 100644 --- a/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py +++ b/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py @@ -1,3 +1,16 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from st2tests.fixturesloader import get_fixtures_packs_base_path PACKS_PATH = get_fixtures_packs_base_path() From 56064a23639152dc9ea2e746a8933f4ff650602b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 27 Jan 2023 13:56:21 -0600 Subject: [PATCH 03/14] drop unused import --- st2common/tests/integration/test_register_content_script.py | 1 - 1 file changed, 1 deletion(-) diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index 0c5f20676f..b0288a910e 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -22,7 +22,6 @@ from st2tests.base import IntegrationTestCase from st2common.util.shell import run_command from st2tests import config as test_config -from st2tests.fixturesloader import get_fixtures_packs_base_path # import this so that pants can infer dependencies for the glob below from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH From 8e84399f5829ba327b836551298b045c1a7ca77c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 28 Jan 2023 23:33:07 -0600 Subject: [PATCH 04/14] add packs_glob pants target --- pants-plugins/pack_metadata/target_types.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index cfadbaab01..4f9d8558f3 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -17,6 +17,7 @@ from pants.core.target_types import ( ResourcesGeneratingSourcesField, ResourcesGeneratorTarget, + GenericTarget, ) @@ -77,3 +78,14 @@ class PackMetadataInGitSubmodule(PackMetadata): "has unmatched globs. It prints instructions on how to checkout git " "submodules." ) + + +class PacksGlob(GenericTarget): + alias = "packs_glob" + core_fields = (*COMMON_TARGET_FIELDS, Dependencies) + help = ( + "Packs glob.\n\n" + "Avoid using this target. It gets automatic dependencies on all " + "subdirectories (packs) except those listed with ! in dependencies. " + "This is unfortunately needed by tests that use a glob to load pack fixtures." + ) From d38f44d494e62ed34b191c989918e71d60c72339 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 00:23:35 -0600 Subject: [PATCH 05/14] pants: stub PacksGlob target --- pants-plugins/pack_metadata/register.py | 19 +++- .../pack_metadata/target_types_rules.py | 88 +++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 pants-plugins/pack_metadata/target_types_rules.py diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 34a6aa85cc..cd651b1183 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,13 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pack_metadata import tailor -from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule +from pack_metadata import tailor, target_types_rules +from pack_metadata.target_types import ( + PackMetadata, + PackMetadataInGitSubmodule, + PacksGlob, +) def rules(): - return tailor.rules() + return [ + tailor.rules(), + target_types_rules.rules(), + ] def target_types(): - return [PackMetadata, PackMetadataInGitSubmodule] + return [ + PackMetadata, + PackMetadataInGitSubmodule, + PacksGlob, + ] diff --git a/pants-plugins/pack_metadata/target_types_rules.py b/pants-plugins/pack_metadata/target_types_rules.py new file mode 100644 index 0000000000..ae96366792 --- /dev/null +++ b/pants-plugins/pack_metadata/target_types_rules.py @@ -0,0 +1,88 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# repurposed from pants.backend.python.target_types_rules +import dataclasses +import os +from dataclasses import dataclass + +from pants.engine.addresses import Address +from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths +from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + ExplicitlyProvidedDependencies, + FieldSet, + InferDependenciesRequest, + InferredDependencies, + InvalidFieldException, +) +from pants.util.logging import LogLevel + +from pack_metadata.target_types import PacksGlobDependencies + + +@dataclass(frozen=True) +class PacksGlobInferenceFieldSet(FieldSet): + required_fields = (PacksGlobDependencies,) + + dependencies: PacksGlobDependencies + + +class InferPacksGlobDependencies(InferDependenciesRequest): + infer_from = PacksGlobInferenceFieldSet + + +@rule( + desc="Inferring packs glob dependencies", + level=LogLevel.DEBUG, +) +async def infer_packs_globs_dependencies( + request: InferPacksGlobDependencies, +) -> InferredDependencies: + address = request.field_set.address + + paths, explicitly_provided_deps = await MultiGet( + Get( + Paths, + PathGlobs( + [os.path.join(address.spec_path, "*")], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=f"{address}'s packs glob", + ), + ), + Get( + ExplicitlyProvidedDependencies, + DependenciesRequest(request.field_set.dependencies), + ), + ) + + # explicitly_provided_deps.includes: FrozenOrderedSet[Address] + # explicitly_provided_deps.ignores: FrozenOrderedSet[Address] + + implicit_packs = {Address(f"{pack}/") for pack in paths.dirs} + + inferred_deps = ( + implicit_packs + - explicitly_provided_deps.ignores + - explicitly_provided_deps.includes + ) + return InferredDependencies(inferred_deps) + + +def rules(): + return [ + *collect_rules(), + UnionRule(InferDependenciesRequest, InferPacksGlobDependencies), + ] From b574dffd08df8fed30a1f5194cb0d07abab94eaf Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 00:31:09 -0600 Subject: [PATCH 06/14] add PacksGlobDependencies field --- pants-plugins/pack_metadata/target_types.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 4f9d8558f3..0b2d41e2c2 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -80,9 +80,13 @@ class PackMetadataInGitSubmodule(PackMetadata): ) +class PacksGlobDependencies(Dependencies): + pass + + class PacksGlob(GenericTarget): alias = "packs_glob" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies) + core_fields = (*COMMON_TARGET_FIELDS, PacksGlobDependencies) help = ( "Packs glob.\n\n" "Avoid using this target. It gets automatic dependencies on all " From 6df725095e9573fba1b1d5493718006ce9de894f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 00:41:49 -0600 Subject: [PATCH 07/14] fix rule registration --- pants-plugins/pack_metadata/register.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index cd651b1183..36c11079d9 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -21,8 +21,8 @@ def rules(): return [ - tailor.rules(), - target_types_rules.rules(), + *tailor.rules(), + *target_types_rules.rules(), ] From dac69fc37232b8ea2141b6dd4ee05d3956eaee8c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 00:48:47 -0600 Subject: [PATCH 08/14] flake8 --- pants-plugins/pack_metadata/target_types_rules.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pants-plugins/pack_metadata/target_types_rules.py b/pants-plugins/pack_metadata/target_types_rules.py index ae96366792..6e72614a47 100644 --- a/pants-plugins/pack_metadata/target_types_rules.py +++ b/pants-plugins/pack_metadata/target_types_rules.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # repurposed from pants.backend.python.target_types_rules -import dataclasses import os from dataclasses import dataclass @@ -20,13 +19,11 @@ from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule from pants.engine.target import ( - Dependencies, DependenciesRequest, ExplicitlyProvidedDependencies, FieldSet, InferDependenciesRequest, InferredDependencies, - InvalidFieldException, ) from pants.util.logging import LogLevel From c8a23f863bd308ef16c1dbf938c03d6012532e93 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 10:52:18 -0600 Subject: [PATCH 09/14] Add test for packs_glob target dep inference --- .../pack_metadata/target_types_rules.py | 16 +-- .../pack_metadata/target_types_rules_test.py | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 pants-plugins/pack_metadata/target_types_rules_test.py diff --git a/pants-plugins/pack_metadata/target_types_rules.py b/pants-plugins/pack_metadata/target_types_rules.py index 6e72614a47..a55f2c423e 100644 --- a/pants-plugins/pack_metadata/target_types_rules.py +++ b/pants-plugins/pack_metadata/target_types_rules.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# repurposed from pants.backend.python.target_types_rules import os from dataclasses import dataclass @@ -65,17 +64,14 @@ async def infer_packs_globs_dependencies( ), ) - # explicitly_provided_deps.includes: FrozenOrderedSet[Address] - # explicitly_provided_deps.ignores: FrozenOrderedSet[Address] + implicit_packs_deps = {Address(pack) for pack in paths.dirs} - implicit_packs = {Address(f"{pack}/") for pack in paths.dirs} - - inferred_deps = ( - implicit_packs - - explicitly_provided_deps.ignores - - explicitly_provided_deps.includes + inferred_packs_deps = ( + implicit_packs_deps + - explicitly_provided_deps.ignores # FrozenOrderedSet[Address] + - explicitly_provided_deps.includes # FrozenOrderedSet[Address] ) - return InferredDependencies(inferred_deps) + return InferredDependencies(inferred_packs_deps) def rules(): diff --git a/pants-plugins/pack_metadata/target_types_rules_test.py b/pants-plugins/pack_metadata/target_types_rules_test.py new file mode 100644 index 0000000000..464309a78f --- /dev/null +++ b/pants-plugins/pack_metadata/target_types_rules_test.py @@ -0,0 +1,116 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from textwrap import dedent + +from pants.backend.python.target_types import ( + PythonSourceTarget, + PythonSourcesGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.addresses import Address +from pants.engine.target import InferredDependencies +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .target_types_rules import ( + InferPacksGlobDependencies, + PacksGlobInferenceFieldSet, + rules as pack_metadata_target_types_rules, +) +from .target_types import PacksGlob + + +def test_infer_packs_globs_dependencies() -> None: + rule_runner = RuleRunner( + rules=[ + *python_target_types_rules(), + *pack_metadata_target_types_rules(), + QueryRule(InferredDependencies, (InferPacksGlobDependencies,)), + ], + target_types=[ + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PacksGlob, + ], + ) + rule_runner.write_files( + { + "packs/BUILD": dedent( + """\ + packs_glob( + name="all_packs_glob", + dependencies=[ + "!./configs", # explicit ignore + "./a", # explicit include + ], + ) + """ + ), + "packs/a/BUILD": "python_sources()", + "packs/a/__init__.py": "", + "packs/a/fixture.py": "", + "packs/b/BUILD": dedent( + """\ + python_sources( + dependencies=["packs/configs/b.yaml"], + ) + """ + ), + "packs/b/__init__.py": "", + "packs/b/fixture.py": "", + "packs/c/BUILD": "python_sources()", + "packs/c/__init__.py": "", + "packs/c/fixture.py": "", + "packs/d/BUILD": "python_sources()", + "packs/d/__init__.py": "", + "packs/d/fixture.py": "", + "packs/configs/BUILD": dedent( + """\ + resources( + sources=["*.yaml"], + ) + """ + ), + "packs/configs/b.yaml": dedent( + """\ + --- + # pack config for pack b + """ + ), + } + ) + + def run_dep_inference(address: Address) -> InferredDependencies: + args = [ + "--source-root-patterns=/packs", + ] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + target = rule_runner.get_target(address) + return rule_runner.request( + InferredDependencies, + [InferPacksGlobDependencies(PacksGlobInferenceFieldSet.create(target))], + ) + + assert run_dep_inference( + Address("packs", target_name="all_packs_glob") + ) == InferredDependencies( + [ + # should not have packs/a (explicit dep does not need to be inferred) + # should not have packs/configs (explicitly ignored) + Address("packs/b"), + Address("packs/c"), + Address("packs/d"), + ], + ) From ae5f4eaf57a46565cdf2ff3caad2866e832f9eb2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 11:57:41 -0600 Subject: [PATCH 10/14] use packs_glob for all_packs --- st2tests/st2tests/fixtures/packs/BUILD | 50 ++++++-------------------- 1 file changed, 10 insertions(+), 40 deletions(-) diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 665501dd96..f9da807965 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -35,49 +35,19 @@ python_sources( ], ) -# Please make sure to register all of the packs in this dir. -# For each pack, depend on the python_sources target (which depends on pack_metadata). -target( +packs_glob( name="all_packs", dependencies=[ - "./action_chain_tests", + # core is a symlink instead of a dir "./core", - "./dummy_pack_1", - "./dummy_pack_2", - "./dummy_pack_3", - "./dummy_pack_4", - "./dummy_pack_5", - "./dummy_pack_6", - "./dummy_pack_7", - "./dummy_pack_8", - "./dummy_pack_9", - "./dummy_pack_10", - "./dummy_pack_11", - "./dummy_pack_12", - "./dummy_pack_13", - "./dummy_pack_14", - "./dummy_pack_15", - "./dummy_pack_16", - "./dummy_pack_17", - "./dummy_pack_18", - "./dummy_pack_19", - "./dummy_pack_20", - "./dummy_pack_21", - "./dummy_pack_22", - "./dummy_pack_23", - "./dummy_pack_schema_with_additional_items_1", - "./dummy_pack_schema_with_additional_properties_1", - "./dummy_pack_schema_with_nested_object_1", - "./dummy_pack_schema_with_nested_object_2", - "./dummy_pack_schema_with_nested_object_3", - "./dummy_pack_schema_with_nested_object_4", - "./dummy_pack_schema_with_nested_object_5", - "./dummy_pack_schema_with_pattern_and_additional_properties_1", - "./dummy_pack_schema_with_pattern_properties_1", - "./orquesta_tests", - "./pack_dir_name_doesnt_match_ref", - "./pack_invalid_requirements", + # use :test_content_version instead because of the git submodule ":test_content_version", - "./test_library_dependencies", + "!./test_content_version", + "!./test_content_version_fixture", + # these are not packs + "!./configs", + "!./executions", + "!./runners", + "!./all_packs_glob", # the fixture that pulls in this target ], ) From e3ddc8022f0cba171e2e23eff746e479c6b79467 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 11:57:24 -0600 Subject: [PATCH 11/14] revert child_packs_glob --- st2actions/tests/unit/test_actions_registrar.py | 4 ++-- st2tests/st2tests/fixtures/BUILD | 16 ---------------- .../st2tests/fixtures/child_packs_glob/BUILD | 3 --- .../fixtures/child_packs_glob/__init__.py | 16 ---------------- 4 files changed, 2 insertions(+), 37 deletions(-) delete mode 100644 st2tests/st2tests/fixtures/child_packs_glob/BUILD delete mode 100644 st2tests/st2tests/fixtures/child_packs_glob/__init__.py diff --git a/st2actions/tests/unit/test_actions_registrar.py b/st2actions/tests/unit/test_actions_registrar.py index 349ec7fd1b..0f3fe76e32 100644 --- a/st2actions/tests/unit/test_actions_registrar.py +++ b/st2actions/tests/unit/test_actions_registrar.py @@ -30,7 +30,6 @@ PACK_NAME as GENERIC_PACK, PACK_PATH as GENERIC_PACK_PATH, ) -from st2tests.fixtures.child_packs_glob import PACKS_PATH import st2tests.fixturesloader as fixtures_loader MOCK_RUNNER_TYPE_DB = RunnerTypeDB(name="run-local", runner_module="st2.runners.local") @@ -53,8 +52,9 @@ class ActionsRegistrarTest(tests_base.DbTestCase): ) def test_register_all_actions(self): try: + packs_base_path = fixtures_loader.get_fixtures_base_path() all_actions_in_db = Action.get_all() - actions_registrar.register_actions(packs_base_paths=[PACKS_PATH]) + actions_registrar.register_actions(packs_base_paths=[packs_base_path]) except Exception as e: print(six.text_type(e)) self.fail("All actions must be registered without exceptions.") diff --git a/st2tests/st2tests/fixtures/BUILD b/st2tests/st2tests/fixtures/BUILD index f35dd8853c..010e795e35 100644 --- a/st2tests/st2tests/fixtures/BUILD +++ b/st2tests/st2tests/fixtures/BUILD @@ -6,19 +6,3 @@ resources( "./rbac_invalid/**/*.yaml", ], ) - -# Please make sure to register all of the packs in this dir. -# For each pack, depend on the python_sources target (which depends on pack_metadata). -target( - name="child_packs", - dependencies=[ - "./aliases", - "./backstop", - "./descendants", - "./generic", - "./localrunner_pack", - "./rule_enforcements", - "./timers", - "./traces", - ], -) diff --git a/st2tests/st2tests/fixtures/child_packs_glob/BUILD b/st2tests/st2tests/fixtures/child_packs_glob/BUILD deleted file mode 100644 index 9cbfb9b54f..0000000000 --- a/st2tests/st2tests/fixtures/child_packs_glob/BUILD +++ /dev/null @@ -1,3 +0,0 @@ -python_sources( - dependencies=["st2tests/st2tests/fixtures:child_packs"], -) diff --git a/st2tests/st2tests/fixtures/child_packs_glob/__init__.py b/st2tests/st2tests/fixtures/child_packs_glob/__init__.py deleted file mode 100644 index 76d1e6e14b..0000000000 --- a/st2tests/st2tests/fixtures/child_packs_glob/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2023 The StackStorm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from st2tests.fixturesloader import get_fixtures_base_path - -PACKS_PATH = get_fixtures_base_path() From 80a88297df7877434bc81367467c812d500b495a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Jan 2023 12:27:56 -0600 Subject: [PATCH 12/14] pants: make packs_glob target gracefully handle test_content_version --- pants-plugins/pack_metadata/target_types_rules.py | 8 +++++--- pants-plugins/pack_metadata/target_types_rules_test.py | 9 +++++++++ st2tests/st2tests/fixtures/packs/BUILD | 1 - 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pants-plugins/pack_metadata/target_types_rules.py b/pants-plugins/pack_metadata/target_types_rules.py index a55f2c423e..d67555849d 100644 --- a/pants-plugins/pack_metadata/target_types_rules.py +++ b/pants-plugins/pack_metadata/target_types_rules.py @@ -49,11 +49,11 @@ async def infer_packs_globs_dependencies( ) -> InferredDependencies: address = request.field_set.address - paths, explicitly_provided_deps = await MultiGet( + pack_build_paths, explicitly_provided_deps = await MultiGet( Get( Paths, PathGlobs( - [os.path.join(address.spec_path, "*")], + [os.path.join(address.spec_path, "*", "BUILD")], glob_match_error_behavior=GlobMatchErrorBehavior.error, description_of_origin=f"{address}'s packs glob", ), @@ -64,7 +64,9 @@ async def infer_packs_globs_dependencies( ), ) - implicit_packs_deps = {Address(pack) for pack in paths.dirs} + implicit_packs_deps = { + Address(os.path.dirname(path)) for path in pack_build_paths.files + } inferred_packs_deps = ( implicit_packs_deps diff --git a/pants-plugins/pack_metadata/target_types_rules_test.py b/pants-plugins/pack_metadata/target_types_rules_test.py index 464309a78f..28780e0025 100644 --- a/pants-plugins/pack_metadata/target_types_rules_test.py +++ b/pants-plugins/pack_metadata/target_types_rules_test.py @@ -49,6 +49,11 @@ def test_infer_packs_globs_dependencies() -> None: { "packs/BUILD": dedent( """\ + python_sources( + name="git_submodule", + sources=["./git_submodule/*.py"], + ) + packs_glob( name="all_packs_glob", dependencies=[ @@ -76,6 +81,9 @@ def test_infer_packs_globs_dependencies() -> None: "packs/d/BUILD": "python_sources()", "packs/d/__init__.py": "", "packs/d/fixture.py": "", + # imitate a pack in a git submodule (should NOT have a BUILD file) + "packs/git_submodule/__init__.py": "", + "packs/git_submodule/fixture.py": "", "packs/configs/BUILD": dedent( """\ resources( @@ -109,6 +117,7 @@ def run_dep_inference(address: Address) -> InferredDependencies: [ # should not have packs/a (explicit dep does not need to be inferred) # should not have packs/configs (explicitly ignored) + # should not have packs/git_submodule (no BUILD file = no targets to add) Address("packs/b"), Address("packs/c"), Address("packs/d"), diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index f9da807965..bbcae502dd 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -42,7 +42,6 @@ packs_glob( "./core", # use :test_content_version instead because of the git submodule ":test_content_version", - "!./test_content_version", "!./test_content_version_fixture", # these are not packs "!./configs", From 775d67b270a4ea88cd28691e8dccad3c49ae97b5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Feb 2023 21:30:37 -0600 Subject: [PATCH 13/14] add more packs globs --- st2tests/st2tests/fixtures/packs_1/BUILD | 3 +++ st2tests/st2tests/fixtures/packs_invalid/BUILD | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 st2tests/st2tests/fixtures/packs_1/BUILD create mode 100644 st2tests/st2tests/fixtures/packs_invalid/BUILD diff --git a/st2tests/st2tests/fixtures/packs_1/BUILD b/st2tests/st2tests/fixtures/packs_1/BUILD new file mode 100644 index 0000000000..251e480e8a --- /dev/null +++ b/st2tests/st2tests/fixtures/packs_1/BUILD @@ -0,0 +1,3 @@ +packs_glob( + name="all_packs", +) diff --git a/st2tests/st2tests/fixtures/packs_invalid/BUILD b/st2tests/st2tests/fixtures/packs_invalid/BUILD new file mode 100644 index 0000000000..251e480e8a --- /dev/null +++ b/st2tests/st2tests/fixtures/packs_invalid/BUILD @@ -0,0 +1,3 @@ +packs_glob( + name="all_packs", +) From e159a1c8b75a2656258bb759e5dbc0ea65c35a8c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 27 Jan 2023 13:51:50 -0600 Subject: [PATCH 14/14] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7274fdbbdd..a5868c756a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805