From 9a41d9690717b7546db035fcdea2524ddf970bce Mon Sep 17 00:00:00 2001 From: David Gilman Date: Wed, 12 Jan 2022 17:15:07 -0500 Subject: [PATCH 01/15] Update to docker compose v2 --- juniper/__init__.py | 2 +- juniper/actions.py | 12 ++++++------ requirements/dev.txt | 27 +++++++++++++-------------- setup.py | 8 +++----- tests/test_actions.py | 20 ++++++++++---------- 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/juniper/__init__.py b/juniper/__init__.py index a6dfad5..ea0402a 100644 --- a/juniper/__init__.py +++ b/juniper/__init__.py @@ -14,4 +14,4 @@ limitations under the License. """ -__version__ = '0.5.5' +__version__ = '0.6.0' diff --git a/juniper/actions.py b/juniper/actions.py index 70b79f3..7f453f3 100644 --- a/juniper/actions.py +++ b/juniper/actions.py @@ -36,7 +36,7 @@ def build_artifacts(logger, manifest): """ compose_fn = build_compose(logger, manifest) - logger.debug(f'docker-compose -f {compose_fn} --project-directory . run sample-lambda bash') + logger.debug(f'docker compose -f {compose_fn} --project-directory . run sample-lambda bash') try: # Must copy the bin directory to the client's folder structure. This directory # will be promtly cleaned up after the artifacts are built. @@ -46,15 +46,15 @@ def build_artifacts(logger, manifest): # Use docker as a way to pip install dependencies, and copy the business logic # specified in the function definitions. - subprocess.run(["docker-compose", "-f", compose_fn, '--project-directory', '.', 'down']) - subprocess.run(["docker-compose", "-f", compose_fn, '--project-directory', '.', 'up']) + subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down']) + subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'up']) finally: shutil.rmtree('./.juni', ignore_errors=True) def build_compose(logger, manifest): """ - Builds a docker-compose file with the lambda functions defined in the manifest. + Builds a docker compose file with the lambda functions defined in the manifest. The definition of the lambda functions includes the name of the function as well as the set of dependencies to include in the packaging. @@ -63,14 +63,14 @@ def build_compose(logger, manifest): """ compose = _get_compose_template(manifest) - # Returns the name of the temp file that has the docker-compose definition. + # Returns the name of the temp file that has the docker compose definition. return write_tmp_file(compose) def _get_compose_template(manifest): """ Build the service entry for each one of the functions in the given context. - Each docker-compose entry will depend on the same image and it's just a static + Each docker compose entry will depend on the same image and it's just a static definition that gets built from a template. The template is in the artifacts folder. """ diff --git a/requirements/dev.txt b/requirements/dev.txt index a1269b9..5969cd0 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,15 +1,14 @@ -Click==7.0 +attrs==21.4.0 +click==8.0.3 click-log==0.3.2 -ipdb==0.11 -coverage==4.5.1 -coverage-badge==0.2.0 -docker>=5 -docker-compose>=1.29 -pytest==3.5.1 -pytest-mock==1.10.0 -pytest-pythonpath==0.7.2 -pytest-cov==2.5.1 -flake8==3.5.0 -PyYAML==5.3 -Sphinx==1.7.6 -phix==0.6.1 \ No newline at end of file +iniconfig==1.1.1 +Jinja2==3.0.3 +MarkupSafe==2.0.1 +packaging==21.3 +pluggy==1.0.0 +py==1.11.0 +pyparsing==3.0.6 +pytest==6.2.5 +pytest-mock==3.6.1 +PyYAML==6.0 +toml==0.10.2 diff --git a/setup.py b/setup.py index f5e5464..6426290 100644 --- a/setup.py +++ b/setup.py @@ -33,12 +33,10 @@ python_requires='>=3.6', test_suite="tests", install_requires=[ - 'click>=5.1', + 'click>=8.0', 'click-log', - 'PyYAML >= 4.3, <= 5.3', - 'Jinja2>=2.10', - 'docker[ssh] >= 5', - 'docker-compose >= 1.29' + 'PyYAML>=6.0', + 'Jinja2>=3.0', ], setup_requires=["pytest-runner"], tests_require=["pytest"], diff --git a/tests/test_actions.py b/tests/test_actions.py index 2d7c5ae..98a0f43 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -26,7 +26,7 @@ def test_build_compose_writes_compose_definition_to_tmp_file(mocker): """ - The docker-compose file created, is written to a tmp file. Make sure that + The docker compose file created, is written to a tmp file. Make sure that the file is writen and validate that the contents of the file match the expected result. """ @@ -45,9 +45,9 @@ def test_build_compose_writes_compose_definition_to_tmp_file(mocker): def test_build_artifacts_invokes_docker_commands(mocker): """ - Validate that the docker-compose commands are executed with the valid paramters. - Since the docker-compose file was dynamically generated, we must pass the full - path of that file to docker-compose command. Also, set the context of the execution + Validate that the docker compose commands are executed with the valid paramters. + Since the docker compose file was dynamically generated, we must pass the full + path of that file to docker compose command. Also, set the context of the execution to the current path. """ @@ -55,14 +55,14 @@ def test_build_artifacts_invokes_docker_commands(mocker): mock_builder = mocker.patch('juniper.actions.build_compose', return_value=tmp_filename) # Mocking the dependencies of this action. These three high level packages are - # needed to invoke docker-compose in the right context! + # needed to invoke docker compose in the right context! mocker.patch('juniper.actions.os') mocker.patch('juniper.actions.shutil') mock_subprocess_run = mocker.patch('juniper.actions.subprocess.run') compose_cmd_calls = [ - mocker.call(["docker-compose", "-f", tmp_filename, '--project-directory', '.', 'down']), - mocker.call(["docker-compose", "-f", tmp_filename, '--project-directory', '.', 'up']) + mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'down']), + mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'up']) ] processor_ctx = reader('./tests/manifests/processor-test.yml') @@ -74,7 +74,7 @@ def test_build_artifacts_invokes_docker_commands(mocker): def test_build_artifacts_copies_scriopts(mocker): """ - Since the docker-compose command will be executed from within the context + Since the docker compose command will be executed from within the context of where the lambda functions live. We need to make sure that the `package.sh` lives in the right context. @@ -86,7 +86,7 @@ def test_build_artifacts_copies_scriopts(mocker): mock_builder = mocker.patch('juniper.actions.build_compose', return_value=tmp_filename) # Mocking the dependencies of this action. These three high level packages are - # needed to invoke docker-compose in the right context! + # needed to invoke docker compose in the right context! mock_os = mocker.patch('juniper.actions.os') mock_shutil = mocker.patch('juniper.actions.shutil') mocker.patch('juniper.actions.subprocess.run') @@ -213,7 +213,7 @@ def test_build_compose_section_supports_layers(): def test_build_compose_supports_layers(mocker): """ Validate that given a manifest that has both functions and layers builds the - correct docker-compose template. + correct docker compose template. """ tmp_filename = '/var/folders/xw/yk2rrhks1w72y0zr_7t7b851qlt8b3/T/tmp52bd77s3' From 0b9b41044e8385872effd00c28120a1311a08603 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Fri, 14 Jan 2022 15:06:23 -0500 Subject: [PATCH 02/15] Fork package name --- setup.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 6426290..2c3bd82 100644 --- a/setup.py +++ b/setup.py @@ -13,16 +13,14 @@ version = re.search(r'__version__ = \'(.*?)\'', f.read()).group(1) setup( - name="juniper", + name="juniper-aidentified", version=version, - author='EAB Tech', - author_email='eabtech@eab.com', - description="Tool to streamline the build of python lambda functions.", + author='Aidentified LLC', + author_email='dgilman@aidentified.com', + description="Tool to streamline the build of python lambda functions. (fork of juniper)", long_description=readme, project_urls=OrderedDict(( - ('Documentation', 'https://eabglobal.github.io/juniper/'), - ('Code', 'https://github.com/eabglobal/juniper'), - ('Issue tracker', 'https://github.com/eabglobal/juniper/issues'), + ('Code', 'https://github.com/dgilmanAIDENTIFIED/juniper'), )), license='Apache Software License', packages=find_packages(), From c9551846d56e8d5e7c572503bc13a9b44983625b Mon Sep 17 00:00:00 2001 From: David Gilman Date: Tue, 12 Apr 2022 18:11:45 -0400 Subject: [PATCH 03/15] Ignore tests in packaging --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2c3bd82..b442c1c 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ ('Code', 'https://github.com/dgilmanAIDENTIFIED/juniper'), )), license='Apache Software License', - packages=find_packages(), + packages=find_packages(exclude=("tests",)), include_package_data=True, entry_points={ 'console_scripts': ['juni=juniper.cli:main'], From c56d0dc70f4a065639e547e6f9d922dfdb18be54 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Tue, 12 Apr 2022 18:12:51 -0400 Subject: [PATCH 04/15] Clean up containers/networks before and after lambda assembly --- juniper/actions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/juniper/actions.py b/juniper/actions.py index 7f453f3..3917a37 100644 --- a/juniper/actions.py +++ b/juniper/actions.py @@ -46,8 +46,9 @@ def build_artifacts(logger, manifest): # Use docker as a way to pip install dependencies, and copy the business logic # specified in the function definitions. - subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down']) + subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down', '--remove-orphans']) subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'up']) + subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down']) finally: shutil.rmtree('./.juni', ignore_errors=True) From 81452cb86863340e9f7dd57ccd1cf69881b6e9a9 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Tue, 12 Apr 2022 18:13:18 -0400 Subject: [PATCH 05/15] Bump version --- juniper/__init__.py | 2 +- setup.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/juniper/__init__.py b/juniper/__init__.py index ea0402a..1b09f4f 100644 --- a/juniper/__init__.py +++ b/juniper/__init__.py @@ -14,4 +14,4 @@ limitations under the License. """ -__version__ = '0.6.0' +__version__ = '0.7.0' diff --git a/setup.py b/setup.py index b442c1c..cb1a1fe 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,9 @@ 'Programming Language :: Python', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Software Development :: Build Tools', ]) From 5eb2046e61e54c60a608fb2dbf893ecf6fa4bc56 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Fri, 9 Dec 2022 09:32:25 -0500 Subject: [PATCH 06/15] Bump requirements --- requirements/dev.txt | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 5969cd0..87ac4de 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,14 +1,41 @@ attrs==21.4.0 -click==8.0.3 -click-log==0.3.2 +bleach==4.1.0 +build==0.9.0 +certifi==2021.10.8 +charset-normalizer==2.0.10 +click==8.1.3 +click-log==0.4.0 +colorama==0.4.4 +commonmark==0.9.1 +docutils==0.18.1 +exceptiongroup==1.0.4 +idna==3.3 +importlib-metadata==4.10.0 iniconfig==1.1.1 -Jinja2==3.0.3 +Jinja2==3.1.2 +keyring==23.5.0 MarkupSafe==2.0.1 packaging==21.3 +pep517==0.12.0 +pkginfo==1.8.2 pluggy==1.0.0 py==1.11.0 +Pygments==2.11.2 pyparsing==3.0.6 -pytest==6.2.5 +pytest==7.2.0 pytest-mock==3.6.1 +pytest-runner==6.0.0 PyYAML==6.0 +readme-renderer==32.0 +requests==2.27.1 +requests-toolbelt==0.9.1 +rfc3986==2.0.0 +rich==12.2.0 +six==1.16.0 toml==0.10.2 +tomli==2.0.0 +tqdm==4.62.3 +twine==4.0.0 +urllib3==1.26.8 +webencodings==0.5.1 +zipp==3.7.0 From 7c21b3e2d7ef1299b1c819af65011ec9a22557bc Mon Sep 17 00:00:00 2001 From: David Gilman Date: Fri, 9 Dec 2022 09:54:15 -0500 Subject: [PATCH 07/15] Fix tests from cleanup changes --- tests/test_actions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_actions.py b/tests/test_actions.py index 98a0f43..7e0cdb2 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -61,8 +61,9 @@ def test_build_artifacts_invokes_docker_commands(mocker): mock_subprocess_run = mocker.patch('juniper.actions.subprocess.run') compose_cmd_calls = [ + mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'down', "--remove-orphans"]), + mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'up']), mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'down']), - mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'up']) ] processor_ctx = reader('./tests/manifests/processor-test.yml') From d3955a2f5bcc42f192735c56851014388c757e53 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Fri, 9 Dec 2022 10:02:50 -0500 Subject: [PATCH 08/15] Working support for platform: in lambdas and layers --- README.rst | 1 + juniper/actions.py | 30 +++++++----- juniper/artifacts/compose-template.yml | 6 +++ .../expectations/custom-docker-platforms.yml | 47 +++++++++++++++++++ tests/manifests/custom-docker-platforms.yml | 24 ++++++++++ tests/test_build_with_overrides.py | 8 ++++ 6 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 tests/expectations/custom-docker-platforms.yml create mode 100644 tests/manifests/custom-docker-platforms.yml diff --git a/README.rst b/README.rst index 930ad31..366345c 100644 --- a/README.rst +++ b/README.rst @@ -257,6 +257,7 @@ This list defines the entire scope of Juniper. Nothing more, nothing else. * Specify docker image to package lamdba functions using different python runtimes * Define pip command line arguments using a pip.conf file * Packaging of lambda layers +* Support for building arm64 layers and lambdas (Graviton2) via specifying the platform Contributing ************ diff --git a/juniper/actions.py b/juniper/actions.py index 3917a37..c16316d 100644 --- a/juniper/actions.py +++ b/juniper/actions.py @@ -18,6 +18,7 @@ import shutil import subprocess from jinja2 import Template +import functools from juniper.constants import DEFAULT_OUT_DIR, DEFAULT_DOCKER_IMAGE from juniper.io import (get_artifact, write_tmp_file, get_artifact_path) @@ -82,6 +83,7 @@ def build_section(label): { 'name': name, 'image': _get_docker_image(manifest, sls_section), + 'platform': _get_platform(manifest, sls_section), 'volumes': _get_volumes(manifest, sls_section) } for name, sls_section in manifest.get(label, {}).items() @@ -133,21 +135,27 @@ def get_vol(include): return volumes -def _get_docker_image(manifest, sls_function): +def _get_attr(key, default, manifest, sls_function): """ - Get the docker image that will be used to package a given function. Precedence - is as follows: function level override, global image override, default. + Get an attribute from the manifest, looking under the function first, + then global:, and finally using a default. - :params manfiest: The juniper manifest file. + + :params key: The key to retrieve + :params default: The value to return if your key is not defined anywhere + :params manifest: The juniper manifest file. :params sls_function: The serverless function definition. """ + function_value = sls_function.get(key) + if function_value: + return function_value + + global_value = manifest.get('global', {}).get(key) + if global_value: + return global_value - function_image = sls_function.get('image') - if function_image: - return function_image + return default - global_image = manifest.get('global', {}).get('image') - if global_image: - return global_image - return DEFAULT_DOCKER_IMAGE +_get_docker_image = functools.partial(_get_attr, "image", DEFAULT_DOCKER_IMAGE) +_get_platform = functools.partial(_get_attr, "platform", None) diff --git a/juniper/artifacts/compose-template.yml b/juniper/artifacts/compose-template.yml index 13af3c9..33c2d7f 100644 --- a/juniper/artifacts/compose-template.yml +++ b/juniper/artifacts/compose-template.yml @@ -11,6 +11,9 @@ services: - {{volume}} {% endfor %} command: sh /var/task/bin/package.sh {{lambda_fn.name}} +{%- if lambda_fn.platform %} + platform: {{ lambda_fn.platform }} +{%- endif %} {% endfor %} {% for layer in layers %} {{layer.name}}-layer: @@ -22,4 +25,7 @@ services: - {{volume}} {% endfor %} command: sh /var/task/bin/build_layer.sh {{layer.name}} +{%- if layer.platform %} + platform: {{ layer.platform }} +{%- endif %} {% endfor %} diff --git a/tests/expectations/custom-docker-platforms.yml b/tests/expectations/custom-docker-platforms.yml new file mode 100644 index 0000000..8d8e9e2 --- /dev/null +++ b/tests/expectations/custom-docker-platforms.yml @@ -0,0 +1,47 @@ +version: '3.6' + +services: + + default-platform-lambda: + image: python:3.8 + environment: + - AWS_DEFAULT_REGION=us-east-1 + platform: linux/amd64 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./src/edge:/var/task/common/edge + command: sh /var/task/bin/package.sh default-platform + + override-platform-lambda: + image: python:3.6-alpine + platform: linux/arm64 + environment: + - AWS_DEFAULT_REGION=us-east-1 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./src/worker/sequential_worker:/var/task/common/sequential_worker + command: sh /var/task/bin/package.sh override-platform + + default-platform-layer-layer: + image: python:3.6-alpine + platform: linux/amd64 + environment: + - AWS_DEFAULT_REGION=us-east-1 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./requirements/default.txt:/var/task/common/requirements.txt + command: sh /var/task/bin/build_layer.sh default-platform-layer + + override-platform-layer-layer: + image: python:3.6-alpine + platform: linux/arm64 + environment: + - AWS_DEFAULT_REGION=us-east-1 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./requirements/override.txt:/var/task/common/requirements.txt + command: sh /var/task/bin/build_layer.sh override-platform-layer diff --git a/tests/manifests/custom-docker-platforms.yml b/tests/manifests/custom-docker-platforms.yml new file mode 100644 index 0000000..16ab124 --- /dev/null +++ b/tests/manifests/custom-docker-platforms.yml @@ -0,0 +1,24 @@ +global: + image: python:3.6-alpine + include: + - ./src/libs/ + - ./src/common/ + platform: linux/amd64 + +functions: + default-platform: + image: python:3.8 + include: + - ./src/edge/ + + override-platform: + include: + - ./src/worker/sequential_worker + platform: linux/arm64 + +layers: + default-platform-layer: + requirements: ./requirements/default.txt + override-platform-layer: + requirements: ./requirements/override.txt + platform: linux/arm64 diff --git a/tests/test_build_with_overrides.py b/tests/test_build_with_overrides.py index 5a4063d..39a5ffa 100644 --- a/tests/test_build_with_overrides.py +++ b/tests/test_build_with_overrides.py @@ -89,6 +89,14 @@ def test_volumes_includes_pipconf(): assert './path/pip.conf:/etc/pip.conf' in volumes +def test_platform_default_override(): + docker_ctx = reader('./tests/manifests/custom-docker-platforms.yml') + result = actions._get_compose_template(docker_ctx) + + expected = read_file('./tests/expectations/custom-docker-platforms.yml') + assert yaml.safe_load(result) == yaml.safe_load(expected) + + def read_file(file_name): with open(file_name, 'r') as f: return f.read() From 725d0abbd43fec2ec2d24212e10c0116c6227d52 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Fri, 9 Dec 2022 10:32:30 -0500 Subject: [PATCH 09/15] Bump version --- juniper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/juniper/__init__.py b/juniper/__init__.py index 1b09f4f..0a5d529 100644 --- a/juniper/__init__.py +++ b/juniper/__init__.py @@ -14,4 +14,4 @@ limitations under the License. """ -__version__ = '0.7.0' +__version__ = '0.8.0' From ef94defbd7f67c02bca7804dfd7e542d7cd434e8 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Fri, 9 Dec 2022 10:33:51 -0500 Subject: [PATCH 10/15] Update twine --- requirements/dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 87ac4de..846a7d0 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -26,7 +26,7 @@ pytest==7.2.0 pytest-mock==3.6.1 pytest-runner==6.0.0 PyYAML==6.0 -readme-renderer==32.0 +readme-renderer==37.3 requests==2.27.1 requests-toolbelt==0.9.1 rfc3986==2.0.0 @@ -35,7 +35,7 @@ six==1.16.0 toml==0.10.2 tomli==2.0.0 tqdm==4.62.3 -twine==4.0.0 +twine==4.0.2 urllib3==1.26.8 webencodings==0.5.1 zipp==3.7.0 From f668773a32461a68125cb9d65f2876eb227cd0e3 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Sat, 19 Aug 2023 12:18:13 -0400 Subject: [PATCH 11/15] Remove depreciated pkg_resources usage --- juniper/actions.py | 6 ++++-- juniper/io.py | 14 +++++++++----- tests/test_actions.py | 16 ++++++++++------ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/juniper/actions.py b/juniper/actions.py index c16316d..ea85449 100644 --- a/juniper/actions.py +++ b/juniper/actions.py @@ -42,8 +42,10 @@ def build_artifacts(logger, manifest): # Must copy the bin directory to the client's folder structure. This directory # will be promtly cleaned up after the artifacts are built. os.makedirs('./.juni/bin', exist_ok=True) - shutil.copy(get_artifact_path('package.sh'), './.juni/bin/') - shutil.copy(get_artifact_path('build_layer.sh'), './.juni/bin/') + with get_artifact_path('package.sh') as path: + shutil.copy(path, './.juni/bin/') + with get_artifact_path('build_layer.sh') as path: + shutil.copy(path, './.juni/bin/') # Use docker as a way to pip install dependencies, and copy the business logic # specified in the function definitions. diff --git a/juniper/io.py b/juniper/io.py index a67bfb0..e327fab 100644 --- a/juniper/io.py +++ b/juniper/io.py @@ -18,7 +18,9 @@ import os import yaml import tempfile -import pkg_resources +import importlib.resources +import contextlib +import pathlib def reader(file_name): @@ -39,12 +41,14 @@ def write_tmp_file(content): def get_artifact(template_name): fn_template_path = get_artifact_path(template_name) - with open(fn_template_path, 'r') as f: - return f.read() + with fn_template_path as path: + with path.open("r") as fd: + return fd.read() -def get_artifact_path(artifact_name): +def get_artifact_path(artifact_name) -> contextlib.AbstractContextManager[pathlib.Path]: """ Reads an artifact out of the rio-tools project. """ - return pkg_resources.resource_filename(__name__, os.path.join('artifacts', artifact_name)) + files = importlib.resources.files("juniper") + return importlib.resources.as_file(files.joinpath(os.path.join('artifacts', artifact_name))) diff --git a/tests/test_actions.py b/tests/test_actions.py index 7e0cdb2..29e6537 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -17,8 +17,8 @@ import yaml from juniper import actions -from unittest.mock import MagicMock, call -from juniper.io import reader, get_artifact_path +from unittest.mock import MagicMock +from juniper.io import reader logger = MagicMock() @@ -98,10 +98,14 @@ def test_build_artifacts_copies_scriopts(mocker): # Validate that this three step process is correctly executed. mock_os.makedirs.assert_called_with('./.juni/bin', exist_ok=True) - mock_shutil.copy.assert_has_calls([ - call(get_artifact_path('package.sh'), './.juni/bin/'), - call(get_artifact_path('build_layer.sh'), './.juni/bin/'), - ]) + assert len(mock_shutil.copy.call_args_list) == 2 + assert len(mock_shutil.copy.call_args_list[0][0]) == 2 + + assert mock_shutil.copy.call_args_list[0][0][0].name == "package.sh" + assert mock_shutil.copy.call_args_list[0][0][1] == "./.juni/bin/" + + assert mock_shutil.copy.call_args_list[1][0][0].name == "build_layer.sh" + assert mock_shutil.copy.call_args_list[1][0][1] == "./.juni/bin/" mock_shutil.rmtree.assert_called_with('./.juni', ignore_errors=True) mock_builder.assert_called_once() From 76799b9dbaefc45631f47c417228ef5387e178fe Mon Sep 17 00:00:00 2001 From: David Gilman Date: Sat, 19 Aug 2023 12:20:44 -0400 Subject: [PATCH 12/15] Remove pytest-runner, switch to pip-tools dependency management --- Makefile | 8 +- requirements/dev.txt | 134 ++++++++++++++++++++++--------- requirements/requirements-dev.in | 5 ++ requirements/requirements.in | 4 + requirements/requirements.txt | 18 +++++ setup.py | 18 ++--- 6 files changed, 136 insertions(+), 51 deletions(-) create mode 100644 requirements/requirements-dev.in create mode 100644 requirements/requirements.in create mode 100644 requirements/requirements.txt diff --git a/Makefile b/Makefile index 1798a39..3680bee 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ install-dev: pip install -q -e .[venv3] pip install -r requirements/dev.txt +deps: + pip-compile -U --allow-unsafe -o requirements/dev.txt requirements/requirements-dev.in requirements/requirements.in + pip-compile -U --allow-unsafe -o requirements/requirements.txt requirements/requirements.in + test: clean-pyc python -m pytest --cov . @@ -36,14 +40,12 @@ gh-pages: release: rm -rf ./dist - python3 -m pip install --upgrade build python3 -m build twine check dist/* twine upload dist/* test-release: rm -rf ./dist - python3 -m pip install --upgrade build python3 -m build twine check dist/* python3 -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* @@ -51,4 +53,4 @@ test-release: clean-pyc: find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + \ No newline at end of file + find . -name '*~' -exec rm -f {} + diff --git a/requirements/dev.txt b/requirements/dev.txt index 846a7d0..68bb177 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,41 +1,103 @@ -attrs==21.4.0 -bleach==4.1.0 -build==0.9.0 -certifi==2021.10.8 -charset-normalizer==2.0.10 -click==8.1.3 +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --output-file=requirements/dev.txt requirements/requirements-dev.in requirements/requirements.in +# +bleach==6.0.0 + # via readme-renderer +build==0.10.0 + # via + # -r requirements/requirements-dev.in + # pip-tools +certifi==2023.7.22 + # via requests +charset-normalizer==3.2.0 + # via requests +click==8.1.7 + # via + # -r requirements/requirements.in + # click-log + # pip-tools click-log==0.4.0 -colorama==0.4.4 -commonmark==0.9.1 -docutils==0.18.1 -exceptiongroup==1.0.4 -idna==3.3 -importlib-metadata==4.10.0 -iniconfig==1.1.1 -Jinja2==3.1.2 -keyring==23.5.0 -MarkupSafe==2.0.1 -packaging==21.3 -pep517==0.12.0 -pkginfo==1.8.2 -pluggy==1.0.0 -py==1.11.0 -Pygments==2.11.2 -pyparsing==3.0.6 -pytest==7.2.0 -pytest-mock==3.6.1 -pytest-runner==6.0.0 -PyYAML==6.0 -readme-renderer==37.3 -requests==2.27.1 -requests-toolbelt==0.9.1 + # via -r requirements/requirements.in +docutils==0.20.1 + # via readme-renderer +idna==3.4 + # via requests +importlib-metadata==6.8.0 + # via + # keyring + # twine +iniconfig==2.0.0 + # via pytest +jaraco-classes==3.3.0 + # via keyring +jinja2==3.1.2 + # via -r requirements/requirements.in +keyring==24.2.0 + # via twine +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.3 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==10.1.0 + # via jaraco-classes +packaging==23.1 + # via + # build + # pytest +pip-tools==7.3.0 + # via -r requirements/requirements-dev.in +pkginfo==1.9.6 + # via twine +pluggy==1.2.0 + # via pytest +pygments==2.16.1 + # via + # readme-renderer + # rich +pyproject-hooks==1.0.0 + # via build +pytest==7.4.0 + # via + # -r requirements/requirements-dev.in + # pytest-mock +pytest-mock==3.11.1 + # via -r requirements/requirements-dev.in +pyyaml==6.0.1 + # via -r requirements/requirements.in +readme-renderer==41.0 + # via twine +requests==2.31.0 + # via + # requests-toolbelt + # twine +requests-toolbelt==1.0.0 + # via twine rfc3986==2.0.0 -rich==12.2.0 + # via twine +rich==13.5.2 + # via twine six==1.16.0 -toml==0.10.2 -tomli==2.0.0 -tqdm==4.62.3 + # via bleach twine==4.0.2 -urllib3==1.26.8 + # via -r requirements/requirements-dev.in +urllib3==2.0.4 + # via + # requests + # twine webencodings==0.5.1 -zipp==3.7.0 + # via bleach +wheel==0.41.1 + # via pip-tools +zipp==3.16.2 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +pip==23.2.1 + # via pip-tools +setuptools==68.1.2 + # via pip-tools diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in new file mode 100644 index 0000000..252d994 --- /dev/null +++ b/requirements/requirements-dev.in @@ -0,0 +1,5 @@ +pytest==7.4.0 +pip-tools +pytest-mock +twine +build diff --git a/requirements/requirements.in b/requirements/requirements.in new file mode 100644 index 0000000..c17ee0f --- /dev/null +++ b/requirements/requirements.in @@ -0,0 +1,4 @@ +click>=8.0 +click-log +PyYAML>=6.0 +Jinja2>=3.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 0000000..932f0ee --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --output-file=requirements/requirements.txt requirements/requirements.in +# +click==8.1.7 + # via + # -r requirements/requirements.in + # click-log +click-log==0.4.0 + # via -r requirements/requirements.in +jinja2==3.1.2 + # via -r requirements/requirements.in +markupsafe==2.1.3 + # via jinja2 +pyyaml==6.0.1 + # via -r requirements/requirements.in diff --git a/setup.py b/setup.py index cb1a1fe..e07f547 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,9 @@ with io.open('juniper/__init__.py', 'rt', encoding='utf8') as f: version = re.search(r'__version__ = \'(.*?)\'', f.read()).group(1) +with open('requirements/requirements.in', 'r') as fd: + install_requires=fd.readlines() + setup( name="juniper-aidentified", version=version, @@ -28,27 +31,18 @@ entry_points={ 'console_scripts': ['juni=juniper.cli:main'], }, - python_requires='>=3.6', + python_requires='>=3.9', test_suite="tests", - install_requires=[ - 'click>=8.0', - 'click-log', - 'PyYAML>=6.0', - 'Jinja2>=3.0', - ], - setup_requires=["pytest-runner"], - tests_require=["pytest"], + install_requires=install_requires, classifiers=[ 'Environment :: Web Environment', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Software Development :: Build Tools', ]) From b8646bccc8ad98ad5eea2775a23a3b4b328b0a73 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Sat, 19 Aug 2023 12:39:34 -0400 Subject: [PATCH 13/15] Version 0.8.1 --- juniper/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/juniper/__init__.py b/juniper/__init__.py index 0a5d529..65a64c8 100644 --- a/juniper/__init__.py +++ b/juniper/__init__.py @@ -14,4 +14,4 @@ limitations under the License. """ -__version__ = '0.8.0' +__version__ = '0.8.1' From ab31259585573385f43a8b39762f1357e50d3094 Mon Sep 17 00:00:00 2001 From: David Gilman Date: Thu, 24 Aug 2023 15:19:25 -0400 Subject: [PATCH 14/15] Include requirements/* for setup.py --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 1ed0f12..5731df1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include Makefile CHANGES.rst LICENSE AUTHORS juniper/artifacts/* +include Makefile CHANGES.rst LICENSE AUTHORS juniper/artifacts/* requirements/* From 41c90b4ae9f95d4e129b60b072807baffbeb34bb Mon Sep 17 00:00:00 2001 From: David Gilman Date: Thu, 24 Aug 2023 15:21:47 -0400 Subject: [PATCH 15/15] Set long_description_content_type --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index e07f547..5b1cc1b 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ author_email='dgilman@aidentified.com', description="Tool to streamline the build of python lambda functions. (fork of juniper)", long_description=readme, + long_description_content_type="text/x-rst", project_urls=OrderedDict(( ('Code', 'https://github.com/dgilmanAIDENTIFIED/juniper'), )),