From de6720e67e973ec24f12c9ed42b6c0a91df38713 Mon Sep 17 00:00:00 2001 From: umar-leadpages Date: Wed, 19 Nov 2025 19:05:43 +0500 Subject: [PATCH 01/19] Code migrations for python 3.11 compatibility --- gcloud_requests/credentials_watcher.py | 2 +- gcloud_requests/proxy.py | 15 +++++++++++---- setup.py | 1 + tox.ini | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/gcloud_requests/credentials_watcher.py b/gcloud_requests/credentials_watcher.py index 1a236fd..8d62031 100644 --- a/gcloud_requests/credentials_watcher.py +++ b/gcloud_requests/credentials_watcher.py @@ -17,7 +17,7 @@ class CredentialsWatcher(Thread): def __init__(self): super(CredentialsWatcher, self).__init__() - self.setDaemon(True) + self.daemon = True self.watch_list_updated = Condition() self.watch_list = [] self.logger = logging.getLogger("gcloud_requests.CredentialsWatcher") diff --git a/gcloud_requests/proxy.py b/gcloud_requests/proxy.py index bd4b5c9..f2762ba 100644 --- a/gcloud_requests/proxy.py +++ b/gcloud_requests/proxy.py @@ -34,10 +34,17 @@ class RequestsProxy(object): TIMEOUT_CONFIG = (3.05, 30) #: Determines how retries should be handled by this proxy. - RETRY_CONFIG = Retry( - total=10, connect=10, read=5, - method_whitelist=Retry.DEFAULT_METHOD_WHITELIST | frozenset(["POST"]) - ) + # Handle compatibility between old urllib3 (method_whitelist) and new (allowed_methods) + _retry_kwargs = { + "total": 10, + "connect": 10, + "read": 5, + } + if hasattr(Retry, "DEFAULT_METHOD_WHITELIST"): + _retry_kwargs["method_whitelist"] = Retry.DEFAULT_METHOD_WHITELIST | frozenset(["POST"]) + elif hasattr(Retry, "DEFAULT_ALLOWED_METHODS"): + _retry_kwargs["allowed_methods"] = Retry.DEFAULT_ALLOWED_METHODS | frozenset(["POST"]) + RETRY_CONFIG = Retry(**_retry_kwargs) #: The number of connections to pool per Session. CONNECTION_POOL_SIZE = 32 diff --git a/setup.py b/setup.py index f2a45f5..b2a3023 100644 --- a/setup.py +++ b/setup.py @@ -46,5 +46,6 @@ def parse_dependencies(filename): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.11", ] ) diff --git a/tox.ini b/tox.ini index 6277e4d..4757e7c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist= - py{27,35,36,37,38}-cpython + py{27,35,36,37,38,311}-cpython flake8 [testenv] From 69f74c1119a6949981df1a5e28959159f3a76831 Mon Sep 17 00:00:00 2001 From: umar-leadpages Date: Mon, 24 Nov 2025 20:39:41 +0500 Subject: [PATCH 02/19] Add support for python 3.9 --- setup.py | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2a3023..3613932 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ def parse_dependencies(filename): "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.11", ] ) diff --git a/tox.ini b/tox.ini index 4757e7c..0f4ad77 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist= - py{27,35,36,37,38,311}-cpython + py{27,35,36,37,38,39,311}-cpython flake8 [testenv] From 399007afaa8b1524ff01568716bbab4d1b742208 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Wed, 26 Nov 2025 08:21:47 -0800 Subject: [PATCH 03/19] added cloudbuild pipeline --- .gitignore | 6 ++++ Makefile | 54 ++++++++++++++++++++++++++++++++++ ci/buildwheel.sh | 64 ++++++++++++++++++++++++++++++++++++++++ ci/cloudbuild.yaml | 73 ++++++++++++++++++++++++++++++++++++++++++++++ ci/runtests.sh | 45 ++++++++++++++++++++++++++++ 5 files changed, 242 insertions(+) create mode 100644 Makefile create mode 100644 ci/buildwheel.sh create mode 100644 ci/cloudbuild.yaml create mode 100644 ci/runtests.sh diff --git a/.gitignore b/.gitignore index b28e93d..fdd86a8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,9 @@ __pycache__ /dist /gcloud_requests.egg-info /htmlcov +cloudbuild_pypirc +netrc +pip.conf +gcloud +build +dist diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c7f4969 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +DOCKER != which docker +MKDIR != which mkdir +RM != which rm + +PROJECT=gcloud_requests + +SUPPORTED_PY_VERSIONS=2.7 3.6 3.7 3.8 3.9 3.11 + +.PHONY: test ci-tests + +netrc: + cp ${HOME}/.netrc netrc + +pip.conf: + cp ${HOME}/.pip/pip.conf pip.conf + +cloudbuild_pypirc: + cp ${HOME}/.pypirc cloudbuild_pypirc + +gcloud: + cp ${HOME}/.config/gcloud gcloud + +test: netrc pip.conf cloudbuild_pypirc gcloud + $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ + -v $(CURDIR):/workspace \ + python:$(PY_VERSION)-alpine sh /workspace/ci/runtests.sh + +ci-tests: + $(foreach pyversion, $(SUPPORTED_PY_VERSIONS), $(MAKE) PY_VERSION=$(pyversion) test;) + +bdist_wheel: netrc pip.conf cloudbuild_pypirc gcloud + # THIS WILL PUBLISH THE LIBRARY! HAVE YOU SET THE gcloud_requests/__init__.py versions TO A DEV VERSION??? + @if [ x"$(BUILD_TYPE)" == x"release" ]; then \ + if grep __version__ gcloud_requests/__init__.py | grep -q '[0-9]*\.[0-9]*[\.\-]dev'; then \ + $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ + -v $(CURDIR):/workspace \ + --env BUILD_TYPE=$(BUILD_TYPE) \ + python:3.11-alpine /bin/sh /workspace/ci/buildwheel.sh; \ + else \ + echo "The __version__ in gcloud_requests/__init__.py does not contain 'dev' designator"; \ + fi \ + else \ + $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ + -v $(CURDIR):/workspace \ + --env BUILD_TYPE=local \ + python:3.11-alpine /bin/sh /workspace/ci/buildwheel.sh; \ + fi + +clean: + -$(RM) netrc + -$(RM) pip.conf + -$(RM) cloudbuild_pypirc + -$(RM) -rf build + -$(RM) -rf dist \ No newline at end of file diff --git a/ci/buildwheel.sh b/ci/buildwheel.sh new file mode 100644 index 0000000..1fdcf95 --- /dev/null +++ b/ci/buildwheel.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +function setup_root() { + mkdir /root/.pip && ln -s /workspace/pip.conf /root/.pip/pip.conf + ln -s /workspace/netrc /root/.netrc + ln -s /workspace/cloudbuild_pypirc /root/.pypirc +} + +function setup_app() { + mkdir /app && cd /app + # copy, because we wanna be able to toss out __pychache__ and *.pyc files + cp -ar /workspace/gcloud_requests /app/gcloud_requests + cp -ar /workspace/tests /app/tests + # copy, don't link, because we change it with `sed` + cp -v /workspace/requirements.txt /app/requirements.txt + ln -s /workspace/requirements_dev.txt /app/requirements_dev.txt + ln -s /workspace/setup.cfg /app/setup.cfg + ln -s /workspace/setup.py /app/setup.py +} + +function build_wheel() { + cd /app + export HOME=/root # Damn you cloudbuild, presets to /home/builder/home ... + sed -i requirements.txt -e "s:^six$:six==1.15:" + pip install -r requirements.txt + pip install wheel + + python setup.py bdist_wheel + + ls -la build + ls -la dist + ls -la . +} + +function upload_lib() { + # twine needs a compiler + cd /app + apk update && apk upgrade && apk add build-base libffi-dev + pip install twine + # "local" refers to the [local] section in pypirc which specifies the URL + echo "Uplaod library to Artifactory" + twine upload \ + --repository local \ + dist/* \ + --config-file /root/.pypirc +} + +function main() { + setup_root + setup_app + build_wheel + # additional safegard against uploading + if [ x"${BUILD_TYPE}" == x"release" ]; then + upload_lib + elif [ x"${BUILD_TYPE}" == x"local" ]; then + echo " copy into /workspace dir, which is a docker volume mount" + rm -rf /workspace/{build,dist} + cp -avr /app/build /app/dist /workspace/ + else + echo "Discard builded library!" + fi +} + +main $@ \ No newline at end of file diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml new file mode 100644 index 0000000..ab19e63 --- /dev/null +++ b/ci/cloudbuild.yaml @@ -0,0 +1,73 @@ +steps: + # Pulls down the netrc and pip conf files we need to access our private npm registry. + # This file is stored within the context of this build, and becomes accessible to the CI image + # via a volume mount. + - name: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine + id: "Prepare: Copy over npm/pip/net config files from bucket" + entrypoint: "gsutil" + args: + - cp + - gs://center-builds/pypi/netrc + - gs://center-builds/pypi/pip.conf + - gs://center-builds/pypi/cloudbuild_pypirc + - . + + - name: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine + id: "Test the git checkout" + entrypoint: "bash" + args: + - -c + - | + ls -la / + ls -la /workspace + git log + + - name: python:2.7-alpine + id: "Test: Inside Python-2.7" + entrypoint: sh + args: + - /workspace/ci/runtests.sh + + - name: python:3.6-alpine + id: "Test: Inside Python-3.6" + entrypoint: sh + args: + - /workspace/ci/runtests.sh + + - name: python:3.7-alpine + id: "Test: Inside Python-3.7" + entrypoint: sh + args: + - /workspace/ci/runtests.sh + + - name: python:3.8-alpine + id: "Test: Inside Python-3.8" + entrypoint: sh + args: + - /workspace/ci/runtests.sh + + - name: python:3.9-alpine + id: "Test: Inside Python-3.9" + entrypoint: sh + args: + - /workspace/ci/runtests.sh + + - name: python:3.11-alpine + id: "Test: Inside Python-3.11" + entrypoint: sh + args: + - /workspace/ci/runtests.sh + + - name: python:3.11-alpine + id: "Build and Publish: create and upload library" + entrypoint: sh + env: + - 'BUILD_TYPE=$_BUILD_TYPE' # Safe-Guard inside buildwheel.sh + args: + - -c + - | + if [ x"$_BUILD_TYPE" == x"release" ]; then + /workspace/ci/buildwheel.sh + else + echo "_BUILD_TYPE is <$_BUILD_TYPE> -> DO NOT build a wheel publish." + fi \ No newline at end of file diff --git a/ci/runtests.sh b/ci/runtests.sh new file mode 100644 index 0000000..d2d755e --- /dev/null +++ b/ci/runtests.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +function setup_root() { + mkdir /root/.pip && ln -s /workspace/pip.conf /root/.pip/pip.conf + ln -s /workspace/netrc /root/.netrc + ln -s /workspace/cloudbuild_pypirc /root/.pypirc +} + +function setup_app() { + mkdir /app && cd /app + # copy, because we wanna be able to toss out __pychache__ and *.pyc files + cp -ar /workspace/gcloud_requests /app/gcloud_requests + cp -ar /workspace/tests /app/tests + # copy, don't link, because we changeit with `sed` + cp -v /workspace/requirements.txt /app/requirements.txt + ln -s /workspace/requirements_dev.txt /app/requirements_dev.txt + ln -s /workspace/setup.cfg /app/setup.cfg + ln -s /workspace/setup.py /app/setup.py +} + +function run_tests() { + cd /app + + # Runtime with venv -> 3m:23s, without venv -> 2m:43s + # this is a throwaway container, venv not needed! + #pip install virtualenv + #virtualenv .venv + #. .venv/bin/activate + + export HOME=/root # this is a cloudbuild problem!!!! + + sed -i requirements.txt -e "s:^six$:six==1.15:" + ls -la + cat requirements.txt requirements_dev.txt + pip install -r requirements_dev.txt + py.test tests +} + +function main() { + setup_root + setup_app + run_tests +} + +main $@ \ No newline at end of file From e300cd4bef6f52d3947828ebda992d83ea4cd253 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Wed, 26 Nov 2025 10:19:28 -0800 Subject: [PATCH 04/19] small update for runtests and makefile for robustness --- Makefile | 32 ++++++++++++++++++++++++-------- ci/buildwheel.sh | 2 +- ci/runtests.sh | 27 +++++++++++++-------------- 3 files changed, 38 insertions(+), 23 deletions(-) mode change 100644 => 100755 ci/buildwheel.sh mode change 100644 => 100755 ci/runtests.sh diff --git a/Makefile b/Makefile index c7f4969..da36ec9 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,11 @@ DOCKER != which docker MKDIR != which mkdir RM != which rm -PROJECT=gcloud_requests +PROJECT=gcloud-requests -SUPPORTED_PY_VERSIONS=2.7 3.6 3.7 3.8 3.9 3.11 +LATEST_SUPPORTED_PY_VERSION=3.11 + +SUPPORTED_PY_VERSIONS=2.7 3.6 3.7 3.8 3.9 $(LATEST_SUPPORTED_PY_VERSION) .PHONY: test ci-tests @@ -17,33 +19,47 @@ pip.conf: cloudbuild_pypirc: cp ${HOME}/.pypirc cloudbuild_pypirc -gcloud: - cp ${HOME}/.config/gcloud gcloud +${HOME}/.config/gcloud/application_default_credentials.json: + gcloud auth application-default login + +run: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application_default_credentials.json + $(DOCKER) run -it --rm=true --name=$(PROJECT)_python$(PY_VERSION) \ + -v $(CURDIR):/workspace \ + -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ + -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ + python:$(PY_VERSION)-alpine \ + /bin/sh -c "/workspace/ci/runtests.sh onlysetup; exec /bin/sh" -test: netrc pip.conf cloudbuild_pypirc gcloud +test: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application_default_credentials.json $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ -v $(CURDIR):/workspace \ + -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ + -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ python:$(PY_VERSION)-alpine sh /workspace/ci/runtests.sh ci-tests: $(foreach pyversion, $(SUPPORTED_PY_VERSIONS), $(MAKE) PY_VERSION=$(pyversion) test;) -bdist_wheel: netrc pip.conf cloudbuild_pypirc gcloud +bdist_wheel: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application_default_credentials.json # THIS WILL PUBLISH THE LIBRARY! HAVE YOU SET THE gcloud_requests/__init__.py versions TO A DEV VERSION??? @if [ x"$(BUILD_TYPE)" == x"release" ]; then \ if grep __version__ gcloud_requests/__init__.py | grep -q '[0-9]*\.[0-9]*[\.\-]dev'; then \ $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ -v $(CURDIR):/workspace \ + -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ + -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ --env BUILD_TYPE=$(BUILD_TYPE) \ - python:3.11-alpine /bin/sh /workspace/ci/buildwheel.sh; \ + python:$(LATEST_SUPPORTED_PY_VERSION)-alpine /bin/sh /workspace/ci/buildwheel.sh; \ else \ echo "The __version__ in gcloud_requests/__init__.py does not contain 'dev' designator"; \ fi \ else \ $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ -v $(CURDIR):/workspace \ + -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ + -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ --env BUILD_TYPE=local \ - python:3.11-alpine /bin/sh /workspace/ci/buildwheel.sh; \ + python:$(LATEST_SUPPORTED_PY_VERSION)-alpine /bin/sh /workspace/ci/buildwheel.sh; \ fi clean: diff --git a/ci/buildwheel.sh b/ci/buildwheel.sh old mode 100644 new mode 100755 index 1fdcf95..90f068d --- a/ci/buildwheel.sh +++ b/ci/buildwheel.sh @@ -13,7 +13,7 @@ function setup_app() { cp -ar /workspace/tests /app/tests # copy, don't link, because we change it with `sed` cp -v /workspace/requirements.txt /app/requirements.txt - ln -s /workspace/requirements_dev.txt /app/requirements_dev.txt + ln -s /workspace/requirements-dev.txt /app/requirements-dev.txt ln -s /workspace/setup.cfg /app/setup.cfg ln -s /workspace/setup.py /app/setup.py } diff --git a/ci/runtests.sh b/ci/runtests.sh old mode 100644 new mode 100755 index d2d755e..40b9fde --- a/ci/runtests.sh +++ b/ci/runtests.sh @@ -13,33 +13,32 @@ function setup_app() { cp -ar /workspace/tests /app/tests # copy, don't link, because we changeit with `sed` cp -v /workspace/requirements.txt /app/requirements.txt - ln -s /workspace/requirements_dev.txt /app/requirements_dev.txt + ln -s /workspace/requirements-dev.txt /app/requirements-dev.txt ln -s /workspace/setup.cfg /app/setup.cfg ln -s /workspace/setup.py /app/setup.py } -function run_tests() { +function setup_env() { cd /app - - # Runtime with venv -> 3m:23s, without venv -> 2m:43s # this is a throwaway container, venv not needed! - #pip install virtualenv - #virtualenv .venv - #. .venv/bin/activate - - export HOME=/root # this is a cloudbuild problem!!!! - sed -i requirements.txt -e "s:^six$:six==1.15:" ls -la - cat requirements.txt requirements_dev.txt - pip install -r requirements_dev.txt - py.test tests + cat requirements.txt requirements-dev.txt + pip install -r requirements-dev.txt } function main() { + local onlysetup=$1 + export HOME=/root # this is a cloudbuild problem!!!! setup_root setup_app - run_tests + setup_env + if [ x"${onlysetup}" != x"onlysetup" ]; then + echo "RUNNING TESTS: onlysetup=<${onlysetup}>" + py.test -sv tests + else + echo "ONLY PREPARE ENV: onlysetup=<${onlysetup}>" + fi } main $@ \ No newline at end of file From 32269d80a74a225292329c9c68f1c7a3699477ca Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Wed, 26 Nov 2025 10:33:36 -0800 Subject: [PATCH 05/19] added linting step --- Makefile | 5 +++++ ci/cloudbuild.yaml | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/Makefile b/Makefile index da36ec9..301edb0 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,11 @@ run: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application_default python:$(PY_VERSION)-alpine \ /bin/sh -c "/workspace/ci/runtests.sh onlysetup; exec /bin/sh" +lint: + $(DOCKER) run --rm \ + -v $(CURDIR):/workspace \ + python:3.11-alpine sh -c "pip install flake8 && flake8 /workspace/gcloud_requests /workspace/tests" + test: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application_default_credentials.json $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ -v $(CURDIR):/workspace \ diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index ab19e63..bad1a64 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -22,6 +22,27 @@ steps: ls -la /workspace git log + - name: python:3.11-alpine + id: "Lint" + entrypoint: sh + args: + - -c + - | + # Create pip and auth dirs + mkdir -p /root/.pip + mkdir -p /root + + # Link workspace configs into expected locations + ln -sf /workspace/pip.conf /root/.pip/pip.conf + ln -sf /workspace/netrc /root/.netrc + ln -sf /workspace/cloudbuild_pypirc /root/.pypirc + + # Install flake8 using the correct pip config + pip install flake8 + + # Run linting + flake8 gcloud_requests tests + - name: python:2.7-alpine id: "Test: Inside Python-2.7" entrypoint: sh From 55d37225b2bbdef5277d5ffcead5d53624bba129 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Wed, 26 Nov 2025 10:41:42 -0800 Subject: [PATCH 06/19] added subs var --- ci/cloudbuild.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index bad1a64..e5314f3 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -1,3 +1,6 @@ +substitutions: + _LATEST_SUPPORTED_PY_VERSION: 3.11 + steps: # Pulls down the netrc and pip conf files we need to access our private npm registry. # This file is stored within the context of this build, and becomes accessible to the CI image @@ -22,7 +25,7 @@ steps: ls -la /workspace git log - - name: python:3.11-alpine + - name: python:${_LATEST_SUPPORTED_PY_VERSION}-alpine id: "Lint" entrypoint: sh args: @@ -73,13 +76,13 @@ steps: args: - /workspace/ci/runtests.sh - - name: python:3.11-alpine - id: "Test: Inside Python-3.11" + - name: python:${_LATEST_SUPPORTED_PY_VERSION}-alpine + id: "Test: Inside Python-${_LATEST_SUPPORTED_PY_VERSION}" entrypoint: sh args: - /workspace/ci/runtests.sh - - name: python:3.11-alpine + - name: python:${_LATEST_SUPPORTED_PY_VERSION}-alpine id: "Build and Publish: create and upload library" entrypoint: sh env: From 33ddf7185bab45b50d0c9bb1a29b38c4c7dea813 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Mon, 1 Dec 2025 20:34:40 -0800 Subject: [PATCH 07/19] Get all tests to work with datastore, get old libs like grcpio to work with old python without pre-built whjeels for arm64 --- Makefile | 50 +++++++++++++++++++++++++++++++++++--------- README.md | 29 +++++++++++++++++++++++++ ci/cloudbuild.yaml | 39 ++++++++++++++++++++++++++++++++-- ci/runtests.sh | 2 ++ requirements-dev.txt | 2 +- 5 files changed, 109 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 301edb0..967adb3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -DOCKER != which docker -MKDIR != which mkdir -RM != which rm +DOCKER ?= docker +MKDIR ?= mkdir +RM ?= rm PROJECT=gcloud-requests @@ -8,6 +8,28 @@ LATEST_SUPPORTED_PY_VERSION=3.11 SUPPORTED_PY_VERSIONS=2.7 3.6 3.7 3.8 3.9 $(LATEST_SUPPORTED_PY_VERSION) +# Determine platform flag (only needed for python 2.7) +# grpcio HAS NO PRE_BUILT WHEELS FOR ARM64 FOR PYTHON 2, SO FORCE PLATFORM TO linuc/amd64 +# For grpcio We also need to rebuild from source with the alpine image, so for python 2.7, we dont use the full debian +define PLATFORM_ARG +$(if $(filter 2.7,$(PY_VERSION)),--platform=linux/amd64,) +endef +PY_IMAGE = python:$(PY_VERSION)-alpine +ifeq ($(PY_VERSION),2.7) + PY_IMAGE = python:2.7 +endif +# Addtionally, the python 2.7 debian image will need bash to work with runtests.sh and buildwheel.sh +SHELL_CMD = sh +ifeq ($(PY_VERSION),2.7) + SHELL_CMD = /bin/bash +endif + +# NOT FULLY Mocked, setting project ID as lp-infra-lab +GOOGLE_CLOUD_PROJECT ?= lp-infra-lab +DATASTORE_PROJECT_ID ?= $(GOOGLE_CLOUD_PROJECT) +DATASTORE_DATASET ?= $(GOOGLE_CLOUD_PROJECT) +GCLOUD_PROJECT ?= $(GOOGLE_CLOUD_PROJECT) + .PHONY: test ci-tests netrc: @@ -23,12 +45,16 @@ ${HOME}/.config/gcloud/application_default_credentials.json: gcloud auth application-default login run: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application_default_credentials.json - $(DOCKER) run -it --rm=true --name=$(PROJECT)_python$(PY_VERSION) \ + $(DOCKER) run $(PLATFORM_ARG) -it --rm=true --name=$(PROJECT)_python$(PY_VERSION) \ -v $(CURDIR):/workspace \ -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ - python:$(PY_VERSION)-alpine \ - /bin/sh -c "/workspace/ci/runtests.sh onlysetup; exec /bin/sh" + -e DATASTORE_PROJECT_ID=$(DATASTORE_PROJECT_ID) \ + -e DATASTORE_DATASET=$(DATASTORE_DATASET) \ + -e GOOGLE_CLOUD_PROJECT=$(GOOGLE_CLOUD_PROJECT) \ + -e GCLOUD_PROJECT=$(GCLOUD_PROJECT) \ + $(PY_IMAGE) \ + $(SHELL_CMD) -c "/workspace/ci/runtests.sh onlysetup; exec /bin/sh" lint: $(DOCKER) run --rm \ @@ -36,11 +62,15 @@ lint: python:3.11-alpine sh -c "pip install flake8 && flake8 /workspace/gcloud_requests /workspace/tests" test: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application_default_credentials.json - $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ + $(DOCKER) run $(PLATFORM_ARG) -it --rm=true --name=$(PROJECT)_$@ \ -v $(CURDIR):/workspace \ -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ - python:$(PY_VERSION)-alpine sh /workspace/ci/runtests.sh + -e DATASTORE_PROJECT_ID=$(DATASTORE_PROJECT_ID) \ + -e DATASTORE_DATASET=$(DATASTORE_DATASET) \ + -e GOOGLE_CLOUD_PROJECT=$(GOOGLE_CLOUD_PROJECT) \ + -e GCLOUD_PROJECT=$(GCLOUD_PROJECT) \ + $(PY_IMAGE) $(SHELL_CMD) /workspace/ci/runtests.sh ci-tests: $(foreach pyversion, $(SUPPORTED_PY_VERSIONS), $(MAKE) PY_VERSION=$(pyversion) test;) @@ -49,7 +79,7 @@ bdist_wheel: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application # THIS WILL PUBLISH THE LIBRARY! HAVE YOU SET THE gcloud_requests/__init__.py versions TO A DEV VERSION??? @if [ x"$(BUILD_TYPE)" == x"release" ]; then \ if grep __version__ gcloud_requests/__init__.py | grep -q '[0-9]*\.[0-9]*[\.\-]dev'; then \ - $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ + $(DOCKER) run $(PLATFORM_ARG) -it --rm=true --name=$(PROJECT)_$@ \ -v $(CURDIR):/workspace \ -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ @@ -59,7 +89,7 @@ bdist_wheel: netrc pip.conf cloudbuild_pypirc ${HOME}/.config/gcloud/application echo "The __version__ in gcloud_requests/__init__.py does not contain 'dev' designator"; \ fi \ else \ - $(DOCKER) run -it --rm=true --name=$(PROJECT)_$@ \ + $(DOCKER) run $(PLATFORM_ARG) -it --rm=true --name=$(PROJECT)_$@ \ -v $(CURDIR):/workspace \ -v ${HOME}/.config/gcloud:/root/.config/gcloud:ro \ -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json \ diff --git a/README.md b/README.md index 45b3752..66d4bfa 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,35 @@ bucket = client.get_bucket("my-bucket") *Note*: This will run the tests against whatever GCP project you're currently logged into via the gcloud tool. +# Running comprehensive tests + +req is a library that is released for multiple versions of python. When +build in CI we run the tests against every version of python the library is +supposed to work with. This can be simulated in development by running the +the tests inside docker containers. The process is simplified by just +running `make ci-tests`. If you wanna run a test suite locally against a +specific version of python only you can run `make PY_VERSION=3.11 test`. + +*Note*: This will run the tests against whatever GCP project `lp-infra-lab`. + +# Adding new python versions to supported roaster + +If you would like to add a new version of Python to the supported versions +it must be handled in three locations: +1. For local changes the variable `SUPPORTED_PY_VERSIONS' in the Makefile + must be adjusted to include (or exclude) the versions in question. +2. in `ci/cloudbuild.yaml` steps need to be added (or removed) to address + the different versions of Python as well. +3. in `ci/cloudbuild.yaml` the last step (build wheel and publish) shall use + the highest version of Python that is supported in the library. + +# Building a Pre-Release version + +Running `make bdist_wheel` will build the library instide a container and +copy the results into `./build/` and `./dist/`. It will not actually +publish the wheel +To Publish the wheel from the local setup, run `make BUILD_TYPE=release bdist_wheel` + ## Authors `gcloud_requests` was authored at [Leadpages][leadpages]. You can diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index e5314f3..929b5c1 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -1,5 +1,9 @@ substitutions: _LATEST_SUPPORTED_PY_VERSION: 3.11 + _DATASTORE_PROJECT_ID: lp-infra-lab + _DATASTORE_DATASET: lp-infra-lab + _GOOGLE_CLOUD_PROJECT: lp-infra-lab + _GCLOUD_PROJECT: lp-infra-lab steps: # Pulls down the netrc and pip conf files we need to access our private npm registry. @@ -46,41 +50,72 @@ steps: # Run linting flake8 gcloud_requests tests - - name: python:2.7-alpine + # We need to rebuild from source with alpine image, so for python 2.7, we dont use alpine + - name: python:2.7 id: "Test: Inside Python-2.7" - entrypoint: sh + entrypoint: bash args: - /workspace/ci/runtests.sh + env: + - DATASTORE_PROJECT_ID=${_DATASTORE_PROJECT_ID} + - DATASTORE_DATASET=${_DATASTORE_DATASET} + - GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT} + - GCLOUD_PROJECT=${_GCLOUD_PROJECT} - name: python:3.6-alpine id: "Test: Inside Python-3.6" entrypoint: sh args: - /workspace/ci/runtests.sh + env: + - DATASTORE_PROJECT_ID=${_DATASTORE_PROJECT_ID} + - DATASTORE_DATASET=${_DATASTORE_DATASET} + - GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT} + - GCLOUD_PROJECT=${_GCLOUD_PROJECT} - name: python:3.7-alpine id: "Test: Inside Python-3.7" entrypoint: sh args: - /workspace/ci/runtests.sh + env: + - DATASTORE_PROJECT_ID=${_DATASTORE_PROJECT_ID} + - DATASTORE_DATASET=${_DATASTORE_DATASET} + - GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT} + - GCLOUD_PROJECT=${_GCLOUD_PROJECT} - name: python:3.8-alpine id: "Test: Inside Python-3.8" entrypoint: sh args: - /workspace/ci/runtests.sh + env: + - DATASTORE_PROJECT_ID=${_DATASTORE_PROJECT_ID} + - DATASTORE_DATASET=${_DATASTORE_DATASET} + - GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT} + - GCLOUD_PROJECT=${_GCLOUD_PROJECT} - name: python:3.9-alpine id: "Test: Inside Python-3.9" entrypoint: sh args: - /workspace/ci/runtests.sh + env: + - DATASTORE_PROJECT_ID=${_DATASTORE_PROJECT_ID} + - DATASTORE_DATASET=${_DATASTORE_DATASET} + - GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT} + - GCLOUD_PROJECT=${_GCLOUD_PROJECT} - name: python:${_LATEST_SUPPORTED_PY_VERSION}-alpine id: "Test: Inside Python-${_LATEST_SUPPORTED_PY_VERSION}" entrypoint: sh args: - /workspace/ci/runtests.sh + env: + - DATASTORE_PROJECT_ID=${_DATASTORE_PROJECT_ID} + - DATASTORE_DATASET=${_DATASTORE_DATASET} + - GOOGLE_CLOUD_PROJECT=${_GOOGLE_CLOUD_PROJECT} + - GCLOUD_PROJECT=${_GCLOUD_PROJECT} - name: python:${_LATEST_SUPPORTED_PY_VERSION}-alpine id: "Build and Publish: create and upload library" diff --git a/ci/runtests.sh b/ci/runtests.sh index 40b9fde..f59981e 100755 --- a/ci/runtests.sh +++ b/ci/runtests.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -e + function setup_root() { mkdir /root/.pip && ln -s /workspace/pip.conf /root/.pip/pip.conf ln -s /workspace/netrc /root/.netrc diff --git a/requirements-dev.txt b/requirements-dev.txt index 91adf33..db7ddd3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,7 +11,7 @@ google-cloud-datastore>=1.6,<2.0 google-cloud-storage>=1.1.1,<2.0 # Testing -futures +futures; python_version < "3" httmock mock pytest>=3 From 8aba35766d387aab6cdcf8594c268e72adada8e6 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Mon, 1 Dec 2025 20:37:48 -0800 Subject: [PATCH 08/19] clean up --- ci/runtests.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/runtests.sh b/ci/runtests.sh index f59981e..40b9fde 100755 --- a/ci/runtests.sh +++ b/ci/runtests.sh @@ -1,7 +1,5 @@ #!/bin/sh -set -e - function setup_root() { mkdir /root/.pip && ln -s /workspace/pip.conf /root/.pip/pip.conf ln -s /workspace/netrc /root/.netrc From cd23c289f6afec756873393850e31f3517af45d7 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 15:33:45 -0800 Subject: [PATCH 09/19] syntax fix --- ci/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index 929b5c1..fee8c02 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -1,5 +1,5 @@ substitutions: - _LATEST_SUPPORTED_PY_VERSION: 3.11 + _LATEST_SUPPORTED_PY_VERSION: "3.11" _DATASTORE_PROJECT_ID: lp-infra-lab _DATASTORE_DATASET: lp-infra-lab _GOOGLE_CLOUD_PROJECT: lp-infra-lab From c4f769cfb3138feb2eaf974c97e13992ff22bd12 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 16:09:21 -0800 Subject: [PATCH 10/19] add logs bucket --- ci/cloudbuild.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index fee8c02..fb30d8a 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -129,4 +129,8 @@ steps: /workspace/ci/buildwheel.sh else echo "_BUILD_TYPE is <$_BUILD_TYPE> -> DO NOT build a wheel publish." - fi \ No newline at end of file + fi + +logsBucket: gs://leadpage-dev_cloudbuild/cloudbuild_logs +options: + logging: GCS_ONLY \ No newline at end of file From bb1ebd884572c7d7926030a579dc00d0339aee42 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 17:02:45 -0800 Subject: [PATCH 11/19] permit test to urlmatch both oauth url as well as metadata server url (for cloudbuild/gce) --- tests/test_datastore_proxy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_datastore_proxy.py b/tests/test_datastore_proxy.py index e9c7985..d54beec 100644 --- a/tests/test_datastore_proxy.py +++ b/tests/test_datastore_proxy.py @@ -170,7 +170,13 @@ def downstream(netloc, request): # RefreshError to be raised refresh_calls = [] - @urlmatch(netloc=r"^(oauth2\.googleapis\.com|accounts\.google\.com)$", path=r"^/(o/oauth2/token|token)$") + # Match BOTH: + # - OAuth2 token endpoints (local/docker) + # - GCE metadata token endpoint (Cloud Build / GCE) + @urlmatch( + netloc=r"^(oauth2\.googleapis\.com|accounts\.google\.com|metadata\.google\.internal)$", + path=r".*token.*", + ) def refresh(netloc, request): refresh_calls.append(1) if sum(refresh_calls) == 1: From 7555b6ef578a38a3a3ad5cceeaf14c81a2ae1324 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 17:31:25 -0800 Subject: [PATCH 12/19] dev ver --- gcloud_requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud_requests/__init__.py b/gcloud_requests/__init__.py index cc9908d..9ded618 100644 --- a/gcloud_requests/__init__.py +++ b/gcloud_requests/__init__.py @@ -4,4 +4,4 @@ from .pubsub import PubSubRequestsProxy # noqa from .storage import CloudStorageRequestsProxy # noqa -__version__ = "2.0.4" +__version__ = "2.0.4-dev" From 3291e75ec85dc685046ef4f396268530d0ad5ee6 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 17:32:20 -0800 Subject: [PATCH 13/19] revert --- gcloud_requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud_requests/__init__.py b/gcloud_requests/__init__.py index 9ded618..cc9908d 100644 --- a/gcloud_requests/__init__.py +++ b/gcloud_requests/__init__.py @@ -4,4 +4,4 @@ from .pubsub import PubSubRequestsProxy # noqa from .storage import CloudStorageRequestsProxy # noqa -__version__ = "2.0.4-dev" +__version__ = "2.0.4" From ee4a4808f2438286f598457762280421ac0683d2 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 17:42:42 -0800 Subject: [PATCH 14/19] add powerful machine for cloudbuild --- ci/cloudbuild.yaml | 3 ++- gcloud_requests/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index fb30d8a..c5ad4ed 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -133,4 +133,5 @@ steps: logsBucket: gs://leadpage-dev_cloudbuild/cloudbuild_logs options: - logging: GCS_ONLY \ No newline at end of file + machineType: 'E2_HIGHCPU_8' # We need these extra cores to run these unit tests for python 2, else it takes 20 minutes + logging: GCS_ONLY diff --git a/gcloud_requests/__init__.py b/gcloud_requests/__init__.py index cc9908d..9ded618 100644 --- a/gcloud_requests/__init__.py +++ b/gcloud_requests/__init__.py @@ -4,4 +4,4 @@ from .pubsub import PubSubRequestsProxy # noqa from .storage import CloudStorageRequestsProxy # noqa -__version__ = "2.0.4" +__version__ = "2.0.4-dev" From 0c14b4392eef6b8fb6e6a19c533791853921aad5 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 17:59:03 -0800 Subject: [PATCH 15/19] revert --- gcloud_requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud_requests/__init__.py b/gcloud_requests/__init__.py index 9ded618..cc9908d 100644 --- a/gcloud_requests/__init__.py +++ b/gcloud_requests/__init__.py @@ -4,4 +4,4 @@ from .pubsub import PubSubRequestsProxy # noqa from .storage import CloudStorageRequestsProxy # noqa -__version__ = "2.0.4-dev" +__version__ = "2.0.4" From e161fbfb60fa28f5e88b766e13f3f5045a252983 Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Tue, 2 Dec 2025 18:05:00 -0800 Subject: [PATCH 16/19] update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 66d4bfa..ea3cde6 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ it must be handled in three locations: must be adjusted to include (or exclude) the versions in question. 2. in `ci/cloudbuild.yaml` steps need to be added (or removed) to address the different versions of Python as well. -3. in `ci/cloudbuild.yaml` the last step (build wheel and publish) shall use - the highest version of Python that is supported in the library. +3. in `ci/cloudbuild.yaml`, update the `_LATEST_SUPPORTED_PY_VERSION` to the + latest supporeted python version. # Building a Pre-Release version From c0c4a49e8a196be427b4d76f6793db7255bd2ebc Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Fri, 5 Dec 2025 08:06:05 -0800 Subject: [PATCH 17/19] apply cosmetic fixes --- ci/cloudbuild.yaml | 1 - ci/runtests.sh | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/cloudbuild.yaml b/ci/cloudbuild.yaml index c5ad4ed..eca0a73 100644 --- a/ci/cloudbuild.yaml +++ b/ci/cloudbuild.yaml @@ -37,7 +37,6 @@ steps: - | # Create pip and auth dirs mkdir -p /root/.pip - mkdir -p /root # Link workspace configs into expected locations ln -sf /workspace/pip.conf /root/.pip/pip.conf diff --git a/ci/runtests.sh b/ci/runtests.sh index 40b9fde..59e2d9a 100755 --- a/ci/runtests.sh +++ b/ci/runtests.sh @@ -9,10 +9,10 @@ function setup_root() { function setup_app() { mkdir /app && cd /app # copy, because we wanna be able to toss out __pychache__ and *.pyc files - cp -ar /workspace/gcloud_requests /app/gcloud_requests - cp -ar /workspace/tests /app/tests - # copy, don't link, because we changeit with `sed` - cp -v /workspace/requirements.txt /app/requirements.txt + ln -s /workspace/gcloud_requests /app/gcloud_requests + ln -s /workspace/tests /app/tests + + ln -s /workspace/requirements.txt /app/requirements.txt ln -s /workspace/requirements-dev.txt /app/requirements-dev.txt ln -s /workspace/setup.cfg /app/setup.cfg ln -s /workspace/setup.py /app/setup.py From 71a56b8e4dc8704993d42d1ca0d46ec8e4cb22fb Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Fri, 5 Dec 2025 08:06:24 -0800 Subject: [PATCH 18/19] update dev ver --- gcloud_requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud_requests/__init__.py b/gcloud_requests/__init__.py index cc9908d..9ded618 100644 --- a/gcloud_requests/__init__.py +++ b/gcloud_requests/__init__.py @@ -4,4 +4,4 @@ from .pubsub import PubSubRequestsProxy # noqa from .storage import CloudStorageRequestsProxy # noqa -__version__ = "2.0.4" +__version__ = "2.0.4-dev" From 8dff75aa3a69dc97a23c6e4006ff9034cf17db3d Mon Sep 17 00:00:00 2001 From: adith-rai-97 Date: Fri, 5 Dec 2025 08:29:26 -0800 Subject: [PATCH 19/19] Update __init__.py - fix version name --- gcloud_requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud_requests/__init__.py b/gcloud_requests/__init__.py index 9ded618..cc9908d 100644 --- a/gcloud_requests/__init__.py +++ b/gcloud_requests/__init__.py @@ -4,4 +4,4 @@ from .pubsub import PubSubRequestsProxy # noqa from .storage import CloudStorageRequestsProxy # noqa -__version__ = "2.0.4-dev" +__version__ = "2.0.4"