Add custom source suffix and HTTP Headers for custom urls with authentication support via env var interpolation

portable-python.yml

@@ -29,7 +29,18 @@ cpython-additional-packages:
 # Uncomment to override cpython or a dependency source URL
 # Note: string "$version" will be replaced with version string (e.g. 1.2.3)
 # cpython-url: https://my-cpython-mirror/cpython-$version.tar.gz
+# cpython-http-headers:
+#  - Authorization: Bearer ${GITHUB_TOKEN}
 # zlib-url: https://my-zlib-mirror/zlib-$version.tar.gz
+# zlib-http-headers:
+#  - Authorization: Bearer ${GITHUB_TOKEN}
+#
+# The .tar.gz in projects public releases has additional files not present the tarball of the git tag
+# uuid-url: https://my-github-enterprise/api/v3/repos/opensource/libuuid/releases/assets/48151
+# uuid-src-suffix: .tar.gz
+# uuid-http-headers:
+#  - Authorization: Bearer ${GITHUB_TOKEN}
+#    Accept: application/octet-stream
 
 # Uncomment to override the ./configure arguments for a dependency
 # Note: this will replace the default arguments, not extend them

pyproject.toml

@@ -63,7 +63,7 @@ ignore = [
 order-by-type = false
 
 [tool.ruff.lint.mccabe]
-max-complexity = 14
+max-complexity = 18
 
 [tool.ruff.lint.pydocstyle]
 convention = "numpy"

src/portable_python/__init__.py

@@ -492,10 +492,23 @@ def is_usable_module(self, name):
     def cfg_version(self, default):
         return PPG.config.get_value("%s-version" % self.m_name) or default
 
+    def cfg_http_headers(self):
+        if config_http_headers := PPG.config.get_value("%s-http-headers" % self.m_name):
+            expanded_http_headers = {}
+            for header_dict in config_http_headers:
+                for key, value in header_dict.items():
+                    expanded_http_headers[os.path.expandvars(key)] = os.path.expandvars(value)
+
+            return expanded_http_headers
+
     def cfg_url(self, version):
         if config_url := PPG.config.get_value("%s-url" % self.m_name):
             url_template = Template(config_url)
-            return url_template.substitute(version=version)
+            url_subbed = url_template.substitute(version=version)
+            return os.path.expandvars(url_subbed)
+
+    def cfg_src_suffix(self):
+        return PPG.config.get_value("%s-src-suffix" % self.m_name)
 
     def cfg_configure(self, deps_lib_dir, deps_lib64_dir):
         if configure := PPG.config.get_value("%s-configure" % self.m_name):
@@ -510,6 +523,16 @@ def url(self):
         """Url of source tarball, if any"""
         return ""
 
+    @property
+    def headers(self):
+        """Headers for connecting to source url, if any"""
+        return self.cfg_http_headers()
+
+    @property
+    def src_suffix(self):
+        """Suffix of src archive for when URL doesn't end in the file extension"""
+        return self.cfg_src_suffix()
+
     @property
     def version(self):
         """Version to use"""
@@ -632,8 +655,16 @@ def compile(self):
                         self._finalize()
                         return
 
+                # Some URL's may not end in file extension, such as with redirects.
+                # Github releases asset endpoint is this way .../releases/assets/48151
+
                 # Split on '#' for urls that include a checksum, such as #sha256=... fragment
                 basename = runez.basename(self.url, extension_marker="#")
+                if not basename.endswith((".zip", ".tar.gz")):
+                    suffix = self.src_suffix or ".tar.gz"
+                    suffix = ".%s" % (suffix.strip("."))  # Ensure it starts with a dot (in case config forgot leading dot)
+                    basename = f"{self.m_name}-{self.version}{suffix}"
+
                 path = self.setup.folders.sources / basename
                 if not path.exists():
                     proxies = {}
@@ -643,7 +674,8 @@ def compile(self):
                     https_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy")
                     if https_proxy:
                         proxies["https"] = https_proxy
-                    RestClient().download(self.url, path, proxies=proxies)
+
+                    RestClient().download(self.url, path, proxies=proxies, headers=self.headers)
 
                 runez.decompress(path, self.m_src_build, simplify=True)
 

src/portable_python/external/__init__.py

@@ -6,7 +6,7 @@ class GettextTiny(ModuleBuilder):
 
     @property
     def url(self):
-        return f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz"
+        return self.cfg_url(self.version) or f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz"
 
     @property
     def version(self):

tests/sample-config1.yml

@@ -40,3 +40,11 @@ cpython-pep668-externally-managed:
 
 cpython-configure:
   - --enable-shared
+
+# Exercise custom url
+bzip2-url: https://my-enterprise/.../assets/bzip2/123#sha256=123...def
+bzip2-version: 1.2.3
+bzip2-src-suffix: tar.gz  # Forgot leading dot on purpose
+bzip2-http-headers:
+  - Authorization: Bearer foo
+    Accept: application/octet-stream

tests/test_setup.py

@@ -9,8 +9,15 @@ def test_config(cli):
     with pytest.raises(runez.system.AbortException):
         PPG.config.parsed_yaml("a: b\ninvalid line", "testing")
 
+    # Exercise custom url in config
+    cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mbzip2")
+    assert cli.succeeded
+    assert "Would download https://my-enterprise/.../assets/bzip2/123#sha256=123...def\n" in cli.logged
+    assert "Would untar build/sources/bzip2-1.2.3.tar.gz -> build/components/bzip2\n" in cli.logged
+
     cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mnone")
     assert cli.succeeded
+
     assert " -mpip install --no-cache-dir --upgrade my-additional-package" in cli.logged
     assert "env MACOSX_DEPLOYMENT_TARGET=12" in cli.logged  # Comes from more specific macos-arm64.yml
     assert " -> dist/cpython-3.9.7-macos-arm64.tar.xz" in cli.logged  # Comes from macos.yml (not defined in macos-arm64.yml)