Capture diffs of dist tarballs vs auto-generated source archives from GitHub

autospec/git.py

@@ -113,6 +113,7 @@ def commit_to_git(config, name, success):
     call("git add whatrequires", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add description", check=False, stderr=subprocess.DEVNULL, cwd=path)
     call("git add attrs", check=False, stderr=subprocess.DEVNULL, cwd=path)
+    call("git add archive.diff", check=False, stderr=subprocess.DEVNULL, cwd=path)
 
     # remove deprecated config files
     call("git rm make_install_append", check=False, stderr=subprocess.DEVNULL, cwd=path)

autospec/tarball.py

@@ -26,7 +26,7 @@
 import zipfile
 
 import download
-from util import do_regex, get_sha1sum, print_fatal, write_out
+from util import call, do_regex, get_sha1sum, print_fatal, write_out
 
 
 class Source():
@@ -184,6 +184,7 @@ def __init__(self, url, name, version, archives, config, base_path):
         self.prefixes = dict()
         self.config = config
         self.base_path = base_path
+        self.autogenerated_tarball = None
 
     def write_upstream(self, sha, tarfile, mode="w"):
         """Write the upstream hash to the upstream file."""
@@ -213,6 +214,14 @@ def process_main_source(self, url):
         main_src = Source(url, '', src_path, self.config.default_pattern)
         return main_src
 
+    def process_autogenerated_source(self, url):
+        """Download any autogenerated source tarball for comparison."""
+        autogenerated_src = None
+        if url:
+            src_path = self.check_or_get_file(url, os.path.basename(url), mode="")
+            autogenerated_src = Source(url, '../autogenerated-tmp', src_path, self.config.default_pattern)
+        return autogenerated_src
+
     def print_header(self):
         """Print header for autospec run."""
         print("\n")
@@ -295,6 +304,9 @@ def name_and_version(self, filemanager):
                     name = re.sub(r"release-", '', name)
                     name = re.sub(r"\d*$", '', name)
                 self.rawname = name
+                # Identify the auto-generated tarball URL for comparison
+                if "/releases/download/" in self.url:
+                    self.autogenerated_tarball = "https://github.com/" + match.group(1).strip() + "/" + self.repo + "/archive/refs/tags/" + match.group(3).strip() + ".tar.gz"
                 version = match.group(3).replace(name, '')
                 if "/archive/" not in self.url:
                     version = re.sub(r"^[-_.a-zA-Z]+", "", version)
@@ -427,3 +439,15 @@ def process(self, filemanager):
         archives_src = self.process_archives()
         # Extract all sources
         self.extract_sources(main_src, archives_src)
+        # Download and process any auto-generated source-tree archive for comparison
+        autogenerated_src = self.process_autogenerated_source(self.autogenerated_tarball)
+        # Extract autogenerated source for comparison
+        if autogenerated_src:
+            autogenerated_src.extract(os.path.join(self.base_path, 'autogenerated-tmp'))
+            # Move the autogenerated source to a non-version-named directory for consistent diffs
+            call(f"mv autogenerated-tmp/{autogenerated_src.prefix} autogenerated", check=True, cwd=self.base_path)
+            call("diff -u -r --unidirectional-new-file ../autogenerated ./",
+                 logfile="archive.diff.in", check=False, cwd=os.path.join(self.base_path, main_src.prefix))
+            call("grep -A14 -E '^(diff|Only in)' archive.diff.in",
+                 logfile="archive.diff", check=False, cwd=os.getcwd())
+            call("rm archive.diff.in", check=False, cwd=os.getcwd())

autospec/util.py

@@ -150,8 +150,9 @@ def binary_in_path(binary):
 
 def write_out(filename, content, mode="w"):
     """File.write convenience wrapper."""
-    with open_auto(filename, mode) as require_f:
-        require_f.write(content)
+    if mode:
+        with open_auto(filename, mode) as require_f:
+            require_f.write(content)
 
 
 def open_auto(*args, **kwargs):