From 298082ecd447b42a74d7d5f7144709eb01162874 Mon Sep 17 00:00:00 2001
From: Trevor Cooper <tcooper@rockylinux.org>
Date: Tue, 16 Nov 2021 21:14:48 -0800
Subject: [PATCH 1/4] update rpminspect_wrapper.sh

---
 rpminspect/rpminspect_wrapper.sh | 148 ++++++++++++++++++++-----------
 1 file changed, 94 insertions(+), 54 deletions(-)
 mode change 100644 => 100755 rpminspect/rpminspect_wrapper.sh

diff --git a/rpminspect/rpminspect_wrapper.sh b/rpminspect/rpminspect_wrapper.sh
old mode 100644
new mode 100755
index d9e20a6..f496241
--- a/rpminspect/rpminspect_wrapper.sh
+++ b/rpminspect/rpminspect_wrapper.sh
@@ -4,48 +4,52 @@
 # - host under test must be installed with ${target} ${releasever}
 # - dnf reposync for ${target} stored at repos/${target}/${releasever}/ relative
 #   to this script to complete analysis.
-# - reruns will skip if existing JSON output found, remove .json file(s) to
-#   redo analysis
+# - reruns will skip if existing json output found, remove to re-analyze
 # - rpminspect --verbose output is captured in .rpminspect.out file and contains
 #   general pass vs FAIL state of each test.
 # - rpminspect JSON output is stored alongside the .rpminspect.out file and
 #   contains full JSON output of all tests.
-# - stderr of rpminspect goes to console, some packages may generate errors
-#   during analysis and they should be manually checked.
+# - stderr of rpminspect is stored alongside the rpminspect.out file as
+#   rpminspect.err. Some packages may generate errors during analysis and they
+#   should be manually checked.
 #
 #
 # ==============================================================================
-# Sample Rocky output (extras repo only)...
+# Sample Rocky output (plus, nfv and tr repos only)...
 #
 # [vagrant@rocky8u4 data]$ ./rpminspect_wrapper.sh
-# Rocky Linux extras repo has...
-#	     4 noarch packages ....
-#
-# ==============================================================================
-# Sample CentOS output (extras repo only)...
-#
-# [vagrant@centos8u4 data]$ ./rpminspect_wrapper.sh
-# CentOS Linux extras repo has...
-# 	     2 x86_64 packages .
-# 	cpaste-1.0.0.3.el8.x86_64 fail
-# 	    32 noarch packages .....
-# 	centos-release-configmanagement-1.1.el8.noarch fail ..........
-# 	centos-release-openstack-train-2.1.el8.noarch fail
-# 	centos-release-openstack-ussuri-1.5.el8.noarch fail
-# 	centos-release-openstack-victoria-1.2.el8.noarch fail
-# 	centos-release-opstools-1.10.el8.noarch fail .......
-# 	centos-release-storage-common-2.2.el8.noarch fail
-# 	centos-release-virt-common-1.2.el8.noarch fail ...
-#
-# ==============================================================================
-# Sample RHEL output (rhel-8-for-x86_64-supplementary-rpms repo only)...
-#
-# [vagrant@rhel8u4 data]$ ./rpminspect_wrapper.sh
-# Red Hat Enterprise Linux supplementary repo has...
-# 	     9 x86_64 packages ....
-# 		java-1.8.0-ibm-headless skipped
-# 		....
-# 	     2 noarch packages ..
+#Rocky Linux plus repo has...
+#	     3 x86_64 packages -
+#		openldap-servers-2.4.46-17.el8_4.x86_64 fail \
+#		openldap-servers-2.4.46-18.el8.x86_64 fail |
+#		thunderbird-91.3.0-2.el8.plus.x86_64 fail
+#Rocky Linux nfv repo has...
+#	    38 x86_64 packages /
+#	     7 noarch packages \
+#Rocky Linux rt repo has...
+#	    26 x86_64 packages -
+#		kernel-rt-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail \
+#		kernel-rt-core-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail |
+#		kernel-rt-debug-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail /
+#		kernel-rt-debug-core-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail -
+#		kernel-rt-debug-devel-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail \
+#		kernel-rt-debug-modules-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail |
+#		kernel-rt-debug-modules-extra-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail /
+#		kernel-rt-devel-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail -
+#		kernel-rt-modules-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail \
+#		kernel-rt-modules-extra-4.18.0-305.25.1.rt7.97.el8_4.x86_64 fail |
+#		kernel-rt-4.18.0-348.rt7.130.el8.0.2.x86_64 fail /
+#		kernel-rt-core-4.18.0-348.rt7.130.el8.0.2.x86_64 fail -
+#		kernel-rt-debug-4.18.0-348.rt7.130.el8.0.2.x86_64 fail \
+#		kernel-rt-debug-core-4.18.0-348.rt7.130.el8.0.2.x86_64 fail |
+#		kernel-rt-debug-devel-4.18.0-348.rt7.130.el8.0.2.x86_64 fail /
+#		kernel-rt-devel-4.18.0-348.rt7.130.el8.0.2.x86_64 fail -
+#		rteval-loads-1.4-11.el8.x86_64 fail /
+#		stress-ng-0.12.06-2.el8.x86_64 fail -
+#		stress-ng-0.11.10-4.el8.x86_64 fail
+#	     4 noarch packages |
+#		rteval-3.1-5.el8_4.noarch fail \
+#		tuned-profiles-realtime-2.15.0-2.el8_4.1.noarch fail
 #
 
 set -e
@@ -58,18 +62,17 @@ rpminspect_base=$(pwd)/rpminspect
 . /etc/os-release
 
 target="${ID:-rocky}"
-releasever="${VERSION_ID:-8.4}"
+releasever="${VERSION_ID:-8.5}"
 name="${NAME:-Rocky Linux}"
 
 case $target in
 	rocky)
 		releasever="$(rpm -q --queryformat="%{VERSION}" rocky-release)"
-		target_repos=(appstream baseos extras ha powertools resilient-storage)
+		target_repos=(appstream baseos extras ha powertools resilient-storage plus nfs rt)
 		;;
 	centos)
 		releasever="$(rpm -q --queryformat="%{VERSION}" centos-linux-release)"
-		#target_repos=(appstream baseos extras ha plus powertools)
-		target_repos=(extras)
+		target_repos=(appstream baseos extras ha plus powertools)
 		;;
 	rhel)
 		releasever="$(rpm -q --queryformat="%{VERSION}" redhat-release)"
@@ -98,7 +101,7 @@ do
 		esac
 	fi
 
-	printf "%s %s repo has..." "${name}" "${repo}"
+	printf "%s %s repo has... " "${name}" "${repo}"
 	for arch in "${arch_list[@]}"
 	do
 
@@ -111,54 +114,91 @@ do
 		if [[ ${#local_packages[@]} -gt 0 ]]
 		then
 			mkdir -p "${rpminspect_base}/${target}/${releasever}/${repo}"
-			printf "\n\t%6d %s packages " "${#local_packages[@]}" "${arch}"
+			printf "\b\n\t%6d %s packages\t" "${#local_packages[@]}" "${arch}"
 
+			i=1
+			sp="/-\|"
 			for pkg in "${local_packages[@]}"
 			do
+				# shellcheck disable=SC2059
+				printf "\b${sp:i++%${#sp}:1}"
 				rpm_name=$(basename "${pkg}")
 				pkg_name=$(rpm -q --queryformat="%{NAME}\n" "$pkg" 2>/dev/null)
-				nvra_pkg_name=$(rpm -q --queryformat="%{NAME}-%{VERSION}.%{RELEASE}.%{ARCH}\n" "$pkg" 2>/dev/null)
+				#nvra_pkg_name=$(rpm -q --queryformat="%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}\n" "$pkg" 2>/dev/null)
+				#nvra_pkg_name=$(basename -s .rpm "$pkg" 2>/dev/null)
+				nvra_pkg_name=$(rpm -qp "$pkg" 2>/dev/null)
 
 				set +e
-				# don't know why the python3-azure-sdk package hangs rpminspect... ignore it.
-				# java-1.8.0-ibm-headless coredumps on runpath test... ignore it.
+				# not sure why the python3-azure-sdk package takes so long (29m25.703s)
+				# and it's exists in two repositories. See the single custom run results and ignore it.
+				# java-1.8.0-ibm-headless coredumps on runpath test... ignore it. NOTE: It's gone in 8.5 but leve here until we never test 8.4 again.
 				if [[ "${pkg_name}" = "python3-azure-sdk" || "${pkg_name}" = "java-1.8.0-ibm-headless" ]]
 				then
-					printf "\n\t\t%s skipped\n\t\t" "${pkg_name}"
+					# shellcheck disable=SC2059
+					printf "\b\n\t\t%s skipped\n\t\t" "${pkg_name}"
 				else
+					rpminspect_bin=rpminspect-fedora
+
+					# these only make sense for src rpm validation
+					#rpminspect_srpm_only="disttag,files,patches,specname"
+
+					# these only make sense when comparing before to after builds
+					#rpminspect_before_after_only="addedfiles,changedfiles,files,filesize,movedfiles,removedfiles"
+
+					# desktop excluded due to huge number of desktop files messages requiring validation, round 2?
+					# license excluded due to Redhat commercial license and no redhat.yaml file (no rpminspect-data-redhat even though it's their tool)
+					# metadata excluded due to the fact that Redhat build servers are unknown
+					# virus excluded for expedience, round 2?
+					#rpminspect_exclude="${rpminspect_srpm_only},${rpminspect_before_after_only},desktop,license,metadata,virus"
+
+					# alternately, provide an explicit list of test to include
+					rpminspect_include="emptyrpm,manpage,xml,elf,modularity,javabytecode,ownership,shellsyntax,annocheck,permissions,capabilities,pathmigration,lto,symlinks,politics,badfuncs,runpath"
+
 					if [[ ! -f "${rpminspect_base}/${target}/${releasever}/${repo}/${nvra_pkg_name}.json" ]]
 					then
 						case "${target}" in
+							centos)
+								local_package_full_path="${local_package_path}/${repo}/Packages/${rpm_name}"
+								;;
 							rhel)
 								rpm_first_char=$(echo "$rpm_name" | head -c 1)
 								local_package_full_path="${local_package_path}/${repo}/Packages/${rpm_first_char}/${rpm_name}"
 								;;
-							*)
-								local_package_full_path="${local_package_path}/${repo}/Packages/${rpm_name}"
+							rocky)
+								rpminspect_bin=rpminspect-rocky
+								#rpminspect_exclude="addedfiles,files,patches,virus,disttag,specname"
+								rpm_first_char=$(echo "$rpm_name" | head -c 1)
+								local_package_full_path="${local_package_path}/${repo}/Packages/${rpm_first_char}/${rpm_name}"
 								;;
+							*)
+						 		;;
 						esac
-						rpminspect-fedora -v -E metadata,files,patches,virus,javabytecode,disttag,specname \
-							--format=json \
-							--output="${rpminspect_base}/${target}/${releasever}/${repo}/${nvra_pkg_name}.json" \
-							"${local_package_full_path}" \
-							>"${rpminspect_base}/${target}/${releasever}/${repo}/${nvra_pkg_name}.rpminspect.out"
-
+							#"${rpminspect_bin}" -v -E "${rpminspect_exclude}" \
+							"${rpminspect_bin}" -v -T "${rpminspect_include}" \
+								--format=json \
+								--output="${rpminspect_base}/${target}/${releasever}/${repo}/${nvra_pkg_name}.json" \
+								"${local_package_full_path}" \
+								>"${rpminspect_base}/${target}/${releasever}/${repo}/${nvra_pkg_name}.rpminspect.out" \
+							2>"${rpminspect_base}/${target}/${releasever}/${repo}/${nvra_pkg_name}.rpminspect.err"
 						ret=$?
 						set -e
 						if [[ $ret -ne 0 ]]
 						then
-							printf "\n\t%s fail " "${nvra_pkg_name}"
+							printf "\b\b\n\t\t%s fail  " "${nvra_pkg_name}"
 						else
-							printf "."
+							# shellcheck disable=SC2059
+							printf "\b${sp:i++%${#sp}:1}"
 						fi
 					else
-						printf "."
+						# shellcheck disable=SC2059
+						printf "\b${sp:i++%${#sp}:1}"
 					fi
 				fi
 			done
 		fi
 	done
 	printf "\n"
+	touch ".$(basename -s .sh "$0").${target}.${releasever}.${repo}.touch"
 done
 
 exit

From 43942f7e961bc142056ba4da162f35fca0e44cfc Mon Sep 17 00:00:00 2001
From: Trevor Cooper <tcooper@rockylinux.org>
Date: Tue, 16 Nov 2021 23:37:44 -0800
Subject: [PATCH 2/4] update compare_rpminspect.sh

---
 rpminspect/compare_rpminspect.sh | 223 ++++++++++++++++++++++++++++---
 1 file changed, 205 insertions(+), 18 deletions(-)

diff --git a/rpminspect/compare_rpminspect.sh b/rpminspect/compare_rpminspect.sh
index c5aabe3..2df8d96 100755
--- a/rpminspect/compare_rpminspect.sh
+++ b/rpminspect/compare_rpminspect.sh
@@ -1,22 +1,209 @@
 #!/usr/bin/env bash
 
-src="${SRC:-rocky}"
-tgt="${TGT:-rhel}"
+set -e
+#set -x
 
-while read -r pkg test status
+repos_base=$(pwd)/repos
+rpminspect_base=$(pwd)/rpminspect
+
+# shellcheck disable=SC1091
+. /etc/os-release
+
+src="${SRC:-rhel}"
+tgt="${ID:-rocky}"
+releasever="${VERSION_ID:-8.5}"
+
+state="${1:fail}"
+state_msg=""
+if [[ "${state}" == "pass" ]]
+then
+	state_msg="PASSES"
+else
+	state_msg="FAILURES"
+fi
+
+diff_out="${src}_vs_${tgt}_${releasever}.${state}.diffs"
+match_out="${src}_vs_${tgt}_${releasever}.${state}.matches"
+
+printf "=====\n%s started at %s for %s\n=====\n" "$0" "$(date)" "${state_msg}" | \
+	tee "${diff_out}" | \
+	tee "${match_out}"
+
+no_pkg_found="false"
+last_pkg=""
+while read -r repo pkg test status
 do
-  src_file=$(find "rpminspect/${src}/8.4" -name "${pkg}")
-  tgt_file=$(find "rpminspect/${tgt}/8.4" -name "${pkg}")
-
-  if [[ -f "${tgt_file}" ]]
-  then
-    diff -q "${src_file}" "${tgt_file}" >/dev/null 2>&1
-    if [[ $? -eq 1 ]]
-    then
-      echo "$test $status in $src_file and $tgt_file"
-      sdiff -s "${src_file}" "${tgt_file}"
-    fi
-  else
-    echo "${pkg} absent in ${tgt}"
-  fi
-done < <(grep -v '^ *#' < "${src}_fail")
+	if [[ "${state,,}" != "${status,,}" ]]
+	then
+					continue
+	fi
+
+	if [[ "${last_pkg}" == "${pkg}" && "${no_pkg_found}" == "true" ]]
+	then
+		continue
+	fi
+	rpm_pkg=$(find "${repos_base}/${tgt}/${releasever}/${repo}" -name "${pkg}*")
+	#pkg_n_v_r_a=$(rpm -q --queryformat="%{NAME} %{VERSION} %{RELEASE} %{ARCH}\n" -p "${rpm_pkg}")
+	pkg_nvra=$(rpm -q --queryformat="%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}\n" -p "${rpm_pkg}")
+	#pkg_nvr=$(rpm -q --queryformat="%{NAME}-%{VERSION}-%{RELEASE}\n" -p "${rpm_pkg}")
+	pkg_nv=$(rpm -q --queryformat="%{NAME}-%{VERSION}\n" -p "${rpm_pkg}")
+	#pkg_n=$(rpm -q --queryformat="%{NAME}\n" -p "${rpm_pkg}")
+	#pkg_v=$(rpm -q --queryformat="%{VERSION}\n" -p "${rpm_pkg}")
+	pkg_r=$(rpm -q --queryformat="%{RELEASE}\n" -p "${rpm_pkg}")
+	declare -a pkg_r_parts
+	readarray -d . -t pkg_r_parts<<<"$pkg_r"
+		pkg_a=$(rpm -q --queryformat="%{ARCH}" -p "${rpm_pkg}")
+		tgt_repo="${repo}"
+		src_repo=""
+
+	case $tgt_repo in
+		extras)
+			src_repo="*"
+			;;
+		powertools)
+			src_repo="codeready"
+			;;
+		codeready)
+			src_repo="powertools"
+			;;
+		ha)
+			src_repo="highavailability"
+			;;
+		highavailability)
+			src_repo="ha"
+			;;
+		resilient-storage)
+			src_repo="resilientstorage"
+			;;
+		resilientstorage)
+			src_repo="resilient-storage"
+			;;
+		*)
+			src_repo="$tgt_repo"
+			;;
+	esac
+
+	set -e
+	tgt_file=$(find "${rpminspect_base}/${tgt}/${releasever}/${tgt_repo}" -name "${pkg_nvra}.json")
+	set +e
+	src_file=$(find "${rpminspect_base}/${src}/${releasever}/${src_repo}" -name "${pkg_nvra}.json" 2>/dev/null)
+	src_file_cnt=$( (($(printf "%s\n" "${src_file}" | wc -w) + 0)) )
+	if [[ $src_file_cnt -gt 1 ]]
+	then
+
+		# TODO: We may get more than one match here and we need to sensibly pick one to compare against
+		#       How???? Maybe compare the lates modular package with the same version based on BUILDTIME?
+		#
+		# foo=$(find /home/vagrant/data/repos/rhel/8.5/appstream -name 'bea-stax-api-1.2.0-16*noarch.rpm')
+		# rpm -q --queryformat="%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm %{BUILDTIME}\n" $(printf "%s\n" ${foo} | sort -V)
+		# bea-stax-api-1.2.0-16.module+el8+2468+c564cec5.noarch.rpm 1544713419
+		# bea-stax-api-1.2.0-16.module+el8.0.0+3892+c903d3f0.noarch.rpm 1565144771
+		# bea-stax-api-1.2.0-16.module+el8.1.0+3366+6dfb954c.noarch.rpm 1560355487
+
+		# FOR NOW... we'll bail
+		echo "${tgt}/${releasever}/${tgt_repo}/$(basename -s .json "${tgt_file}").rpm MULTIPLE_TARGET_PACKAGES_IN ${tgt}/${releasever}/${tgt_repo}"
+		continue
+	fi
+
+	if test -z "${tgt_file}"
+	then
+		# For modular packages add only the first nibble of the ${RELEASE} identifier and try to find a match
+		src_file=$(find "${rpminspect_base}/${src}/${releasever}/${src_repo}" -name "${pkg_nv}-${pkg_r_parts[0]}*${pkg_a}.json" 2>/dev/null)
+		if test -z "${tgt_file}"
+		then
+			src_file=$(find "${rpminspect_base}/${src}/${releasever}" -name "${pkg_nv}*${pkg_a}.json" 2>/dev/null)
+			src_file_cnt=$( (( $(printf "%s\n" "${src_file}" | wc -w) + 0 )) )
+			if [[ $src_file_cnt -gt 1 ]]
+			then
+				echo "${tgt}/${releasever}/${tgt_repo}/$(basename -s .json "${tgt_file}").rpm MULTIPLE_TARGET_PACKAGES_IN ${tgt}/${releasever}/${tgt_repo}"
+				continue
+			fi
+			# If we climbed back up the tree we aren't searching in a specific tgt_repo anymore...
+			tgt_repo=""
+		fi
+	fi
+
+	tgt_basename="$(basename -s .json "${tgt_file}")"
+	src_basename="$(basename -s .json "${src_file}")"
+
+	# Check to make sure we have a target to compare to...
+	if [[ -f "${src_file}" ]]
+	then
+		# At this point we may have climbed back up the (inverted) target repo tree to find a match, for
+		# example...
+		#
+		# tgt_file: /home/vagrant/data/rpminspect/rocky/8.5/plus/openldap-servers-2.4.46-17.el8_4.x86_64.json
+		# src_file: /home/vagrant/data/rpminspect/rhel/8.5/codeready/openldap-servers-2.4.46-9.el8.x86_64.json
+		#
+		# ...we need to extract the ${src_repo} from the ${src_file} path...
+		new_src_repo=$(basename "$(dirname "${src_file}")")
+		if [ "${new_src_repo}" != "${src_repo}" ]
+		then
+			src_repo="${new_src_repo}"
+		fi
+
+		# Here we don't compare the whole file because we know it will not be the same as full path
+		# to package is included, rpminspect version may different in rocky and rhel, ...
+		#result=$(diff -s <(jq --sort-keys .""${test}"" "${tgt_file}") <(jq --sort-keys .""${test}"" "${src_file}"))
+		#result=$(diff -s -w <(jq -r --sort-keys .""${test}"" "${tgt_file}" | grep -Ev "annocheck: Version") <(jq -r --sort-keys .""${test}"" "${src_file}" | grep -Ev "annocheck: Version"))
+		#result=$(diff -s <(jq --sort-keys .""${test}"" "${tgt_file}") <(jq --sort-keys .""${test}"" "${src_file}"))
+
+		jqls=$(mktemp -p /dev/shm)
+		jqrs=$(mktemp -p /dev/shm)
+
+		case $test in
+			javabytecode)
+				jq --arg test "${test}" -r --sort-keys '.[$test] | sort_by(.message)' "${tgt_file}" | grep -Ev "annocheck: Version" > "$jqls"
+				jq --arg test "${test}" -r --sort-keys '.[$test] | sort_by(.message)' "${src_file}" | grep -Ev "annocheck: Version" > "$jqrs"
+				;;
+			*)
+				jq --arg test "${test}" -r --sort-keys '.[$test]' "${tgt_file}" | grep -Ev "annocheck: Version" > "$jqls"
+				jq --arg test "${test}" -r --sort-keys '.[$test]' "${src_file}" | grep -Ev "annocheck: Version" > "$jqrs"
+				;;
+		esac
+
+		# compare the JSON of the two tests while ignoring white space
+		diff -s -w "${jqls}" "${jqrs}" >/dev/null 2>&1
+		ret=$?
+		if [[ $ret -ne 0 ]]
+		then
+			printf "\n=====\n\t%s test %s are DIFFERENT between...\n" "${test}" "${state_msg}" >> "${diff_out}"
+			printf "%s %s %s_DIFFERENT %s\n" \
+				"${tgt}/${releasever}/${tgt_repo}/${tgt_basename}.rpm" \
+				"${test}" "${state_msg}" \
+				"${src}/${releasever}/${src_repo}/${src_basename}.rpm"
+			printf "  src_file: %s\n  tgt_file: %s\n\n" "${src_file}" "${tgt_file}" >> "${diff_out}"
+
+			#diff --suppress-common-lines -w <(jq -r --sort-keys .""${test}"" "${tgt_file}" | grep -Ev \"annocheck: Version\") <(jq -r --sort-keys .""${test}"" "${src_file}" | grep -Ev "annocheck: Version") | \
+			#We were filtering before
+			#grep -Ev "PASS:|skip:|info:|debug:|annobin plugin was built" \
+
+			diff --suppress-common-lines -w "${jqls}" "${jqrs}" | \
+				sed 's,\\n,\n,g' | \
+				grep -Ev "skip:|info:|debug:|annobin plugin was built" \
+				>>"${diff_out}"
+		else
+			printf "%s %s %s_MATCH %s\n" \
+				"${tgt}/${releasever}/${tgt_repo}/${tgt_basename}.rpm" \
+				"${test}" \
+				"${state_msg}" \
+				"${src}/${releasever}/${src_repo}/${src_basename}.rpm" | \
+				tee -a "${match_out}"
+		fi
+		rm "${jqls}" "${jqrs}"
+		no_pkg_found="false"
+	else
+		printf "%s %s %s\n" \
+			"${tgt}/${releasever}/${tgt_repo}/${tgt_basename}.rpm" \
+			"NO_TARGET_PACKAGE_IN" \
+			"${src}/${releasever}/${src_repo}"
+		no_pkg_found="true"
+	fi
+	last_pkg="${pkg}"
+done < <(grep -v '^ *#' < "${rpminspect_base}/${tgt}/${releasever}/${tgt}_${state}")
+
+printf "=====\n%s ended at %s for %s\n=====" "$0" "$(date)" "${state_msg}" | \
+	tee "${diff_out}" | \
+	tee "${match_out}"
+
+touch ".$(basename -s .sh "$0").${tgt}.${releasever}.${state}.touch"

From 4c242a9f90a7727625bcd8aff696b9736ddb56a6 Mon Sep 17 00:00:00 2001
From: Trevor Cooper <tcooper@rockylinux.org>
Date: Tue, 16 Nov 2021 21:11:33 -0800
Subject: [PATCH 3/4] additional rpminspect processing scripts

---
 rpminspect/parse_rpminspect.sh   | 30 ++++++++++++++++++
 rpminspect/repo_sync.sh          | 45 +++++++++++++++++++++++++++
 rpminspect/summary_rpminspect.sh | 52 ++++++++++++++++++++++++++++++++
 3 files changed, 127 insertions(+)
 create mode 100755 rpminspect/parse_rpminspect.sh
 create mode 100755 rpminspect/repo_sync.sh
 create mode 100755 rpminspect/summary_rpminspect.sh

diff --git a/rpminspect/parse_rpminspect.sh b/rpminspect/parse_rpminspect.sh
new file mode 100755
index 0000000..abc5870
--- /dev/null
+++ b/rpminspect/parse_rpminspect.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Notes:
+# - host under test must be installed with ${target} ${releasever}
+# - rpminspect JSON output is stored alongside the .rpminspect.out file and
+#   contains full JSON output of all tests.
+#
+
+set -e
+#set -x
+
+#repos_base=$(pwd)/repos
+rpminspect_base=$(pwd)/rpminspect
+
+# shellcheck disable=SC1091
+. /etc/os-release
+
+target="${ID:-rocky}"
+releasever="${VERSION_ID:-8.5}"
+
+for state in pass fail
+do
+	grep -iH "${state}" "${rpminspect_base}/${target}/${releasever}/*/*.out" | \
+		tr ':' ' ' | tr '/' ' ' | sed 's/inspect.out//g' | \
+		awk '{print $7,$8,$10,$12}' > \
+		"${rpminspect_base}/${target}/${releasever}/${target}_${state}"
+	touch ".$(basename -s .sh "$0").${target}.${releasever}.${state}"
+done
+wc -l "${rpminspect_base}/${target}/${releasever}/${target}*"
+
diff --git a/rpminspect/repo_sync.sh b/rpminspect/repo_sync.sh
new file mode 100755
index 0000000..8d9ede4
--- /dev/null
+++ b/rpminspect/repo_sync.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -e
+#set -x
+
+repos_base=$(pwd)/repos
+#rpminspect_base=$(pwd)/rpminspect
+#urls_base=$(pwd)/urls
+
+# shellcheck disable=SC1091
+. /etc/os-release
+
+tgt="${ID}"
+releasever="${VERSION_ID}"
+
+dnf clean all
+
+case $tgt in
+	rocky)
+		tgt_repos=(appstream baseos devel extras ha powertools resilient-storage plus nfv rt)
+		;;
+	centos)
+		releasever="$(rpm -q --queryformat="%{VERSION}" centos-linux-release)"
+		tgt_repos=(appstream baseos extras ha plus powertools)
+		;;
+	rhel)
+		printf "repos for %s must be sync'd on an entitled system" "${tgt}"
+		#tgt_repos=(appstream baseos extras ha powertools resilient-storage)
+		exit 1
+		;;
+	*)
+		;;
+esac
+
+printf "Syncing %s Repos..." "${NAME}"
+
+for repo in "${tgt_repos[@]}"
+do
+	printf " %s" "${repo}"
+	dnf reposync --repoid="${repo}" --download-metadata --newest-only --download-path="${repos_base}/${tgt}/${releasever}/" >/dev/null
+	touch ".$(basename -s .sh "$0").${tgt}.${releasever}.${repo}.touch"
+done
+printf "\n"
+
+exit
diff --git a/rpminspect/summary_rpminspect.sh b/rpminspect/summary_rpminspect.sh
new file mode 100755
index 0000000..6f1b8e4
--- /dev/null
+++ b/rpminspect/summary_rpminspect.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Notes:
+# - host under test must be installed with ${tgt} ${releasever}
+# - rpminspect JSON output is stored alongside the .rpminspect.out file and
+#   contains full JSON output of all tests.
+#
+
+set -e
+#set -x
+
+repos_base=$(pwd)/repos
+rpminspect_base=$(pwd)/rpminspect
+
+# shellcheck disable=SC1091
+. /etc/os-release
+
+tgt="${ID:-rocky}"
+releasever="${VERSION_ID:-8.5}"
+
+printf "\n=====\n Summary of all tests for %s version %s...\n\n" "${tgt}" "${releasever}"
+wc -l "${rpminspect_base}/${tgt}/${releasever}/${tgt}_pass"
+wc -l "${rpminspect_base}/${tgt}/${releasever}/${tgt}_fail"
+
+for state in fail pass
+do
+	printf "\n=====\n Summary of %sing tests...\n" "${state}"
+	awk '{c[$1]++;} END {for (i in c) print c[i],i}' "${rpminspect_base}/${tgt}/${releasever}/${tgt}_${state}" | \
+		sort -k1,1nr | \
+		while read -r total repo; do
+			printf "\n-----\n%s had %d %sing tests with the following breakdown...\n" "${repo}" "${total}" "${state}"
+			grep -E "^${repo}[[:space:]]" "${rpminspect_base}/${tgt}/${releasever}/${tgt}_${state}" | \
+				awk '{c[$3]++;} END {for (i in c) printf("\t%d %s\n",c[i],i)}' | \
+				sort -k1,1nr
+		done
+done
+
+repos=$(cd "${repos_base}/${tgt}/${releasever}" && for d in $(find . -mindepth 1 -maxdepth 1 -type d | sort); do printf "%s " "$(basename "$d")"; done; printf "\n")
+for state in fail pass
+do
+	for repo in ${repos}
+	do
+		printf "\n-----\n %s repository %sing tests...\n-----\n" "${repo}" "${state}"
+		grep -E "[[:space:]]${repo}[[:space:]]" "${rpminspect_base}/${tgt}/${releasever}/${tgt}_${state}" | \
+			awk '{t[$3]++} END {for (i in t) print t[i],i;}' \
+			"${rpminspect_base}/${tgt}/${releasever}/${tgt}_${state}" | \
+			sort -k1,1nr | \
+			column -t
+	done
+done
+
+touch ".$(basename -s .sh "$0").${tgt}.${releasever}.touch"

From c1214ffb616abcf9e6b05eb18895633b8949504a Mon Sep 17 00:00:00 2001
From: Trevor Cooper <tcooper@rockylinux.org>
Date: Thu, 18 Nov 2021 21:27:12 -0800
Subject: [PATCH 4/4] add rocky8u5 rpminspect processing summary

---
 rpminspect/Rocky_8u5.md | 608 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 608 insertions(+)
 create mode 100644 rpminspect/Rocky_8u5.md

diff --git a/rpminspect/Rocky_8u5.md b/rpminspect/Rocky_8u5.md
new file mode 100644
index 0000000..d12e04a
--- /dev/null
+++ b/rpminspect/Rocky_8u5.md
@@ -0,0 +1,608 @@
+# Testing/QA Team Package Inspection and Comparison for Rocky 8.5
+
+## Overview
+
+As part of the Testing / QA process when validating a new release for promotion
+from Release Candidate (RC) to General Availability (GA) the Testing team will
+compare the Release Engineering build products with those available from the
+upstream release where applicable in an attempt to identify any issues with the
+Rocky build which result in the production of packages that are not bug-for-bug
+reproductions of the packages imported from upstream.
+
+
+## Scope
+
+Packages that are unique to Rocky are not compared to upstream as they are,
+well, unique. The same can be said for packages that are unique in upstream. No
+effort is made with this workflow to identify missing packages from Rocky
+that exist in upstream. We have other tools for that.
+
+
+## Requirements
+
+- Rocky Linux 8.5 installed in system under test (SUT)
+- The following packages and their dependencies are required:
+    - `rpminspect`
+    - `rpminspect-data-rocky`
+    - `jsondiff`
+    - `jq`
+- A local clone of the upstream repositories to be compared
+
+
+## Conventions
+
+The following terms are used throughout the workflow and deserve definition:
+
+- `{src}` - The source distribution
+- `{tgt}` - The target distribution
+- `{releasever}` - The version number stored as `*TBD*` in `/etc/os-release` on
+`${src}` system.
+-
+
+
+## Layout
+
+Portions of this workflow depend on a specific layout of code and content in
+order to function without modification. That layout is depicted below...
+
+
+
+The data directories can be local to the system or provided in some other
+mechanism such as an NFS our bind mount. If not located at the default location
+then the workflow may need to be modified to accomodate their use.
+
+
+## Workflow
+
+The basic workflow sequence is...
+
+-   (re) Create source and target local repository clones for Rocky and RHEL
+-   Run rpminspect against ALL packages in source and target repository clones
+-   Parse the rpminspect output to identify tests as pass and fail
+-   Summarize the parsed rpminspect output
+-   Compare
+
+## sync repositories locally
+
+```
+[vagrant@rocky8u5 data]$ ./repo_sync.sh
+75 files removed
+Syncing Rocky Linux Repos... appstream baseos devel extras ha powertools resilient-storage plus nfv rt
+```
+
+## (re)run rpminspect_wrapper.sh
+
+*NOTE: If you want/need to replace previous reports/analysis be sure to clean out JSON reports.*
+
+### Optional clean out previous reports...
+
+```
+[vagrant@rocky8u5 data]$ find rpminspect/rocky/8.5 -name "*.json" | wc -l
+10259
+
+[vagrant@rocky8u5 data]$ find rpminspect/rocky/8.5 -name "*.json" -print0 | xargs -0 /bin/rm
+
+[vagrant@rocky8u5 data]$ find rpminspect/rocky/8.5 -name "*.json" | wc -l
+0
+```
+
+```
+[vagrant@rocky8u5 data]$ time ./rpminspect_wrapper.sh 2>&1 | tee rpminspect_wrapper.log
+Rocky Linux appstream repo has...
+	  2941 x86_64 packages -
+		389-ds-base-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64 fail
+		389-ds-base-legacy-tools-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64 fail
+...
+		resource-agents-4.1.1-98.el8.x86_64 fail
+	     4 i686 packages
+		libnozzle1-1.18-1.el8.i686 fail
+	    26 noarch packages
+		python3-azure-sdk skipped
+
+real	95m56.557s
+user	57m49.724s
+sys	26m55.865s
+```
+
+If you don't update any packages in the locally sync'd repos the inspection will skip analysis for any packages that have output JSON files. So, updates to the local repository clone can be reprocessed with minimal time required.
+
+For example...
+
+```
+[vagrant@rocky8u5 data]$ time ./rpminspect_wrapper.sh
+Rocky Linux appstream repo has...
+	  2941 x86_64 packages
+	   983 i686 packages
+	  2240 noarch packages
+		subscription-manager-migration-data-2.0.51-1.noarch fail
+Rocky Linux baseos repo has...
+	  1110 x86_64 packages
+	   403 i686 packages
+	   230 noarch packages
+		rocky-obsolete-packages-8-4.noarch fail
+Rocky Linux extras repo has...
+	     1 x86_64 packages
+	    36 noarch packages
+Rocky Linux ha repo has...
+	    31 x86_64 packages
+	     4 i686 packages
+	    26 noarch packages
+		python3-azure-sdk skipped
+Rocky Linux powertools repo has...
+	   759 x86_64 packages
+	   625 i686 packages
+	   811 noarch packages
+Rocky Linux resilient-storage repo has...
+	    33 x86_64 packages
+	     4 i686 packages
+	    26 noarch packages
+		python3-azure-sdk skipped
+
+real	4m25.786s
+user	1m30.924s
+sys	1m12.632s
+```
+
+## parse_rpminspect step
+
+```
+[vagrant@rocky8u5 data]$ ./parse_rpminspect.sh
+    1973 /home/vagrant/data/rpminspect/rocky/8.5/rocky_fail
+  172923 /home/vagrant/data/rpminspect/rocky/8.5/rocky_pass
+  174896 total
+```
+
+## summarize rpminspect step
+
+```
+[vagrant@rocky8u5 data]$ ./summary_rpminspect.sh
+
+=====
+ Summary of all tests for rocky version 8.5...
+
+  172923 /home/vagrant/data/rpminspect/rocky/8.5/rocky_pass
+    1973 /home/vagrant/data/rpminspect/rocky/8.5/rocky_fail
+  174896 total
+
+=====
+ Summary of failing tests...
+
+-----
+appstream had 1088 failing tests with the following breakdown...
+	366 annocheck
+	232 badfuncs
+	184 javabytecode
+	146 runpath
+	63 emptyrpm
+	39 xml
+	16 shellsyntax
+	10 pathmigration
+	8 ownership
+	7 capabilities
+	7 lto
+	4 permissions
+	3 elf
+	2 manpage
+	1 symlinks
+
+-----
+powertools had 501 failing tests with the following breakdown...
+	298 javabytecode
+	77 annocheck
+	48 runpath
+	30 xml
+	28 badfuncs
+	7 shellsyntax
+	6 pathmigration
+	4 emptyrpm
+	1 elf
+	1 lto
+	1 permissions
+
+-----
+baseos had 355 failing tests with the following breakdown...
+	137 badfuncs
+	70 annocheck
+	60 runpath
+	46 pathmigration
+	12 elf
+	8 emptyrpm
+	6 ownership
+	5 capabilities
+	5 xml
+	2 javabytecode
+	2 shellsyntax
+	1 permissions
+	1 symlinks
+
+-----
+ha had 13 failing tests with the following breakdown...
+	4 badfuncs
+	4 emptyrpm
+	2 annocheck
+	2 xml
+	1 shellsyntax
+
+-----
+resilient-storage had 13 failing tests with the following breakdown...
+	4 badfuncs
+	4 emptyrpm
+	2 annocheck
+	2 xml
+	1 shellsyntax
+
+-----
+plus had 3 failing tests with the following breakdown...
+	2 badfuncs
+	1 annocheck
+
+=====
+ Summary of passing tests...
+
+-----
+appstream had 103683 passing tests with the following breakdown...
+	6163 modularity
+	6163 politics
+	6162 symlinks
+	6161 manpage
+	6160 elf
+	6159 permissions
+	6156 capabilities
+	6156 lto
+	6155 ownership
+	6153 pathmigration
+	6147 shellsyntax
+	6124 xml
+	6100 emptyrpm
+	6017 runpath
+	5979 javabytecode
+	5931 badfuncs
+	5797 annocheck
+
+-----
+powertools had 36814 passing tests with the following breakdown...
+	2195 capabilities
+	2195 manpage
+	2195 modularity
+	2195 ownership
+	2195 politics
+	2195 symlinks
+	2194 elf
+	2194 lto
+	2194 permissions
+	2191 emptyrpm
+	2189 pathmigration
+	2188 shellsyntax
+	2167 badfuncs
+	2165 xml
+	2147 runpath
+	2118 annocheck
+	1897 javabytecode
+
+-----
+baseos had 29259 passing tests with the following breakdown...
+	1742 lto
+	1742 manpage
+	1742 modularity
+	1742 politics
+	1741 permissions
+	1741 symlinks
+	1740 javabytecode
+	1740 shellsyntax
+	1737 capabilities
+	1737 xml
+	1736 ownership
+	1734 emptyrpm
+	1730 elf
+	1696 pathmigration
+	1682 runpath
+	1672 annocheck
+	1605 badfuncs
+
+-----
+resilient-storage had 1041 passing tests with the following breakdown...
+	62 capabilities
+	62 elf
+	62 javabytecode
+	62 lto
+	62 manpage
+	62 modularity
+	62 ownership
+	62 pathmigration
+	62 permissions
+	62 politics
+	62 runpath
+	62 symlinks
+	61 shellsyntax
+	60 annocheck
+	60 xml
+	58 badfuncs
+	58 emptyrpm
+
+-----
+ha had 1007 passing tests with the following breakdown...
+	60 capabilities
+	60 elf
+	60 javabytecode
+	60 lto
+	60 manpage
+	60 modularity
+	60 ownership
+	60 pathmigration
+	60 permissions
+	60 politics
+	60 runpath
+	60 symlinks
+	59 shellsyntax
+	58 annocheck
+	58 xml
+	56 badfuncs
+	56 emptyrpm
+
+-----
+extras had 629 passing tests with the following breakdown...
+	37 annocheck
+	37 badfuncs
+	37 capabilities
+	37 elf
+	37 emptyrpm
+	37 javabytecode
+	37 lto
+	37 manpage
+	37 modularity
+	37 ownership
+	37 pathmigration
+	37 permissions
+	37 politics
+	37 runpath
+	37 shellsyntax
+	37 symlinks
+	37 xml
+
+-----
+nfv had 459 passing tests with the following breakdown...
+	27 annocheck
+	27 badfuncs
+	27 capabilities
+	27 elf
+	27 emptyrpm
+	27 javabytecode
+	27 lto
+	27 manpage
+	27 modularity
+	27 ownership
+	27 pathmigration
+	27 permissions
+	27 politics
+	27 runpath
+	27 shellsyntax
+	27 symlinks
+	27 xml
+
+-----
+plus had 31 passing tests with the following breakdown...
+	2 capabilities
+	2 elf
+	2 emptyrpm
+	2 javabytecode
+	2 lto
+	2 manpage
+	2 modularity
+	2 ownership
+	2 pathmigration
+	2 permissions
+	2 politics
+	2 runpath
+	2 shellsyntax
+	2 symlinks
+	2 xml
+	1 annocheck
+```
+
+
+## compare_rpminspect step
+
+```
+[vagrant@rocky8u5 data]$ time ./compare_rpminspect.sh fail 2>"compare_rpminspect.fail.$(date -Isec).err" | tee "compare_rpminspect.fail.$(date -Isec).out"
+=====
+./compare_rpminspect.sh started at Wed Nov 17 08:14:02 UTC 2021 for FAILURES
+=====
+rocky/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64.rpm annocheck FAILURES_MATCH rhel/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+12398+47000435.x86_64.rpm
+rocky/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64.rpm runpath FAILURES_MATCH rhel/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+12398+47000435.x86_64.rpm
+rocky/8.5/appstream/389-ds-base-legacy-tools-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64.rpm runpath FAILURES_MATCH rhel/8.5/appstream/389-ds-base-legacy-tools-1.4.3.23-10.module+el8.5.0+12398+47000435.x86_64.rpm
+
+...
+
+rocky/8.5/rt/kernel-rt-debug-devel-4.18.0-348.rt7.130.el8.0.2.x86_64.rpm NO_TARGET_PACKAGE_IN rhel/8.5/rt
+rocky/8.5/rt/kernel-rt-devel-4.18.0-348.rt7.130.el8.0.2.x86_64.rpm NO_TARGET_PACKAGE_IN rhel/8.5/rt
+rocky/8.5/rt/stress-ng-0.12.06-2.el8.x86_64.rpm NO_TARGET_PACKAGE_IN rhel/8.5/rt
+=====
+./compare_rpminspect.sh ended at Wed Nov 17 09:11:32 UTC 2021 for FAILURES
+=====
+real	57m30.305s
+user	42m10.831s
+sys	8m26.707s
+```
+
+Optionally, you can repeat the analysis for rpminspect tests deemed to have passed as well.
+
+*NOTE: This will take a lot longer to complete.*
+
+```
+[vagrant@rocky8u5 data]$ time ./compare_rpminspect.sh pass 2>"compare_rpminspect.pass.$(date -Isec).err" | tee "compare_rpminspect.pass.$(date -Isec).out"
+=====
+./compare_rpminspect.sh started at Wed Nov 17 16:10:28 UTC 2021 for PASSES
+=====
+rocky/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64.rpm emptyrpm PASSES_MATCH rhel/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+12398+47000435.x86_64.rpm
+rocky/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64.rpm manpage PASSES_MATCH rhel/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+12398+47000435.x86_64.rpm
+rocky/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+700+370e33d5.x86_64.rpm xml PASSES_MATCH rhel/8.5/appstream/389-ds-base-1.4.3.23-10.module+el8.5.0+12398+47000435.x86_64.rpm
+...
+
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm emptyrpm PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm manpage PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm xml PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm elf PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm modularity PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm javabytecode PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm ownership PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm shellsyntax PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm annocheck PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm permissions PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm capabilities PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm pathmigration PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm lto PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm symlinks PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm politics PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm badfuncs PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+rocky/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm runpath PASSES_MATCH rhel/8.5/appstream/abrt-2.10.9-21.el8.x86_64.rpm
+
+...
+
+rocky/8.5/rt/kernel-rt-modules-extra-4.18.0-348.rt7.130.el8.0.2.x86_64.rpm NO_TARGET_PACKAGE_IN rhel/8.5/rt
+rocky/8.5/rt/rteval-3.2-2.el8.noarch.rpm NO_TARGET_PACKAGE_IN rhel/8.5/rt
+rocky/8.5/rt/rteval-loads-1.4-13.el8.x86_64.rpm NO_TARGET_PACKAGE_IN rhel/8.5/rt
+
+real	611m31.049s  <--- you do NOT want to do this often. Maybe only repeat on/if there are PASSES_DIFF found.
+user	101m13.507s
+sys	88m19.529s
+```
+*NOTE: As can be seen for abrt above, there are 17 PASSES_MATCH entries for each 100% successful comparison of non-modular packages.*
+
+*NOTE: Modular packages may have failure due only to package version being listed in the message for a test.*
+
+Sample PASSES_DIFFERENT examination
+
+```
+[vagrant@rocky8u5 data]$ diff /home/vagrant/data/rpminspect/rocky/8.5/powertools/xorg-x11-drv-wacom-devel-0.38.0-1.el8.x86_64.rpminspect.out /home/vagrant/data/rpminspect/rhel/8.5/codeready/xorg-x11-drv-wacom-devel-0.38.0-1.el8.x86_64.rpminspect.out
+9c9
+< Running annocheck inspection...      pass
+---
+> Running annocheck inspection...      FAIL
+```
+
+
+
+
+```
+[vagrant@rocky8u5 data]$ jsondiff <(jq -r --sort-keys '.["annocheck"]' /home/vagrant/data/rpminspect/rocky/8.5/powertools/xorg-x11-drv-wacom-devel-0.38.0-1.el8.x86_64.json) <(jq -r --sort-keys '.["annocheck"]' /home/vagrant/data/rpminspect/rhel/8.5/codeready/xorg-x11-drv-wacom-devel-0.38.0-1.el8.x86_64.json) -i 2 -s symmetric
+{
+  "0": {
+    "details": [
+      "annocheck: Version 9.65.\nHardened: /usr/bin/isdv4-serial-debugger: PASS: pie test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: cf-protection test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: property-note test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: writeable-got test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: dynamic-segment test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: bind-now test \nHardened: /usr/bin/isdv4-serial-debugger: info: set binary producer to Gas version 2.\nHardened: /usr/bin/isdv4-serial-debugger: info: notes produced by assembler plugin version 1\nHardened: /usr/bin/isdv4-serial-debugger: info: set binary producer to GCC version 9.\nHardened: /usr/bin/isdv4-serial-debugger: info: notes produced by gcc plugin version 9.23\nHardened: /usr/bin/isdv4-serial-debugger: PASS: stack-prot test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: pic test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: fortify test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: glibcxx-assertions test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: optimization test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: warnings test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: stack-clash test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: entry test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: gnu-stack test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: gnu-relro test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: notes test because no gaps found \nHardened: /usr/bin/isdv4-serial-debugger: skip: branch-protection test because not an AArch64 binary \nHardened: /usr/bin/isdv4-serial-debugger: skip: dynamic-tags test because AArch64 specific \nHardened: /usr/bin/isdv4-serial-debugger: skip: go-revision test because no GO compiled code found \nHardened: /usr/bin/isdv4-serial-debugger: PASS: lto test \nHardened: /usr/bin/isdv4-serial-debugger: skip: only-go test because no GO compiled code found \nHardened: /usr/bin/isdv4-serial-debugger: PASS: run-path test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: rwx-seg test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: short-enum test \nHardened: /usr/bin/isdv4-serial-debugger: skip: stack-realign test because not an x86 executable \nHardened: /usr/bin/isdv4-serial-debugger: PASS: textrel test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: threads test ",
+      "annocheck: Version 9.72.\nHardened: /usr/bin/isdv4-serial-debugger: PASS: pie test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: cf-protection test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: property-note test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: writeable-got test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: dynamic-segment test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: bind-now test \nHardened: Internal Failure: /usr/bin/isdv4-serial-debugger: Unrecognised annobin note type 0.\nHardened: /usr/bin/isdv4-serial-debugger: PASS: entry test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: gnu-stack test because stack segment exists with the correct permissions \nHardened: /usr/bin/isdv4-serial-debugger: PASS: gnu-relro test \nHardened: /usr/bin/isdv4-serial-debugger: Not checking for gaps (binary created by a tool without an annobin plugin).\nHardened: /usr/bin/isdv4-serial-debugger: PASS: notes test \nHardened: /usr/bin/isdv4-serial-debugger: skip: branch-protection test because not an AArch64 binary \nHardened: /usr/bin/isdv4-serial-debugger: skip: dynamic-tags test because AArch64 specific \nHardened: /usr/bin/isdv4-serial-debugger: MAYB: test: fortify because no valid notes found regarding this test \nHardened: /usr/bin/isdv4-serial-debugger: MAYB: test: glibcxx-assertions because no valid notes found regarding this test \nHardened: /usr/bin/isdv4-serial-debugger: skip: go-revision test because no GO compiled code found \nHardened: /usr/bin/isdv4-serial-debugger: PASS: lto test \nHardened: /usr/bin/isdv4-serial-debugger: skip: only-go test because no GO compiled code found \nHardened: /usr/bin/isdv4-serial-debugger: skip: optimization test because not compiled code \nHardened: /usr/bin/isdv4-serial-debugger: skip: pic test because not compiled code \nHardened: /usr/bin/isdv4-serial-debugger: PASS: production test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: run-path test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: rwx-seg test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: short-enum test \nHardened: /usr/bin/isdv4-serial-debugger: MAYB: test: stack-clash because no notes found regarding this test \nHardened: /usr/bin/isdv4-serial-debugger: skip: stack-prot test because not compiled code \nHardened: /usr/bin/isdv4-serial-debugger: skip: stack-realign test because not an x86 executable \nHardened: /usr/bin/isdv4-serial-debugger: PASS: textrel test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: threads test \nHardened: /usr/bin/isdv4-serial-debugger: PASS: unicode test \nHardened: /usr/bin/isdv4-serial-debugger: skip: warnings test because compiling in LTO mode hides preprocessor and warning options "
+    ],
+    "message": [
+      "annocheck 'hardened' test passes for /usr/bin/isdv4-serial-debugger on x86_64",
+      "annocheck 'hardened' test fails for /usr/bin/isdv4-serial-debugger on x86_64"
+    ],
+    "result": [
+      "INFO",
+      "VERIFY"
+    ]
+  },
+  "$delete": [
+    [
+      1,
+      {
+        "result": "OK",
+        "waiver authorization": "Not Waivable"
+      }
+    ]
+  ]
+}
+```
+
+Summary of DIFF for PASSES...
+
+```
+[vagrant@rocky8u5 data]$ grep DIFF compare_rpminspect.pass.2021-11-17T16:10:28+00:00.out | wc -l
+204
+
+[vagrant@rocky8u5 data]$ grep DIFF compare_rpminspect.pass.2021-11-17T16:10:28+00:00.out | grep annocheck | wc -l
+203
+
+[vagrant@rocky8u5 data]$ grep DIFF compare_rpminspect.pass.2021-11-17T16:10:28+00:00.out | grep -v annocheck
+rocky/8.5/appstream/python36-devel-3.6.8-38.module+el8.5.0+671+195e4563.x86_64.rpm symlinks PASSES_DIFFERENT rhel/8.5/appstream/python36-devel-3.6.8-38.module+el8.5.0+12207+5c5719bc.x86_64.rpm
+
+```
+
+This is actually a problem with rhel...
+
+```
+[vagrant@rocky8u5 data]$ jsondiff <(jq -r --sort-keys '.["symlinks"]' /home/vagrant/data/rpminspect/rocky/8.5/appstream/python36-devel-3.6.8-38.module+el8.5.0+671+195e4563.x86_64.json) <(jq -r --sort-keys '.["symlinks"]' /home/vagrant/data/rpminspect/rhel/8.5/appstream/python36-devel-3.6.8-38.module+el8.5.0+12207+5c5719bc.x86_64.json) | jq '.'
+{
+  "$insert": [
+    [
+      0,
+      {
+        "message": "symbolic link /usr/bin/python3.6-config is a dangling symbolic link in python36-devel on x86_64",
+        "remedy": "Make sure symlinks point to a valid destination in one of the subpackages of the build; dangling symlinks are not allowed.  If you are comparing builds and have a non-symlink turn in to a symlink, ensure this is deliberate.  NOTE:  You cannot turn a directory in to a symlink due to RPM limitations.",
+        "result": "INFO",
+        "waiver authorization": "Not Waivable"
+      }
+    ]
+  ]
+}
+```
+
+```
+[vagrant@rocky8u5 data]$ diff <(jq -r --sort-keys '.["symlinks"]' /home/vagrant/data/rpminspect/rocky/8.5/appstream/python36-devel-3.6.8-38.module+el8.5.0+671+195e4563.x86_64.json) <(jq -r --sort-keys '.["symlinks"]' /home/vagrant/data/rpminspect/rhel/8.5/appstream/python36-devel-3.6.8-38.module+el8.5.0+12207+5c5719bc.x86_64.json)
+2a3,8
+>     "message": "symbolic link /usr/bin/python3.6-config is a dangling symbolic link in python36-devel on x86_64",
+>     "remedy": "Make sure symlinks point to a valid destination in one of the subpackages of the build; dangling symlinks are not allowed.  If you are comparing builds and have a non-symlink turn in to a symlink, ensure this is deliberate.  NOTE:  You cannot turn a directory in to a symlink due to RPM limitations.",
+>     "result": "INFO",
+>     "waiver authorization": "Not Waivable"
+>   },
+>   {
+```
+
+##
+
+Principle is to compare failed test in rocky and rhel to see if they fail for the same reason...
+
+### example baseos tar package has a badfuncs failure
+
+```
+[vagrant@rocky8u5 data]$ grep -i fail rpminspect/rocky/8.5/rocky_fail | grep baseos | grep "baseos tar"
+baseos tar-1.30-5.el8.x86_64.rpm badfuncs FAIL
+
+[vagrant@rocky8u5 data]$ grep -i fail rpminspect/rhel/8.5/rhel_fail | grep baseos | grep "baseos tar"
+baseos tar-1.30-5.el8.x86_64.rpm badfuncs FAIL
+```
+
+### extract the badfuncs results
+
+```
+[vagrant@rocky8u5 data]$ jq --sort-keys ."badfuncs" rpminspect/rocky/8.5/baseos/tar-1.30-5.el8.x86_64.json
+[
+  {
+    "details": "Forbidden function symbols found:\n\tgethostbyname\n",
+    "message": "/usr/bin/tar may use forbidden functions on x86_64",
+    "remedy": "Forbidden symbols were found in an ELF file in the package.  The configuration settings for rpminspect indicate the named symbols are forbidden in packages.  If this is deliberate, you may want to disable the badfuncs inspection.  If it is not deliberate, check the man pages for the named symbols to see what API functions have replaced the forbidden symbols.  Usually a function is marked as deprecated but still provided in order to allow for backwards compatibility.  Whenever possible the deprecated functions should not be used.",
+    "result": "VERIFY",
+    "waiver authorization": "Anyone"
+  }
+]
+```
+
+```
+[vagrant@rocky8u5 data]$ jq --sort-keys ."badfuncs" rpminspect/rhel/8.5/baseos/tar-1.30-5.el8.x86_64.json
+[
+  {
+    "details": "Forbidden function symbols found:\n\tgethostbyname\n",
+    "message": "/usr/bin/tar may use forbidden functions on x86_64",
+    "remedy": "Forbidden symbols were found in an ELF file in the package.  The configuration settings for rpminspect indicate the named symbols are forbidden in packages.  If this is deliberate, you may want to disable the badfuncs inspection.  If it is not deliberate, check the man pages for the named symbols to see what API functions have replaced the forbidden symbols.  Usually a function is marked as deprecated but still provided in order to allow for backwards compatibility.  Whenever possible the deprecated functions should not be used.",
+    "result": "VERIFY",
+    "waiver authorization": "Anyone"
+  }
+]
+```
+
+### diff the extract badfuncs results
+
+```
+[vagrant@rocky8u5 data]$ diff -s <(jq --sort-keys ."badfuncs" rpminspect/rocky/8.5/baseos/tar-1.30-5.el8.x86_64.json) <(jq --sort-keys ."badfuncs" rpminspect/rhel/8.5/baseos/tar-1.30-5.el8.x86_64.json)
+Files /dev/fd/63 and /dev/fd/62 are identical
+```
+
+If the badfuncs failure in rocky matches that in rhel the we are "bug-for-bug compatible".