From da5d75bbf4398e0aee9256a2a57eef2d02400eb7 Mon Sep 17 00:00:00 2001 From: juanmichelini Date: Tue, 13 Jan 2026 22:59:45 -0300 Subject: [PATCH 1/2] Fix SDK submodule update to handle commit SHAs - Fetch all from origin first to ensure commit is available - Then checkout the specific commit/ref - This allows passing both branch names and commit SHAs Co-authored-by: openhands --- .github/workflows/build-swebenchmultimodal-images.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-swebenchmultimodal-images.yml b/.github/workflows/build-swebenchmultimodal-images.yml index b0fc1f92..c2dd6265 100644 --- a/.github/workflows/build-swebenchmultimodal-images.yml +++ b/.github/workflows/build-swebenchmultimodal-images.yml @@ -161,8 +161,12 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' && inputs.sdk-commit != '' }} run: | cd vendor/software-agent-sdk - git fetch origin ${{ inputs.sdk-commit }} - git checkout FETCH_HEAD + # Fetch from origin (handles both branch names and commit SHAs) + git fetch origin + # Try to fetch the specific ref (works for branches/tags) + git fetch origin ${{ inputs.sdk-commit }} 2>/dev/null || true + # Checkout the commit/ref (works for both branches and commit SHAs) + git checkout ${{ inputs.sdk-commit }} SDK_SHA=$(git rev-parse HEAD) cd ../.. # Stage the submodule reference update so make build uses it From 08ee26163eb9c515e2910e4d1710ec482e94d6e8 Mon Sep 17 00:00:00 2001 From: juanmichelini Date: Wed, 14 Jan 2026 00:25:30 -0300 Subject: [PATCH 2/2] fix(swebenchmultimodal): pass selected_instances_file to collect_unique_base_images The build_images.py script was ignoring the --select parameter and building all instances in the split instead of just the selected ones. This caused builds to take 61 minutes for 102 instances instead of ~2 minutes for 5. This fix matches the implementation in the regular swebench build_images.py. --- benchmarks/swebenchmultimodal/build_images.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/benchmarks/swebenchmultimodal/build_images.py b/benchmarks/swebenchmultimodal/build_images.py index d32b5dc6..4bc43cc2 100644 --- a/benchmarks/swebenchmultimodal/build_images.py +++ b/benchmarks/swebenchmultimodal/build_images.py @@ -57,9 +57,17 @@ def extract_custom_tag(base_image: str) -> str: return name -def collect_unique_base_images(dataset, split, n_limit): +def collect_unique_base_images( + dataset, + split, + n_limit, + selected_instances_file: str | None = None, +): df = get_dataset( - dataset_name=dataset, split=split, eval_limit=n_limit if n_limit else None + dataset_name=dataset, + split=split, + eval_limit=n_limit if n_limit else None, + selected_instances_file=selected_instances_file, ) return sorted( {get_official_docker_image(str(row["instance_id"])) for _, row in df.iterrows()} @@ -71,7 +79,10 @@ def main(argv: list[str]) -> int: args = parser.parse_args(argv) base_images: list[str] = collect_unique_base_images( - args.dataset, args.split, args.n_limit + args.dataset, + args.split, + args.n_limit, + args.select, ) build_dir = default_build_output_dir(args.dataset, args.split) return build_all_images(