From ae0ec776df9a767125609edb86d897ce36c7cf5c Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 06:23:13 +0000
Subject: [PATCH 01/57] cloudbuild for perf benchmarks

---
 cloudbuild/benchmarks-cloudbuild.yaml         | 231 ++++++++++++++++++
 cloudbuild/run_benchmarks.sh                  | 190 ++++++++++++++
 .../perf/microbenchmarks/listing/configs.yaml |  18 --
 .../perf/microbenchmarks/read/configs.yaml    |   4 +-
 .../perf/microbenchmarks/write/configs.yaml   |   7 +-
 5 files changed, 426 insertions(+), 24 deletions(-)
 create mode 100644 cloudbuild/benchmarks-cloudbuild.yaml
 create mode 100644 cloudbuild/run_benchmarks.sh

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
new file mode 100644
index 00000000..beb5841a
--- /dev/null
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -0,0 +1,231 @@
+substitutions:
+  _PROJECT_ID: "gcs-aiml-clients-testing-101"
+  _ZONE: "us-central1-a"
+  _SHORT_BUILD_ID: ${BUILD_ID:0:8}
+  _RUN_ID: ${BUILD_ID}
+  _BUCKET_PREFIX: "gcsfs-perf"
+  _SERVICE_ACCOUNT: "gcsfs-perf-ci-runner@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
+  _REPO_URL: "https://github.com/fsspec/gcsfs.git"
+  _BRANCH: "main"
+  # Fanout Configuration: Space-separated list of "group:config_name" or "group:" (for all configs in group).
+  # Example: "read:read_seq,read_rand write: listing:"
+  _BENCHMARK_FANOUT_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process,read_rand_multi_process write:write_seq,write_seq_multi_process listing: delete: rename:"
+
+steps:
+  # 1. Generate a persistent SSH key for this build run.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "generate-ssh-key"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        mkdir -p /workspace/.ssh
+        ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb
+        cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub
+    waitFor: ["-"]
+
+  # 2. Initialize shared variables.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "init-variables"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        # Define shared variables
+        export PROJECT_ID="${_PROJECT_ID}"
+        export ZONE="${_ZONE}"
+        export SHORT_BUILD_ID="${_SHORT_BUILD_ID}"
+        export RUN_ID="${_RUN_ID}"
+        export SERVICE_ACCOUNT="${_SERVICE_ACCOUNT}"
+        export REPO_URL="${_REPO_URL}"
+        export BRANCH="${_BRANCH}"
+        export BENCHMARK_FANOUT_CONFIG="${_BENCHMARK_FANOUT_CONFIG}"
+        export COMMIT_SHA="${COMMIT_SHA}"
+
+        export REGIONAL_BUCKET="${_BUCKET_PREFIX}-regional-${_SHORT_BUILD_ID}"
+        export ZONAL_BUCKET="${_BUCKET_PREFIX}-zonal-${_SHORT_BUILD_ID}"
+        export HNS_BUCKET="${_BUCKET_PREFIX}-hns-${_SHORT_BUILD_ID}"
+        export RESULTS_BUCKET="${_BUCKET_PREFIX}-results-${_PROJECT_ID}"
+        export TEMPLATE_NAME="gcsfs-perf-tmpl-${_SHORT_BUILD_ID}"
+        export MIG_NAME="gcsfs-perf-mig-${_SHORT_BUILD_ID}"
+
+        # Save variables to workspace for other steps
+        {
+          echo "export PROJECT_ID='$PROJECT_ID'"
+          echo "export ZONE='$ZONE'"
+          echo "export SHORT_BUILD_ID='$SHORT_BUILD_ID'"
+          echo "export RUN_ID='$RUN_ID'"
+          echo "export SERVICE_ACCOUNT='$SERVICE_ACCOUNT'"
+          echo "export REPO_URL='$REPO_URL'"
+          echo "export BRANCH='$BRANCH'"
+          echo "export BENCHMARK_FANOUT_CONFIG='$BENCHMARK_FANOUT_CONFIG'"
+          echo "export COMMIT_SHA='$COMMIT_SHA'"
+          echo "export REGIONAL_BUCKET='$REGIONAL_BUCKET'"
+          echo "export ZONAL_BUCKET='$ZONAL_BUCKET'"
+          echo "export HNS_BUCKET='$HNS_BUCKET'"
+          echo "export RESULTS_BUCKET='$RESULTS_BUCKET'"
+          echo "export TEMPLATE_NAME='$TEMPLATE_NAME'"
+          echo "export MIG_NAME='$MIG_NAME'"
+        } > /workspace/build_vars.env
+    waitFor: ["-"]
+
+  # 3. Create all necessary GCS buckets in parallel.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "create-buckets"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        set -e
+        source /workspace/build_vars.env
+
+        echo "--- Creating Regional bucket: $REGIONAL_BUCKET ---"
+        gcloud storage buckets create gs://$REGIONAL_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} &
+
+        echo "--- Creating Zonal bucket: $ZONAL_BUCKET ---"
+        gcloud storage buckets create gs://$ZONAL_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} --placement=$ZONE --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
+
+        echo "--- Creating HNS bucket: $HNS_BUCKET ---"
+        gcloud storage buckets create gs://$HNS_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+
+        # Create HNS Results Bucket (Persistent)
+        if ! gcloud storage buckets describe gs://$RESULTS_BUCKET --project=$$PROJECT_ID >/dev/null 2>&1; then
+          echo "--- Creating Results HNS bucket: $RESULTS_BUCKET ---"
+          gcloud storage buckets create gs://$RESULTS_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        else
+          echo "--- Results bucket gs://$RESULTS_BUCKET already exists ---"
+        fi
+
+        wait
+    waitFor: ["init-variables"]
+
+  # 4. Calculate number of VMs needed.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "count-configs"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        source /workspace/build_vars.env
+        IFS=' ' read -r -a CONFIG_ARRAY <<< "$BENCHMARK_FANOUT_CONFIG"
+        NUM_VMS=${#CONFIG_ARRAY[@]}
+        echo "export NUM_VMS=$NUM_VMS" >> /workspace/build_vars.env
+        echo "Calculated NUM_VMS: $NUM_VMS"
+    waitFor: ["init-variables"]
+
+  # 5. Create Instance Group based on count.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "create-mig"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        set -e
+        source /workspace/build_vars.env
+
+        echo "Creating Instance Template: $TEMPLATE_NAME"
+        gcloud compute instance-templates create "$TEMPLATE_NAME" \
+            --project="$$PROJECT_ID" \
+            --machine-type="c4-standard-192" \
+            --image-family="ubuntu-2204-lts" \
+            --image-project="ubuntu-os-cloud" \
+            --boot-disk-type="hyperdisk-balanced" \
+            --boot-disk-size="100GB" \
+            --network-interface="network-tier=PREMIUM,nic-type=GVNIC" \
+            --network-performance-configs="total-egress-bandwidth-tier=TIER_1" \
+            --scopes="https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/devstorage.read_write" \
+            --metadata="enable-oslogin=TRUE" \
+            --service-account="$SERVICE_ACCOUNT" \
+            --tags="allow-ssh" \
+            --labels="build-id=$SHORT_BUILD_ID" \
+            --quiet
+
+        echo "Creating Managed Instance Group: $MIG_NAME with size $NUM_VMS"
+        gcloud compute instance-groups managed create "$MIG_NAME" \
+            --project="$$PROJECT_ID" \
+            --zone="$ZONE" \
+            --template="$TEMPLATE_NAME" \
+            --size="$NUM_VMS" \
+            --quiet
+
+        echo "Waiting for Instance Group to be stable..."
+        gcloud compute instance-groups managed wait-until "$MIG_NAME" \
+            --project="$$PROJECT_ID" \
+            --zone="$ZONE" \
+            --stable \
+            --quiet
+
+        # Save instance names for the next step
+        gcloud compute instance-groups managed list-instances "$MIG_NAME" \
+            --project="$$PROJECT_ID" \
+            --zone="$ZONE" \
+            --format="value(instance)" > /workspace/instances.txt
+    waitFor: ["count-configs"]
+
+  # 6. Run Benchmarks.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "run-benchmarks"
+    entrypoint: "bash"
+    args: ["cloudbuild/run_benchmarks.sh"]
+    waitFor:
+      - "create-mig"
+      - "create-buckets"
+      - "generate-ssh-key"
+
+  # --- Cleanup Steps ---
+
+  # Clean up the SSH key.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "cleanup-ssh-key"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        echo "--- Removing SSH key from OS Login profile ---"
+        gcloud compute os-login ssh-keys remove \
+          --key-file=/workspace/gcb_ssh_key.pub || true
+    waitFor:
+      - "run-benchmarks"
+
+  # Delete MIG and Template.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "cleanup-mig"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        source /workspace/build_vars.env
+        echo "--- Cleaning up Instance Group and Template ---"
+
+        gcloud compute instance-groups managed delete "$MIG_NAME" \
+            --project="$$PROJECT_ID" \
+            --zone="$ZONE" \
+            --quiet || true
+
+        gcloud compute instance-templates delete "$TEMPLATE_NAME" \
+            --project="$$PROJECT_ID" \
+            --quiet || true
+    waitFor:
+      - "cleanup-ssh-key"
+
+  # Delete all GCS buckets.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "delete-buckets"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        set -e
+        source /workspace/build_vars.env
+        echo "--- Deleting test buckets ---"
+        gcloud storage rm --recursive gs://$REGIONAL_BUCKET &
+        gcloud storage rm --recursive gs://$ZONAL_BUCKET &
+        gcloud storage rm --recursive gs://$HNS_BUCKET &
+        wait
+    waitFor:
+      - "run-benchmarks"
+
+timeout: "7200s" # 2 hours
+
+options:
+  logging: CLOUD_LOGGING_ONLY
diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
new file mode 100644
index 00000000..80b222a7
--- /dev/null
+++ b/cloudbuild/run_benchmarks.sh
@@ -0,0 +1,190 @@
+#!/bin/bash
+set -e
+
+# Load build variables (buckets, etc.)
+source /workspace/build_vars.env
+
+# Waits for SSH to become available on the VM.
+# Arguments:
+#   vm_name: The name of the VM instance.
+#   log_file: Path to the log file for this VM.
+wait_for_ssh() {
+  local vm_name=$1
+  local log_file=$2
+
+  # Wait for SSH (MIG is stable, but SSH might take a moment)
+  echo "[$vm_name] Waiting for SSH..." >> "$log_file"
+  for i in {1..20}; do
+    if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo ready" >> "$log_file" 2>&1; then
+      return 0
+    fi
+    sleep 10
+  done
+  return 1
+}
+
+# Sets up the VM by installing dependencies and cloning the repository.
+# Arguments:
+#   vm_name: The name of the VM instance.
+#   log_file: Path to the log file for this VM.
+setup_vm() {
+  local vm_name=$1
+  local log_file=$2
+
+  # Setup VM (Install deps, Clone repo)
+  echo "[$vm_name] Setting up VM..." >> "$log_file"
+  local SETUP_SCRIPT="
+    set -e
+    export DEBIAN_FRONTEND=noninteractive
+    sudo apt-get update > /dev/null
+    sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null
+
+    # Clone Repo
+    echo 'Cloning repo...'
+    git clone ${REPO_URL} gcsfs
+    cd gcsfs
+    if [ -n '${COMMIT_SHA}' ]; then
+      git checkout ${COMMIT_SHA}
+    else
+      git checkout ${BRANCH}
+    fi
+
+    # Install Python deps
+    python3 -m venv env
+    source env/bin/activate
+    pip install --upgrade pip > /dev/null
+    pip install -e . > /dev/null
+    pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
+  "
+
+  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$SETUP_SCRIPT" >> "$log_file" 2>&1
+}
+
+# Runs the benchmark on the VM and uploads the results.
+# Arguments:
+#   vm_name: The name of the VM instance.
+#   group: The benchmark group.
+#   config: The benchmark config.
+#   log_file: Path to the log file for this VM.
+run_benchmark() {
+  local vm_name=$1
+  local group=$2
+  local config=$3
+  local log_file=$4
+
+  # Run Benchmark
+  echo "[$vm_name] Running benchmark..." >> "$log_file"
+
+  # Construct config arg
+  local CONFIG_ARG=""
+  if [ -n "$config" ]; then
+    CONFIG_ARG="--config=$config"
+  fi
+
+  local RUN_CMD="
+    source gcsfs/env/bin/activate
+    python gcsfs/gcsfs/tests/perf/microbenchmarks/run.py \
+      --group=$group \
+      $CONFIG_ARG \
+      --regional-bucket='${REGIONAL_BUCKET}' \
+      --zonal-bucket='${ZONAL_BUCKET}' \
+      --hns-bucket='${HNS_BUCKET}' \
+      --log=true \
+      --log-level=INFO
+
+    # Upload results
+    echo '--- Uploading Results ---'
+    DATE_DIR=\$(date +%d%m%Y)
+    RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'
+    if [ -d \"\$RESULTS_DIR\" ]; then
+      echo \"Uploading from \$RESULTS_DIR to gs://${RESULTS_BUCKET}/\${DATE_DIR}/${RUN_ID}/\"
+      cd \"\$RESULTS_DIR\" && gcloud storage cp --recursive . gs://${RESULTS_BUCKET}/\${DATE_DIR}/${RUN_ID}/
+    else
+      echo \"No results directory found at \$RESULTS_DIR\"
+    fi
+  "
+
+  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$RUN_CMD" >> "$log_file" 2>&1
+}
+
+# Orchestrates the execution of a single benchmark job: wait for SSH, setup VM, and run benchmark.
+# Arguments:
+#   group: The benchmark group.
+#   config: The benchmark config.
+#   vm_name: The name of the VM instance.
+run_job() {
+  local group=$1
+  local config=$2
+  local vm_name=$3
+  local log_file="/workspace/${vm_name}.log"
+
+  echo "[$vm_name] Starting job for Group: $group, Config: $config" > "$log_file"
+
+  if ! wait_for_ssh "$vm_name" "$log_file"; then
+    echo "[$vm_name] SSH wait failed." >> "$log_file"
+    return 1
+  fi
+
+  if ! setup_vm "$vm_name" "$log_file"; then
+    echo "[$vm_name] Setup failed." >> "$log_file"
+    return 1
+  fi
+
+  if ! run_benchmark "$vm_name" "$group" "$config" "$log_file"; then
+    echo "[$vm_name] Benchmark failed." >> "$log_file"
+    return 1
+  fi
+
+  echo "[$vm_name] Benchmark finished successfully." >> "$log_file"
+  return 0
+}
+
+# Main function to orchestrate the parallel execution of benchmarks across VMs.
+main() {
+  # Load instances
+  local INSTANCES=($(cat /workspace/instances.txt))
+
+  # Load configs
+  local CONFIG_ARRAY
+  IFS=' ' read -r -a CONFIG_ARRAY <<< "${BENCHMARK_FANOUT_CONFIG}"
+
+  local NUM_VMS=${#CONFIG_ARRAY[@]}
+  if [ "${#INSTANCES[@]}" -ne "$NUM_VMS" ]; then
+      echo "Error: Number of instances (${#INSTANCES[@]}) does not match number of configs ($NUM_VMS)."
+      exit 1
+  fi
+
+  # Main Loop
+  local pids=""
+
+  for i in "${!CONFIG_ARRAY[@]}"; do
+    local entry="${CONFIG_ARRAY[$i]}"
+    local vm_name="${INSTANCES[$i]}"
+
+    IFS=':' read -r group config <<< "$entry"
+
+    echo "Launching job for $group:$config on $vm_name"
+
+    # Run in background
+    run_job "$group" "$config" "$vm_name" &
+    pids="$pids $!"
+  done
+
+  # Wait for all jobs
+  local failures=0
+  for pid in $pids; do
+    wait $pid || failures=$((failures+1))
+  done
+
+  # Print logs
+  echo "--- Benchmark Logs ---"
+  cat /workspace/*.log || true
+
+  if [ $failures -ne 0 ]; then
+    echo "$failures benchmark jobs failed."
+    exit 1
+  fi
+  echo "All benchmarks completed successfully."
+}
+
+main
diff --git a/gcsfs/tests/perf/microbenchmarks/listing/configs.yaml b/gcsfs/tests/perf/microbenchmarks/listing/configs.yaml
index 5ce0c770..8f586383 100644
--- a/gcsfs/tests/perf/microbenchmarks/listing/configs.yaml
+++ b/gcsfs/tests/perf/microbenchmarks/listing/configs.yaml
@@ -9,49 +9,31 @@ common:
 
 scenarios:
   - name: "list_flat"
-    threads: [1]
-    processes: [1]
     depth: 0
     pattern: "ls"
 
   - name: "list_recursive"
-    threads: [1]
-    processes: [1]
     depth: 8
     pattern: "ls"
 
   - name: "list_recursive_deep"
-    threads: [1]
-    processes: [1]
     depth: 24
     pattern: "ls"
 
-  - name: "list_recursive_multi_thread"
-    threads: [4, 8]
-    processes: [1]
-    pattern: "ls"
-
   - name: "list_recursive_multi_process"
-    threads: [1, 2]
     processes: [4, 8]
     pattern: "ls"
 
   - name: "find_flat"
-    threads: [1]
-    processes: [1]
     depth: 0
     pattern: "find"
 
   - name: "find_recursive"
-    threads: [1]
-    processes: [1]
     depth: 8
     folders: [100]
     pattern: "find"
 
   - name: "find_recursive_deep"
-    threads: [1]
-    processes: [1]
     depth: 24
     folders: [2050]
     pattern: "find"
diff --git a/gcsfs/tests/perf/microbenchmarks/read/configs.yaml b/gcsfs/tests/perf/microbenchmarks/read/configs.yaml
index 85dd30c6..7ac0a5ab 100644
--- a/gcsfs/tests/perf/microbenchmarks/read/configs.yaml
+++ b/gcsfs/tests/perf/microbenchmarks/read/configs.yaml
@@ -14,11 +14,11 @@ scenarios:
 
   - name: "read_seq_multi_process"
     pattern: "seq"
-    processes: [16,40,96,144]
+    processes: [16,48]
 
   - name: "read_rand"
     pattern: "rand"
 
   - name: "read_rand_multi_process"
     pattern: "rand"
-    processes: [16,40,96,144]
+    processes: [16,48]
diff --git a/gcsfs/tests/perf/microbenchmarks/write/configs.yaml b/gcsfs/tests/perf/microbenchmarks/write/configs.yaml
index b16a0515..fbcba3bc 100644
--- a/gcsfs/tests/perf/microbenchmarks/write/configs.yaml
+++ b/gcsfs/tests/perf/microbenchmarks/write/configs.yaml
@@ -5,12 +5,11 @@ common:
   file_sizes_mb:
     - 1024 # 1GB
   chunk_sizes_mb:
-    - 16
-    - 64
+    - 32
   rounds: 3
 
 scenarios:
   - name: "write_seq"
 
-  - name: "write_seq_multi_process_single_thread"
-    processes: [16,40,96]
+  - name: "write_seq_multi_process"
+    processes: [16,48]

From 44656ba299736489a3f50a1f31fa891f0f8d1469 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 07:46:25 +0000
Subject: [PATCH 02/57] Added option to skip steps for debugging

---
 cloudbuild/benchmarks-cloudbuild.yaml | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index beb5841a..434f2db6 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -10,6 +10,7 @@ substitutions:
   # Fanout Configuration: Space-separated list of "group:config_name" or "group:" (for all configs in group).
   # Example: "read:read_seq,read_rand write: listing:"
   _BENCHMARK_FANOUT_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process,read_rand_multi_process write:write_seq,write_seq_multi_process listing: delete: rename:"
+  _SKIP_STEPS: ""
 
 steps:
   # 1. Generate a persistent SSH key for this build run.
@@ -76,6 +77,10 @@ steps:
     args:
       - "-c"
       - |
+        if [[ ",${_SKIP_STEPS}," == *",create-buckets,"* ]]; then
+          echo "--- Skipping step: create-buckets ---"
+          exit 0
+        fi
         set -e
         source /workspace/build_vars.env
 
@@ -120,6 +125,10 @@ steps:
     args:
       - "-c"
       - |
+        if [[ ",${_SKIP_STEPS}," == *",create-mig,"* ]]; then
+          echo "--- Skipping step: create-mig ---"
+          exit 0
+        fi
         set -e
         source /workspace/build_vars.env
 
@@ -166,7 +175,14 @@ steps:
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "run-benchmarks"
     entrypoint: "bash"
-    args: ["cloudbuild/run_benchmarks.sh"]
+    args:
+      - "-c"
+      - |
+        if [[ ",${_SKIP_STEPS}," == *",run-benchmarks,"* ]]; then
+          echo "--- Skipping step: run-benchmarks ---"
+          exit 0
+        fi
+        bash cloudbuild/run_benchmarks.sh
     waitFor:
       - "create-mig"
       - "create-buckets"
@@ -194,6 +210,10 @@ steps:
     args:
       - "-c"
       - |
+        if [[ ",${_SKIP_STEPS}," == *",cleanup-mig,"* ]]; then
+          echo "--- Skipping step: cleanup-mig ---"
+          exit 0
+        fi
         source /workspace/build_vars.env
         echo "--- Cleaning up Instance Group and Template ---"
 
@@ -215,6 +235,10 @@ steps:
     args:
       - "-c"
       - |
+        if [[ ",${_SKIP_STEPS}," == *",delete-buckets,"* ]]; then
+          echo "--- Skipping step: delete-buckets ---"
+          exit 0
+        fi
         set -e
         source /workspace/build_vars.env
         echo "--- Deleting test buckets ---"

From 742c1dd2d0eda27c9818a9947b3feaa9f0e642b8 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 07:51:15 +0000
Subject: [PATCH 03/57] Added logging pool

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 434f2db6..64053cc9 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -253,3 +253,5 @@ timeout: "7200s" # 2 hours
 
 options:
   logging: CLOUD_LOGGING_ONLY
+  pool:
+    name: "projects/${_PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"

From cd21ef3088eb4285f1082dbac0207986c8b89bdb Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:27:40 +0000
Subject: [PATCH 04/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 77 +++++++++++----------------
 1 file changed, 31 insertions(+), 46 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 64053cc9..487191d2 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -33,34 +33,20 @@ steps:
       - "-c"
       - |
         # Define shared variables
-        export PROJECT_ID="${_PROJECT_ID}"
-        export ZONE="${_ZONE}"
-        export SHORT_BUILD_ID="${_SHORT_BUILD_ID}"
-        export RUN_ID="${_RUN_ID}"
-        export SERVICE_ACCOUNT="${_SERVICE_ACCOUNT}"
-        export REPO_URL="${_REPO_URL}"
-        export BRANCH="${_BRANCH}"
-        export BENCHMARK_FANOUT_CONFIG="${_BENCHMARK_FANOUT_CONFIG}"
-        export COMMIT_SHA="${COMMIT_SHA}"
+        REGIONAL_BUCKET="${_BUCKET_PREFIX}-regional-${_SHORT_BUILD_ID}"
+        ZONAL_BUCKET="${_BUCKET_PREFIX}-zonal-${_SHORT_BUILD_ID}"
+        HNS_BUCKET="${_BUCKET_PREFIX}-hns-${_SHORT_BUILD_ID}"
+        RESULTS_BUCKET="${_BUCKET_PREFIX}-results-${_PROJECT_ID}"
 
-        export REGIONAL_BUCKET="${_BUCKET_PREFIX}-regional-${_SHORT_BUILD_ID}"
-        export ZONAL_BUCKET="${_BUCKET_PREFIX}-zonal-${_SHORT_BUILD_ID}"
-        export HNS_BUCKET="${_BUCKET_PREFIX}-hns-${_SHORT_BUILD_ID}"
-        export RESULTS_BUCKET="${_BUCKET_PREFIX}-results-${_PROJECT_ID}"
-        export TEMPLATE_NAME="gcsfs-perf-tmpl-${_SHORT_BUILD_ID}"
-        export MIG_NAME="gcsfs-perf-mig-${_SHORT_BUILD_ID}"
-
-        # Save variables to workspace for other steps
+        # Save variables to workspace
         {
-          echo "export PROJECT_ID='$PROJECT_ID'"
-          echo "export ZONE='$ZONE'"
-          echo "export SHORT_BUILD_ID='$SHORT_BUILD_ID'"
-          echo "export RUN_ID='$RUN_ID'"
-          echo "export SERVICE_ACCOUNT='$SERVICE_ACCOUNT'"
-          echo "export REPO_URL='$REPO_URL'"
-          echo "export BRANCH='$BRANCH'"
-          echo "export BENCHMARK_FANOUT_CONFIG='$BENCHMARK_FANOUT_CONFIG'"
-          echo "export COMMIT_SHA='$COMMIT_SHA'"
+          echo "export PROJECT_ID='${_PROJECT_ID}'"
+          echo "export ZONE='${_ZONE}'"
+          echo "export REPO_URL='${_REPO_URL}'"
+          echo "export BRANCH='${_BRANCH}'"
+          echo "export BENCHMARK_FANOUT_CONFIG='${_BENCHMARK_FANOUT_CONFIG}'"
+          echo "export COMMIT_SHA='${COMMIT_SHA}'"
+          echo "export RUN_ID='${_RUN_ID}'"
           echo "export REGIONAL_BUCKET='$REGIONAL_BUCKET'"
           echo "export ZONAL_BUCKET='$ZONAL_BUCKET'"
           echo "export HNS_BUCKET='$HNS_BUCKET'"
@@ -85,18 +71,18 @@ steps:
         source /workspace/build_vars.env
 
         echo "--- Creating Regional bucket: $REGIONAL_BUCKET ---"
-        gcloud storage buckets create gs://$REGIONAL_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} &
+        gcloud storage buckets create gs://$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} &
 
         echo "--- Creating Zonal bucket: $ZONAL_BUCKET ---"
-        gcloud storage buckets create gs://$ZONAL_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} --placement=$ZONE --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
+        gcloud storage buckets create gs://$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
 
         echo "--- Creating HNS bucket: $HNS_BUCKET ---"
-        gcloud storage buckets create gs://$HNS_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        gcloud storage buckets create gs://$HNS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
 
         # Create HNS Results Bucket (Persistent)
-        if ! gcloud storage buckets describe gs://$RESULTS_BUCKET --project=$$PROJECT_ID >/dev/null 2>&1; then
+        if ! gcloud storage buckets describe gs://$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
           echo "--- Creating Results HNS bucket: $RESULTS_BUCKET ---"
-          gcloud storage buckets create gs://$RESULTS_BUCKET --project=$$PROJECT_ID --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+          gcloud storage buckets create gs://$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
         else
           echo "--- Results bucket gs://$RESULTS_BUCKET already exists ---"
         fi
@@ -111,8 +97,7 @@ steps:
     args:
       - "-c"
       - |
-        source /workspace/build_vars.env
-        IFS=' ' read -r -a CONFIG_ARRAY <<< "$BENCHMARK_FANOUT_CONFIG"
+        IFS=' ' read -r -a CONFIG_ARRAY <<< "$_BENCHMARK_FANOUT_CONFIG"
         NUM_VMS=${#CONFIG_ARRAY[@]}
         echo "export NUM_VMS=$NUM_VMS" >> /workspace/build_vars.env
         echo "Calculated NUM_VMS: $NUM_VMS"
@@ -134,7 +119,7 @@ steps:
 
         echo "Creating Instance Template: $TEMPLATE_NAME"
         gcloud compute instance-templates create "$TEMPLATE_NAME" \
-            --project="$$PROJECT_ID" \
+            --project="${_PROJECT_ID}" \
             --machine-type="c4-standard-192" \
             --image-family="ubuntu-2204-lts" \
             --image-project="ubuntu-os-cloud" \
@@ -144,30 +129,30 @@ steps:
             --network-performance-configs="total-egress-bandwidth-tier=TIER_1" \
             --scopes="https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/devstorage.read_write" \
             --metadata="enable-oslogin=TRUE" \
-            --service-account="$SERVICE_ACCOUNT" \
+            --service-account="${_SERVICE_ACCOUNT}" \
             --tags="allow-ssh" \
-            --labels="build-id=$SHORT_BUILD_ID" \
+            --labels="build-id=${_SHORT_BUILD_ID}" \
             --quiet
 
         echo "Creating Managed Instance Group: $MIG_NAME with size $NUM_VMS"
         gcloud compute instance-groups managed create "$MIG_NAME" \
-            --project="$$PROJECT_ID" \
-            --zone="$ZONE" \
+            --project="${_PROJECT_ID}" \
+            --zone="${_ZONE}" \
             --template="$TEMPLATE_NAME" \
             --size="$NUM_VMS" \
             --quiet
 
         echo "Waiting for Instance Group to be stable..."
         gcloud compute instance-groups managed wait-until "$MIG_NAME" \
-            --project="$$PROJECT_ID" \
-            --zone="$ZONE" \
+            --project="${_PROJECT_ID}" \
+            --zone="${_ZONE}" \
             --stable \
             --quiet
 
         # Save instance names for the next step
         gcloud compute instance-groups managed list-instances "$MIG_NAME" \
-            --project="$$PROJECT_ID" \
-            --zone="$ZONE" \
+            --project="${_PROJECT_ID}" \
+            --zone="${_ZONE}" \
             --format="value(instance)" > /workspace/instances.txt
     waitFor: ["count-configs"]
 
@@ -218,12 +203,12 @@ steps:
         echo "--- Cleaning up Instance Group and Template ---"
 
         gcloud compute instance-groups managed delete "$MIG_NAME" \
-            --project="$$PROJECT_ID" \
-            --zone="$ZONE" \
+            --project="${_PROJECT_ID}" \
+            --zone="${_ZONE}" \
             --quiet || true
 
         gcloud compute instance-templates delete "$TEMPLATE_NAME" \
-            --project="$$PROJECT_ID" \
+            --project="${_PROJECT_ID}" \
             --quiet || true
     waitFor:
       - "cleanup-ssh-key"
@@ -254,4 +239,4 @@ timeout: "7200s" # 2 hours
 options:
   logging: CLOUD_LOGGING_ONLY
   pool:
-    name: "projects/${_PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"
+    name: "projects/${_PROJECT_ID}/locations/${ZONE::-2}/workerPools/cloud-build-worker-pool"

From c911331220461cdc7ddce2ce93bf2faf984ab025 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:33:14 +0000
Subject: [PATCH 05/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 487191d2..fd0d89f4 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -70,8 +70,8 @@ steps:
         set -e
         source /workspace/build_vars.env
 
-        echo "--- Creating Regional bucket: $REGIONAL_BUCKET ---"
-        gcloud storage buckets create gs://$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} &
+        echo "--- Creating Regional bucket: $$REGIONAL_BUCKET ---"
+        gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} &
 
         echo "--- Creating Zonal bucket: $ZONAL_BUCKET ---"
         gcloud storage buckets create gs://$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
@@ -227,7 +227,7 @@ steps:
         set -e
         source /workspace/build_vars.env
         echo "--- Deleting test buckets ---"
-        gcloud storage rm --recursive gs://$REGIONAL_BUCKET &
+        gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
         gcloud storage rm --recursive gs://$ZONAL_BUCKET &
         gcloud storage rm --recursive gs://$HNS_BUCKET &
         wait

From f2763ea241c3715e5fbd689fa367ddcb2dcba91e Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:43:22 +0000
Subject: [PATCH 06/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 34 ++++++++++-----------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index fd0d89f4..9fa9e885 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -33,27 +33,19 @@ steps:
       - "-c"
       - |
         # Define shared variables
-        REGIONAL_BUCKET="${_BUCKET_PREFIX}-regional-${_SHORT_BUILD_ID}"
-        ZONAL_BUCKET="${_BUCKET_PREFIX}-zonal-${_SHORT_BUILD_ID}"
-        HNS_BUCKET="${_BUCKET_PREFIX}-hns-${_SHORT_BUILD_ID}"
-        RESULTS_BUCKET="${_BUCKET_PREFIX}-results-${_PROJECT_ID}"
-
-        # Save variables to workspace
-        {
-          echo "export PROJECT_ID='${_PROJECT_ID}'"
-          echo "export ZONE='${_ZONE}'"
-          echo "export REPO_URL='${_REPO_URL}'"
-          echo "export BRANCH='${_BRANCH}'"
-          echo "export BENCHMARK_FANOUT_CONFIG='${_BENCHMARK_FANOUT_CONFIG}'"
-          echo "export COMMIT_SHA='${COMMIT_SHA}'"
-          echo "export RUN_ID='${_RUN_ID}'"
-          echo "export REGIONAL_BUCKET='$REGIONAL_BUCKET'"
-          echo "export ZONAL_BUCKET='$ZONAL_BUCKET'"
-          echo "export HNS_BUCKET='$HNS_BUCKET'"
-          echo "export RESULTS_BUCKET='$RESULTS_BUCKET'"
-          echo "export TEMPLATE_NAME='$TEMPLATE_NAME'"
-          echo "export MIG_NAME='$MIG_NAME'"
-        } > /workspace/build_vars.env
+        echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
+        echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
+        echo "export REPO_URL=${_REPO_URL}" >> /workspace/build_vars.env
+        echo "export BRANCH=${_BRANCH}" >> /workspace/build_vars.env
+        echo "export BENCHMARK_FANOUT_CONFIG=${_BENCHMARK_FANOUT_CONFIG}" >> /workspace/build_vars.env
+        echo "export COMMIT_SHA=${COMMIT_SHA}" >> /workspace/build_vars.env
+        echo "export RUN_ID=${_RUN_ID}" >> /workspace/build_vars.env
+        echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-${_SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export ZONAL_BUCKET=${_BUCKET_PREFIX}-zonal-${_SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export HNS_BUCKET=${_BUCKET_PREFIX}-hns-${_SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export RESULTS_BUCKET=${_BUCKET_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
+        echo "export TEMPLATE_NAME='gcsfs-perf-tmpl-${_SHORT_BUILD_ID}'" >> /workspace/build_vars.env
+        echo "export MIG_NAME='gcsfs-perf-mig-${_SHORT_BUILD_ID}'" >> /workspace/build_vars.env
     waitFor: ["-"]
 
   # 3. Create all necessary GCS buckets in parallel.

From 9dc2364a05c5201e117507c1e0d7b6215d126115 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:45:42 +0000
Subject: [PATCH 07/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 38 +++++++++++++--------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 9fa9e885..e99a7d3d 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -65,18 +65,18 @@ steps:
         echo "--- Creating Regional bucket: $$REGIONAL_BUCKET ---"
         gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} &
 
-        echo "--- Creating Zonal bucket: $ZONAL_BUCKET ---"
-        gcloud storage buckets create gs://$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
+        echo "--- Creating Zonal bucket: $$ZONAL_BUCKET ---"
+        gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
 
-        echo "--- Creating HNS bucket: $HNS_BUCKET ---"
-        gcloud storage buckets create gs://$HNS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        echo "--- Creating HNS bucket: $$HNS_BUCKET ---"
+        gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
 
         # Create HNS Results Bucket (Persistent)
-        if ! gcloud storage buckets describe gs://$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
-          echo "--- Creating Results HNS bucket: $RESULTS_BUCKET ---"
-          gcloud storage buckets create gs://$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        if ! gcloud storage buckets describe gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
+          echo "--- Creating Results HNS bucket: $$RESULTS_BUCKET ---"
+          gcloud storage buckets create gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
         else
-          echo "--- Results bucket gs://$RESULTS_BUCKET already exists ---"
+          echo "--- Results bucket gs://$$RESULTS_BUCKET already exists ---"
         fi
 
         wait
@@ -109,8 +109,8 @@ steps:
         set -e
         source /workspace/build_vars.env
 
-        echo "Creating Instance Template: $TEMPLATE_NAME"
-        gcloud compute instance-templates create "$TEMPLATE_NAME" \
+        echo "Creating Instance Template: $$TEMPLATE_NAME"
+        gcloud compute instance-templates create "$$TEMPLATE_NAME" \
             --project="${_PROJECT_ID}" \
             --machine-type="c4-standard-192" \
             --image-family="ubuntu-2204-lts" \
@@ -126,23 +126,23 @@ steps:
             --labels="build-id=${_SHORT_BUILD_ID}" \
             --quiet
 
-        echo "Creating Managed Instance Group: $MIG_NAME with size $NUM_VMS"
-        gcloud compute instance-groups managed create "$MIG_NAME" \
+        echo "Creating Managed Instance Group: $$MIG_NAME with size $NUM_VMS"
+        gcloud compute instance-groups managed create "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
-            --template="$TEMPLATE_NAME" \
+            --template="$$TEMPLATE_NAME" \
             --size="$NUM_VMS" \
             --quiet
 
         echo "Waiting for Instance Group to be stable..."
-        gcloud compute instance-groups managed wait-until "$MIG_NAME" \
+        gcloud compute instance-groups managed wait-until "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
             --stable \
             --quiet
 
         # Save instance names for the next step
-        gcloud compute instance-groups managed list-instances "$MIG_NAME" \
+        gcloud compute instance-groups managed list-instances "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
             --format="value(instance)" > /workspace/instances.txt
@@ -194,12 +194,12 @@ steps:
         source /workspace/build_vars.env
         echo "--- Cleaning up Instance Group and Template ---"
 
-        gcloud compute instance-groups managed delete "$MIG_NAME" \
+        gcloud compute instance-groups managed delete "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
             --quiet || true
 
-        gcloud compute instance-templates delete "$TEMPLATE_NAME" \
+        gcloud compute instance-templates delete "$$TEMPLATE_NAME" \
             --project="${_PROJECT_ID}" \
             --quiet || true
     waitFor:
@@ -220,8 +220,8 @@ steps:
         source /workspace/build_vars.env
         echo "--- Deleting test buckets ---"
         gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
-        gcloud storage rm --recursive gs://$ZONAL_BUCKET &
-        gcloud storage rm --recursive gs://$HNS_BUCKET &
+        gcloud storage rm --recursive gs://$$ZONAL_BUCKET &
+        gcloud storage rm --recursive gs://$$HNS_BUCKET &
         wait
     waitFor:
       - "run-benchmarks"

From 3eda30c227abc8a9df8c750a1ce1a6f98d6b8de7 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:46:35 +0000
Subject: [PATCH 08/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index e99a7d3d..52b3e3bc 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -126,12 +126,12 @@ steps:
             --labels="build-id=${_SHORT_BUILD_ID}" \
             --quiet
 
-        echo "Creating Managed Instance Group: $$MIG_NAME with size $NUM_VMS"
+        echo "Creating Managed Instance Group: $$MIG_NAME with size $$NUM_VMS"
         gcloud compute instance-groups managed create "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
             --template="$$TEMPLATE_NAME" \
-            --size="$NUM_VMS" \
+            --size="$$NUM_VMS" \
             --quiet
 
         echo "Waiting for Instance Group to be stable..."

From f6089becaf6de9470c1e1fcee17717243d3f7553 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:48:53 +0000
Subject: [PATCH 09/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 52b3e3bc..d6e428b8 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -90,9 +90,8 @@ steps:
       - "-c"
       - |
         IFS=' ' read -r -a CONFIG_ARRAY <<< "$_BENCHMARK_FANOUT_CONFIG"
-        NUM_VMS=${#CONFIG_ARRAY[@]}
-        echo "export NUM_VMS=$NUM_VMS" >> /workspace/build_vars.env
-        echo "Calculated NUM_VMS: $NUM_VMS"
+        echo "export NUM_VMS=${#CONFIG_ARRAY[@]}" >> /workspace/build_vars.env
+        echo "Calculated NUM_VMS: ${#CONFIG_ARRAY[@]}"
     waitFor: ["init-variables"]
 
   # 5. Create Instance Group based on count.

From 1a73e69bad9d11c85b8090b19698269f2b0b2259 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:49:36 +0000
Subject: [PATCH 10/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index d6e428b8..23980e45 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -230,4 +230,4 @@ timeout: "7200s" # 2 hours
 options:
   logging: CLOUD_LOGGING_ONLY
   pool:
-    name: "projects/${_PROJECT_ID}/locations/${ZONE::-2}/workerPools/cloud-build-worker-pool"
+    name: "projects/${_PROJECT_ID}/locations/${_ZONE::-2}/workerPools/cloud-build-worker-pool"

From 5183d79e0cfa83e96ab89fc74d238ffbb36df719 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 08:50:35 +0000
Subject: [PATCH 11/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 23980e45..48386263 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -89,9 +89,9 @@ steps:
     args:
       - "-c"
       - |
-        IFS=' ' read -r -a CONFIG_ARRAY <<< "$_BENCHMARK_FANOUT_CONFIG"
-        echo "export NUM_VMS=${#CONFIG_ARRAY[@]}" >> /workspace/build_vars.env
-        echo "Calculated NUM_VMS: ${#CONFIG_ARRAY[@]}"
+        IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_FANOUT_CONFIG}"
+        echo "export NUM_VMS=$${#CONFIG_ARRAY[@]}" >> /workspace/build_vars.env
+        echo "Calculated NUM_VMS: $${#CONFIG_ARRAY[@]}"
     waitFor: ["init-variables"]
 
   # 5. Create Instance Group based on count.
@@ -230,4 +230,4 @@ timeout: "7200s" # 2 hours
 options:
   logging: CLOUD_LOGGING_ONLY
   pool:
-    name: "projects/${_PROJECT_ID}/locations/${_ZONE::-2}/workerPools/cloud-build-worker-pool"
+    name: "projects/${_PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"

From 428085938e020aec98a3417175bc08cddf530cc0 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 10:26:03 +0000
Subject: [PATCH 12/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 47 ++++++++++++++-------------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 48386263..65e39a92 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -1,10 +1,9 @@
 substitutions:
   _PROJECT_ID: "gcs-aiml-clients-testing-101"
+  _LOCATION: "us-central1""
   _ZONE: "us-central1-a"
-  _SHORT_BUILD_ID: ${BUILD_ID:0:8}
-  _RUN_ID: ${BUILD_ID}
   _BUCKET_PREFIX: "gcsfs-perf"
-  _SERVICE_ACCOUNT: "gcsfs-perf-ci-runner@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
+  _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
   _REPO_URL: "https://github.com/fsspec/gcsfs.git"
   _BRANCH: "main"
   # Fanout Configuration: Space-separated list of "group:config_name" or "group:" (for all configs in group).
@@ -29,9 +28,12 @@ steps:
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "init-variables"
     entrypoint: "bash"
+    env:
+      - "BUILD_ID=${BUILD_ID}"
     args:
       - "-c"
       - |
+        SHORT_BUILD_ID=$${BUILD_ID:0:8}
         # Define shared variables
         echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
         echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
@@ -39,13 +41,14 @@ steps:
         echo "export BRANCH=${_BRANCH}" >> /workspace/build_vars.env
         echo "export BENCHMARK_FANOUT_CONFIG=${_BENCHMARK_FANOUT_CONFIG}" >> /workspace/build_vars.env
         echo "export COMMIT_SHA=${COMMIT_SHA}" >> /workspace/build_vars.env
-        echo "export RUN_ID=${_RUN_ID}" >> /workspace/build_vars.env
-        echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-${_SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export ZONAL_BUCKET=${_BUCKET_PREFIX}-zonal-${_SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export HNS_BUCKET=${_BUCKET_PREFIX}-hns-${_SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
+        echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export ZONAL_BUCKET=${_BUCKET_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export HNS_BUCKET=${_BUCKET_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export RESULTS_BUCKET=${_BUCKET_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
-        echo "export TEMPLATE_NAME='gcsfs-perf-tmpl-${_SHORT_BUILD_ID}'" >> /workspace/build_vars.env
-        echo "export MIG_NAME='gcsfs-perf-mig-${_SHORT_BUILD_ID}'" >> /workspace/build_vars.env
+        echo "export TEMPLATE_NAME='gcsfs-perf-tmpl-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
+        echo "export MIG_NAME='gcsfs-perf-mig-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
     waitFor: ["-"]
 
   # 3. Create all necessary GCS buckets in parallel.
@@ -62,21 +65,21 @@ steps:
         set -e
         source /workspace/build_vars.env
 
-        echo "--- Creating Regional bucket: $$REGIONAL_BUCKET ---"
-        gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} &
+        echo "--- Creating Regional bucket: $${REGIONAL_BUCKET} ---"
+        gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} &
 
-        echo "--- Creating Zonal bucket: $$ZONAL_BUCKET ---"
-        gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
+        echo "--- Creating Zonal bucket: $${ZONAL_BUCKET} ---"
+        gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
 
-        echo "--- Creating HNS bucket: $$HNS_BUCKET ---"
-        gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        echo "--- Creating HNS bucket: $${HNS_BUCKET} ---"
+        gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
 
         # Create HNS Results Bucket (Persistent)
         if ! gcloud storage buckets describe gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
-          echo "--- Creating Results HNS bucket: $$RESULTS_BUCKET ---"
-          gcloud storage buckets create gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${ZONE::-2} --enable-hierarchical-namespace --uniform-bucket-level-access &
+          echo "--- Creating Results HNS bucket: $${RESULTS_BUCKET} ---"
+          gcloud storage buckets create gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
         else
-          echo "--- Results bucket gs://$$RESULTS_BUCKET already exists ---"
+          echo "--- Results bucket gs://$${RESULTS_BUCKET} already exists ---"
         fi
 
         wait
@@ -108,7 +111,7 @@ steps:
         set -e
         source /workspace/build_vars.env
 
-        echo "Creating Instance Template: $$TEMPLATE_NAME"
+        echo "Creating Instance Template: $${TEMPLATE_NAME}"
         gcloud compute instance-templates create "$$TEMPLATE_NAME" \
             --project="${_PROJECT_ID}" \
             --machine-type="c4-standard-192" \
@@ -122,10 +125,10 @@ steps:
             --metadata="enable-oslogin=TRUE" \
             --service-account="${_SERVICE_ACCOUNT}" \
             --tags="allow-ssh" \
-            --labels="build-id=${_SHORT_BUILD_ID}" \
+            --labels="build-id=$$SHORT_BUILD_ID" \
             --quiet
 
-        echo "Creating Managed Instance Group: $$MIG_NAME with size $$NUM_VMS"
+        echo "Creating Managed Instance Group: $${MIG_NAME} with size $$NUM_VMS"
         gcloud compute instance-groups managed create "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
@@ -230,4 +233,4 @@ timeout: "7200s" # 2 hours
 options:
   logging: CLOUD_LOGGING_ONLY
   pool:
-    name: "projects/${_PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"
+    name: "projects/${_PROJECT_ID}/locations/${_LOCATION}/workerPools/cloud-build-worker-pool"

From 04223bffde5d87b4c2f0ec5626423e52f9b17d69 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 10:26:34 +0000
Subject: [PATCH 13/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 65e39a92..cb807f46 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -1,6 +1,6 @@
 substitutions:
   _PROJECT_ID: "gcs-aiml-clients-testing-101"
-  _LOCATION: "us-central1""
+  _LOCATION: "us-central1"
   _ZONE: "us-central1-a"
   _BUCKET_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"

From 19d64e10cd87925d5a707e28b51c6cc364eff4a1 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 11:00:37 +0000
Subject: [PATCH 14/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 18 ++++--------------
 cloudbuild/run_benchmarks.sh          | 18 +++++++-----------
 2 files changed, 11 insertions(+), 25 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index cb807f46..f70083b6 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -4,12 +4,10 @@ substitutions:
   _ZONE: "us-central1-a"
   _BUCKET_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
-  _REPO_URL: "https://github.com/fsspec/gcsfs.git"
-  _BRANCH: "main"
+  _SKIP_STEPS: ""
   # Fanout Configuration: Space-separated list of "group:config_name" or "group:" (for all configs in group).
   # Example: "read:read_seq,read_rand write: listing:"
   _BENCHMARK_FANOUT_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process,read_rand_multi_process write:write_seq,write_seq_multi_process listing: delete: rename:"
-  _SKIP_STEPS: ""
 
 steps:
   # 1. Generate a persistent SSH key for this build run.
@@ -37,10 +35,7 @@ steps:
         # Define shared variables
         echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
         echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
-        echo "export REPO_URL=${_REPO_URL}" >> /workspace/build_vars.env
-        echo "export BRANCH=${_BRANCH}" >> /workspace/build_vars.env
         echo "export BENCHMARK_FANOUT_CONFIG=${_BENCHMARK_FANOUT_CONFIG}" >> /workspace/build_vars.env
-        echo "export COMMIT_SHA=${COMMIT_SHA}" >> /workspace/build_vars.env
         echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
         echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
@@ -65,18 +60,13 @@ steps:
         set -e
         source /workspace/build_vars.env
 
-        echo "--- Creating Regional bucket: $${REGIONAL_BUCKET} ---"
+        # Create Test Buckets, in parallel
         gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} &
-
-        echo "--- Creating Zonal bucket: $${ZONAL_BUCKET} ---"
         gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
-
-        echo "--- Creating HNS bucket: $${HNS_BUCKET} ---"
         gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
 
         # Create HNS Results Bucket (Persistent)
         if ! gcloud storage buckets describe gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
-          echo "--- Creating Results HNS bucket: $${RESULTS_BUCKET} ---"
           gcloud storage buckets create gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
         else
           echo "--- Results bucket gs://$${RESULTS_BUCKET} already exists ---"
@@ -194,7 +184,6 @@ steps:
           exit 0
         fi
         source /workspace/build_vars.env
-        echo "--- Cleaning up Instance Group and Template ---"
 
         gcloud compute instance-groups managed delete "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
@@ -220,7 +209,8 @@ steps:
         fi
         set -e
         source /workspace/build_vars.env
-        echo "--- Deleting test buckets ---"
+
+        # Delete Test Buckets, in parallel
         gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
         gcloud storage rm --recursive gs://$$ZONAL_BUCKET &
         gcloud storage rm --recursive gs://$$HNS_BUCKET &
diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index 80b222a7..db126ee5 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -31,28 +31,24 @@ setup_vm() {
   local vm_name=$1
   local log_file=$2
 
-  # Setup VM (Install deps, Clone repo)
-  echo "[$vm_name] Setting up VM..." >> "$log_file"
+  # Copy source code
+  echo "[$vm_name] Copying source code..." >> "$log_file"
+  gcloud compute scp --recurse . "${vm_name}:~/gcsfs" --project="${PROJECT_ID}" --zone="${ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine >> "$log_file" 2>&1
+
   local SETUP_SCRIPT="
     set -e
+
     export DEBIAN_FRONTEND=noninteractive
     sudo apt-get update > /dev/null
     sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null
 
-    # Clone Repo
-    echo 'Cloning repo...'
-    git clone ${REPO_URL} gcsfs
     cd gcsfs
-    if [ -n '${COMMIT_SHA}' ]; then
-      git checkout ${COMMIT_SHA}
-    else
-      git checkout ${BRANCH}
-    fi
 
-    # Install Python deps
     python3 -m venv env
     source env/bin/activate
+
     pip install --upgrade pip > /dev/null
+    pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null
     pip install -e . > /dev/null
     pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
   "

From 55f667d92d45a8360f05b1ccc48c9833db74d24c Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 12:19:29 +0000
Subject: [PATCH 15/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index f70083b6..419f554e 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -137,7 +137,7 @@ steps:
         gcloud compute instance-groups managed list-instances "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
-            --format="value(instance)" > /workspace/instances.txt
+            --format="value(name)" > /workspace/instances.txt
     waitFor: ["count-configs"]
 
   # 6. Run Benchmarks.

From 476ae2a4ad0a9c1d554012af6d4d6c69faf916d5 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 13:20:36 +0000
Subject: [PATCH 16/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 419f554e..ff0f07ce 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -35,7 +35,7 @@ steps:
         # Define shared variables
         echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
         echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
-        echo "export BENCHMARK_FANOUT_CONFIG=${_BENCHMARK_FANOUT_CONFIG}" >> /workspace/build_vars.env
+        echo "export BENCHMARK_FANOUT_CONFIG='${_BENCHMARK_FANOUT_CONFIG}'" >> /workspace/build_vars.env
         echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
         echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env

From 07876587be361dc51f28f49bc44af056fe492dd7 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 13:33:38 +0000
Subject: [PATCH 17/57] updating variable management

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index ff0f07ce..0289fd31 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -133,7 +133,7 @@ steps:
             --stable \
             --quiet
 
-        # Save instance names for the next step
+        echo "Saving Instance Names..."
         gcloud compute instance-groups managed list-instances "$$MIG_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \

From 54bb87bcb72dd3e10957fd5b1955abba545241e5 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 14:08:18 +0000
Subject: [PATCH 18/57] disable infra tests

---
 cloudbuild/run_benchmarks.sh                | 41 ++++++++-------------
 gcsfs/tests/conftest.py                     | 14 ++++++-
 gcsfs/tests/perf/microbenchmarks/configs.py |  3 ++
 3 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index db126ee5..eb0a03f4 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -7,15 +7,13 @@ source /workspace/build_vars.env
 # Waits for SSH to become available on the VM.
 # Arguments:
 #   vm_name: The name of the VM instance.
-#   log_file: Path to the log file for this VM.
 wait_for_ssh() {
   local vm_name=$1
-  local log_file=$2
 
   # Wait for SSH (MIG is stable, but SSH might take a moment)
-  echo "[$vm_name] Waiting for SSH..." >> "$log_file"
+  echo "[$vm_name] Waiting for SSH..."
   for i in {1..20}; do
-    if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo ready" >> "$log_file" 2>&1; then
+    if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo ready"; then
       return 0
     fi
     sleep 10
@@ -26,14 +24,12 @@ wait_for_ssh() {
 # Sets up the VM by installing dependencies and cloning the repository.
 # Arguments:
 #   vm_name: The name of the VM instance.
-#   log_file: Path to the log file for this VM.
 setup_vm() {
   local vm_name=$1
-  local log_file=$2
 
   # Copy source code
-  echo "[$vm_name] Copying source code..." >> "$log_file"
-  gcloud compute scp --recurse . "${vm_name}:~/gcsfs" --project="${PROJECT_ID}" --zone="${ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine >> "$log_file" 2>&1
+  echo "[$vm_name] Copying source code..."
+  gcloud compute scp --recurse . "${vm_name}:~/gcsfs" --project="${PROJECT_ID}" --zone="${ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
 
   local SETUP_SCRIPT="
     set -e
@@ -53,7 +49,7 @@ setup_vm() {
     pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
   "
 
-  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$SETUP_SCRIPT" >> "$log_file" 2>&1
+  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$SETUP_SCRIPT"
 }
 
 # Runs the benchmark on the VM and uploads the results.
@@ -61,15 +57,13 @@ setup_vm() {
 #   vm_name: The name of the VM instance.
 #   group: The benchmark group.
 #   config: The benchmark config.
-#   log_file: Path to the log file for this VM.
 run_benchmark() {
   local vm_name=$1
   local group=$2
   local config=$3
-  local log_file=$4
 
   # Run Benchmark
-  echo "[$vm_name] Running benchmark..." >> "$log_file"
+  echo "[$vm_name] Running benchmark..."
 
   # Construct config arg
   local CONFIG_ARG=""
@@ -100,7 +94,7 @@ run_benchmark() {
     fi
   "
 
-  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$RUN_CMD" >> "$log_file" 2>&1
+  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$RUN_CMD"
 }
 
 # Orchestrates the execution of a single benchmark job: wait for SSH, setup VM, and run benchmark.
@@ -112,26 +106,25 @@ run_job() {
   local group=$1
   local config=$2
   local vm_name=$3
-  local log_file="/workspace/${vm_name}.log"
 
-  echo "[$vm_name] Starting job for Group: $group, Config: $config" > "$log_file"
+  echo "[$vm_name] Starting job for Group: $group, Config: $config"
 
-  if ! wait_for_ssh "$vm_name" "$log_file"; then
-    echo "[$vm_name] SSH wait failed." >> "$log_file"
+  if ! wait_for_ssh "$vm_name"; then
+    echo "[$vm_name] SSH wait failed."
     return 1
   fi
 
-  if ! setup_vm "$vm_name" "$log_file"; then
-    echo "[$vm_name] Setup failed." >> "$log_file"
+  if ! setup_vm "$vm_name"; then
+    echo "[$vm_name] Setup failed."
     return 1
   fi
 
-  if ! run_benchmark "$vm_name" "$group" "$config" "$log_file"; then
-    echo "[$vm_name] Benchmark failed." >> "$log_file"
+  if ! run_benchmark "$vm_name" "$group" "$config"; then
+    echo "[$vm_name] Benchmark failed."
     return 1
   fi
 
-  echo "[$vm_name] Benchmark finished successfully." >> "$log_file"
+  echo "[$vm_name] Benchmark finished successfully."
   return 0
 }
 
@@ -172,10 +165,6 @@ main() {
     wait $pid || failures=$((failures+1))
   done
 
-  # Print logs
-  echo "--- Benchmark Logs ---"
-  cat /workspace/*.log || true
-
   if [ $failures -ne 0 ]; then
     echo "$failures benchmark jobs failed."
     exit 1
diff --git a/gcsfs/tests/conftest.py b/gcsfs/tests/conftest.py
index 25d52e96..184b4a75 100644
--- a/gcsfs/tests/conftest.py
+++ b/gcsfs/tests/conftest.py
@@ -448,13 +448,13 @@ def pytest_addoption(parser):
         "--run-benchmarks",
         action="store_true",
         default=False,
-        help="run benchmark tests",
+        help="run only perf benchmark tests",
     )
     parser.addoption(
         "--run-benchmarks-infra",
         action="store_true",
         default=False,
-        help="run benchmark infrastructure tests",
+        help="run only benchmark infrastructure tests",
     )
 
 
@@ -478,4 +478,14 @@ def pytest_ignore_collect(collection_path, config):
             if benchmark_subdirs.intersection(path_parts):
                 return True
 
+        # If only --run-benchmarks is passed, ignore the unit tests.
+        if config.getoption("--run-benchmarks") and not config.getoption(
+            "--run-benchmarks-infra"
+        ):
+            if os.path.basename(path_str).startswith("test_"):
+                benchmark_subdirs = {"delete", "listing", "read", "rename", "write"}
+                path_parts = set(path_str.replace(os.sep, "/").split("/"))
+                if not benchmark_subdirs.intersection(path_parts):
+                    return True
+
     return None
diff --git a/gcsfs/tests/perf/microbenchmarks/configs.py b/gcsfs/tests/perf/microbenchmarks/configs.py
index 77a983e2..8184a82f 100644
--- a/gcsfs/tests/perf/microbenchmarks/configs.py
+++ b/gcsfs/tests/perf/microbenchmarks/configs.py
@@ -19,6 +19,9 @@ def _load_config(self):
         scenarios = config["scenarios"]
 
         if BENCHMARK_FILTER:
+            logging.info(
+                f"Filtering the scenarios based on BENCHMARK_FILTER: {BENCHMARK_FILTER}"
+            )
             filter_names = [
                 name.strip().lower() for name in BENCHMARK_FILTER.split(",")
             ]

From b243c9b2cdaae9354d2509e7fab1fd586b9513ab Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 14:11:49 +0000
Subject: [PATCH 19/57] updating variable management

---
 cloudbuild/run_benchmarks.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index eb0a03f4..45e5e818 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -12,11 +12,13 @@ wait_for_ssh() {
 
   # Wait for SSH (MIG is stable, but SSH might take a moment)
   echo "[$vm_name] Waiting for SSH..."
-  for i in {1..20}; do
-    if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo ready"; then
+  for i in {1..10}; do
+    if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" ; then
+      echo "[$vm_name] SSH available."
       return 0
     fi
-    sleep 10
+    echo "Waiting for VM to become available... (attempt $$i/10)"
+    sleep 1
   done
   return 1
 }

From 3d086854798f0e0549e3bd36fcb4c091151347c9 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 14:21:49 +0000
Subject: [PATCH 20/57] SSH issue debug

---
 cloudbuild/run_benchmarks.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index 45e5e818..b8e3a21d 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -12,13 +12,14 @@ wait_for_ssh() {
 
   # Wait for SSH (MIG is stable, but SSH might take a moment)
   echo "[$vm_name] Waiting for SSH..."
+  cat /workspace/.ssh/google_compute_engine.pub
   for i in {1..10}; do
     if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" ; then
       echo "[$vm_name] SSH available."
       return 0
     fi
-    echo "Waiting for VM to become available... (attempt $$i/10)"
-    sleep 1
+    echo "Waiting for VM to become available... (attempt $i/10)"
+    sleep 15
   done
   return 1
 }

From 28edde3544d3ea081d604f273caa9aefa709834b Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 19:39:58 +0000
Subject: [PATCH 21/57] Removed mig

---
 cloudbuild/benchmarks-cloudbuild.yaml | 77 +++++++--------------------
 cloudbuild/run_benchmarks.sh          | 68 +++++++----------------
 2 files changed, 39 insertions(+), 106 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 0289fd31..fe8ec850 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -5,9 +5,7 @@ substitutions:
   _BUCKET_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
   _SKIP_STEPS: ""
-  # Fanout Configuration: Space-separated list of "group:config_name" or "group:" (for all configs in group).
-  # Example: "read:read_seq,read_rand write: listing:"
-  _BENCHMARK_FANOUT_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process,read_rand_multi_process write:write_seq,write_seq_multi_process listing: delete: rename:"
+  _BENCHMARK_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process read:read_rand_multi_process"
 
 steps:
   # 1. Generate a persistent SSH key for this build run.
@@ -35,15 +33,14 @@ steps:
         # Define shared variables
         echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
         echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
-        echo "export BENCHMARK_FANOUT_CONFIG='${_BENCHMARK_FANOUT_CONFIG}'" >> /workspace/build_vars.env
+        echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/build_vars.env
         echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
         echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export ZONAL_BUCKET=${_BUCKET_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export HNS_BUCKET=${_BUCKET_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export RESULTS_BUCKET=${_BUCKET_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
-        echo "export TEMPLATE_NAME='gcsfs-perf-tmpl-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
-        echo "export MIG_NAME='gcsfs-perf-mig-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
+        echo "export VM_NAME='${_BUCKET_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
     waitFor: ["-"]
 
   # 3. Create all necessary GCS buckets in parallel.
@@ -75,35 +72,24 @@ steps:
         wait
     waitFor: ["init-variables"]
 
-  # 4. Calculate number of VMs needed.
+  # 4. Create a single VM for benchmarks.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "count-configs"
+    id: "create-vm"
     entrypoint: "bash"
     args:
       - "-c"
       - |
-        IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_FANOUT_CONFIG}"
-        echo "export NUM_VMS=$${#CONFIG_ARRAY[@]}" >> /workspace/build_vars.env
-        echo "Calculated NUM_VMS: $${#CONFIG_ARRAY[@]}"
-    waitFor: ["init-variables"]
-
-  # 5. Create Instance Group based on count.
-  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "create-mig"
-    entrypoint: "bash"
-    args:
-      - "-c"
-      - |
-        if [[ ",${_SKIP_STEPS}," == *",create-mig,"* ]]; then
-          echo "--- Skipping step: create-mig ---"
+        if [[ ",${_SKIP_STEPS}," == *",create-vm,"* ]]; then
+          echo "--- Skipping step: create-vm ---"
           exit 0
         fi
         set -e
         source /workspace/build_vars.env
 
-        echo "Creating Instance Template: $${TEMPLATE_NAME}"
-        gcloud compute instance-templates create "$$TEMPLATE_NAME" \
+        echo "Creating VM: $${VM_NAME}"
+        gcloud compute instances create "$$VM_NAME" \
             --project="${_PROJECT_ID}" \
+            --zone="${_ZONE}" \
             --machine-type="c4-standard-192" \
             --image-family="ubuntu-2204-lts" \
             --image-project="ubuntu-os-cloud" \
@@ -117,30 +103,9 @@ steps:
             --tags="allow-ssh" \
             --labels="build-id=$$SHORT_BUILD_ID" \
             --quiet
+    waitFor: ["init-variables"]
 
-        echo "Creating Managed Instance Group: $${MIG_NAME} with size $$NUM_VMS"
-        gcloud compute instance-groups managed create "$$MIG_NAME" \
-            --project="${_PROJECT_ID}" \
-            --zone="${_ZONE}" \
-            --template="$$TEMPLATE_NAME" \
-            --size="$$NUM_VMS" \
-            --quiet
-
-        echo "Waiting for Instance Group to be stable..."
-        gcloud compute instance-groups managed wait-until "$$MIG_NAME" \
-            --project="${_PROJECT_ID}" \
-            --zone="${_ZONE}" \
-            --stable \
-            --quiet
-
-        echo "Saving Instance Names..."
-        gcloud compute instance-groups managed list-instances "$$MIG_NAME" \
-            --project="${_PROJECT_ID}" \
-            --zone="${_ZONE}" \
-            --format="value(name)" > /workspace/instances.txt
-    waitFor: ["count-configs"]
-
-  # 6. Run Benchmarks.
+  # 5. Run Benchmarks.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "run-benchmarks"
     entrypoint: "bash"
@@ -153,7 +118,7 @@ steps:
         fi
         bash cloudbuild/run_benchmarks.sh
     waitFor:
-      - "create-mig"
+      - "create-vm"
       - "create-buckets"
       - "generate-ssh-key"
 
@@ -172,27 +137,23 @@ steps:
     waitFor:
       - "run-benchmarks"
 
-  # Delete MIG and Template.
+  # Delete VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "cleanup-mig"
+    id: "cleanup-vm"
     entrypoint: "bash"
     args:
       - "-c"
       - |
-        if [[ ",${_SKIP_STEPS}," == *",cleanup-mig,"* ]]; then
-          echo "--- Skipping step: cleanup-mig ---"
+        if [[ ",${_SKIP_STEPS}," == *",cleanup-vm,"* ]]; then
+          echo "--- Skipping step: cleanup-vm ---"
           exit 0
         fi
         source /workspace/build_vars.env
 
-        gcloud compute instance-groups managed delete "$$MIG_NAME" \
+        gcloud compute instances delete "$$VM_NAME" \
             --project="${_PROJECT_ID}" \
             --zone="${_ZONE}" \
             --quiet || true
-
-        gcloud compute instance-templates delete "$$TEMPLATE_NAME" \
-            --project="${_PROJECT_ID}" \
-            --quiet || true
     waitFor:
       - "cleanup-ssh-key"
 
@@ -218,7 +179,7 @@ steps:
     waitFor:
       - "run-benchmarks"
 
-timeout: "7200s" # 2 hours
+timeout: "14400s" # 2 hours
 
 options:
   logging: CLOUD_LOGGING_ONLY
diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index b8e3a21d..eacfe1eb 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -91,7 +91,7 @@ run_benchmark() {
     RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'
     if [ -d \"\$RESULTS_DIR\" ]; then
       echo \"Uploading from \$RESULTS_DIR to gs://${RESULTS_BUCKET}/\${DATE_DIR}/${RUN_ID}/\"
-      cd \"\$RESULTS_DIR\" && gcloud storage cp --recursive . gs://${RESULTS_BUCKET}/\${DATE_DIR}/${RUN_ID}/
+      cd \"\$RESULTS_DIR\" && gcloud storage cp --recursive . gs://${RESULTS_BUCKET}/\${DATE_DIR}/${RUN_ID}/ && rm -rf *
     else
       echo \"No results directory found at \$RESULTS_DIR\"
     fi
@@ -100,75 +100,47 @@ run_benchmark() {
   gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$RUN_CMD"
 }
 
-# Orchestrates the execution of a single benchmark job: wait for SSH, setup VM, and run benchmark.
-# Arguments:
-#   group: The benchmark group.
-#   config: The benchmark config.
-#   vm_name: The name of the VM instance.
-run_job() {
-  local group=$1
-  local config=$2
-  local vm_name=$3
+# Main function to orchestrate the sequential execution of benchmarks on a single VM.
+main() {
+  local vm_name="${VM_NAME}"
 
-  echo "[$vm_name] Starting job for Group: $group, Config: $config"
+  echo "[$vm_name] Starting benchmarks..."
 
   if ! wait_for_ssh "$vm_name"; then
     echo "[$vm_name] SSH wait failed."
-    return 1
+    exit 1
   fi
 
   if ! setup_vm "$vm_name"; then
     echo "[$vm_name] Setup failed."
-    return 1
-  fi
-
-  if ! run_benchmark "$vm_name" "$group" "$config"; then
-    echo "[$vm_name] Benchmark failed."
-    return 1
+    exit 1
   fi
 
-  echo "[$vm_name] Benchmark finished successfully."
-  return 0
-}
-
-# Main function to orchestrate the parallel execution of benchmarks across VMs.
-main() {
-  # Load instances
-  local INSTANCES=($(cat /workspace/instances.txt))
-
   # Load configs
   local CONFIG_ARRAY
-  IFS=' ' read -r -a CONFIG_ARRAY <<< "${BENCHMARK_FANOUT_CONFIG}"
-
-  local NUM_VMS=${#CONFIG_ARRAY[@]}
-  if [ "${#INSTANCES[@]}" -ne "$NUM_VMS" ]; then
-      echo "Error: Number of instances (${#INSTANCES[@]}) does not match number of configs ($NUM_VMS)."
-      exit 1
-  fi
+  IFS=',' read -r -a CONFIG_ARRAY <<< "${BENCHMARK_CONFIG}"
 
-  # Main Loop
-  local pids=""
+  local failures=0
 
-  for i in "${!CONFIG_ARRAY[@]}"; do
-    local entry="${CONFIG_ARRAY[$i]}"
-    local vm_name="${INSTANCES[$i]}"
+  for entry in "${CONFIG_ARRAY[@]}"; do
+    # Trim whitespace
+    entry=$(echo "$entry" | xargs)
+    if [ -z "$entry" ]; then continue; fi
 
     IFS=':' read -r group config <<< "$entry"
 
     echo "Launching job for $group:$config on $vm_name"
 
-    # Run in background
-    run_job "$group" "$config" "$vm_name" &
-    pids="$pids $!"
-  done
+    if ! run_benchmark "$vm_name" "$group" "$config"; then
+        echo "Benchmark $group:$config failed."
+        failures=$((failures+1))
+    fi
 
-  # Wait for all jobs
-  local failures=0
-  for pid in $pids; do
-    wait $pid || failures=$((failures+1))
+    echo "Sleeping for 30 seconds..."
+    sleep 30
   done
 
-  if [ $failures -ne 0 ]; then
+  if [ "${failures:-0}" -ne 0 ]; then
     echo "$failures benchmark jobs failed."
     exit 1
   fi

From 74fae2ac12eefd166941cc64c1f1620f07c382dc Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 19:42:15 +0000
Subject: [PATCH 22/57] Removed mig

---
 cloudbuild/run_benchmarks.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index eacfe1eb..87e6c69e 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -118,7 +118,7 @@ main() {
 
   # Load configs
   local CONFIG_ARRAY
-  IFS=',' read -r -a CONFIG_ARRAY <<< "${BENCHMARK_CONFIG}"
+  IFS=' ' read -r -a CONFIG_ARRAY <<< "${BENCHMARK_CONFIG}"
 
   local failures=0
 

From e8d54406c55c725df91ff93051f6d72bdd17deef Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 19:46:59 +0000
Subject: [PATCH 23/57] Failures as warning to ensure cleanup

---
 cloudbuild/run_benchmarks.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index 87e6c69e..c56a54fb 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -13,12 +13,12 @@ wait_for_ssh() {
   # Wait for SSH (MIG is stable, but SSH might take a moment)
   echo "[$vm_name] Waiting for SSH..."
   cat /workspace/.ssh/google_compute_engine.pub
-  for i in {1..10}; do
+  for i in {1..4}; do
     if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" ; then
       echo "[$vm_name] SSH available."
       return 0
     fi
-    echo "Waiting for VM to become available... (attempt $i/10)"
+    echo "Waiting for VM to become available... (attempt $i/4)"
     sleep 15
   done
   return 1
@@ -141,10 +141,10 @@ main() {
   done
 
   if [ "${failures:-0}" -ne 0 ]; then
-    echo "$failures benchmark jobs failed."
-    exit 1
+    echo "WARNING: $failures benchmark jobs failed. Returning success to proceed with cleanup."
+  else
+    echo "All benchmarks completed successfully."
   fi
-  echo "All benchmarks completed successfully."
 }
 
 main

From 09271c1b020f126e6ff7430af9a573bd1481a80a Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 20:16:18 +0000
Subject: [PATCH 24/57] bucket types introduced

---
 cloudbuild/benchmarks-cloudbuild.yaml | 47 ++++++++++++---------------
 cloudbuild/run_benchmarks.sh          | 26 ++++++++++-----
 2 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index fe8ec850..d54fe906 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -4,8 +4,8 @@ substitutions:
   _ZONE: "us-central1-a"
   _BUCKET_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
-  _SKIP_STEPS: ""
   _BENCHMARK_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process read:read_rand_multi_process"
+  _BUCKET_TYPES: "regional zonal hns"
 
 steps:
   # 1. Generate a persistent SSH key for this build run.
@@ -34,6 +34,7 @@ steps:
         echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
         echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
         echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/build_vars.env
+        echo "export BUCKET_TYPES='${_BUCKET_TYPES}'" >> /workspace/build_vars.env
         echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
         echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
@@ -50,17 +51,19 @@ steps:
     args:
       - "-c"
       - |
-        if [[ ",${_SKIP_STEPS}," == *",create-buckets,"* ]]; then
-          echo "--- Skipping step: create-buckets ---"
-          exit 0
-        fi
         set -e
         source /workspace/build_vars.env
 
         # Create Test Buckets, in parallel
-        gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} &
-        gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
-        gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        if [[ " ${BUCKET_TYPES} " =~ " regional " ]]; then
+          gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} &
+        fi
+        if [[ " ${BUCKET_TYPES} " =~ " zonal " ]]; then
+          gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
+        fi
+        if [[ " ${BUCKET_TYPES} " =~ " hns " ]]; then
+          gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        fi
 
         # Create HNS Results Bucket (Persistent)
         if ! gcloud storage buckets describe gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
@@ -79,10 +82,6 @@ steps:
     args:
       - "-c"
       - |
-        if [[ ",${_SKIP_STEPS}," == *",create-vm,"* ]]; then
-          echo "--- Skipping step: create-vm ---"
-          exit 0
-        fi
         set -e
         source /workspace/build_vars.env
 
@@ -112,10 +111,6 @@ steps:
     args:
       - "-c"
       - |
-        if [[ ",${_SKIP_STEPS}," == *",run-benchmarks,"* ]]; then
-          echo "--- Skipping step: run-benchmarks ---"
-          exit 0
-        fi
         bash cloudbuild/run_benchmarks.sh
     waitFor:
       - "create-vm"
@@ -144,10 +139,6 @@ steps:
     args:
       - "-c"
       - |
-        if [[ ",${_SKIP_STEPS}," == *",cleanup-vm,"* ]]; then
-          echo "--- Skipping step: cleanup-vm ---"
-          exit 0
-        fi
         source /workspace/build_vars.env
 
         gcloud compute instances delete "$$VM_NAME" \
@@ -164,17 +155,19 @@ steps:
     args:
       - "-c"
       - |
-        if [[ ",${_SKIP_STEPS}," == *",delete-buckets,"* ]]; then
-          echo "--- Skipping step: delete-buckets ---"
-          exit 0
-        fi
         set -e
         source /workspace/build_vars.env
 
         # Delete Test Buckets, in parallel
-        gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
-        gcloud storage rm --recursive gs://$$ZONAL_BUCKET &
-        gcloud storage rm --recursive gs://$$HNS_BUCKET &
+        if [[ " ${BUCKET_TYPES} " =~ " regional " ]]; then
+          gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
+        fi
+        if [[ " ${BUCKET_TYPES} " =~ " zonal " ]]; then
+          gcloud storage rm --recursive gs://$$ZONAL_BUCKET &
+        fi
+        if [[ " ${BUCKET_TYPES} " =~ " hns " ]]; then
+          gcloud storage rm --recursive gs://$$HNS_BUCKET &
+        fi
         wait
     waitFor:
       - "run-benchmarks"
diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index c56a54fb..5c9eb8ac 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -13,7 +13,7 @@ wait_for_ssh() {
   # Wait for SSH (MIG is stable, but SSH might take a moment)
   echo "[$vm_name] Waiting for SSH..."
   cat /workspace/.ssh/google_compute_engine.pub
-  for i in {1..4}; do
+  for i in {1..1}; do
     if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" ; then
       echo "[$vm_name] SSH available."
       return 0
@@ -74,14 +74,24 @@ run_benchmark() {
     CONFIG_ARG="--config=$config"
   fi
 
+  # Construct bucket args based on BUCKET_TYPES
+  local BUCKET_ARGS=""
+  if [[ " ${BUCKET_TYPES} " =~ " regional " ]]; then
+    BUCKET_ARGS="${BUCKET_ARGS} --regional-bucket='${REGIONAL_BUCKET}'"
+  fi
+  if [[ " ${BUCKET_TYPES} " =~ " zonal " ]]; then
+    BUCKET_ARGS="${BUCKET_ARGS} --zonal-bucket='${ZONAL_BUCKET}'"
+  fi
+  if [[ " ${BUCKET_TYPES} " =~ " hns " ]]; then
+    BUCKET_ARGS="${BUCKET_ARGS} --hns-bucket='${HNS_BUCKET}'"
+  fi
+
   local RUN_CMD="
     source gcsfs/env/bin/activate
     python gcsfs/gcsfs/tests/perf/microbenchmarks/run.py \
       --group=$group \
       $CONFIG_ARG \
-      --regional-bucket='${REGIONAL_BUCKET}' \
-      --zonal-bucket='${ZONAL_BUCKET}' \
-      --hns-bucket='${HNS_BUCKET}' \
+      $BUCKET_ARGS \
       --log=true \
       --log-level=INFO
 
@@ -107,13 +117,13 @@ main() {
   echo "[$vm_name] Starting benchmarks..."
 
   if ! wait_for_ssh "$vm_name"; then
-    echo "[$vm_name] SSH wait failed."
-    exit 1
+    echo "WARNING: [$vm_name] SSH wait failed."
+    exit 0
   fi
 
   if ! setup_vm "$vm_name"; then
-    echo "[$vm_name] Setup failed."
-    exit 1
+    echo "WARNING: [$vm_name] Setup failed."
+    exit 0
   fi
 
   # Load configs

From 37cc2642a5d92d8eeccba2e3e978030771f139fe Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 20:19:58 +0000
Subject: [PATCH 25/57] bucket types introduced

---
 cloudbuild/benchmarks-cloudbuild.yaml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index d54fe906..27f88fcf 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -2,7 +2,7 @@ substitutions:
   _PROJECT_ID: "gcs-aiml-clients-testing-101"
   _LOCATION: "us-central1"
   _ZONE: "us-central1-a"
-  _BUCKET_PREFIX: "gcsfs-perf"
+  _INFRA_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
   _BENCHMARK_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process read:read_rand_multi_process"
   _BUCKET_TYPES: "regional zonal hns"
@@ -37,11 +37,11 @@ steps:
         echo "export BUCKET_TYPES='${_BUCKET_TYPES}'" >> /workspace/build_vars.env
         echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
         echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export REGIONAL_BUCKET=${_BUCKET_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export ZONAL_BUCKET=${_BUCKET_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export HNS_BUCKET=${_BUCKET_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export RESULTS_BUCKET=${_BUCKET_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
-        echo "export VM_NAME='${_BUCKET_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
+        echo "export REGIONAL_BUCKET=${_INFRA_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export ZONAL_BUCKET=${_INFRA_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export HNS_BUCKET=${_INFRA_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export RESULTS_BUCKET=${_INFRA_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
+        echo "export VM_NAME='${_INFRA_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
     waitFor: ["-"]
 
   # 3. Create all necessary GCS buckets in parallel.
@@ -55,13 +55,13 @@ steps:
         source /workspace/build_vars.env
 
         # Create Test Buckets, in parallel
-        if [[ " ${BUCKET_TYPES} " =~ " regional " ]]; then
+        if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
           gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} &
         fi
-        if [[ " ${BUCKET_TYPES} " =~ " zonal " ]]; then
+        if [[ " ${_BUCKET_TYPES} " =~ " zonal " ]]; then
           gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
         fi
-        if [[ " ${BUCKET_TYPES} " =~ " hns " ]]; then
+        if [[ " ${_BUCKET_TYPES} " =~ " hns " ]]; then
           gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
         fi
 
@@ -159,13 +159,13 @@ steps:
         source /workspace/build_vars.env
 
         # Delete Test Buckets, in parallel
-        if [[ " ${BUCKET_TYPES} " =~ " regional " ]]; then
+        if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
           gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
         fi
-        if [[ " ${BUCKET_TYPES} " =~ " zonal " ]]; then
+        if [[ " ${_BUCKET_TYPES} " =~ " zonal " ]]; then
           gcloud storage rm --recursive gs://$$ZONAL_BUCKET &
         fi
-        if [[ " ${BUCKET_TYPES} " =~ " hns " ]]; then
+        if [[ " ${_BUCKET_TYPES} " =~ " hns " ]]; then
           gcloud storage rm --recursive gs://$$HNS_BUCKET &
         fi
         wait

From 0eb4b822cdad89f36706d5b48b2b46eaba78b5d5 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Fri, 23 Jan 2026 20:27:22 +0000
Subject: [PATCH 26/57] bucket types introduced

---
 cloudbuild/benchmarks-cloudbuild.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 27f88fcf..c090ddd1 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -5,7 +5,7 @@ substitutions:
   _INFRA_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
   _BENCHMARK_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process read:read_rand_multi_process"
-  _BUCKET_TYPES: "regional zonal hns"
+  _BUCKET_TYPES: "zonal"
 
 steps:
   # 1. Generate a persistent SSH key for this build run.
@@ -69,7 +69,7 @@ steps:
         if ! gcloud storage buckets describe gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
           gcloud storage buckets create gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
         else
-          echo "--- Results bucket gs://$${RESULTS_BUCKET} already exists ---"
+          echo "Results bucket gs://$${RESULTS_BUCKET} already exists"
         fi
 
         wait
@@ -126,7 +126,6 @@ steps:
     args:
       - "-c"
       - |
-        echo "--- Removing SSH key from OS Login profile ---"
         gcloud compute os-login ssh-keys remove \
           --key-file=/workspace/gcb_ssh_key.pub || true
     waitFor:
@@ -158,7 +157,6 @@ steps:
         set -e
         source /workspace/build_vars.env
 
-        # Delete Test Buckets, in parallel
         if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
           gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
         fi

From b9c29c7fb319e733a3d79668629eafcbc9a4e6ea Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 09:39:24 +0000
Subject: [PATCH 27/57] SSH in cloudbuild

---
 cloudbuild/benchmarks-cloudbuild.yaml | 27 +++++++++++++++++++++++++--
 cloudbuild/run_benchmarks.sh          | 25 -------------------------
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index c090ddd1..c1b17318 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -104,7 +104,30 @@ steps:
             --quiet
     waitFor: ["init-variables"]
 
-  # 5. Run Benchmarks.
+  # 5. Wait for SSH.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "wait-for-ssh"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        set -e
+        source /workspace/build_vars.env
+
+        echo "Waiting for SSH on VM: $${VM_NAME}"
+        for i in {1..3}; do
+          if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
+            echo "SSH is ready."
+            exit 0
+          fi
+          echo "Waiting for VM to become available... (attempt $$i/3)"
+          sleep 15
+        done
+        echo "Timeout waiting for SSH."
+        exit 1
+    waitFor: ["create-vm"]
+
+  # 6. Run Benchmarks.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "run-benchmarks"
     entrypoint: "bash"
@@ -113,7 +136,7 @@ steps:
       - |
         bash cloudbuild/run_benchmarks.sh
     waitFor:
-      - "create-vm"
+      - "wait-for-ssh"
       - "create-buckets"
       - "generate-ssh-key"
 
diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index 5c9eb8ac..3fdf6092 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -4,26 +4,6 @@ set -e
 # Load build variables (buckets, etc.)
 source /workspace/build_vars.env
 
-# Waits for SSH to become available on the VM.
-# Arguments:
-#   vm_name: The name of the VM instance.
-wait_for_ssh() {
-  local vm_name=$1
-
-  # Wait for SSH (MIG is stable, but SSH might take a moment)
-  echo "[$vm_name] Waiting for SSH..."
-  cat /workspace/.ssh/google_compute_engine.pub
-  for i in {1..1}; do
-    if gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" ; then
-      echo "[$vm_name] SSH available."
-      return 0
-    fi
-    echo "Waiting for VM to become available... (attempt $i/4)"
-    sleep 15
-  done
-  return 1
-}
-
 # Sets up the VM by installing dependencies and cloning the repository.
 # Arguments:
 #   vm_name: The name of the VM instance.
@@ -116,11 +96,6 @@ main() {
 
   echo "[$vm_name] Starting benchmarks..."
 
-  if ! wait_for_ssh "$vm_name"; then
-    echo "WARNING: [$vm_name] SSH wait failed."
-    exit 0
-  fi
-
   if ! setup_vm "$vm_name"; then
     echo "WARNING: [$vm_name] Setup failed."
     exit 0

From 17e39d70c658580847d44e37cdbe146dcf749574 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 10:00:32 +0000
Subject: [PATCH 28/57] undo script and move to cloudbuild

---
 cloudbuild/benchmarks-cloudbuild.yaml |  95 +++++++++++++++++-
 cloudbuild/run_benchmarks.sh          | 134 +-------------------------
 2 files changed, 93 insertions(+), 136 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index c1b17318..c7bc5135 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -127,21 +127,108 @@ steps:
         exit 1
     waitFor: ["create-vm"]
 
-  # 6. Run Benchmarks.
+  # 6. Setup VM.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "setup-vm"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        set -e
+        source /workspace/build_vars.env
+
+        echo "[$${VM_NAME}] Copying source code..."
+        gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
+
+        SETUP_SCRIPT="
+          set -e
+          export DEBIAN_FRONTEND=noninteractive
+          sudo apt-get update > /dev/null
+          sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null
+          cd gcsfs
+          python3 -m venv env
+          source env/bin/activate
+          pip install --upgrade pip > /dev/null
+          pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null
+          pip install -e . > /dev/null
+          pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
+        "
+
+        gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT"
+    waitFor: ["wait-for-ssh"]
+
+  # 7. Run Benchmarks.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "run-benchmarks"
     entrypoint: "bash"
     args:
       - "-c"
       - |
-        bash cloudbuild/run_benchmarks.sh
+        set -e
+        source /workspace/build_vars.env
+
+        IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_CONFIG}"
+        failures=0
+
+        for entry in "$${CONFIG_ARRAY[@]}"; do
+          entry=$$(echo "$$entry" | xargs)
+          if [ -z "$$entry" ]; then continue; fi
+
+          IFS=':' read -r group config <<< "$$entry"
+          echo "Launching job for $$group:$$config on $$VM_NAME"
+
+          CONFIG_ARG=""
+          if [ -n "$$config" ]; then
+            CONFIG_ARG="--config=$$config"
+          fi
+
+          BUCKET_ARGS=""
+          if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
+            BUCKET_ARGS="$${BUCKET_ARGS} --regional-bucket='$${REGIONAL_BUCKET}'"
+          fi
+          if [[ " ${_BUCKET_TYPES} " =~ " zonal " ]]; then
+            BUCKET_ARGS="$${BUCKET_ARGS} --zonal-bucket='$${ZONAL_BUCKET}'"
+          fi
+          if [[ " ${_BUCKET_TYPES} " =~ " hns " ]]; then
+            BUCKET_ARGS="$${BUCKET_ARGS} --hns-bucket='$${HNS_BUCKET}'"
+          fi
+
+          RUN_CMD="
+            source gcsfs/env/bin/activate
+            python gcsfs/gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
+            echo '--- Uploading Results ---'
+            DATE_DIR=\$(date +%d%m%Y)
+            RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'
+            if [ -d \\"\$RESULTS_DIR\\" ]; then
+              echo \\"Uploading from \$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/\\"
+              cd \\"\$RESULTS_DIR\\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/ && rm -rf *
+            else
+              echo \\"No results directory found at \$RESULTS_DIR\\"
+            fi
+          "
+
+          if ! gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_CMD"; then
+            echo "Benchmark $$group:$$config failed."
+            failures=$$((failures+1))
+          fi
+
+          echo "Sleeping for 30 seconds..."
+          sleep 30
+        done
+
+        if [ "$${failures:-0}" -ne 0 ]; then
+          echo "WARNING: $$failures benchmark jobs failed. Returning success to proceed with cleanup."
+        else
+          echo "All benchmarks completed successfully."
+        fi
     waitFor:
-      - "wait-for-ssh"
+      - "setup-vm"
       - "create-buckets"
       - "generate-ssh-key"
 
   # --- Cleanup Steps ---
 
+  # 8. Clean up the SSH key.
   # Clean up the SSH key.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "cleanup-ssh-key"
@@ -154,6 +241,7 @@ steps:
     waitFor:
       - "run-benchmarks"
 
+  # 9. Delete VM.
   # Delete VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "cleanup-vm"
@@ -170,6 +258,7 @@ steps:
     waitFor:
       - "cleanup-ssh-key"
 
+  # 10. Delete all GCS buckets.
   # Delete all GCS buckets.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "delete-buckets"
diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
index 3fdf6092..153ace24 100644
--- a/cloudbuild/run_benchmarks.sh
+++ b/cloudbuild/run_benchmarks.sh
@@ -1,135 +1,3 @@
 #!/bin/bash
 set -e
-
-# Load build variables (buckets, etc.)
-source /workspace/build_vars.env
-
-# Sets up the VM by installing dependencies and cloning the repository.
-# Arguments:
-#   vm_name: The name of the VM instance.
-setup_vm() {
-  local vm_name=$1
-
-  # Copy source code
-  echo "[$vm_name] Copying source code..."
-  gcloud compute scp --recurse . "${vm_name}:~/gcsfs" --project="${PROJECT_ID}" --zone="${ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
-
-  local SETUP_SCRIPT="
-    set -e
-
-    export DEBIAN_FRONTEND=noninteractive
-    sudo apt-get update > /dev/null
-    sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null
-
-    cd gcsfs
-
-    python3 -m venv env
-    source env/bin/activate
-
-    pip install --upgrade pip > /dev/null
-    pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null
-    pip install -e . > /dev/null
-    pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
-  "
-
-  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$SETUP_SCRIPT"
-}
-
-# Runs the benchmark on the VM and uploads the results.
-# Arguments:
-#   vm_name: The name of the VM instance.
-#   group: The benchmark group.
-#   config: The benchmark config.
-run_benchmark() {
-  local vm_name=$1
-  local group=$2
-  local config=$3
-
-  # Run Benchmark
-  echo "[$vm_name] Running benchmark..."
-
-  # Construct config arg
-  local CONFIG_ARG=""
-  if [ -n "$config" ]; then
-    CONFIG_ARG="--config=$config"
-  fi
-
-  # Construct bucket args based on BUCKET_TYPES
-  local BUCKET_ARGS=""
-  if [[ " ${BUCKET_TYPES} " =~ " regional " ]]; then
-    BUCKET_ARGS="${BUCKET_ARGS} --regional-bucket='${REGIONAL_BUCKET}'"
-  fi
-  if [[ " ${BUCKET_TYPES} " =~ " zonal " ]]; then
-    BUCKET_ARGS="${BUCKET_ARGS} --zonal-bucket='${ZONAL_BUCKET}'"
-  fi
-  if [[ " ${BUCKET_TYPES} " =~ " hns " ]]; then
-    BUCKET_ARGS="${BUCKET_ARGS} --hns-bucket='${HNS_BUCKET}'"
-  fi
-
-  local RUN_CMD="
-    source gcsfs/env/bin/activate
-    python gcsfs/gcsfs/tests/perf/microbenchmarks/run.py \
-      --group=$group \
-      $CONFIG_ARG \
-      $BUCKET_ARGS \
-      --log=true \
-      --log-level=INFO
-
-    # Upload results
-    echo '--- Uploading Results ---'
-    DATE_DIR=\$(date +%d%m%Y)
-    RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'
-    if [ -d \"\$RESULTS_DIR\" ]; then
-      echo \"Uploading from \$RESULTS_DIR to gs://${RESULTS_BUCKET}/\${DATE_DIR}/${RUN_ID}/\"
-      cd \"\$RESULTS_DIR\" && gcloud storage cp --recursive . gs://${RESULTS_BUCKET}/\${DATE_DIR}/${RUN_ID}/ && rm -rf *
-    else
-      echo \"No results directory found at \$RESULTS_DIR\"
-    fi
-  "
-
-  gcloud compute ssh "$vm_name" --project="${PROJECT_ID}" --zone="${ZONE}" --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$RUN_CMD"
-}
-
-# Main function to orchestrate the sequential execution of benchmarks on a single VM.
-main() {
-  local vm_name="${VM_NAME}"
-
-  echo "[$vm_name] Starting benchmarks..."
-
-  if ! setup_vm "$vm_name"; then
-    echo "WARNING: [$vm_name] Setup failed."
-    exit 0
-  fi
-
-  # Load configs
-  local CONFIG_ARRAY
-  IFS=' ' read -r -a CONFIG_ARRAY <<< "${BENCHMARK_CONFIG}"
-
-  local failures=0
-
-  for entry in "${CONFIG_ARRAY[@]}"; do
-    # Trim whitespace
-    entry=$(echo "$entry" | xargs)
-    if [ -z "$entry" ]; then continue; fi
-
-    IFS=':' read -r group config <<< "$entry"
-
-    echo "Launching job for $group:$config on $vm_name"
-
-    if ! run_benchmark "$vm_name" "$group" "$config"; then
-        echo "Benchmark $group:$config failed."
-        failures=$((failures+1))
-    fi
-
-    echo "Sleeping for 30 seconds..."
-    sleep 30
-  done
-
-  if [ "${failures:-0}" -ne 0 ]; then
-    echo "WARNING: $failures benchmark jobs failed. Returning success to proceed with cleanup."
-  else
-    echo "All benchmarks completed successfully."
-  fi
-}
-
-main
+# This script is now empty as the logic has been moved to benchmarks-cloudbuild.yaml

From d32d4aa04246cfdfffb4266374a554ff635f7861 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 10:39:35 +0000
Subject: [PATCH 29/57] fix cloudbuild script and config for testing

---
 cloudbuild/benchmarks-cloudbuild.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index c7bc5135..adecb1a9 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -4,7 +4,7 @@ substitutions:
   _ZONE: "us-central1-a"
   _INFRA_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
-  _BENCHMARK_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process read:read_rand_multi_process"
+  _BENCHMARK_CONFIG: "read:read_seq,read_rand rename:rename_flat"
   _BUCKET_TYPES: "zonal"
 
 steps:
@@ -193,7 +193,7 @@ steps:
             BUCKET_ARGS="$${BUCKET_ARGS} --hns-bucket='$${HNS_BUCKET}'"
           fi
 
-          RUN_CMD="
+          RUN_SCRIPT="
             source gcsfs/env/bin/activate
             python gcsfs/gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
             echo '--- Uploading Results ---'
@@ -207,7 +207,7 @@ steps:
             fi
           "
 
-          if ! gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_CMD"; then
+          if ! gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then
             echo "Benchmark $$group:$$config failed."
             failures=$$((failures+1))
           fi

From cce5a77194b3163954fd9567208896db816db721 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 10:40:40 +0000
Subject: [PATCH 30/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index adecb1a9..06ce45ad 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -199,11 +199,11 @@ steps:
             echo '--- Uploading Results ---'
             DATE_DIR=\$(date +%d%m%Y)
             RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'
-            if [ -d \\"\$RESULTS_DIR\\" ]; then
-              echo \\"Uploading from \$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/\\"
-              cd \\"\$RESULTS_DIR\\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/ && rm -rf *
+            if [ -d \\"\$$RESULTS_DIR\\" ]; then
+              echo \\"Uploading from \$$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/\\"
+              cd \\"\$$RESULTS_DIR\\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/ && rm -rf *
             else
-              echo \\"No results directory found at \$RESULTS_DIR\\"
+              echo \\"No results directory found at \$$RESULTS_DIR\\"
             fi
           "
 

From ffa4566bf9aed349a5e6a9c3b2c029cd7225ed6a Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 10:41:35 +0000
Subject: [PATCH 31/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 06ce45ad..986f9c18 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -200,8 +200,8 @@ steps:
             DATE_DIR=\$(date +%d%m%Y)
             RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'
             if [ -d \\"\$$RESULTS_DIR\\" ]; then
-              echo \\"Uploading from \$$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/\\"
-              cd \\"\$$RESULTS_DIR\\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\${DATE_DIR}/$${RUN_ID}/ && rm -rf *
+              echo \\"Uploading from \$$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/\\"
+              cd \\"\$$RESULTS_DIR\\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/ && rm -rf *
             else
               echo \\"No results directory found at \$$RESULTS_DIR\\"
             fi

From c2da9a01d7b6af9ad01e42555d1ea0afef0ec224 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 11:03:39 +0000
Subject: [PATCH 32/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 74 +++++++++++++++++++--------
 1 file changed, 53 insertions(+), 21 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 986f9c18..c19ee85c 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -104,41 +104,46 @@ steps:
             --quiet
     waitFor: ["init-variables"]
 
-  # 5. Wait for SSH.
+  # 5. Setup VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "wait-for-ssh"
+    id: "setup-vm"
     entrypoint: "bash"
     args:
       - "-c"
       - |
+        if [ -f /workspace/failure ]; then exit 1; fi
         set -e
         source /workspace/build_vars.env
+        trap 'touch /workspace/failure' ERR
 
         echo "Waiting for SSH on VM: $${VM_NAME}"
+        SSH_READY=false
         for i in {1..3}; do
-          if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
+          if gcloud compute ssh $$VM_NAME --project=${_PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
             echo "SSH is ready."
-            exit 0
+            SSH_READY=true
+            break
           fi
           echo "Waiting for VM to become available... (attempt $$i/3)"
           sleep 15
         done
-        echo "Timeout waiting for SSH."
-        exit 1
-    waitFor: ["create-vm"]
 
-  # 6. Setup VM.
-  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "setup-vm"
-    entrypoint: "bash"
-    args:
-      - "-c"
-      - |
-        set -e
-        source /workspace/build_vars.env
+        if [ "$SSH_READY" = false ]; then
+            echo "Timeout waiting for SSH."
+            touch /workspace/failure
+            exit 1
+        fi
 
         echo "[$${VM_NAME}] Copying source code..."
-        gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
+        if gcloud compute scp --recurse . $${VM_NAME}:~/gcsfs --project=${_PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+          echo "Source code copied successfully."
+        else
+          echo "Failed to copy source code."
+          touch /workspace/failure
+          exit 1
+        fi
+
+        echo "[$${VM_NAME}] Installing dependencies..."
 
         SETUP_SCRIPT="
           set -e
@@ -154,18 +159,27 @@ steps:
           pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
         "
 
-        gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT"
-    waitFor: ["wait-for-ssh"]
+        if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT"; then
+          echo "Dependencies installed successfully."
+        else
+          echo "Failed to install dependencies."
+          touch /workspace/failure
+          exit 1
+        fi
+    waitFor: ["create-vm"]
 
-  # 7. Run Benchmarks.
+  # 6. Run Benchmarks.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "run-benchmarks"
+    allowFailure: true
     entrypoint: "bash"
     args:
       - "-c"
       - |
+        if [ -f /workspace/failure ]; then exit 1; fi
         set -e
         source /workspace/build_vars.env
+        trap 'touch /workspace/failure' ERR
 
         IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_CONFIG}"
         failures=0
@@ -217,7 +231,9 @@ steps:
         done
 
         if [ "$${failures:-0}" -ne 0 ]; then
-          echo "WARNING: $$failures benchmark jobs failed. Returning success to proceed with cleanup."
+          echo "ERROR: $$failures benchmark jobs failed."
+          touch /workspace/failure
+          exit 1
         else
           echo "All benchmarks completed successfully."
         fi
@@ -282,6 +298,22 @@ steps:
     waitFor:
       - "run-benchmarks"
 
+  # 11. Check for failures.
+  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
+    id: "check-failure"
+    entrypoint: "bash"
+    args:
+      - "-c"
+      - |
+        if [ -f /workspace/failure ]; then
+          echo "Build failed. See previous steps for details."
+          exit 1
+        fi
+        echo "Build successful."
+    waitFor:
+      - "cleanup-vm"
+      - "delete-buckets"
+
 timeout: "14400s" # 2 hours
 
 options:

From bd0bdd7d2b10a2839910109257762df0145fdf5f Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 11:04:26 +0000
Subject: [PATCH 33/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index c19ee85c..6f5cf328 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -128,7 +128,7 @@ steps:
           sleep 15
         done
 
-        if [ "$SSH_READY" = false ]; then
+        if [ "$$SSH_READY" = false ]; then
             echo "Timeout waiting for SSH."
             touch /workspace/failure
             exit 1

From cf20272156d6d0178f681dc8ec5487634d125069 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 11:22:09 +0000
Subject: [PATCH 34/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 6f5cf328..83cca79e 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -107,6 +107,7 @@ steps:
   # 5. Setup VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "setup-vm"
+    allowFailure: true
     entrypoint: "bash"
     args:
       - "-c"
@@ -258,7 +259,6 @@ steps:
       - "run-benchmarks"
 
   # 9. Delete VM.
-  # Delete VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "cleanup-vm"
     entrypoint: "bash"
@@ -275,7 +275,6 @@ steps:
       - "cleanup-ssh-key"
 
   # 10. Delete all GCS buckets.
-  # Delete all GCS buckets.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "delete-buckets"
     entrypoint: "bash"

From 493a1ef704ff3944f2f8805e10377672385e904c Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 11:34:36 +0000
Subject: [PATCH 35/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 83cca79e..6148ad88 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -117,7 +117,7 @@ steps:
         source /workspace/build_vars.env
         trap 'touch /workspace/failure' ERR
 
-        echo "Waiting for SSH on VM: $${VM_NAME}"
+        echo "Waiting for SSH on VM: $${VM_NAME}... (attempt 1/3)"
         SSH_READY=false
         for i in {1..3}; do
           if gcloud compute ssh $$VM_NAME --project=${_PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
@@ -125,7 +125,7 @@ steps:
             SSH_READY=true
             break
           fi
-          echo "Waiting for VM to become available... (attempt $$i/3)"
+          echo "Waiting for VM to become available... (attempt $${i+1}/3)"
           sleep 15
         done
 
@@ -135,11 +135,13 @@ steps:
             exit 1
         fi
 
-        echo "[$${VM_NAME}] Copying source code..."
-        if gcloud compute scp --recurse . $${VM_NAME}:~/gcsfs --project=${_PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
-          echo "Source code copied successfully."
+        echo "[$${VM_NAME}] Packaging and copying source code..."
+        # Tar source code to avoid copying venv symlinks/artifacts that cause scp errors
+        tar -czf /workspace/source.tar.gz --exclude-vcs --exclude='venv*' --exclude='__pycache__' --exclude='*.pyc' .
+        if gcloud compute scp /workspace/source.tar.gz $${VM_NAME}:~/source.tar.gz --project=${_PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+          echo "Source code archive copied successfully."
         else
-          echo "Failed to copy source code."
+          echo "Failed to copy source code archive."
           touch /workspace/failure
           exit 1
         fi
@@ -148,6 +150,8 @@ steps:
 
         SETUP_SCRIPT="
           set -e
+          mkdir -p ~/gcsfs
+          tar -xzf ~/source.tar.gz -C ~/gcsfs
           export DEBIAN_FRONTEND=noninteractive
           sudo apt-get update > /dev/null
           sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null

From b9a19296f91beb9ea3903793f9e14c8c89eaeb2b Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 11:53:39 +0000
Subject: [PATCH 36/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 6148ad88..33a47537 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -135,13 +135,20 @@ steps:
             exit 1
         fi
 
-        echo "[$${VM_NAME}] Packaging and copying source code..."
-        # Tar source code to avoid copying venv symlinks/artifacts that cause scp errors
-        tar -czf /workspace/source.tar.gz --exclude-vcs --exclude='venv*' --exclude='__pycache__' --exclude='*.pyc' .
-        if gcloud compute scp /workspace/source.tar.gz $${VM_NAME}:~/source.tar.gz --project=${_PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
-          echo "Source code archive copied successfully."
+            exit 1
+        fi
+
+        echo "[$${VM_NAME}] Creating remote directory..."
+        gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
+
+        echo "[$${VM_NAME}] Copying source code..."
+        # Use GLOBIGNORE to exclude venv and other artifacts from wildcard expansion
+        shopt -s extglob dotglob
+        export GLOBIGNORE="venv*:__pycache__:.git:*.pyc:failure"
+        if gcloud compute scp --recurse * $${VM_NAME}:~/gcsfs --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+          echo "Source code copied successfully."
         else
-          echo "Failed to copy source code archive."
+          echo "Failed to copy source code."
           touch /workspace/failure
           exit 1
         fi

From d4ce1671cae1e01c9f21c3333db55b474c9c2f66 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 12:02:23 +0000
Subject: [PATCH 37/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 33a47537..40c71b0a 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -135,20 +135,16 @@ steps:
             exit 1
         fi
 
-            exit 1
-        fi
-
-        echo "[$${VM_NAME}] Creating remote directory..."
-        gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
-
-        echo "[$${VM_NAME}] Copying source code..."
-        # Use GLOBIGNORE to exclude venv and other artifacts from wildcard expansion
-        shopt -s extglob dotglob
-        export GLOBIGNORE="venv*:__pycache__:.git:*.pyc:failure"
-        if gcloud compute scp --recurse * $${VM_NAME}:~/gcsfs --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
-          echo "Source code copied successfully."
+        echo "ls result below"
+        ls
+
+        echo "[$${VM_NAME}] Packaging and copying source code..."
+        # Tar source code (gcsfs folder and setup files) to avoid copying venv/artifacts and "file changed" errors
+        tar -czf source.tar.gz gcsfs setup.py versioneer.py requirements.txt
+        if gcloud compute scp source.tar.gz $${VM_NAME}:~/source.tar.gz --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+          echo "Source code archive copied successfully."
         else
-          echo "Failed to copy source code."
+          echo "Failed to copy source code archive."
           touch /workspace/failure
           exit 1
         fi

From 2b9c4629b636c79b86ca02787623e9f798160630 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 13:09:07 +0000
Subject: [PATCH 38/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 66 +++++++++++++--------------
 1 file changed, 32 insertions(+), 34 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 40c71b0a..552b1e12 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -15,9 +15,10 @@ steps:
     args:
       - "-c"
       - |
+        mkdir -p /workspace/gcsfs-bench-tmp
         mkdir -p /workspace/.ssh
         ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb
-        cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub
+        cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcsfs-bench-tmp/gcb_ssh_key.pub
     waitFor: ["-"]
 
   # 2. Initialize shared variables.
@@ -31,17 +32,17 @@ steps:
       - |
         SHORT_BUILD_ID=$${BUILD_ID:0:8}
         # Define shared variables
-        echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
-        echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
-        echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/build_vars.env
-        echo "export BUCKET_TYPES='${_BUCKET_TYPES}'" >> /workspace/build_vars.env
-        echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
-        echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export REGIONAL_BUCKET=${_INFRA_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export ZONAL_BUCKET=${_INFRA_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export HNS_BUCKET=${_INFRA_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export RESULTS_BUCKET=${_INFRA_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
-        echo "export VM_NAME='${_INFRA_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
+        echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export ZONE=${_ZONE}" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export BUCKET_TYPES='${_BUCKET_TYPES}'" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export RUN_ID=$${BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export REGIONAL_BUCKET=${_INFRA_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export ZONAL_BUCKET=${_INFRA_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export HNS_BUCKET=${_INFRA_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export RESULTS_BUCKET=${_INFRA_PREFIX}-results-${_PROJECT_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export VM_NAME='${_INFRA_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/gcsfs-bench-tmp/build_vars.env
     waitFor: ["-"]
 
   # 3. Create all necessary GCS buckets in parallel.
@@ -52,7 +53,7 @@ steps:
       - "-c"
       - |
         set -e
-        source /workspace/build_vars.env
+        source /workspace/gcsfs-bench-tmp/build_vars.env
 
         # Create Test Buckets, in parallel
         if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
@@ -83,7 +84,7 @@ steps:
       - "-c"
       - |
         set -e
-        source /workspace/build_vars.env
+        source /workspace/gcsfs-bench-tmp/build_vars.env
 
         echo "Creating VM: $${VM_NAME}"
         gcloud compute instances create "$$VM_NAME" \
@@ -112,10 +113,10 @@ steps:
     args:
       - "-c"
       - |
-        if [ -f /workspace/failure ]; then exit 1; fi
+        if [ -f /workspace/gcsfs-bench-tmp/failure ]; then exit 1; fi
         set -e
-        source /workspace/build_vars.env
-        trap 'touch /workspace/failure' ERR
+        source /workspace/gcsfs-bench-tmp/build_vars.env
+        trap 'touch /workspace/gcsfs-bench-tmp/failure' ERR
 
         echo "Waiting for SSH on VM: $${VM_NAME}... (attempt 1/3)"
         SSH_READY=false
@@ -131,21 +132,18 @@ steps:
 
         if [ "$$SSH_READY" = false ]; then
             echo "Timeout waiting for SSH."
-            touch /workspace/failure
+            touch /workspace/gcsfs-bench-tmp/failure
             exit 1
         fi
 
-        echo "ls result below"
-        ls
-
         echo "[$${VM_NAME}] Packaging and copying source code..."
-        # Tar source code (gcsfs folder and setup files) to avoid copying venv/artifacts and "file changed" errors
-        tar -czf source.tar.gz gcsfs setup.py versioneer.py requirements.txt
-        if gcloud compute scp source.tar.gz $${VM_NAME}:~/source.tar.gz --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+        # Tar source code excluding tmp dir and other artifacts
+        tar -czf /workspace/gcsfs-bench-tmp/source.tar.gz --exclude="gcsfs-bench-tmp" --exclude=".git" --exclude=".ssh" --exclude="venv*" --exclude="__pycache__" --exclude="*.pyc" .
+        if gcloud compute scp /workspace/gcsfs-bench-tmp/source.tar.gz $${VM_NAME}:~/source.tar.gz --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
           echo "Source code archive copied successfully."
         else
           echo "Failed to copy source code archive."
-          touch /workspace/failure
+          touch /workspace/gcsfs-bench-tmp/failure
           exit 1
         fi
 
@@ -171,7 +169,7 @@ steps:
           echo "Dependencies installed successfully."
         else
           echo "Failed to install dependencies."
-          touch /workspace/failure
+          touch /workspace/gcsfs-bench-tmp/failure
           exit 1
         fi
     waitFor: ["create-vm"]
@@ -184,10 +182,10 @@ steps:
     args:
       - "-c"
       - |
-        if [ -f /workspace/failure ]; then exit 1; fi
+        if [ -f /workspace/gcsfs-bench-tmp/failure ]; then exit 1; fi
         set -e
-        source /workspace/build_vars.env
-        trap 'touch /workspace/failure' ERR
+        source /workspace/gcsfs-bench-tmp/build_vars.env
+        trap 'touch /workspace/gcsfs-bench-tmp/failure' ERR
 
         IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_CONFIG}"
         failures=0
@@ -240,7 +238,7 @@ steps:
 
         if [ "$${failures:-0}" -ne 0 ]; then
           echo "ERROR: $$failures benchmark jobs failed."
-          touch /workspace/failure
+          touch /workspace/gcsfs-bench-tmp/failure
           exit 1
         else
           echo "All benchmarks completed successfully."
@@ -261,7 +259,7 @@ steps:
       - "-c"
       - |
         gcloud compute os-login ssh-keys remove \
-          --key-file=/workspace/gcb_ssh_key.pub || true
+          --key-file=/workspace/gcsfs-bench-tmp/gcb_ssh_key.pub || true
     waitFor:
       - "run-benchmarks"
 
@@ -272,7 +270,7 @@ steps:
     args:
       - "-c"
       - |
-        source /workspace/build_vars.env
+        source /workspace/gcsfs-bench-tmp/build_vars.env
 
         gcloud compute instances delete "$$VM_NAME" \
             --project="${_PROJECT_ID}" \
@@ -289,7 +287,7 @@ steps:
       - "-c"
       - |
         set -e
-        source /workspace/build_vars.env
+        source /workspace/gcsfs-bench-tmp/build_vars.env
 
         if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
           gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
@@ -311,7 +309,7 @@ steps:
     args:
       - "-c"
       - |
-        if [ -f /workspace/failure ]; then
+        if [ -f /workspace/gcsfs-bench-tmp/failure ]; then
           echo "Build failed. See previous steps for details."
           exit 1
         fi

From 5cf00c3353057bc5d5160c24f3178abd46809880 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 13:14:06 +0000
Subject: [PATCH 39/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 552b1e12..3413f74c 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -32,6 +32,7 @@ steps:
       - |
         SHORT_BUILD_ID=$${BUILD_ID:0:8}
         # Define shared variables
+        touch /workspace/gcsfs-bench-tmp/build_vars.env
         echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/gcsfs-bench-tmp/build_vars.env
         echo "export ZONE=${_ZONE}" >> /workspace/gcsfs-bench-tmp/build_vars.env
         echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/gcsfs-bench-tmp/build_vars.env
@@ -251,7 +252,6 @@ steps:
   # --- Cleanup Steps ---
 
   # 8. Clean up the SSH key.
-  # Clean up the SSH key.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "cleanup-ssh-key"
     entrypoint: "bash"

From 19b7d7b4424eea86a5c5ef9b85ec25ec76de3106 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 13:28:00 +0000
Subject: [PATCH 40/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 73 +++++++++++++--------------
 1 file changed, 35 insertions(+), 38 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 3413f74c..0b015602 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -15,10 +15,9 @@ steps:
     args:
       - "-c"
       - |
-        mkdir -p /workspace/gcsfs-bench-tmp
         mkdir -p /workspace/.ssh
         ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb
-        cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcsfs-bench-tmp/gcb_ssh_key.pub
+        cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub
     waitFor: ["-"]
 
   # 2. Initialize shared variables.
@@ -32,18 +31,17 @@ steps:
       - |
         SHORT_BUILD_ID=$${BUILD_ID:0:8}
         # Define shared variables
-        touch /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export ZONE=${_ZONE}" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export BUCKET_TYPES='${_BUCKET_TYPES}'" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export RUN_ID=$${BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export REGIONAL_BUCKET=${_INFRA_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export ZONAL_BUCKET=${_INFRA_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export HNS_BUCKET=${_INFRA_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export RESULTS_BUCKET=${_INFRA_PREFIX}-results-${_PROJECT_ID}" >> /workspace/gcsfs-bench-tmp/build_vars.env
-        echo "export VM_NAME='${_INFRA_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/gcsfs-bench-tmp/build_vars.env
+        echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
+        echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
+        echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/build_vars.env
+        echo "export BUCKET_TYPES='${_BUCKET_TYPES}'" >> /workspace/build_vars.env
+        echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
+        echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export REGIONAL_BUCKET=${_INFRA_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export ZONAL_BUCKET=${_INFRA_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export HNS_BUCKET=${_INFRA_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
+        echo "export RESULTS_BUCKET=${_INFRA_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
+        echo "export VM_NAME='${_INFRA_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
     waitFor: ["-"]
 
   # 3. Create all necessary GCS buckets in parallel.
@@ -54,7 +52,7 @@ steps:
       - "-c"
       - |
         set -e
-        source /workspace/gcsfs-bench-tmp/build_vars.env
+        source /workspace/build_vars.env
 
         # Create Test Buckets, in parallel
         if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
@@ -85,7 +83,7 @@ steps:
       - "-c"
       - |
         set -e
-        source /workspace/gcsfs-bench-tmp/build_vars.env
+        source /workspace/build_vars.env
 
         echo "Creating VM: $${VM_NAME}"
         gcloud compute instances create "$$VM_NAME" \
@@ -114,10 +112,10 @@ steps:
     args:
       - "-c"
       - |
-        if [ -f /workspace/gcsfs-bench-tmp/failure ]; then exit 1; fi
+        if [ -f /workspace/failure ]; then exit 1; fi
         set -e
-        source /workspace/gcsfs-bench-tmp/build_vars.env
-        trap 'touch /workspace/gcsfs-bench-tmp/failure' ERR
+        source /workspace/build_vars.env
+        trap 'touch /workspace/failure' ERR
 
         echo "Waiting for SSH on VM: $${VM_NAME}... (attempt 1/3)"
         SSH_READY=false
@@ -133,18 +131,19 @@ steps:
 
         if [ "$$SSH_READY" = false ]; then
             echo "Timeout waiting for SSH."
-            touch /workspace/gcsfs-bench-tmp/failure
+            touch /workspace/failure
             exit 1
         fi
 
-        echo "[$${VM_NAME}] Packaging and copying source code..."
-        # Tar source code excluding tmp dir and other artifacts
-        tar -czf /workspace/gcsfs-bench-tmp/source.tar.gz --exclude="gcsfs-bench-tmp" --exclude=".git" --exclude=".ssh" --exclude="venv*" --exclude="__pycache__" --exclude="*.pyc" .
-        if gcloud compute scp /workspace/gcsfs-bench-tmp/source.tar.gz $${VM_NAME}:~/source.tar.gz --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
-          echo "Source code archive copied successfully."
+        echo "[$${VM_NAME}] Creating remote directory..."
+        gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
+
+        echo "[$${VM_NAME}] Copying source code..."
+        if gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}"zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+          echo "Source code copied successfully."
         else
-          echo "Failed to copy source code archive."
-          touch /workspace/gcsfs-bench-tmp/failure
+          echo "Failed to copy source code."
+          touch /workspace/failure
           exit 1
         fi
 
@@ -152,8 +151,6 @@ steps:
 
         SETUP_SCRIPT="
           set -e
-          mkdir -p ~/gcsfs
-          tar -xzf ~/source.tar.gz -C ~/gcsfs
           export DEBIAN_FRONTEND=noninteractive
           sudo apt-get update > /dev/null
           sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null
@@ -170,7 +167,7 @@ steps:
           echo "Dependencies installed successfully."
         else
           echo "Failed to install dependencies."
-          touch /workspace/gcsfs-bench-tmp/failure
+          touch /workspace/failure
           exit 1
         fi
     waitFor: ["create-vm"]
@@ -183,10 +180,10 @@ steps:
     args:
       - "-c"
       - |
-        if [ -f /workspace/gcsfs-bench-tmp/failure ]; then exit 1; fi
+        if [ -f /workspace/failure ]; then exit 1; fi
         set -e
-        source /workspace/gcsfs-bench-tmp/build_vars.env
-        trap 'touch /workspace/gcsfs-bench-tmp/failure' ERR
+        source /workspace/build_vars.env
+        trap 'touch /workspace/failure' ERR
 
         IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_CONFIG}"
         failures=0
@@ -239,7 +236,7 @@ steps:
 
         if [ "$${failures:-0}" -ne 0 ]; then
           echo "ERROR: $$failures benchmark jobs failed."
-          touch /workspace/gcsfs-bench-tmp/failure
+          touch /workspace/failure
           exit 1
         else
           echo "All benchmarks completed successfully."
@@ -259,7 +256,7 @@ steps:
       - "-c"
       - |
         gcloud compute os-login ssh-keys remove \
-          --key-file=/workspace/gcsfs-bench-tmp/gcb_ssh_key.pub || true
+          --key-file=/workspace/gcb_ssh_key.pub || true
     waitFor:
       - "run-benchmarks"
 
@@ -270,7 +267,7 @@ steps:
     args:
       - "-c"
       - |
-        source /workspace/gcsfs-bench-tmp/build_vars.env
+        source /workspace/build_vars.env
 
         gcloud compute instances delete "$$VM_NAME" \
             --project="${_PROJECT_ID}" \
@@ -287,7 +284,7 @@ steps:
       - "-c"
       - |
         set -e
-        source /workspace/gcsfs-bench-tmp/build_vars.env
+        source /workspace/build_vars.env
 
         if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
           gcloud storage rm --recursive gs://$$REGIONAL_BUCKET &
@@ -309,7 +306,7 @@ steps:
     args:
       - "-c"
       - |
-        if [ -f /workspace/gcsfs-bench-tmp/failure ]; then
+        if [ -f /workspace/failure ]; then
           echo "Build failed. See previous steps for details."
           exit 1
         fi

From ccc00828e09ffce73d4e47c03a48af4a389ff766 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 13:42:17 +0000
Subject: [PATCH 41/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 0b015602..b80b85b2 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -139,7 +139,7 @@ steps:
         gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
 
         echo "[$${VM_NAME}] Copying source code..."
-        if gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}"zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+        if gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}" zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
           echo "Source code copied successfully."
         else
           echo "Failed to copy source code."

From c80042eacfec80c972809391b5b030f0e8e94afc Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 13:49:14 +0000
Subject: [PATCH 42/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index b80b85b2..5b1251f6 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -139,7 +139,7 @@ steps:
         gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
 
         echo "[$${VM_NAME}] Copying source code..."
-        if gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}" zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+        if gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
           echo "Source code copied successfully."
         else
           echo "Failed to copy source code."
@@ -151,7 +151,6 @@ steps:
 
         SETUP_SCRIPT="
           set -e
-          export DEBIAN_FRONTEND=noninteractive
           sudo apt-get update > /dev/null
           sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null
           cd gcsfs

From 21c952373ddcc13b47267780676acc7a69981851 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 14:02:49 +0000
Subject: [PATCH 43/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 5b1251f6..ac841914 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -125,7 +125,7 @@ steps:
             SSH_READY=true
             break
           fi
-          echo "Waiting for VM to become available... (attempt $${i+1}/3)"
+          echo "Waiting for VM to become available... (attempt $$((i+1))/3)"
           sleep 15
         done
 

From abbffefbaac5536db6630e7e4df0029b4ee20277 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 14:14:54 +0000
Subject: [PATCH 44/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index ac841914..c16c1c9d 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -151,15 +151,23 @@ steps:
 
         SETUP_SCRIPT="
           set -e
-          sudo apt-get update > /dev/null
-          sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git > /dev/null
+          sudo apt-get update
+          echo '--- Installing python and other dependencies on VM ---'
+          sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git
+          echo '--- Entering GCSFS source folder ---'
           cd gcsfs
+          ls
+
+          echo '--- Creating virtual environment ---'
           python3 -m venv env
           source env/bin/activate
-          pip install --upgrade pip > /dev/null
-          pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null
-          pip install -e . > /dev/null
-          pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
+          echo '--- Installing pip packages ---'
+          pip install --upgrade pip
+          pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage
+          echo '--- Installing GCSFS packages ---'
+          pip install -e .
+          echo '--- Installing requirements for benchmarks ---'
+          pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt
         "
 
         if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT"; then

From 61b279e49fb3e4319359d4f62e5a19585be30a31 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 14:21:53 +0000
Subject: [PATCH 45/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index c16c1c9d..600e95d5 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -153,7 +153,7 @@ steps:
           set -e
           sudo apt-get update
           echo '--- Installing python and other dependencies on VM ---'
-          sudo apt-get install -y python3-pip python3-venv fuse fuse3 libfuse2 git
+          sudo apt-get install -y python3-pip python3-venv git
           echo '--- Entering GCSFS source folder ---'
           cd gcsfs
           ls

From 4fa611269994238b7f186b042f14815357e1555d Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 14:34:58 +0000
Subject: [PATCH 46/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 600e95d5..75e06ea3 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -151,23 +151,19 @@ steps:
 
         SETUP_SCRIPT="
           set -e
-          sudo apt-get update
+          sudo apt-get update > /dev/null
           echo '--- Installing python and other dependencies on VM ---'
-          sudo apt-get install -y python3-pip python3-venv git
-          echo '--- Entering GCSFS source folder ---'
+          sudo apt-get install -y python3-pip python3-venv git > /dev/null
           cd gcsfs
-          ls
-
-          echo '--- Creating virtual environment ---'
           python3 -m venv env
           source env/bin/activate
           echo '--- Installing pip packages ---'
-          pip install --upgrade pip
-          pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage
+          pip install --upgrade pip > /dev/null
+          pip install pytest pytest-timeout pytest-subtests pytest-asyncio fusepy google-cloud-storage > /dev/null
           echo '--- Installing GCSFS packages ---'
-          pip install -e .
+          pip install -e . > /dev/null
           echo '--- Installing requirements for benchmarks ---'
-          pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt
+          pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
         "
 
         if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT"; then
@@ -219,8 +215,12 @@ steps:
           fi
 
           RUN_SCRIPT="
-            source gcsfs/env/bin/activate
-            python gcsfs/gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
+            cd gcsfs
+            source env/bin/activate
+
+            echo "Running Command: python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO"
+            python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
+
             echo '--- Uploading Results ---'
             DATE_DIR=\$(date +%d%m%Y)
             RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'

From 6722790074e536913136775104bcab8aa434a62f Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 17:41:38 +0000
Subject: [PATCH 47/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 75e06ea3..528457a7 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -214,7 +214,10 @@ steps:
             BUCKET_ARGS="$${BUCKET_ARGS} --hns-bucket='$${HNS_BUCKET}'"
           fi
 
+          echo "Running benchmarks with config: $$CONFIG_ARG for bucket(s): $$BUCKET_ARGS"
+
           RUN_SCRIPT="
+            set -e
             cd gcsfs
             source env/bin/activate
 

From 94e128d89a4f20b60f4051f07b86fa97479298a1 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 17:50:34 +0000
Subject: [PATCH 48/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 528457a7..139aa678 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -217,8 +217,12 @@ steps:
           echo "Running benchmarks with config: $$CONFIG_ARG for bucket(s): $$BUCKET_ARGS"
 
           RUN_SCRIPT="
+            echo "Running benchmarks script"
+
             set -e
+            echo "Entering gcsfs directory"
             cd gcsfs
+            echo "Activating virtual environment"
             source env/bin/activate
 
             echo "Running Command: python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO"

From a5e2a574afb9a4accba44a286c0fc4c337fa6dfc Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 18:00:02 +0000
Subject: [PATCH 49/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 9 +++++----
 cloudbuild/run_benchmarks.sh          | 3 ---
 2 files changed, 5 insertions(+), 7 deletions(-)
 delete mode 100644 cloudbuild/run_benchmarks.sh

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 139aa678..71820c74 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -222,6 +222,7 @@ steps:
             set -e
             echo "Entering gcsfs directory"
             cd gcsfs
+            ls
             echo "Activating virtual environment"
             source env/bin/activate
 
@@ -230,7 +231,7 @@ steps:
 
             echo '--- Uploading Results ---'
             DATE_DIR=\$(date +%d%m%Y)
-            RESULTS_DIR='gcsfs/gcsfs/tests/perf/microbenchmarks/__run__'
+            RESULTS_DIR='gcsfs/tests/perf/microbenchmarks/__run__'
             if [ -d \\"\$$RESULTS_DIR\\" ]; then
               echo \\"Uploading from \$$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/\\"
               cd \\"\$$RESULTS_DIR\\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/ && rm -rf *
@@ -239,7 +240,9 @@ steps:
             fi
           "
 
-          if ! gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then
+          if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then
+            echo "All benchmarks completed successfully."
+          else
             echo "Benchmark $$group:$$config failed."
             failures=$$((failures+1))
           fi
@@ -252,8 +255,6 @@ steps:
           echo "ERROR: $$failures benchmark jobs failed."
           touch /workspace/failure
           exit 1
-        else
-          echo "All benchmarks completed successfully."
         fi
     waitFor:
       - "setup-vm"
diff --git a/cloudbuild/run_benchmarks.sh b/cloudbuild/run_benchmarks.sh
deleted file mode 100644
index 153ace24..00000000
--- a/cloudbuild/run_benchmarks.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-set -e
-# This script is now empty as the logic has been moved to benchmarks-cloudbuild.yaml

From db8dca56a32c44e78bd2704fbb4dd2be9e2092e3 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 18:11:37 +0000
Subject: [PATCH 50/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 35 +++++++++------------------
 1 file changed, 12 insertions(+), 23 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 71820c74..2204149b 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -106,7 +106,7 @@ steps:
 
   # 5. Setup VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "setup-vm"
+    id: "setup-and-run-benchmarks"
     allowFailure: true
     entrypoint: "bash"
     args:
@@ -173,21 +173,8 @@ steps:
           touch /workspace/failure
           exit 1
         fi
-    waitFor: ["create-vm"]
-
-  # 6. Run Benchmarks.
-  - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "run-benchmarks"
-    allowFailure: true
-    entrypoint: "bash"
-    args:
-      - "-c"
-      - |
-        if [ -f /workspace/failure ]; then exit 1; fi
-        set -e
-        source /workspace/build_vars.env
-        trap 'touch /workspace/failure' ERR
 
+        # --- Run Benchmarks ---
         IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_CONFIG}"
         failures=0
 
@@ -217,16 +204,16 @@ steps:
           echo "Running benchmarks with config: $$CONFIG_ARG for bucket(s): $$BUCKET_ARGS"
 
           RUN_SCRIPT="
-            echo "Running benchmarks script"
+            echo \"Running benchmarks script\"
 
             set -e
-            echo "Entering gcsfs directory"
+            echo \"Entering gcsfs directory\"
             cd gcsfs
             ls
-            echo "Activating virtual environment"
+            echo \"Activating virtual environment\"
             source env/bin/activate
 
-            echo "Running Command: python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO"
+            echo \"Running Command: python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO\"
             python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
 
             echo '--- Uploading Results ---'
@@ -241,7 +228,7 @@ steps:
           "
 
           if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then
-            echo "All benchmarks completed successfully."
+            echo "Benchmark $$group:$$config completed successfully."
           else
             echo "Benchmark $$group:$$config failed."
             failures=$$((failures+1))
@@ -255,9 +242,11 @@ steps:
           echo "ERROR: $$failures benchmark jobs failed."
           touch /workspace/failure
           exit 1
+        else
+          echo "All benchmarks completed successfully."
         fi
     waitFor:
-      - "setup-vm"
+      - "create-vm"
       - "create-buckets"
       - "generate-ssh-key"
 
@@ -273,7 +262,7 @@ steps:
         gcloud compute os-login ssh-keys remove \
           --key-file=/workspace/gcb_ssh_key.pub || true
     waitFor:
-      - "run-benchmarks"
+      - "setup-and-run-benchmarks"
 
   # 9. Delete VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
@@ -312,7 +301,7 @@ steps:
         fi
         wait
     waitFor:
-      - "run-benchmarks"
+      - "setup-and-run-benchmarks"
 
   # 11. Check for failures.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"

From 2a758ab5a5c21c288ba64aa02f1472be424f15ab Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Tue, 27 Jan 2026 18:25:13 +0000
Subject: [PATCH 51/57] fix cloudbuild script

---
 cloudbuild/benchmarks-cloudbuild.yaml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 2204149b..8ae91918 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -217,14 +217,6 @@ steps:
             python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
 
             echo '--- Uploading Results ---'
-            DATE_DIR=\$(date +%d%m%Y)
-            RESULTS_DIR='gcsfs/tests/perf/microbenchmarks/__run__'
-            if [ -d \\"\$$RESULTS_DIR\\" ]; then
-              echo \\"Uploading from \$$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/\\"
-              cd \\"\$$RESULTS_DIR\\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/ && rm -rf *
-            else
-              echo \\"No results directory found at \$$RESULTS_DIR\\"
-            fi
           "
 
           if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then

From b5dc005c8cf8917e3d2f1c811f0d12c815afe616 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Wed, 28 Jan 2026 02:58:48 +0000
Subject: [PATCH 52/57] SSH in cloudbuild

---
 cloudbuild/benchmarks-cloudbuild.yaml         | 26 ++++++++++++++++---
 .../perf/microbenchmarks/read/configs.yaml    |  2 +-
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 8ae91918..4eac200a 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -4,7 +4,7 @@ substitutions:
   _ZONE: "us-central1-a"
   _INFRA_PREFIX: "gcsfs-perf"
   _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
-  _BENCHMARK_CONFIG: "read:read_seq,read_rand rename:rename_flat"
+  _BENCHMARK_CONFIG: "read:read_seq read:read_seq_multi_process write:write_seq write:write_seq_multi_process"
   _BUCKET_TYPES: "zonal"
 
 steps:
@@ -215,8 +215,6 @@ steps:
 
             echo \"Running Command: python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO\"
             python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
-
-            echo '--- Uploading Results ---'
           "
 
           if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then
@@ -230,6 +228,28 @@ steps:
           sleep 30
         done
 
+        echo "Uploading results..."
+        UPLOAD_SCRIPT="
+            set -e
+            cd gcsfs
+            echo '--- Uploading Results ---'
+            DATE_DIR=\$(date +%d%m%Y)
+            RESULTS_DIR='gcsfs/tests/perf/microbenchmarks/__run__'
+            if [ -d \"\$$RESULTS_DIR\" ]; then
+              echo \"Uploading from \$$RESULTS_DIR to gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/\"
+              cd \"\$$RESULTS_DIR\" && gcloud storage cp --recursive . gs://$${RESULTS_BUCKET}/\$${DATE_DIR}/$${RUN_ID}/ && rm -rf *
+            else
+              echo \"No results directory found at \$$RESULTS_DIR\"
+            fi
+        "
+        if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$UPLOAD_SCRIPT"; then
+          echo "Results uploaded successfully."
+        else
+          echo "Failed to upload results."
+          touch /workspace/failure
+          exit 1
+        fi
+
         if [ "$${failures:-0}" -ne 0 ]; then
           echo "ERROR: $$failures benchmark jobs failed."
           touch /workspace/failure
diff --git a/gcsfs/tests/perf/microbenchmarks/read/configs.yaml b/gcsfs/tests/perf/microbenchmarks/read/configs.yaml
index 7ac0a5ab..ae7d9070 100644
--- a/gcsfs/tests/perf/microbenchmarks/read/configs.yaml
+++ b/gcsfs/tests/perf/microbenchmarks/read/configs.yaml
@@ -5,7 +5,7 @@ common:
   file_sizes_mb:
     - 1024 # 1GB
   block_sizes_mb:
-    - 1
+    - 100
   rounds: 3
 
 scenarios:

From 9ff78d9dad196d4bd5704ff62708dc3b4f6f9f34 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Wed, 28 Jan 2026 09:44:02 +0000
Subject: [PATCH 53/57] test cleanup fixes

---
 cloudbuild/benchmarks-cloudbuild.yaml        | 43 ++++++++------------
 gcsfs/tests/conftest.py                      | 12 +++---
 gcsfs/tests/perf/microbenchmarks/conftest.py | 14 +------
 3 files changed, 25 insertions(+), 44 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 4eac200a..52cdaf37 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -1,9 +1,7 @@
 substitutions:
-  _PROJECT_ID: "gcs-aiml-clients-testing-101"
-  _LOCATION: "us-central1"
   _ZONE: "us-central1-a"
   _INFRA_PREFIX: "gcsfs-perf"
-  _SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
+  _VM_SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
   _BENCHMARK_CONFIG: "read:read_seq read:read_seq_multi_process write:write_seq write:write_seq_multi_process"
   _BUCKET_TYPES: "zonal"
 
@@ -31,16 +29,11 @@ steps:
       - |
         SHORT_BUILD_ID=$${BUILD_ID:0:8}
         # Define shared variables
-        echo "export PROJECT_ID=${_PROJECT_ID}" > /workspace/build_vars.env
-        echo "export ZONE=${_ZONE}" >> /workspace/build_vars.env
-        echo "export BENCHMARK_CONFIG='${_BENCHMARK_CONFIG}'" >> /workspace/build_vars.env
-        echo "export BUCKET_TYPES='${_BUCKET_TYPES}'" >> /workspace/build_vars.env
         echo "export RUN_ID=$${BUILD_ID}" >> /workspace/build_vars.env
-        echo "export SHORT_BUILD_ID=$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export REGIONAL_BUCKET=${_INFRA_PREFIX}-regional-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export ZONAL_BUCKET=${_INFRA_PREFIX}-zonal-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
         echo "export HNS_BUCKET=${_INFRA_PREFIX}-hns-$${SHORT_BUILD_ID}" >> /workspace/build_vars.env
-        echo "export RESULTS_BUCKET=${_INFRA_PREFIX}-results-${_PROJECT_ID}" >> /workspace/build_vars.env
+        echo "export RESULTS_BUCKET=${_INFRA_PREFIX}-run-results" >> /workspace/build_vars.env
         echo "export VM_NAME='${_INFRA_PREFIX}-vm-$${SHORT_BUILD_ID}'" >> /workspace/build_vars.env
     waitFor: ["-"]
 
@@ -56,18 +49,18 @@ steps:
 
         # Create Test Buckets, in parallel
         if [[ " ${_BUCKET_TYPES} " =~ " regional " ]]; then
-          gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} &
+          gcloud storage buckets create gs://$$REGIONAL_BUCKET --project=${PROJECT_ID} --location=${LOCATION} &
         fi
         if [[ " ${_BUCKET_TYPES} " =~ " zonal " ]]; then
-          gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
+          gcloud storage buckets create gs://$$ZONAL_BUCKET --project=${PROJECT_ID} --location=${LOCATION} --placement=${_ZONE} --default-storage-class=RAPID --enable-hierarchical-namespace --uniform-bucket-level-access &
         fi
         if [[ " ${_BUCKET_TYPES} " =~ " hns " ]]; then
-          gcloud storage buckets create gs://$$HNS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
+          gcloud storage buckets create gs://$$HNS_BUCKET --project=${PROJECT_ID} --location=${LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
         fi
 
         # Create HNS Results Bucket (Persistent)
-        if ! gcloud storage buckets describe gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} >/dev/null 2>&1; then
-          gcloud storage buckets create gs://$$RESULTS_BUCKET --project=${_PROJECT_ID} --location=${_LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
+        if ! gcloud storage buckets describe gs://$$RESULTS_BUCKET --project=${PROJECT_ID} >/dev/null 2>&1; then
+          gcloud storage buckets create gs://$$RESULTS_BUCKET --project=${PROJECT_ID} --location=${LOCATION} --enable-hierarchical-namespace --uniform-bucket-level-access &
         else
           echo "Results bucket gs://$${RESULTS_BUCKET} already exists"
         fi
@@ -87,7 +80,7 @@ steps:
 
         echo "Creating VM: $${VM_NAME}"
         gcloud compute instances create "$$VM_NAME" \
-            --project="${_PROJECT_ID}" \
+            --project="${PROJECT_ID}" \
             --zone="${_ZONE}" \
             --machine-type="c4-standard-192" \
             --image-family="ubuntu-2204-lts" \
@@ -98,9 +91,9 @@ steps:
             --network-performance-configs="total-egress-bandwidth-tier=TIER_1" \
             --scopes="https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/devstorage.read_write" \
             --metadata="enable-oslogin=TRUE" \
-            --service-account="${_SERVICE_ACCOUNT}" \
+            --service-account="${_VM_SERVICE_ACCOUNT}" \
             --tags="allow-ssh" \
-            --labels="build-id=$$SHORT_BUILD_ID" \
+            --labels="build-id=$$RUN_ID" \
             --quiet
     waitFor: ["init-variables"]
 
@@ -120,7 +113,7 @@ steps:
         echo "Waiting for SSH on VM: $${VM_NAME}... (attempt 1/3)"
         SSH_READY=false
         for i in {1..3}; do
-          if gcloud compute ssh $$VM_NAME --project=${_PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
+          if gcloud compute ssh $$VM_NAME --project=${PROJECT_ID} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
             echo "SSH is ready."
             SSH_READY=true
             break
@@ -136,10 +129,10 @@ steps:
         fi
 
         echo "[$${VM_NAME}] Creating remote directory..."
-        gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
+        gcloud compute ssh "$$VM_NAME" --project="${PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
 
         echo "[$${VM_NAME}] Copying source code..."
-        if gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${_PROJECT_ID}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
+        if gcloud compute scp --recurse . "$${VM_NAME}:~/gcsfs" --project="${PROJECT_ID}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine; then
           echo "Source code copied successfully."
         else
           echo "Failed to copy source code."
@@ -166,7 +159,7 @@ steps:
           pip install -r gcsfs/tests/perf/microbenchmarks/requirements.txt > /dev/null
         "
 
-        if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT"; then
+        if gcloud compute ssh "$$VM_NAME" --project="${PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$SETUP_SCRIPT"; then
           echo "Dependencies installed successfully."
         else
           echo "Failed to install dependencies."
@@ -217,7 +210,7 @@ steps:
             python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO
           "
 
-          if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then
+          if gcloud compute ssh "$$VM_NAME" --project="${PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$RUN_SCRIPT"; then
             echo "Benchmark $$group:$$config completed successfully."
           else
             echo "Benchmark $$group:$$config failed."
@@ -242,7 +235,7 @@ steps:
               echo \"No results directory found at \$$RESULTS_DIR\"
             fi
         "
-        if gcloud compute ssh "$$VM_NAME" --project="${_PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$UPLOAD_SCRIPT"; then
+        if gcloud compute ssh "$$VM_NAME" --project="${PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="$$UPLOAD_SCRIPT"; then
           echo "Results uploaded successfully."
         else
           echo "Failed to upload results."
@@ -286,7 +279,7 @@ steps:
         source /workspace/build_vars.env
 
         gcloud compute instances delete "$$VM_NAME" \
-            --project="${_PROJECT_ID}" \
+            --project="${PROJECT_ID}" \
             --zone="${_ZONE}" \
             --quiet || true
     waitFor:
@@ -336,4 +329,4 @@ timeout: "14400s" # 2 hours
 options:
   logging: CLOUD_LOGGING_ONLY
   pool:
-    name: "projects/${_PROJECT_ID}/locations/${_LOCATION}/workerPools/cloud-build-worker-pool"
+    name: "projects/${PROJECT_ID}/locations/${LOCATION}/workerPools/cloud-build-worker-pool"
diff --git a/gcsfs/tests/conftest.py b/gcsfs/tests/conftest.py
index 184b4a75..56619ece 100644
--- a/gcsfs/tests/conftest.py
+++ b/gcsfs/tests/conftest.py
@@ -162,14 +162,14 @@ def gcs(gcs_factory, buckets_to_delete, populate_bucket):
             # managed externally and should not be deleted by the tests.
             buckets_to_delete.add(TEST_BUCKET)
         else:
-            _cleanup_gcs(gcs, bucket=TEST_BUCKET)
+            _cleanup_gcs(gcs, bucket=TEST_BUCKET, bucket_populated=populate_bucket)
 
         if populate_bucket:
             gcs.pipe({TEST_BUCKET + "/" + k: v for k, v in allfiles.items()})
         gcs.invalidate_cache()
         yield gcs
     finally:
-        _cleanup_gcs(gcs)
+        _cleanup_gcs(gcs, bucket_populated=populate_bucket)
 
 
 @pytest.fixture
@@ -186,7 +186,7 @@ def factory(**kwargs):
     yield factory
 
     for fs in created_instances:
-        _cleanup_gcs(fs)
+        _cleanup_gcs(fs, bucket_populated=populate_bucket)
 
 
 @pytest.fixture
@@ -197,13 +197,13 @@ def extended_gcsfs(gcs_factory, buckets_to_delete, populate_bucket):
     try:
         yield extended_gcsfs
     finally:
-        _cleanup_gcs(extended_gcsfs)
+        _cleanup_gcs(extended_gcsfs, bucket_populated=populate_bucket)
 
 
-def _cleanup_gcs(gcs, bucket=TEST_BUCKET):
+def _cleanup_gcs(gcs, bucket=TEST_BUCKET, bucket_populated=True):
     """Clean the bucket contents, logging a warning on failure."""
     try:
-        if gcs.exists(bucket):
+        if bucket_populated and gcs.exists(bucket):
             files_to_delete = gcs.find(bucket)
             if files_to_delete:
                 gcs.rm(files_to_delete)
diff --git a/gcsfs/tests/perf/microbenchmarks/conftest.py b/gcsfs/tests/perf/microbenchmarks/conftest.py
index c70fdefc..f6d5b7f9 100644
--- a/gcsfs/tests/perf/microbenchmarks/conftest.py
+++ b/gcsfs/tests/perf/microbenchmarks/conftest.py
@@ -173,19 +173,7 @@ def gcsfs_benchmark_listing(extended_gcs_factory, request):
     # --- Teardown ---
     logging.info(f"Tearing down benchmark '{params.name}': deleting files and folders.")
     try:
-        gcs.rm(file_paths)
-        if params.bucket_type != "regional":
-            # Sort by length descending to delete children first
-            for d in sorted(target_dirs, key=len, reverse=True):
-                try:
-                    gcs.rmdir(d)
-                except Exception:
-                    pass
-        else:
-            try:
-                gcs.rmdir(prefix)
-            except Exception:
-                pass
+        gcs.rm(file_paths, recursive=True)
     except Exception as e:
         logging.error(f"Failed to clean up benchmark files: {e}")
 

From 9b5db26d13a5c4ff7d1734cacc3adf2092d66dde Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Wed, 28 Jan 2026 10:16:42 +0000
Subject: [PATCH 54/57] test cleanup fixes

---
 cloudbuild/benchmarks-cloudbuild.yaml | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 52cdaf37..066f086e 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -86,7 +86,7 @@ steps:
             --image-family="ubuntu-2204-lts" \
             --image-project="ubuntu-os-cloud" \
             --boot-disk-type="hyperdisk-balanced" \
-            --boot-disk-size="100GB" \
+            --boot-disk-size="30GB" \
             --network-interface="network-tier=PREMIUM,nic-type=GVNIC" \
             --network-performance-configs="total-egress-bandwidth-tier=TIER_1" \
             --scopes="https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/devstorage.read_write" \
@@ -97,9 +97,9 @@ steps:
             --quiet
     waitFor: ["init-variables"]
 
-  # 5. Setup VM.
+  # 5. Run benchmarks.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "setup-and-run-benchmarks"
+    id: "run-benchmarks"
     allowFailure: true
     entrypoint: "bash"
     args:
@@ -110,6 +110,7 @@ steps:
         source /workspace/build_vars.env
         trap 'touch /workspace/failure' ERR
 
+        # --- Setup SSH ---
         echo "Waiting for SSH on VM: $${VM_NAME}... (attempt 1/3)"
         SSH_READY=false
         for i in {1..3}; do
@@ -128,6 +129,7 @@ steps:
             exit 1
         fi
 
+        # --- Copy source code ---
         echo "[$${VM_NAME}] Creating remote directory..."
         gcloud compute ssh "$$VM_NAME" --project="${PROJECT_ID}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="mkdir -p ~/gcsfs"
 
@@ -140,6 +142,7 @@ steps:
           exit 1
         fi
 
+        # --- Install dependencies and packages ---
         echo "[$${VM_NAME}] Installing dependencies..."
 
         SETUP_SCRIPT="
@@ -167,7 +170,7 @@ steps:
           exit 1
         fi
 
-        # --- Run Benchmarks ---
+        # --- Run benchmarks ---
         IFS=' ' read -r -a CONFIG_ARRAY <<< "${_BENCHMARK_CONFIG}"
         failures=0
 
@@ -176,7 +179,7 @@ steps:
           if [ -z "$$entry" ]; then continue; fi
 
           IFS=':' read -r group config <<< "$$entry"
-          echo "Launching job for $$group:$$config on $$VM_NAME"
+          echo "[$$VM_NAME]: Launching benchmark run for $$group:$$config"
 
           CONFIG_ARG=""
           if [ -n "$$config" ]; then
@@ -194,16 +197,9 @@ steps:
             BUCKET_ARGS="$${BUCKET_ARGS} --hns-bucket='$${HNS_BUCKET}'"
           fi
 
-          echo "Running benchmarks with config: $$CONFIG_ARG for bucket(s): $$BUCKET_ARGS"
-
           RUN_SCRIPT="
-            echo \"Running benchmarks script\"
-
             set -e
-            echo \"Entering gcsfs directory\"
             cd gcsfs
-            ls
-            echo \"Activating virtual environment\"
             source env/bin/activate
 
             echo \"Running Command: python gcsfs/tests/perf/microbenchmarks/run.py --group=$$group $$CONFIG_ARG $$BUCKET_ARGS --log=true --log-level=INFO\"
@@ -217,11 +213,11 @@ steps:
             failures=$$((failures+1))
           fi
 
-          echo "Sleeping for 30 seconds..."
+          echo "Sleeping for 30 seconds for cool down..."
           sleep 30
         done
 
-        echo "Uploading results..."
+        # --- Upload results ---
         UPLOAD_SCRIPT="
             set -e
             cd gcsfs

From 5d196e046edef524435206bb9c444d18a10250e3 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Wed, 28 Jan 2026 10:18:29 +0000
Subject: [PATCH 55/57] test cleanup fixes

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 066f086e..90e4760c 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -263,7 +263,7 @@ steps:
         gcloud compute os-login ssh-keys remove \
           --key-file=/workspace/gcb_ssh_key.pub || true
     waitFor:
-      - "setup-and-run-benchmarks"
+      - "run-benchmarks"
 
   # 9. Delete VM.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"

From 4fcf6909d1e13151044ebf59d80e245fece5c91a Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Wed, 28 Jan 2026 10:19:39 +0000
Subject: [PATCH 56/57] test cleanup fixes

---
 cloudbuild/benchmarks-cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 90e4760c..7c23a0d4 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -302,7 +302,7 @@ steps:
         fi
         wait
     waitFor:
-      - "setup-and-run-benchmarks"
+      - "run-benchmarks"
 
   # 11. Check for failures.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"

From 37a9e590adea121a40a7d5deaf660c52fb63cde6 Mon Sep 17 00:00:00 2001
From: jasha26 <jasha@google.com>
Date: Wed, 28 Jan 2026 10:40:49 +0000
Subject: [PATCH 57/57] fixing python version warning

---
 cloudbuild/benchmarks-cloudbuild.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cloudbuild/benchmarks-cloudbuild.yaml b/cloudbuild/benchmarks-cloudbuild.yaml
index 7c23a0d4..6313d392 100644
--- a/cloudbuild/benchmarks-cloudbuild.yaml
+++ b/cloudbuild/benchmarks-cloudbuild.yaml
@@ -2,7 +2,7 @@ substitutions:
   _ZONE: "us-central1-a"
   _INFRA_PREFIX: "gcsfs-perf"
   _VM_SERVICE_ACCOUNT: "gcsfs-zonal-vm-sc@gcs-aiml-clients-testing-101.iam.gserviceaccount.com"
-  _BENCHMARK_CONFIG: "read:read_seq read:read_seq_multi_process write:write_seq write:write_seq_multi_process"
+  _BENCHMARK_CONFIG: "read:read_seq,read_rand read:read_seq_multi_process write:write_seq write:write_seq_multi_process"
   _BUCKET_TYPES: "zonal"
 
 steps:
@@ -83,7 +83,7 @@ steps:
             --project="${PROJECT_ID}" \
             --zone="${_ZONE}" \
             --machine-type="c4-standard-192" \
-            --image-family="ubuntu-2204-lts" \
+            --image-family="ubuntu-2404-lts-amd64" \
             --image-project="ubuntu-os-cloud" \
             --boot-disk-type="hyperdisk-balanced" \
             --boot-disk-size="30GB" \