Unstable and custom release support

.github/actions/apply-docker-version/apply-docker-version.sh

@@ -57,19 +57,19 @@ update_dockerfile() {
     echo "Updating $dockerfile..."
 
     # Update REDIS_DOWNLOAD_URL
-    if grep -q "^ENV REDIS_DOWNLOAD_URL=" "$dockerfile"; then
-        sed -i "s|^ENV REDIS_DOWNLOAD_URL=.*|ENV REDIS_DOWNLOAD_URL=$REDIS_ARCHIVE_URL|" "$dockerfile"
+    if grep -q "^ARG REDIS_DOWNLOAD_URL=" "$dockerfile"; then
+        sed -i "s|^ARG REDIS_DOWNLOAD_URL=.*|ARG REDIS_DOWNLOAD_URL=$REDIS_ARCHIVE_URL|" "$dockerfile"
     else
-        echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_URL not found"
+        echo "Cannot update $dockerfile, ARG REDIS_DOWNLOAD_URL not found"
         return 1
     fi
 
 
     # Update REDIS_DOWNLOAD_SHA
-    if grep -q "^ENV REDIS_DOWNLOAD_SHA=" "$dockerfile"; then
-        sed -i "s|^ENV REDIS_DOWNLOAD_SHA=.*|ENV REDIS_DOWNLOAD_SHA=$REDIS_ARCHIVE_SHA|" "$dockerfile"
+    if grep -q "^ARG REDIS_DOWNLOAD_SHA=" "$dockerfile"; then
+        sed -i "s|^ARG REDIS_DOWNLOAD_SHA=.*|ARG REDIS_DOWNLOAD_SHA=$REDIS_ARCHIVE_SHA|" "$dockerfile"
     else
-        echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_SHA not found"
+        echo "Cannot update $dockerfile, ARG REDIS_DOWNLOAD_SHA not found"
         return 1
     fi
 }

.github/actions/build-and-tag-locally/action.yml

@@ -1,4 +1,5 @@
 name: Build and Test
+description: Build and test redis docker image
 
 inputs:
   distribution:
@@ -22,6 +23,24 @@ inputs:
   release_tag:
     description: 'Release tag to build'
     required: false
+  redisearch_version:
+    description: 'Redisearch version to build'
+    required: false
+  redisjson_version:
+    description: 'Redisjson version to build'
+    required: false
+  redisbloom_version:
+    description: 'Redisbloom version to build'
+    required: false
+  redistimeseries_version:
+    description: 'Redistimeseries version to build'
+    required: false
+  custom_ref_type:
+    description: 'Type of custom release, branch tag or commit. Empty for release'
+    required: false
+  run_type:
+    description: 'Run type, either release, pr, nightly, unstable or custom'
+    required: true
 
 runs:
   using: "composite"
@@ -87,28 +106,90 @@ runs:
         username: ${{ inputs.registry_username }}
         password: ${{ inputs.registry_password }}
 
+    - name: Redis release archive
+      shell: bash
+      if: inputs.run_type != 'release' && inputs.run_type != 'pr'
+      run: |
+        if [[ "${{ inputs.custom_ref_type }}" == "branch" ]]; then
+            echo "REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/heads/${{ inputs.release_tag }}.tar.gz" | tee -a "$GITHUB_ENV"
+        elif [[ "${{ inputs.custom_ref_type }}" == "tag" ]]; then
+            echo "REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/${{ inputs.release_tag }}.tar.gz" | tee -a "$GITHUB_ENV"
+        elif [[ "${{ inputs.custom_ref_type }}" == "commit" ]]; then
+            echo "REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/${{ inputs.release_tag }}.tar.gz" | tee -a "$GITHUB_ENV"
+        else
+            echo "Wrong custom_build type, available options are tag, branch and commit"
+            exit 1
+        fi
+
     - name: Build
+      id: build
       uses: docker/build-push-action@v6
       with:
         context: ${{ inputs.distribution }}
         push: false
         load: true
         platforms: ${{ inputs.platform }}
         tags: ${{ github.sha }}:${{ steps.platform.outputs.display_name }}
-        cache-from: type=gha,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}
-        cache-to: type=gha,mode=max,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}
+        cache-from: ${{ inputs.run_type == 'release' && format('type=gha,scope={0}-{1}', inputs.distribution, steps.platform.outputs.display_name) || '' }}
+        cache-to: ${{ inputs.run_type == 'release' && format('type=gha,mode=max,scope={0}-{1}', inputs.distribution, steps.platform.outputs.display_name) || '' }}
+        build-args: |
+          REDISEARCH_VERSION=${{ inputs.redisearch_version }}
+          REDISJSON_VERSION=${{ inputs.redisjson_version }}
+          REDISBLOOM_VERSION=${{ inputs.redisbloom_version }}
+          REDISTIMESERIES_VERSION=${{ inputs.redistimeseries_version }}
+          ${{ (inputs.run_type != 'release' && inputs.run_type != 'pr') && 'CUSTOM_BUILD=true' || '' }}
+          ${{ (inputs.run_type != 'release' && inputs.run_type != 'pr') && format('REDIS_DOWNLOAD_URL={0}', env.REDIS_DOWNLOAD_URL) || '' }}
 
-    - name: Save image
+    - name: Capture build logs on failure
       shell: bash
+      if: failure() && steps.build.outcome == 'failure' && inputs.run_type == 'nightly'
       run: |
-        docker save -o /tmp/image-${{ steps.platform.outputs.display_name }}.tar ${{ github.sha }}:${{ steps.platform.outputs.display_name }}
+        mkdir -p /tmp/build-logs
+
+        echo "Build failed for ${{ inputs.distribution }} on ${{ inputs.platform }}"
+        echo "Capturing detailed logs for troubleshooting..."
+
+        # Get docker history for the built image (might not exist if build failed early)
+        docker history ${{ github.sha }}:${{ steps.platform.outputs.display_name }} > /tmp/build-logs/image-history.log 2>&1 || echo "Failed to get image history"
+
+        # Get docker inspect output (might not exist if build failed early)
+        docker inspect ${{ github.sha }}:${{ steps.platform.outputs.display_name }} > /tmp/build-logs/image-inspect.json 2>&1 || echo "Failed to inspect image"
 
-    - name: Upload image
+        # Get docker build cache info
+        docker buildx du > /tmp/build-logs/buildx-du.log 2>&1 || echo "Failed to get build cache info"
+
+        # Get system info
+        uname -a > /tmp/build-logs/system-info.log 2>&1
+        docker info > /tmp/build-logs/docker-info.log 2>&1
+
+        # Create a summary file
+        {
+          echo "Build failure summary for ${{ inputs.distribution }} on ${{ inputs.platform }}"
+          echo "Date: $(date)"
+          echo "GitHub SHA: ${{ github.sha }}"
+          echo "Platform: ${{ steps.platform.outputs.display_name }}"
+          echo "Distribution: ${{ inputs.distribution }}"
+        } > /tmp/build-logs/failure-summary.txt
+
+        # Try to extract error information from the build logs
+        echo "Analyzing build failure..."
+
+        # Check for common error patterns
+        if docker buildx build --no-cache ${{ inputs.distribution }} --platform=${{ inputs.platform }} 2>&1 | tee /tmp/build-logs/build-error.log | grep -q "ERROR"; then
+          echo "Found ERROR in build output"
+          grep -A 10 -B 5 "ERROR" /tmp/build-logs/build-error.log > /tmp/build-logs/error-context.log || true
+        fi
+
+        echo "Log capture complete"
+
+    - name: Upload build failure logs
+      if: failure() && steps.build.outcome == 'failure' && inputs.run_type == 'nightly'
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ steps.platform.outputs.display_name }}-${{ inputs.distribution }}-docker-image.tar
-        path: /tmp/image-${{ steps.platform.outputs.display_name }}.tar
-        retention-days: 45
+        name: build-failure-${{ steps.platform.outputs.display_name }}-${{ inputs.distribution }}
+        path: /tmp/build-logs/
+        retention-days: 30
+        if-no-files-found: warn
 
     - name: Run container
       shell: bash
@@ -226,25 +307,59 @@ runs:
 
     - name: Format registry tag
       id: format-registry-tag
+      if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }}
       shell: bash
       run: |
-        printf "tag=%s:%s%s-%s-%s" \
+        if [[ "${{ inputs.run_type }}" == "release" ]]; then
+          tag_prefix="${{ inputs.release_tag }}"
+        elif [[ "${{ inputs.run_type }}" == "unstable" || "${{ inputs.run_type }}" == "nightly" ]]; then
+          tag_prefix="unstable"
+        elif [[ "${{ inputs.run_type }}" == "custom" ]]; then
+          tag_prefix="custom"
+        else
+          echo "Wrong run_type value, available options are release, nightly, unstable and custom"
+          exit 1
+        fi
+
+        printf "tag=%s:%s-%s-%s-%s" \
           "${{ inputs.registry_repository }}" \
-          "${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}" \
-          "${{ github.sha }}" \
+          "${tag_prefix}" \
+          "${{ github.run_id }}" \
           "${{ inputs.distribution }}" \
           "${{ steps.platform.outputs.display_name }}" \
           | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT"
 
+    - name: Image labels
+      if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }}
+      id: get-image-labels
+      uses: ./.github/actions/create-image-labels
+      with:
+          redis_version: ${{ inputs.release_tag }}
+          redisearch_version: ${{ inputs.redisearch_version }}
+          redisjson_version: ${{ inputs.redisjson_version }}
+          redisbloom_version: ${{ inputs.redisbloom_version }}
+          redistimeseries_version: ${{ inputs.redistimeseries_version }}
+
     - name: Push image
       uses: docker/build-push-action@v6
       if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }}
       with:
         context: ${{ inputs.distribution }}
         push: true
         tags: ${{ steps.format-registry-tag.outputs.tag }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
+        cache-from: ${{ inputs.run_type == 'release' && 'type=gha' || '' }}
+        cache-to: ${{ inputs.run_type == 'release' && 'type=gha,mode=max' || '' }}
+        build-args: |
+          REDISEARCH_VERSION=${{ inputs.redisearch_version }}
+          REDISJSON_VERSION=${{ inputs.redisjson_version }}
+          REDISBLOOM_VERSION=${{ inputs.redisbloom_version }}
+          REDISTIMESERIES_VERSION=${{ inputs.redistimeseries_version }}
+          ${{ inputs.run_type != 'release' && 'CUSTOM_BUILD=true' || '' }}
+          ${{ inputs.run_type != 'release' && format('REDIS_DOWNLOAD_URL={0}', env.REDIS_DOWNLOAD_URL) || '' }}
+        labels: |
+            ${{ steps.get-image-labels.outputs.image-labels }}
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
 
     - name: Save image metadata to artifact
       shell: bash

.github/actions/create-image-labels/action.yml

@@ -0,0 +1,114 @@
+name: Construct docker image labels
+description: Construct docker image labels for io.redis
+
+inputs:
+  redis_version:
+    description: 'Redis version to build'
+    required: true
+  redisjson_version:
+    description: 'redisjson to build'
+    required: false
+  redisbloom_version:
+    description: 'redisbloom to build'
+    required: false
+  redistimeseries_version:
+    description: 'redistimeseries to build'
+    required: false
+  redisearch_version:
+    description: 'redisearch to build'
+    required: false
+
+outputs:
+  image-labels:
+    description: 'JSON list of built snap files'
+    value: ${{ steps.get-labels.outputs.image_labels }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout Redis repository
+      uses: actions/checkout@v4
+      with:
+        repository: redis/redis
+        path: redis
+        ref: ${{ inputs.redis_version }}
+
+    - name: Get modules versions
+      if: inputs.redisearch_version == '' || inputs.redisjson_version == '' || inputs.redistimeseries_version == '' || inputs.redisbloom_version == ''
+      shell: bash
+      run: |
+        get_module_version() {
+          local module="$1"
+          grep MODULE_VERSION modules/$module/Makefile | cut -d '=' -f2 | tr -d ' '
+        }
+        cd redis
+        if [ -z "${{ inputs.redisearch_version }}" ]; then
+          echo redisearch_version=$(get_module_version redisearch) >> $GITHUB_ENV
+        fi
+        if [ -z "${{ inputs.redisjson_version }}" ]; then
+          echo redisjson_version=$(get_module_version redisjson) >> $GITHUB_ENV
+        fi
+        if [ -z "${{ inputs.redisbloom_version }}" ]; then
+          echo redisbloom_version=$(get_module_version redisbloom) >> $GITHUB_ENV
+        fi
+        if [ -z "${{ inputs.redistimeseries_version }}" ]; then
+          echo redistimeseries_version=$(get_module_version redistimeseries) >> $GITHUB_ENV
+        fi
+
+    - name: Checkout Redisearch repository
+      uses: actions/checkout@v4
+      with:
+        repository: RediSearch/RediSearch
+        path: redisearch
+        ref: ${{ inputs.redisearch_version != '' && inputs.redisearch_version || env.redisearch_version }}
+
+    - name: Checkout Redisjson repository
+      uses: actions/checkout@v4
+      with:
+        repository: redisjson/redisjson
+        path: redisjson
+        ref: ${{ inputs.redisjson_version != '' && inputs.redisjson_version || env.redisjson_version }}
+
+    - name: Checkout RedisTimeSeries repository
+      uses: actions/checkout@v4
+      with:
+        repository: RedisTimeSeries/RedisTimeSeries
+        path: redistimeseries
+        ref: ${{ inputs.redistimeseries_version != '' && inputs.redistimeseries_version || env.redistimeseries_version }}
+
+    - name: Checkout RedisBloom repository
+      uses: actions/checkout@v4
+      with:
+        repository: RedisBloom/RedisBloom
+        path: redisbloom
+        ref: ${{ inputs.redisbloom_version != '' && inputs.redisbloom_version || env.redisbloom_version }}
+
+    - name: Get image labels
+      id: get-labels
+      shell: bash
+      run: |
+        labels=""
+        created_timestamp=$(date -u --rfc-3339=seconds)
+        labels+="org.opencontainers.image.created=$created_timestamp"$'\n'
+        # Modules revisions labels
+        modules_repos=(redisearch redisjson redistimeseries redisbloom)
+        for module in "${modules_repos[@]}"; do
+          revision=$(git -C $module rev-parse HEAD)
+          labels+="io.redis.$module.revision=$revision"$'\n'
+        done
+
+        # Redis labels
+        cd redis
+        redis_revision=$(git rev-parse HEAD)
+        labels+="io.redis.revision=$redis_revision"$'\n'
+        VER=$(sed -n 's/^.* REDIS_VERSION "\(.*\)"$/\1/g p' < src/version.h)
+        if [ "$VER" = "255.255.255" ]; then
+          labels+="io.redis.version=unstable"$'\n'
+        else
+          labels+="io.redis.version=$VER"$'\n'
+        fi
+        labels+="io.redis.source=https://github.com/redis/redis"$'\n'
+        echo "labels: $labels"
+        echo "image_labels<<EOF" >> $GITHUB_OUTPUT
+        echo "$labels" >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT