Flyte 2 WIP

.dockerignore

@@ -0,0 +1,37 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+# CI/CD
+.github
+
+# Generated files
+gen/
+!gen/go/**
+
+# Dependencies
+node_modules/
+target/
+**/Cargo.lock
+*.pyc
+__pycache__/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+
+# Test coverage
+coverage/
+*.coverage
+.pytest_cache/

.github/actions/setup-python-env/action.yml

@@ -0,0 +1,47 @@
+name: "Setup Python Environment"
+description: "Set up Python environment for the given Python version"
+
+inputs:
+  python-version:
+    description: "Python version to use"
+    required: true
+    default: "3.12"
+  uv-version:
+    description: "uv version to use"
+    required: true
+    default: "0.8.4"
+  working-directory:
+    description: "Default working directory for all steps"
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      id: setup-uv
+      with:
+        version: ${{ inputs.uv-version }}
+        enable-cache: 'true'
+        cache-suffix: ${{ inputs.python-version }}
+        cache-dependency-glob: |
+          pyproject.toml
+          uv.lock
+        working-directory: ${{ inputs.working-directory }}
+
+    - name: Install Python dependencies
+      if: steps.setup-uv.outputs.cache-hit == 'false'
+      working-directory: ${{ inputs.working-directory }}
+      run: uv sync --all-groups --frozen
+      shell: bash
+
+    - name: List Python dependencies
+      if: steps.setup-uv.outputs.cache-hit == 'true'
+      working-directory: ${{ inputs.working-directory }}
+      run: uv pip list
+      shell: bash

.github/workflows/build-ci-image.yml

@@ -0,0 +1,158 @@
+name: Build and Publish CI Docker Image
+
+on:
+  push:
+    branches:
+      - main
+      - v2
+    paths:
+      - 'gen.Dockerfile'
+      - '.github/workflows/build-ci-image.yml'
+  pull_request:
+    paths:
+      - 'gen.Dockerfile'
+      - '.github/workflows/build-ci-image.yml'
+  workflow_dispatch:
+    inputs:
+      force_rebuild:
+        description: 'Force rebuild of the image'
+        required: false
+        default: 'false'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}/ci
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      pull-requests: write
+
+    outputs:
+      image-tag: ${{ steps.set-tag.outputs.tag }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            # Use branch name for branch pushes
+            type=ref,event=branch
+            # Use PR number for pull requests
+            type=ref,event=pr
+            # Use 'latest' tag for main branch
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            # Use 'v2' tag for v2 branch
+            type=raw,value=v2,enable=${{ github.ref == 'refs/heads/v2' }}
+            # Add git sha as tag
+            type=sha,prefix=${{ github.head_ref }}-,enable=${{ github.ref != 'refs/heads/v2' && github.ref != 'refs/heads/main' }}
+
+      - name: Set image tag output
+        id: set-tag
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "tag=pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+          elif [ "${{ github.ref }}" == "refs/heads/v2" ]; then
+            echo "tag=v2" >> $GITHUB_OUTPUT
+          elif [ "${{ github.ref }}" == "refs/heads/main" ]; then
+            echo "tag=latest" >> $GITHUB_OUTPUT
+          else
+            echo "tag=$(echo ${{ github.ref }} | sed 's/refs\/heads\///')" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./gen.Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
+          # Multi-layer caching strategy for speed
+          cache-from: |
+            type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+            type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.set-tag.outputs.tag }}
+            type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:v2
+            type=gha
+          cache-to: |
+            type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+            type=gha,mode=max
+          # Enable BuildKit features for better caching
+          build-args: |
+            BUILDKIT_INLINE_CACHE=1
+
+      - name: Image digest
+        run: echo "Image built with digest ${{ steps.meta.outputs.digest }}"
+
+      - name: Comment on PR with image tag
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const tag = '${{ steps.set-tag.outputs.tag }}';
+            const registry = '${{ env.REGISTRY }}';
+            const imageName = '${{ env.IMAGE_NAME }}';
+            const fullImage = `${registry}/${imageName}:${tag}`;
+
+            const comment = `## 🐳 Docker CI Image Built
+
+            The CI Docker image has been built and pushed for this PR!
+
+            **Image:** \`${fullImage}\`
+
+            This image will be automatically used by CI workflows in this PR.
+
+            To test locally:
+            \`\`\`bash
+            make gen DOCKER_CI_IMAGE=${fullImage}
+            \`\`\`
+            `;
+
+            // Find existing comment
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(comment =>
+              comment.user.type === 'Bot' &&
+              comment.body.includes('🐳 Docker CI Image Built')
+            );
+
+            if (botComment) {
+              // Update existing comment
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: comment
+              });
+            } else {
+              // Create new comment
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: comment
+              });
+            }

.github/workflows/check-generate.yml

@@ -0,0 +1,114 @@
+name: Check Generated Files
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  check-generate:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      actions: read
+      packages: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Determine Docker image
+        id: docker-image
+        run: |
+          # Check if gen.Dockerfile or build workflow was modified
+          git fetch origin ${{ github.base_ref }}
+          if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -E '^(gen\.Dockerfile|Dockerfile\.ci|\.github/workflows/build-ci-image\.yml)$'; then
+            echo "modified=true" >> $GITHUB_OUTPUT
+            echo "image=ghcr.io/flyteorg/flyte/ci:pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+            echo "📦 gen.Dockerfile modified - will use PR-specific image"
+          else
+            echo "modified=false" >> $GITHUB_OUTPUT
+            echo "image=ghcr.io/flyteorg/flyte/ci:v2" >> $GITHUB_OUTPUT
+            echo "📦 Using default v2 image"
+          fi
+
+      - name: Wait for Docker image build workflow
+        if: steps.docker-image.outputs.modified == 'true'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const maxAttempts = 60; // 20 minutes max
+            const delaySeconds = 20;
+
+            console.log('⏳ Waiting for Docker image build workflow to complete...');
+
+            for (let attempt = 0; attempt < maxAttempts; attempt++) {
+              // Get workflow runs for this PR
+              const { data: runs } = await github.rest.actions.listWorkflowRuns({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                workflow_id: 'build-ci-image.yml',
+                event: 'pull_request',
+                per_page: 10
+              });
+
+              // Find the run for this PR
+              const prRun = runs.workflow_runs.find(run =>
+                run.head_sha === context.payload.pull_request.head.sha
+              );
+
+              if (prRun) {
+                console.log(`Found workflow run: ${prRun.html_url}`);
+                console.log(`Status: ${prRun.status}, Conclusion: ${prRun.conclusion}`);
+
+                if (prRun.status === 'completed') {
+                  if (prRun.conclusion === 'success') {
+                    console.log('✅ Docker image build completed successfully!');
+                    return;
+                  } else {
+                    core.setFailed(`❌ Docker image build failed with conclusion: ${prRun.conclusion}`);
+                    return;
+                  }
+                }
+
+                console.log(`Attempt ${attempt + 1}/${maxAttempts}: Build still running, waiting ${delaySeconds} seconds...`);
+              } else {
+                console.log(`Attempt ${attempt + 1}/${maxAttempts}: Build not started yet, waiting ${delaySeconds} seconds...`);
+              }
+
+              await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000));
+            }
+
+            core.setFailed('❌ Timeout waiting for Docker image build to complete');
+
+      - name: Login to GHCR
+        if: steps.docker-image.outputs.modified == 'true'
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Pull Docker image
+        if: steps.docker-image.outputs.modified == 'true'
+        run: |
+          IMAGE="${{ steps.docker-image.outputs.image }}"
+          echo "📦 Pulling image: $IMAGE"
+          docker pull "$IMAGE"
+
+      - name: Run checks in container
+        run: |
+          IMAGE="${{ steps.docker-image.outputs.image }}"
+          echo "Using image: $IMAGE"
+
+          docker run --rm \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace \
+            -e SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 \
+            -e UV_PROJECT_ENVIRONMENT=/tmp/flyte-venv \
+            "$IMAGE" \
+            bash -c "
+              git config --global --add safe.directory /workspace &&
+              cd gen/python && uv sync --all-groups --frozen && cd ../.. &&
+              make gen-local &&
+              git diff --exit-code || (echo 'Generated files are out of date. Run \`make gen\` and commit changes.' && exit 1) &&
+              make build-crate
+            "
+
+