(iggy) branch master updated: fix(ci): simplify use_latest_ci to download script from master directly (#2141)

[hgruszecki]

This is an automated email from the ASF dual-hosted git repository.

hgruszecki pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git


The following commit(s) were added to refs/heads/master by this push:
     new ccfff470 fix(ci): simplify use_latest_ci to download script from 
master directly (#2141)
ccfff470 is described below

commit ccfff4707b01ce1fb5f7f36fe8d2a0a9336a80f9
Author: Hubert Gruszecki <h.grusze...@gmail.com>
AuthorDate: Mon Sep 1 12:20:24 2025 +0200

    fix(ci): simplify use_latest_ci to download script from master directly 
(#2141)
    
    - Download copy-latest-from-master.sh from master to /tmp before use
    - Fixes publishing older commits that don't have the script or config
    files
    - Ensures .github/config is copied for plan job to load publish.yml
    - Apply same pattern to both publish.yml and _build_python_wheels.yml
---
 .github/workflows/_build_python_wheels.yml |  60 ++++++++--------
 .github/workflows/publish.yml              | 111 ++++++++++++++++++-----------
 scripts/copy-latest-from-master.sh         |  41 +++++++----
 3 files changed, 130 insertions(+), 82 deletions(-)

diff --git a/.github/workflows/_build_python_wheels.yml 
b/.github/workflows/_build_python_wheels.yml
index a5afcde2..257ca3fa 100644
--- a/.github/workflows/_build_python_wheels.yml
+++ b/.github/workflows/_build_python_wheels.yml
@@ -45,19 +45,20 @@ jobs:
       matrix:
         target: [x86_64, aarch64]
     steps:
-      - name: Save latest CI and scripts from master
+      - name: Download latest copy script from master
         if: inputs.use_latest_ci
         run: |
-          # Clone master and save CI and scripts
-          bash scripts/copy-latest-from-master.sh save \
-            .github \
-            scripts
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
 
       - uses: actions/checkout@v4
 
-      - name: Apply latest CI and scripts
+      - name: Save and apply latest CI from master
         if: inputs.use_latest_ci
-        run: bash scripts/copy-latest-from-master.sh apply
+        run: |
+          /tmp/copy-latest-from-master.sh save .github scripts
+          /tmp/copy-latest-from-master.sh apply
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -100,19 +101,20 @@ jobs:
       matrix:
         target: [x86_64, aarch64]
     steps:
-      - name: Save latest CI and scripts from master
+      - name: Download latest copy script from master
         if: inputs.use_latest_ci
         run: |
-          # Clone master and save CI and scripts
-          bash scripts/copy-latest-from-master.sh save \
-            .github \
-            scripts
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
 
       - uses: actions/checkout@v4
 
-      - name: Apply latest CI and scripts
+      - name: Save and apply latest CI from master
         if: inputs.use_latest_ci
-        run: bash scripts/copy-latest-from-master.sh apply
+        run: |
+          /tmp/copy-latest-from-master.sh save .github scripts
+          /tmp/copy-latest-from-master.sh apply
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -138,21 +140,22 @@ jobs:
   windows:
     runs-on: windows-latest
     steps:
-      - name: Save latest CI and scripts from master
+      - name: Download latest copy script from master
         if: inputs.use_latest_ci
         shell: bash
         run: |
-          # Clone master and save CI and scripts
-          bash scripts/copy-latest-from-master.sh save \
-            .github \
-            scripts
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
 
       - uses: actions/checkout@v4
 
-      - name: Apply latest CI and scripts
+      - name: Save and apply latest CI from master
         if: inputs.use_latest_ci
         shell: bash
-        run: bash scripts/copy-latest-from-master.sh apply
+        run: |
+          /tmp/copy-latest-from-master.sh save .github scripts
+          /tmp/copy-latest-from-master.sh apply
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -179,19 +182,20 @@ jobs:
   sdist:
     runs-on: ubuntu-latest
     steps:
-      - name: Save latest CI and scripts from master
+      - name: Download latest copy script from master
         if: inputs.use_latest_ci
         run: |
-          # Clone master and save CI and scripts
-          bash scripts/copy-latest-from-master.sh save \
-            .github \
-            scripts
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
 
       - uses: actions/checkout@v4
 
-      - name: Apply latest CI and scripts
+      - name: Save and apply latest CI from master
         if: inputs.use_latest_ci
-        run: bash scripts/copy-latest-from-master.sh apply
+        run: |
+          /tmp/copy-latest-from-master.sh save .github scripts
+          /tmp/copy-latest-from-master.sh apply
 
       - name: Build sdist
         uses: PyO3/maturin-action@v1
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 004e7996..6d688987 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -139,10 +139,30 @@ jobs:
       has_python: ${{ steps.mk.outputs.has_python }}
       has_rust_crates: ${{ steps.mk.outputs.has_rust_crates }}
     steps:
+      - name: Download latest copy script from master
+        if: inputs.use_latest_ci
+        run: |
+          # Download the copy script from master branch
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
+          echo "✅ Downloaded latest copy script from master"
+
       - uses: actions/checkout@v4
         with:
           ref: ${{ needs.validate.outputs.commit }}
 
+      - name: Save and apply latest CI from master
+        if: inputs.use_latest_ci
+        run: |
+          # Save latest files from master (including config)
+          /tmp/copy-latest-from-master.sh save \
+            .github \
+            scripts
+
+          # Apply them to current checkout
+          /tmp/copy-latest-from-master.sh apply
+
       - name: Load publish config
         id: cfg
         run: |
@@ -270,18 +290,13 @@ jobs:
     if: needs.validate.outputs.has_targets == 'true' && 
fromJson(needs.plan.outputs.targets).include[0].key != 'noop'
     runs-on: ubuntu-latest
     steps:
-      - name: Save latest CI and scripts from master
+      - name: Download latest copy script from master
         if: inputs.use_latest_ci
         run: |
-          # Clone master and save CI, scripts, and Dockerfiles
-          bash scripts/copy-latest-from-master.sh save \
-            .github \
-            scripts \
-            web/Dockerfile \
-            core/server/Dockerfile \
-            core/ai/mcp/Dockerfile \
-            core/connectors/runtime/Dockerfile \
-            core/bench/dashboard/server/Dockerfile
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
+          echo "✅ Downloaded latest copy script from master"
 
       - name: Checkout at commit
         uses: actions/checkout@v4
@@ -289,9 +304,19 @@ jobs:
           ref: ${{ needs.validate.outputs.commit }}
           fetch-depth: 0
 
-      - name: Apply latest CI and scripts
+      - name: Save and apply latest CI from master
         if: inputs.use_latest_ci
-        run: bash scripts/copy-latest-from-master.sh apply
+        run: |
+          /tmp/copy-latest-from-master.sh save \
+            .github \
+            scripts \
+            web/Dockerfile \
+            core/server/Dockerfile \
+            core/ai/mcp/Dockerfile \
+            core/connectors/runtime/Dockerfile \
+            core/bench/dashboard/server/Dockerfile
+
+          /tmp/copy-latest-from-master.sh apply
 
       - name: Setup yq
         run: |
@@ -441,18 +466,13 @@ jobs:
     outputs:
       status: ${{ steps.final-status.outputs.status }}
     steps:
-      - name: Save latest CI and scripts from master
+      - name: Download latest copy script from master
         if: inputs.use_latest_ci
         run: |
-          # Clone master and save CI, scripts, and Dockerfiles
-          bash scripts/copy-latest-from-master.sh save \
-            .github \
-            scripts \
-            web/Dockerfile \
-            core/server/Dockerfile \
-            core/ai/mcp/Dockerfile \
-            core/connectors/runtime/Dockerfile \
-            core/bench/dashboard/server/Dockerfile
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
+          echo "✅ Downloaded latest copy script from master"
 
       - name: Checkout at commit
         uses: actions/checkout@v4
@@ -460,9 +480,19 @@ jobs:
           ref: ${{ needs.validate.outputs.commit }}
           fetch-depth: 0
 
-      - name: Apply latest CI and scripts
+      - name: Save and apply latest CI from master
         if: inputs.use_latest_ci
-        run: bash scripts/copy-latest-from-master.sh apply
+        run: |
+          /tmp/copy-latest-from-master.sh save \
+            .github \
+            scripts \
+            web/Dockerfile \
+            core/server/Dockerfile \
+            core/ai/mcp/Dockerfile \
+            core/connectors/runtime/Dockerfile \
+            core/bench/dashboard/server/Dockerfile
+
+          /tmp/copy-latest-from-master.sh apply
 
       - name: Setup Rust with cache
         uses: ./.github/actions/utils/setup-rust-with-cache
@@ -598,22 +628,13 @@ jobs:
       version: ${{ steps.ver.outputs.version }}
       tag: ${{ steps.ver.outputs.tag }}
     steps:
-      - name: Save latest CI and scripts from master
+      - name: Download latest copy script from master
         if: inputs.use_latest_ci
         run: |
-          # Clone master and save CI, scripts, and Dockerfiles
-          bash scripts/copy-latest-from-master.sh save \
-            .github \
-            scripts \
-            web/Dockerfile \
-            core/server/Dockerfile \
-            core/ai/mcp/Dockerfile \
-            core/connectors/runtime/Dockerfile \
-            core/bench/dashboard/server/Dockerfile
-          echo "Scripts:"
-          ls -la /tmp/latest-scripts/
-          echo ".github:"
-          ls -la /tmp/latest-github/
+          curl -sSL "https://raw.githubusercontent.com/${{ github.repository 
}}/master/scripts/copy-latest-from-master.sh" \
+            -o /tmp/copy-latest-from-master.sh
+          chmod +x /tmp/copy-latest-from-master.sh
+          echo "✅ Downloaded latest copy script from master"
 
       - name: Checkout at commit
         uses: actions/checkout@v4
@@ -621,9 +642,19 @@ jobs:
           ref: ${{ needs.validate.outputs.commit }}
           fetch-depth: 0
 
-      - name: Apply latest CI and scripts
+      - name: Save and apply latest CI from master
         if: inputs.use_latest_ci
-        run: bash scripts/copy-latest-from-master.sh apply
+        run: |
+          /tmp/copy-latest-from-master.sh save \
+            .github \
+            scripts \
+            web/Dockerfile \
+            core/server/Dockerfile \
+            core/ai/mcp/Dockerfile \
+            core/connectors/runtime/Dockerfile \
+            core/bench/dashboard/server/Dockerfile
+
+          /tmp/copy-latest-from-master.sh apply
 
       - name: Ensure version extractor is executable
         run: |
diff --git a/scripts/copy-latest-from-master.sh 
b/scripts/copy-latest-from-master.sh
index 79ab9bee..51625736 100755
--- a/scripts/copy-latest-from-master.sh
+++ b/scripts/copy-latest-from-master.sh
@@ -51,13 +51,26 @@ case "$COMMAND" in
             echo "Usage: $0 save <path1> [path2] ..."
             exit 1
         fi
-        
+
         echo "📦 Cloning master branch..."
         rm -rf "$TMP_REPO" "$TMP_SAVED"
+
+        # Use GITHUB_REPOSITORY if available, otherwise try to detect from git 
remote
+        if [ -z "${GITHUB_REPOSITORY:-}" ]; then
+            # Try to extract from git remote
+            REPO_URL=$(git remote get-url origin 2>/dev/null || echo "")
+            if [[ "$REPO_URL" =~ github\.com[:/]([^/]+/[^/.]+)(\.git)?$ ]]; 
then
+                GITHUB_REPOSITORY="${BASH_REMATCH[1]}"
+            else
+                echo "Error: Could not determine repository. Set 
GITHUB_REPOSITORY environment variable."
+                exit 1
+            fi
+        fi
+
         git clone --depth 1 --branch master 
"https://github.com/${GITHUB_REPOSITORY}.git"; "$TMP_REPO"
-        
+
         mkdir -p "$TMP_SAVED"
-        
+
         echo "💾 Saving specified files from master..."
         for path in "$@"; do
             if [ -e "$TMP_REPO/$path" ]; then
@@ -66,7 +79,7 @@ case "$COMMAND" in
                 if [ "$parent_dir" != "." ]; then
                     mkdir -p "$TMP_SAVED/$parent_dir"
                 fi
-                
+
                 # Copy the file or directory
                 cp -r "$TMP_REPO/$path" "$TMP_SAVED/$path"
                 echo "  ✅ Saved: $path"
@@ -74,50 +87,50 @@ case "$COMMAND" in
                 echo "  ⚠️  Not found in master: $path"
             fi
         done
-        
+
         echo ""
         echo "📋 Summary of saved files:"
         find "$TMP_SAVED" -type f | sed "s|$TMP_SAVED/|  - |"
-        
+
         # Clean up the cloned repo
         rm -rf "$TMP_REPO"
         ;;
-        
+
     apply)
         if [ ! -d "$TMP_SAVED" ]; then
             echo "Error: No saved files found. Run 'save' command first."
             exit 1
         fi
-        
+
         echo "📝 Applying saved files to current directory..."
-        
+
         # Iterate through all saved files
         find "$TMP_SAVED" -type f | while read -r saved_file; do
             # Get relative path by removing the tmp prefix
             rel_path="${saved_file#$TMP_SAVED/}"
             dest_dir=$(dirname "$rel_path")
-            
+
             # Check if destination directory exists
             if [ "$dest_dir" != "." ] && [ ! -d "$dest_dir" ]; then
                 echo "  ℹ️  Skipping $rel_path - destination directory 
'$dest_dir' does not exist"
                 continue
             fi
-            
+
             # Copy the file
             cp "$saved_file" "$rel_path"
             echo "  ✅ Applied: $rel_path"
-            
+
             # Make scripts executable if they're in scripts/ directory
             if [[ "$rel_path" == scripts/*.sh ]]; then
                 chmod +x "$rel_path"
                 echo "     Made executable: $rel_path"
             fi
         done
-        
+
         echo ""
         echo "✅ All applicable files have been applied"
         ;;
-        
+
     *)
         echo "Error: Unknown command '$COMMAND'"
         echo "Valid commands: save, apply"