This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ray.git
The following commit(s) were added to refs/heads/main by this push:
new 74ec45c Adding Datafusion Ray tests on an Kind cluster using the Helm
chart (#35)
74ec45c is described below
commit 74ec45c01196e81835c470f5d3ba4c2bc94d9e1c
Author: Edmondo Porcu <[email protected]>
AuthorDate: Tue Oct 22 11:27:46 2024 -0400
Adding Datafusion Ray tests on an Kind cluster using the Helm chart (#35)
* Adding Datafusion Ray tests on an Kind cluster using the Helm chart
* Using hardcoded hashes for actions
* Pinning dependencies for main
* Cheating about the semver regexp
* Restored previous version of the action
* Empty commit
* Updated to latest version of free disk space
---
.github/workflows/k8s.yml | 64 ++++++++++++++++++++++++++++++++++++++++++++++-
k8s/kind-config.yaml | 19 ++++++++++++++
2 files changed, 82 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml
index 7f05cc6..7b4d0bd 100644
--- a/.github/workflows/k8s.yml
+++ b/.github/workflows/k8s.yml
@@ -20,6 +20,11 @@ on:
push:
pull_request:
+env:
+ RAY_HELM_VERSION: 1.1.0
+ PYTHON_VERSION: 3.9
+ IMAGE_REPOSITORY: ghcr.io/apache/datafusion-ray
+
jobs:
build:
runs-on: ubuntu-latest
@@ -32,7 +37,7 @@ jobs:
uses: docker/metadata-action@v5
with:
images: |
- ghcr.io/apache/datafusion-ray
+ ${{ env.IMAGE_REPOSITORY }}
tags: |
type=ref,event=branch
type=ref,event=pr
@@ -40,6 +45,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=sha
+
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@@ -59,3 +65,59 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
push: ${{ startsWith(github.ref, 'refs/tags/') }}
+ load: true
+
+ - name: Kind Cluster
+ uses: helm/[email protected]
+ with:
+ config: ./k8s/kind-config.yaml
+
+ - name: Free Disk Space (Ubuntu)
+ uses: jlumbroso/[email protected]
+ with:
+ # this might remove tools that are actually needed,
+ # when set to "true" but frees about 6 GB
+ tool-cache: true
+ docker-images: false
+
+ - name: Extract short SHA
+ run: |
+ SHORT_SHA=$(echo "${GITHUB_SHA}" | cut -c1-7)
+ echo "IMAGE_TAG=sha-${SHORT_SHA}" >> "$GITHUB_ENV"
+ echo "Extracted short SHA tag: sha-${SHORT_SHA}"
+
+ - name: Load Docker image into Kind cluster
+ run: |
+ kind load docker-image ${{ env.IMAGE_REPOSITORY }}:${{ env.IMAGE_TAG
}} -n chart-testing
+
+ - name: Install Helm
+ uses: azure/[email protected]
+ with:
+ version: "3.16.0"
+
+ - name: Deploy helm chart
+ run: |
+ helm repo add kuberay https://ray-project.github.io/kuberay-helm/
+ helm install kuberay-operator kuberay/kuberay-operator --version ${{
env.RAY_HELM_VERSION }}
+ helm install raycluster kuberay/ray-cluster --version ${{
env.RAY_HELM_VERSION }} \
+ --set image.repository=${{ env.IMAGE_REPOSITORY }} \
+ --set image.tag=${{ env.IMAGE_TAG }} \
+ --set imagePullPolicy=Always
+ echo "Deployed Ray cluster with image repository: ${{
env.IMAGE_REPOSITORY }} and tag: ${{ env.IMAGE_TAG }}"
+
+ - name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ env.PYTHON_VERSION }}
+
+ - name: Install python dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements-in.txt
+
+ - name: Submit an example ray Job
+ run: |
+ kubectl port-forward service/raycluster-kuberay-head-svc 8265:8265 &
+ export RAY_ADDRESS="http://127.0.0.1:8265";
+ pip install "ray[default]"
+ ray job submit --working-dir ./examples/ -- python3 tips.py
diff --git a/k8s/kind-config.yaml b/k8s/kind-config.yaml
new file mode 100644
index 0000000..0979d75
--- /dev/null
+++ b/k8s/kind-config.yaml
@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+ - role: control-plane
+ extraMounts:
+ - hostPath: /tmp/kind-node-disk
+ containerPath: /var/lib/containerd
+ - role: worker
+ extraMounts:
+ - hostPath: /tmp/kind-node-disk1
+ containerPath: /var/lib/containerd
+ - role: worker
+ extraMounts:
+ - hostPath: /tmp/kind-node-disk2
+ containerPath: /var/lib/containerd
+ - role: worker
+ extraMounts:
+ - hostPath: /tmp/kind-node-disk3
+ containerPath: /var/lib/containerd
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
