This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch gha-arc-rebase in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit c4e8315489c19f7d94c96bf9e437d99c1ec8e354 Author: hussein-awala <[email protected]> AuthorDate: Fri Aug 30 01:28:39 2024 +0200 switch to Autoscaling Runner Scale Sets mode --- README.rst | 2 +- helm/{ci-helmfile.yaml => ci.yaml} | 22 ++-- helm/infra-helmfile.yaml | 9 ++ helm/runners-config/templates/runners.yaml | 71 ------------- helm/runners-config/values.yaml | 117 --------------------- .../Chart.yaml => values/ebs-csi-dirver.yaml} | 10 +- ...r.yaml => gha-runner-scale-set-controller.yaml} | 4 +- .../gha-runner-scale-sets/common.yaml.gotmpl | 61 +++++++++++ .../runners.yaml} | 42 ++++++-- runner/Dockerfile | 52 +++++---- terraform/eks/eks.tf | 9 +- terraform/eks/iam.tf | 44 +++++++- terraform/eks/variables.tf | 17 +-- 13 files changed, 212 insertions(+), 248 deletions(-) diff --git a/README.rst b/README.rst index aa47573..12fa8ef 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ you need to follow the steps below: 3. Update kubeconfig to access the EKS cluster:: - AWS_PROFILE=airflow aws eks --region us-east-2 update-kubeconfig --name airflow + AWS_PROFILE=airflow aws eks --region us-east-2 update-kubeconfig --name airflow --alias airflow This command will update the kubeconfig file located at `~/.kube/config` to include the EKS cluster. You can verify that the cluster is added by running `kubectl get nodes`. diff --git a/helm/ci-helmfile.yaml b/helm/ci.yaml similarity index 62% rename from helm/ci-helmfile.yaml rename to helm/ci.yaml index eda1b4b..e1da5a0 100644 --- a/helm/ci-helmfile.yaml +++ b/helm/ci.yaml @@ -15,18 +15,24 @@ # specific language governing permissions and limitations # under the License. --- -repositories: - - name: actions-runner-controller - url: https://actions-runner-controller.github.io/actions-runner-controller +environments: + default: + values: + - ./values/gha-runner-scale-sets/runners.yaml releases: - name: actions-runner-controller - chart: actions-runner-controller/actions-runner-controller + chart: oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller namespace: ci - version: 0.23.7 + version: 0.9.3 values: - - ./values/actions-runner-controller.yaml + - ./values/gha-runner-scale-set-controller.yaml - - name: runners-config - chart: ./runners-config + {{- range $key, $value := .Values.runnerScaleSets }} + - name: {{ $key }} + chart: oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set namespace: ci + version: 0.9.3 + values: + - ./values/gha-runner-scale-sets/common.yaml.gotmpl + {{- end }} diff --git a/helm/infra-helmfile.yaml b/helm/infra-helmfile.yaml index f69bc41..d8c49e5 100644 --- a/helm/infra-helmfile.yaml +++ b/helm/infra-helmfile.yaml @@ -20,12 +20,21 @@ repositories: url: https://charts.jetstack.io - name: autoscaler url: https://kubernetes.github.io/autoscaler + - name: aws-ebs-csi-driver + url: https://kubernetes-sigs.github.io/aws-ebs-csi-driver releases: - name: cluster-config chart: ./cluster-config namespace: kube-system + # https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/charts/aws-ebs-csi-driver + - name: aws-ebs-csi-driver + chart: aws-ebs-csi-driver/aws-ebs-csi-driver + namespace: kube-system + values: + - ./values/ebs-csi-dirver.yaml + # https://cert-manager.io/docs/installation/helm/ - name: cert-manager chart: jetstack/cert-manager diff --git a/helm/runners-config/templates/runners.yaml b/helm/runners-config/templates/runners.yaml deleted file mode 100644 index b11257f..0000000 --- a/helm/runners-config/templates/runners.yaml +++ /dev/null @@ -1,71 +0,0 @@ -{{/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/}} - -################################################################################ -# Airflow GitHub Actions Runner Deployments and HRAutoscaler -################################################################################ -{{ $common := .Values.common }} -{{ $values := .Values }} -{{ range $runner := $values.runners }} ---- -apiVersion: actions.summerwind.dev/v1alpha1 -kind: RunnerDeployment -metadata: - name: {{ $runner.name }}-runners-deployment -spec: - template: - spec: - repository: {{ $values.githubRepository }} - nodeSelector: - {{- $runner.nodeSelector | toYaml | nindent 8 }} - tolerations: - {{- $runner.tolerations | toYaml | nindent 8 }} - labels: - {{- $common.runnersLabels | toYaml | nindent 8 }} - {{- $runner.labels | toYaml | nindent 8 }} - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: runner-deployment-name - operator: In - values: - - {{ $runner.name }}-runners-deployment - topologyKey: kubernetes.io/hostname - env: - - name: DISABLE_RUNNER_UPDATE - value: "false" # TODO: Set to true to disable runner update after patching the latest version ---- -apiVersion: actions.summerwind.dev/v1alpha1 -kind: HorizontalRunnerAutoscaler -metadata: - name: {{ $runner.name }}-runners-deployment-autoscaler -spec: - scaleDownDelaySecondsAfterScaleOut: {{ $values.scaleDownDelaySecondsAfterScaleOut }} - scaleTargetRef: - kind: RunnerDeployment - name: {{ $runner.name }}-runners-deployment - minReplicas: {{ $runner.minReplicas }} - maxReplicas: {{ $runner.maxReplicas }} - metrics: - - type: TotalNumberOfQueuedAndInProgressWorkflowRuns - repositoryNames: - - {{ $values.githubRepository }} -{{ end }} diff --git a/helm/runners-config/values.yaml b/helm/runners-config/values.yaml deleted file mode 100644 index e22ed43..0000000 --- a/helm/runners-config/values.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. ---- -githubRepository: apache/airflow - -scaleDownDelaySecondsAfterScaleOut: 30 - -common: - runnersLabels: - - airflow-runner - - eks-runner - -runners: - - name: small - minReplicas: 0 - maxReplicas: 30 - labels: - - small - - arm64 - nodeSelector: - node-type: gha-runners - size: small - arch: arm64 - tolerations: - - key: node-type - operator: Equal - value: gha-runners - effect: NoSchedule - - name: medium - minReplicas: 0 - maxReplicas: 30 - labels: - - medium - - arm64 - nodeSelector: - node-type: gha-runners - size: medium - arch: arm64 - tolerations: - - key: node-type - operator: Equal - value: gha-runners - effect: NoSchedule - - name: large - minReplicas: 0 - maxReplicas: 30 - labels: - - large - - arm64 - nodeSelector: - node-type: gha-runners - size: large - arch: arm64 - tolerations: - - key: node-type - operator: Equal - value: gha-runners - effect: NoSchedule - - name: small-x64 - minReplicas: 0 - maxReplicas: 30 - labels: - - small - - X64 - nodeSelector: - node-type: gha-runners - size: small - arch: x64 - tolerations: - - key: node-type - operator: Equal - value: gha-runners - effect: NoSchedule - - name: medium-x64 - minReplicas: 0 - maxReplicas: 30 - labels: - - medium - - X64 - nodeSelector: - node-type: gha-runners - size: medium - arch: x64 - tolerations: - - key: node-type - operator: Equal - value: gha-runners - effect: NoSchedule - - name: large-x64 - minReplicas: 0 - maxReplicas: 30 - labels: - - large - - X64 - nodeSelector: - node-type: gha-runners - size: large - arch: x64 - tolerations: - - key: node-type - operator: Equal - value: gha-runners - effect: NoSchedule diff --git a/helm/runners-config/Chart.yaml b/helm/values/ebs-csi-dirver.yaml similarity index 84% rename from helm/runners-config/Chart.yaml rename to helm/values/ebs-csi-dirver.yaml index 7113941..7f8f0d0 100644 --- a/helm/runners-config/Chart.yaml +++ b/helm/values/ebs-csi-dirver.yaml @@ -15,9 +15,7 @@ # specific language governing permissions and limitations # under the License. --- -apiVersion: v2 -name: runners-config -description: A Helm chart to manage the ARC runners -type: application -version: 0.1.0 -appVersion: "1.16.0" +controller: + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::827901512104:role/ebs-csi-driver diff --git a/helm/values/actions-runner-controller.yaml b/helm/values/gha-runner-scale-set-controller.yaml similarity index 94% copy from helm/values/actions-runner-controller.yaml copy to helm/values/gha-runner-scale-set-controller.yaml index bb5786f..63ab994 100644 --- a/helm/values/actions-runner-controller.yaml +++ b/helm/values/gha-runner-scale-set-controller.yaml @@ -19,9 +19,7 @@ replicaCount: 1 labels: type: ci - app: actions-runner-controller - -syncPeriod: 10s + app: gha-runner-scale-set-controller nodeSelector: node-type: default diff --git a/helm/values/gha-runner-scale-sets/common.yaml.gotmpl b/helm/values/gha-runner-scale-sets/common.yaml.gotmpl new file mode 100644 index 0000000..9ed5238 --- /dev/null +++ b/helm/values/gha-runner-scale-sets/common.yaml.gotmpl @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +{{ $config := index .Values.runnerScaleSets .Release.Name }} + +githubConfigUrl: https://github.com/apache + +githubConfigSecret: github-config + +runnerGroup: airflow-ci + +minRunners: {{ $config.minRunners }} + +maxRunners: {{ $config.maxRunners }} + +containerMode: + type: "dind" + +template: + spec: + containers: + - name: runner + image: {{ .Values.image }} + command: [ "/home/runner/run.sh" ] + nodeSelector: + node-type: gha-runners + size: {{ $config.size }} + arch: {{ $config.arch }} + tolerations: + - key: node-type + operator: Equal + value: gha-runners + effect: NoSchedule + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: actions.github.com/scale-set-name + operator: In + values: + - {{ .Release.Name }} + topologyKey: kubernetes.io/hostname + +controllerServiceAccount: + name: actions-runner-controller-gha-rs-controller + namespace: {{ .Release.Namespace }} diff --git a/helm/values/actions-runner-controller.yaml b/helm/values/gha-runner-scale-sets/runners.yaml similarity index 52% rename from helm/values/actions-runner-controller.yaml rename to helm/values/gha-runner-scale-sets/runners.yaml index bb5786f..40268eb 100644 --- a/helm/values/actions-runner-controller.yaml +++ b/helm/values/gha-runner-scale-sets/runners.yaml @@ -15,13 +15,35 @@ # specific language governing permissions and limitations # under the License. --- -replicaCount: 1 - -labels: - type: ci - app: actions-runner-controller - -syncPeriod: 10s - -nodeSelector: - node-type: default +image: ghcr.io/apache/airflow-ci-infra/actions-runner:20240830-rc1 +runnerScaleSets: + arc-small-amd: + minRunners: 0 + maxRunners: 30 + size: small + arch: x64 + arc-medium-amd: + minRunners: 0 + maxRunners: 30 + size: medium + arch: x64 + arc-large-amd: + minRunners: 0 + maxRunners: 30 + size: large + arch: x64 + arc-small-arm: + minRunners: 0 + maxRunners: 30 + size: small + arch: arm64 + arc-medium-arm: + minRunners: 0 + maxRunners: 30 + size: medium + arch: arm64 + arc-large-arm: + minRunners: 0 + maxRunners: 30 + size: large + arch: arm64 diff --git a/runner/Dockerfile b/runner/Dockerfile index d1ee441..05daa82 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -1,27 +1,35 @@ -ARG ACTIONS_RUNNER_VERSION=v2.309.0-ubuntu-22.04-ead26ab - -FROM summerwind/actions-runner:${ACTIONS_RUNNER_VERSION} +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +FROM ghcr.io/actions/actions-runner:latest USER root -ARG AIRFLOW_RUNNER_VERSION=2.309.0-airflow11 - -RUN rm -rf $RUNNER_ASSETS_DIR/* \ - && export ARCH=$(arch) \ - && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "x86_64" ] || [ "$ARCH" = "i386" ]; then export ARCH=x64 ; fi \ - && if [ "$ARCH" = "aarch64" ]; then export ARCH=arm64 ; fi \ - && cd "$RUNNER_ASSETS_DIR" \ - && curl -fLo runner.tar.gz "https://github.com/ashb/runner/releases/download/v${AIRFLOW_RUNNER_VERSION}/actions-runner-linux-${ARCH}-${AIRFLOW_RUNNER_VERSION}.tar.gz"; \ - && tar xzf ./runner.tar.gz \ - && rm runner.tar.gz \ - && ./bin/installdependencies.sh \ - && mv ./externals ./externalstmp - -# TODO: Remove this once https://github.com/actions/setup-python/issues/705 is solved -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - curl nodejs wget unzip vim git jq build-essential libssl-dev libffi-dev python3 python3-venv python3-dev python3-pip python-is-python3 \ - && chown -R runner:runner /usr/local/lib/ --recursive \ - && chown -R runner:runner /usr/local/bin/ --recursive +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl nodejs npm wget unzip vim git jq build-essential netcat \ + libssl-dev libffi-dev python3 python3-venv python3-dev python3-pip python-is-python3 \ + && npm install -g yarn \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update \ + && apt-get install -y docker-compose-plugin USER runner \ No newline at end of file diff --git a/terraform/eks/eks.tf b/terraform/eks/eks.tf index adf6fdb..5b4203a 100644 --- a/terraform/eks/eks.tf +++ b/terraform/eks/eks.tf @@ -45,7 +45,14 @@ module "eks" { default_nodes = { name = "default" - instance_types = ["t4g.medium"] + instance_types = [ + "t4g.medium", + "t4g.large", + "t4g.xlarge", + "m7g.medium", + "m7g.large", + "m7g.xlarge", + ] min_size = 0 max_size = 5 diff --git a/terraform/eks/iam.tf b/terraform/eks/iam.tf index 983ee0e..e2c3000 100644 --- a/terraform/eks/iam.tf +++ b/terraform/eks/iam.tf @@ -89,4 +89,46 @@ resource "aws_iam_role" "autoscaler_role" { resource "aws_iam_role_policy_attachment" "autoscaler_policy_attachment" { role = aws_iam_role.autoscaler_role.name policy_arn = aws_iam_policy.autoscaler_policy.arn -} \ No newline at end of file +} + +data "aws_iam_policy_document" "ebs_csi_driver_assume_policy_document" { + statement { + actions = [ + "sts:AssumeRoleWithWebIdentity" + ] + + principals { + type = "Federated" + identifiers = [ + module.eks.oidc_provider_arn + ] + } + + condition { + test = "StringEquals" + variable = "${module.eks.oidc_provider}:sub" + values = [ + "system:serviceaccount:kube-system:ebs-csi-controller-sa" + ] + } + + condition { + test = "StringEquals" + variable = "${module.eks.oidc_provider}:aud" + values = [ + "sts.amazonaws.com" + ] + } + + } +} + +resource "aws_iam_role" "ebs_csi_driver" { + name = "ebs-csi-driver" + assume_role_policy = data.aws_iam_policy_document.ebs_csi_driver_assume_policy_document.json +} + +resource "aws_iam_role_policy_attachment" "ebs_csi_driver" { + role = aws_iam_role.ebs_csi_driver.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" +} diff --git a/terraform/eks/variables.tf b/terraform/eks/variables.tf index d04ce6b..045c496 100644 --- a/terraform/eks/variables.tf +++ b/terraform/eks/variables.tf @@ -24,20 +24,21 @@ variable "region" { variable "runners_node_types" { description = "Node type for the runners" type = list(string) - default = ["t4g"] + default = [ + "t4g", + "m7g", + "m6g", + ] } variable "x64_runners_node_types" { description = "X64 node type for the runners" type = list(string) default = [ - "r6a", - "r5", - "r5ad", - "r5n", - "r5b", - "r5a", - "r6i", + "m7a", + "m6a", + "t3", + "t3a", ] }
