This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch gha-arc-rebase in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit 0819da33c6b4c7360014737c3c94e6c874f1fc3e Author: Hussein Awala <[email protected]> AuthorDate: Tue Oct 24 00:07:26 2023 +0200 Create runners on x86 to simplify the migration and update the labels --- helm/ci-helmfile.yaml | 2 +- helm/cluster-config/values.yaml | 2 +- helm/infra-helmfile.yaml | 4 +- helm/runners-config/templates/runners.yaml | 4 +- helm/runners-config/values.yaml | 58 ++++++++++++++++++- helm/values/actions-runner-controller.yaml | 2 +- helm/values/cert-manager.yaml | 2 +- helm/values/cluster-autoscaler.yaml | 2 +- terraform/eks/eks.tf | 90 ++++++++++++++++++++++++++++++ terraform/eks/variables.tf | 21 +++++++ 10 files changed, 178 insertions(+), 9 deletions(-) diff --git a/helm/ci-helmfile.yaml b/helm/ci-helmfile.yaml index f291977..432b36a 100644 --- a/helm/ci-helmfile.yaml +++ b/helm/ci-helmfile.yaml @@ -23,7 +23,7 @@ releases: - name: actions-runner-controller chart: actions-runner-controller/actions-runner-controller namespace: ci - version: 0.23.3 + version: 0.23.5 values: - ./values/actions-runner-controller.yaml diff --git a/helm/cluster-config/values.yaml b/helm/cluster-config/values.yaml index d6bc559..5152985 100644 --- a/helm/cluster-config/values.yaml +++ b/helm/cluster-config/values.yaml @@ -14,4 +14,4 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ---- \ No newline at end of file +--- diff --git a/helm/infra-helmfile.yaml b/helm/infra-helmfile.yaml index c25a507..1bd9f5e 100644 --- a/helm/infra-helmfile.yaml +++ b/helm/infra-helmfile.yaml @@ -30,7 +30,7 @@ releases: - name: cert-manager chart: jetstack/cert-manager namespace: infra - version: v1.12.2 + version: v1.13.1 values: - ./values/cert-manager.yaml @@ -38,6 +38,6 @@ releases: - name: autoscaler chart: autoscaler/cluster-autoscaler namespace: infra - version: 9.29.1 + version: 9.29.3 values: - ./values/cluster-autoscaler.yaml diff --git a/helm/runners-config/templates/runners.yaml b/helm/runners-config/templates/runners.yaml index 78eaba5..b11257f 100644 --- a/helm/runners-config/templates/runners.yaml +++ b/helm/runners-config/templates/runners.yaml @@ -20,6 +20,7 @@ ################################################################################ # Airflow GitHub Actions Runner Deployments and HRAutoscaler ################################################################################ +{{ $common := .Values.common }} {{ $values := .Values }} {{ range $runner := $values.runners }} --- @@ -36,6 +37,7 @@ spec: tolerations: {{- $runner.tolerations | toYaml | nindent 8 }} labels: + {{- $common.runnersLabels | toYaml | nindent 8 }} {{- $runner.labels | toYaml | nindent 8 }} affinity: podAntiAffinity: @@ -66,4 +68,4 @@ spec: - type: TotalNumberOfQueuedAndInProgressWorkflowRuns repositoryNames: - {{ $values.githubRepository }} -{{ end }} \ No newline at end of file +{{ end }} diff --git a/helm/runners-config/values.yaml b/helm/runners-config/values.yaml index c3f77a4..404cab7 100644 --- a/helm/runners-config/values.yaml +++ b/helm/runners-config/values.yaml @@ -19,15 +19,22 @@ githubRepository: apache/airflow scaleDownDelaySecondsAfterScaleOut: 30 +common: + runnersLabels: + - airflow-runner + - eks-runner + runners: - name: small minReplicas: 0 maxReplicas: 30 labels: - small + - arm64 nodeSelector: node-type: gha-runners size: small + arch: arm64 tolerations: - key: node-type operator: Equal @@ -38,9 +45,11 @@ runners: maxReplicas: 30 labels: - medium + - arm64 nodeSelector: node-type: gha-runners size: medium + arch: arm64 tolerations: - key: node-type operator: Equal @@ -51,11 +60,58 @@ runners: maxReplicas: 30 labels: - large + - arm64 + nodeSelector: + node-type: gha-runners + size: large + arch: arm64 + tolerations: + - key: node-type + operator: Equal + value: gha-runners + effect: NoSchedule + - name: small-x86 + minReplicas: 0 + maxReplicas: 30 + labels: + - small + - x86 + nodeSelector: + node-type: gha-runners + size: small + arch: x86 + tolerations: + - key: node-type + operator: Equal + value: gha-runners + effect: NoSchedule + - name: medium-x86 + minReplicas: 0 + maxReplicas: 30 + labels: + - medium + - x86 + nodeSelector: + node-type: gha-runners + size: medium + arch: x86 + tolerations: + - key: node-type + operator: Equal + value: gha-runners + effect: NoSchedule + - name: large-x86 + minReplicas: 0 + maxReplicas: 30 + labels: + - large + - x86 nodeSelector: node-type: gha-runners size: large + arch: x86 tolerations: - key: node-type operator: Equal value: gha-runners - effect: NoSchedule \ No newline at end of file + effect: NoSchedule diff --git a/helm/values/actions-runner-controller.yaml b/helm/values/actions-runner-controller.yaml index 291e076..a131154 100644 --- a/helm/values/actions-runner-controller.yaml +++ b/helm/values/actions-runner-controller.yaml @@ -28,4 +28,4 @@ nodeSelector: image: actionsRunnerRepositoryAndTag: public.ecr.aws/u9s5q9f7/airflow-gha-runner:2.304.0-airflow8 - dindSidecarRepositoryAndTag: public.ecr.aws/u9s5q9f7/docker:dind \ No newline at end of file + dindSidecarRepositoryAndTag: public.ecr.aws/u9s5q9f7/docker:dind diff --git a/helm/values/cert-manager.yaml b/helm/values/cert-manager.yaml index 668cde0..be548e5 100644 --- a/helm/values/cert-manager.yaml +++ b/helm/values/cert-manager.yaml @@ -17,4 +17,4 @@ --- installCRDs: true nodeSelector: - node-type: default \ No newline at end of file + node-type: default diff --git a/helm/values/cluster-autoscaler.yaml b/helm/values/cluster-autoscaler.yaml index c955a7c..2b4bf57 100644 --- a/helm/values/cluster-autoscaler.yaml +++ b/helm/values/cluster-autoscaler.yaml @@ -32,4 +32,4 @@ rbac: extraArgs: scan-interval: 10s scale-down-delay-after-add: 5m - scale-down-unneeded-time: 5m \ No newline at end of file + scale-down-unneeded-time: 5m diff --git a/terraform/eks/eks.tf b/terraform/eks/eks.tf index 8d657ab..2776204 100644 --- a/terraform/eks/eks.tf +++ b/terraform/eks/eks.tf @@ -58,6 +58,7 @@ module "eks" { } } + # GHA arm runners' nodes GHA_runners_small = { name = "gha-runners-small" @@ -73,6 +74,7 @@ module "eks" { labels = { "node-type" = "gha-runners" "size" = "small" + "arch" = "arm64" } taints = [ @@ -99,6 +101,7 @@ module "eks" { labels = { "node-type" = "gha-runners" "size" = "medium" + "arch" = "arm64" } taints = [ @@ -123,6 +126,93 @@ module "eks" { labels = { "node-type" = "gha-runners" "size" = "large" + "arch" = "arm64" + } + + taints = [ + { + "key" = "node-type" + "value" = "gha-runners" + "effect" = "NO_SCHEDULE" + } + ] + } + + # GHA x86 runners' nodes + x86_GHA_runners_small = { + name = "gha-x86-runners-small" + + ami_type = "AL2_x86_64" + + instance_types = [for node_type in var.x86_runners_node_types: "${node_type}.${var.small_runners_x86_node_size}"] + + min_size = 0 + max_size = 30 + + capacity_type = "SPOT" + + disk_size = 50 + + labels = { + "node-type" = "gha-runners" + "size" = "small" + "arch" = "x86" + } + + taints = [ + { + "key" = "node-type" + "value" = "gha-runners" + "effect" = "NO_SCHEDULE" + } + ] + } + + x86_GHA_runners_medium = { + name = "gha-x86-runners-medium" + + ami_type = "AL2_x86_64" + + instance_types = [for node_type in var.x86_runners_node_types: "${node_type}.${var.medium_runners_node_size}"] + + min_size = 0 + max_size = 30 + + capacity_type = "SPOT" + + disk_size = 50 + + labels = { + "node-type" = "gha-runners" + "size" = "medium" + "arch" = "x86" + } + + taints = [ + { + "key" = "node-type" + "value" = "gha-runners" + "effect" = "NO_SCHEDULE" + } + ] + } + + x86_GHA_runners_large = { + name = "gha-x86-runners-large" + + ami_type = "AL2_x86_64" + + instance_types = [for node_type in var.x86_runners_node_types: "${node_type}.${var.large_runners_node_size}"] + + min_size = 0 + max_size = 30 + + capacity_type = "SPOT" + + labels = { + "node-type" = "gha-runners" + "size" = "large" + "arch" = "x86" } taints = [ diff --git a/terraform/eks/variables.tf b/terraform/eks/variables.tf index a6cdff7..52f0e67 100644 --- a/terraform/eks/variables.tf +++ b/terraform/eks/variables.tf @@ -33,12 +33,33 @@ variable "runners_node_types" { default = ["t4g"] } +variable "x86_runners_node_types" { + description = "X86 node type for the runners" + type = list(string) + default = [ + "r6a", + "r5", + "r5ad", + "r5n", + "r5b", + "r5a", + "r6i", + ] +} + variable "small_runners_node_size" { description = "Node size for the small runners" type = string default = "medium" } +variable "small_runners_x86_node_size" { + # This is a separate variable because the r series doesn't have a medium size + description = "Node size for the small runners" + type = string + default = "large" +} + variable "medium_runners_node_size" { description = "Node size for the medium runners" type = string
