This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch gha-arc-rebase in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit dd4e18cb64b6ea236b5e3cf999b30ecbfbbfdd1b Author: Hussein Awala <[email protected]> AuthorDate: Fri Jul 21 02:49:43 2023 +0200 Create a role for the AS deployment and add tags to the ASGs --- terraform/eks/asg_tags.tf | 67 +++++++++++++++++++++++++++++++++++++++++ terraform/eks/eks.tf | 18 +++++------ terraform/eks/iam.tf | 75 ++++++++++++++++++++++++++++++++++++++++++++++ terraform/eks/provider.tf | 4 +-- terraform/eks/variables.tf | 6 ++++ 5 files changed, 159 insertions(+), 11 deletions(-) diff --git a/terraform/eks/asg_tags.tf b/terraform/eks/asg_tags.tf new file mode 100644 index 0000000..3c05429 --- /dev/null +++ b/terraform/eks/asg_tags.tf @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +###################################################################################################### +# In order to scale the node groups from zero, we need to add the labels and taints to the ASGs tags: +# k8s.io/cluster-autoscaler/node-template/label/<label-key> = <label-value> +# k8s.io/cluster-autoscaler/node-template/taint/<taint-key> = <taint-value>:<taint-effect> +###################################################################################################### + +locals { + + taint_effects = { + NO_SCHEDULE = "NoSchedule" + NO_EXECUTE = "NoExecute" + PREFER_NO_SCHEDULE = "PreferNoSchedule" + } + + cluster_autoscaler_label_tags = merge([ + for name, group in module.eks.eks_managed_node_groups : { + for label_name, label_value in coalesce(group.node_group_labels, {}) : "${name}|label|${label_name}" => { + autoscaling_group = group.node_group_autoscaling_group_names[0], + key = "k8s.io/cluster-autoscaler/node-template/label/${label_name}", + value = label_value, + } + } + ]...) + + cluster_autoscaler_taint_tags = merge([ + for name, group in module.eks.eks_managed_node_groups : { + for taint in coalesce(group.node_group_taints, []) : "${name}|taint|${taint.key}" => { + autoscaling_group = group.node_group_autoscaling_group_names[0], + key = "k8s.io/cluster-autoscaler/node-template/taint/${taint.key}" + value = "${taint.value}:${local.taint_effects[taint.effect]}" + } + } + ]...) + + cluster_autoscaler_asg_tags = merge(local.cluster_autoscaler_label_tags, local.cluster_autoscaler_taint_tags) +} + +resource "aws_autoscaling_group_tag" "cluster_autoscaler_label_tags" { + for_each = local.cluster_autoscaler_asg_tags + + autoscaling_group_name = each.value.autoscaling_group + + tag { + key = each.value.key + value = each.value.value + + propagate_at_launch = true + } +} + diff --git a/terraform/eks/eks.tf b/terraform/eks/eks.tf index e130df7..5f239c8 100644 --- a/terraform/eks/eks.tf +++ b/terraform/eks/eks.tf @@ -28,8 +28,8 @@ module "eks" { cluster_name = local.cluster_name cluster_version = "1.27" - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets cluster_endpoint_public_access = true @@ -44,10 +44,10 @@ module "eks" { instance_types = ["t4g.small"] - min_size = 1 - max_size = 3 + min_size = 1 + max_size = 3 - capacity_type = "SPOT" + capacity_type = "SPOT" labels = { "node-type" = "default" @@ -63,11 +63,11 @@ module "eks" { max_size = 3 desired_size = 0 - capacity_type = "SPOT" + capacity_type = "SPOT" labels = { "node-type" = "gha-runners" - "size" = "small" + "size" = "small" } taints = [ @@ -88,11 +88,11 @@ module "eks" { max_size = 3 desired_size = 0 - capacity_type = "SPOT" + capacity_type = "SPOT" labels = { "node-type" = "gha-runners" - "size" = "medium" + "size" = "medium" } taints = [ diff --git a/terraform/eks/iam.tf b/terraform/eks/iam.tf new file mode 100644 index 0000000..dacc5f5 --- /dev/null +++ b/terraform/eks/iam.tf @@ -0,0 +1,75 @@ +data "aws_iam_policy_document" "autoscaler_policy_document" { + statement { + effect = "Allow" + actions = [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplateVersions" + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:DescribeImages", + "ec2:GetInstanceTypesFromInstanceRequirements", + "eks:DescribeNodegroup" + ] + resources = ["*"] + } +} + +resource "aws_iam_policy" "autoscaler_policy" { + name = "eks-autoscaler-policy" + description = "EKS Autoscaler Policy" + policy = data.aws_iam_policy_document.autoscaler_policy_document.json +} + +data "aws_iam_policy_document" "autoscaler_role_assume_policy_document" { + statement { + actions = [ + "sts:AssumeRoleWithWebIdentity" + ] + + principals { + type = "Federated" + identifiers = [ + module.eks.oidc_provider_arn + ] + } + + condition { + test = "StringEquals" + variable = "${module.eks.oidc_provider}:sub" + values = [ + "system:serviceaccount:infra:cluster-autoscaler" + ] + } + + condition { + test = "StringEquals" + variable = "${module.eks.oidc_provider}:aud" + values = [ + "sts.amazonaws.com" + ] + } + + } +} + +resource "aws_iam_role" "autoscaler_role" { + name = "eks-autoscaler-role" + assume_role_policy = data.aws_iam_policy_document.autoscaler_role_assume_policy_document.json +} + +resource "aws_iam_role_policy_attachment" "autoscaler_policy_attachment" { + role = aws_iam_role.autoscaler_role.name + policy_arn = aws_iam_policy.autoscaler_policy.arn +} \ No newline at end of file diff --git a/terraform/eks/provider.tf b/terraform/eks/provider.tf index c4c55f9..585ac24 100644 --- a/terraform/eks/provider.tf +++ b/terraform/eks/provider.tf @@ -16,6 +16,6 @@ # under the License. provider "aws" { - profile = "airflow" - region = "us-east-2" + profile = "airflow" + region = "us-east-2" } \ No newline at end of file diff --git a/terraform/eks/variables.tf b/terraform/eks/variables.tf index eaf84b6..f39442d 100644 --- a/terraform/eks/variables.tf +++ b/terraform/eks/variables.tf @@ -20,3 +20,9 @@ variable "region" { type = string default = "us-east-2" } + +variable "account_id" { + description = "AWS account ID" + type = string + default = "827901512104" +} \ No newline at end of file
