This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch runs-on-flex-three-zones in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit 5add067f532f51874bd4f3e338713cc63135ff6d Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Oct 31 10:27:21 2026 +0100 Run the RunsOn runners in all three us-east-2 zones from RunsOn's Terraform module The CloudFormation template only uses the first two zones, use2-az1 and use2-az2, whose Spot placement score for the runners' families is 1 against 9 in use2-az3; 5.6% of the Spot runners were reclaimed from 2026-10-25 to 2026-10-30. The template has no parameter for the zones, so a second stack from the Flex module runs beside it until the jobs move with an env=flex label. Generated-by: Claude Opus 5 --- .github/workflows/terraform-runs-on-flex.yml | 116 +++++++++++++ terraform/runs-on-access/runs-on-flex.tf | 239 +++++++++++++++++++++++++++ terraform/runs-on-flex/README.rst | 129 +++++++++++++++ terraform/runs-on-flex/main.tf | 59 +++++++ terraform/runs-on-flex/network.tf | 89 ++++++++++ terraform/runs-on-flex/outputs.tf | 36 ++++ terraform/runs-on-flex/provider.tf | 32 ++++ terraform/runs-on-flex/state.tf | 27 +++ terraform/runs-on-flex/variables.tf | 66 ++++++++ terraform/runs-on-flex/version.tf | 28 ++++ 10 files changed, 821 insertions(+) diff --git a/.github/workflows/terraform-runs-on-flex.yml b/.github/workflows/terraform-runs-on-flex.yml new file mode 100644 index 0000000..9981f49 --- /dev/null +++ b/.github/workflows/terraform-runs-on-flex.yml @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +# Plans terraform/runs-on-flex on pull requests and on main; applies the plan made on main once a +# required reviewer approves the runs-on-prod environment. Same flow as terraform-runs-on.yml; the +# AWS roles live in terraform/runs-on-access and are applied by hand. +name: Terraform (RunsOn Flex) + +on: # yamllint disable-line rule:truthy + pull_request: + paths: + - "terraform/runs-on-flex/*.tf" + - "terraform/runs-on-flex/.terraform.lock.hcl" + - ".github/workflows/terraform-runs-on-flex.yml" + # Pull requests only: a plan proves the plan role still admits pull requests after an account + # admin applied the changed trust policies by hand. On main the change needs no apply. + - "terraform/runs-on-access/*.tf" + push: + branches: [main] + paths: + - "terraform/runs-on-flex/*.tf" + - "terraform/runs-on-flex/.terraform.lock.hcl" + - ".github/workflows/terraform-runs-on-flex.yml" + +permissions: {} + +concurrency: + group: terraform-runs-on-flex-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + AWS_REGION: us-east-2 + TF_IN_AUTOMATION: "true" + TF_INPUT: "false" + +jobs: + plan: + # Fork pull requests get no OIDC token, so they cannot plan. + if: >- + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-slim + permissions: + contents: read + id-token: write + defaults: + run: + working-directory: terraform/runs-on-flex + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: arn:aws:iam::827901512104:role/runs-on-terraform-flex-plan + aws-region: ${{ env.AWS_REGION }} + - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: 1.16.2 + terraform_wrapper: false + - run: terraform init -lockfile=readonly + # The plan role cannot write the lock file; apply takes the lock and refuses a stale plan. + - run: terraform plan -lock=false -out=runs-on-flex.tfplan + - if: github.event_name == 'push' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: runs-on-flex-tfplan + path: terraform/runs-on-flex/runs-on-flex.tfplan + retention-days: 7 + + apply: + needs: plan + if: github.event_name == 'push' + # Not ubuntu-slim: its 15-minute job limit is shorter than creating the stack may take, and a + # killed apply leaves the state lock held. + runs-on: ubuntu-24.04 + # Required reviewers and the main-only branch policy come from .asf.yaml. + environment: runs-on-prod + permissions: + contents: read + id-token: write + defaults: + run: + working-directory: terraform/runs-on-flex + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: arn:aws:iam::827901512104:role/runs-on-terraform-flex-apply + aws-region: ${{ env.AWS_REGION }} + - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: 1.16.2 + terraform_wrapper: false + - run: terraform init -lockfile=readonly + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: runs-on-flex-tfplan + path: terraform/runs-on-flex + - run: terraform apply runs-on-flex.tfplan diff --git a/terraform/runs-on-access/runs-on-flex.tf b/terraform/runs-on-access/runs-on-flex.tf new file mode 100644 index 0000000..0fcbb99 --- /dev/null +++ b/terraform/runs-on-access/runs-on-flex.tf @@ -0,0 +1,239 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Roles of the Terraform (RunsOn Flex) workflow, which applies terraform/runs-on-flex: RunsOn's +# Terraform module instead of its CloudFormation template. No CloudFormation service role sits in +# between, so the apply role itself creates the infrastructure. It gets what the CloudFormation +# service role in main.tf has (AdministratorAccess with denies) and three limits more: it acts +# only in us-east-2 (IAM and Budgets are global), it reaches no S3 bucket but the stack's cache +# buckets and its own state, and every role it creates carries the runs-on-flex-boundary +# permissions boundary, so no role of the stack can be given more than the boundary allows. +locals { + flex_stack = "runs-on-flex" + flex_state = "arn:aws:s3:::airflow-ci-tfstate/tf-state/runs-on-flex.tfstate" + + # Every IAM role and instance profile of the module (v3.5.1) is named <stack>-<purpose>. AWS + # creates the service-linked roles on first use of ECS and Spot. + flex_iam_resources = [ + "arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*", + "arn:aws:iam::${local.account_id}:instance-profile/${local.flex_stack}-*", + "arn:aws:iam::${local.account_id}:role/aws-service-role/*", + ] + # The only AWS-managed policies the module attaches (runner SSM access, ECS task execution). + flex_managed_policies = [ + "arn:aws:iam::aws:policy/AmazonSSMManagedEC2InstanceDefaultPolicy", + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + ] + iam_write_actions = [ + "iam:Add*", "iam:Attach*", "iam:Change*", "iam:Create*", "iam:Deactivate*", "iam:Delete*", + "iam:Detach*", "iam:Enable*", "iam:PassRole", "iam:Put*", "iam:Remove*", "iam:Reset*", + "iam:Set*", "iam:Tag*", "iam:Untag*", "iam:Update*", "iam:Upload*", + ] + tfstate_bucket = ["arn:aws:s3:::airflow-ci-tfstate", "arn:aws:s3:::airflow-ci-tfstate/*"] +} + +# Named outside the stack's runs-on-flex-* prefix: the apply role may write IAM only under that +# prefix, and must not be able to remove its own guardrails. +resource "aws_iam_role" "flex_terraform" { + for_each = local.github_subjects + + name = "runs-on-terraform-flex-${each.key}" + description = "terraform ${each.key} of terraform/runs-on-flex from apache/airflow-ci-infra GitHub Actions" + assume_role_policy = data.aws_iam_policy_document.github_trust[each.key].json +} + +# Refreshing the module's ~150 resources reads across a dozen services, so plans get +# ReadOnlyAccess. Plans also run on pull requests, so the denies keep secrets out of reach: +# ReadOnlyAccess can read every S3 object (the other Terraform states included) and SSM +# parameters, which the aws/ssm key would decrypt. +resource "aws_iam_role_policy_attachment" "flex_plan_read_only" { + role = aws_iam_role.flex_terraform["plan"].name + policy_arn = "arn:aws:iam::aws:policy/ReadOnlyAccess" +} + +data "aws_iam_policy_document" "flex_plan_guardrails" { + statement { + sid = "OnlyThisStateObject" + effect = "Deny" + actions = ["s3:GetObject*"] + not_resources = [local.flex_state] + } + statement { + sid = "NoSsmDecrypt" + effect = "Deny" + actions = ["kms:Decrypt"] + resources = ["*"] + condition { + test = "StringLike" + variable = "kms:ViaService" + values = ["ssm.*.amazonaws.com"] + } + } + statement { + sid = "NoSecretValues" + effect = "Deny" + actions = ["secretsmanager:GetSecretValue", "secretsmanager:BatchGetSecretValue"] + resources = ["*"] + } +} + +resource "aws_iam_role_policy" "flex_plan_guardrails" { + name = "guardrails" + role = aws_iam_role.flex_terraform["plan"].name + policy = data.aws_iam_policy_document.flex_plan_guardrails.json +} + +# RunsOn documents only "admin rights" for deploying the stack, and the module's resource set +# changes between releases, so AdministratorAccess as for the CloudFormation service role. +resource "aws_iam_role_policy_attachment" "flex_apply" { + role = aws_iam_role.flex_terraform["apply"].name + policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess" +} + +data "aws_iam_policy_document" "flex_apply_guardrails" { + statement { + sid = "OnlyTheStackIam" + effect = "Deny" + actions = local.iam_write_actions + not_resources = local.flex_iam_resources + } + # A stack role without the boundary could be given any permission and then be passed to a Lambda + # function or an ECS task, which would lift every deny of this policy. + statement { + sid = "StackRolesKeepTheBoundary" + effect = "Deny" + actions = ["iam:CreateRole", "iam:PutRolePermissionsBoundary", "iam:PutRolePolicy", "iam:AttachRolePolicy"] + resources = ["arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*"] + condition { + test = "StringNotEquals" + variable = "iam:PermissionsBoundary" + values = [aws_iam_policy.flex_boundary.arn] + } + } + statement { + sid = "NoBoundaryRemoval" + effect = "Deny" + actions = ["iam:DeleteRolePermissionsBoundary"] + resources = ["*"] + } + statement { + sid = "OnlyTheModuleManagedPolicies" + effect = "Deny" + actions = ["iam:AttachRolePolicy"] + resources = ["*"] + condition { + test = "ArnNotEquals" + variable = "iam:PolicyARN" + values = local.flex_managed_policies + } + } + statement { + sid = "NoRoleHopping" + effect = "Deny" + actions = ["sts:AssumeRole", "sts:AssumeRoleWithWebIdentity", "sts:AssumeRoleWithSAML"] + resources = ["*"] + } + # Only the stack's cache bucket (bucket_prefix "<stack>-cache-") and this state and its lock file. + statement { + sid = "OnlyTheStackBuckets" + effect = "Deny" + actions = ["s3:*"] + not_resources = [ + "arn:aws:s3:::${local.flex_stack}-cache-*", + local.flex_state, + "${local.flex_state}.tflock", + "arn:aws:s3:::airflow-ci-tfstate", + ] + } + statement { + sid = "NoStateBucketChanges" + effect = "Deny" + actions = ["s3:Put*", "s3:Delete*"] + resources = ["arn:aws:s3:::airflow-ci-tfstate"] + } + statement { + sid = "OnlyTheRunnersRegion" + effect = "Deny" + not_actions = ["iam:*", "budgets:*", "sts:*"] + resources = ["*"] + condition { + test = "StringNotEquals" + variable = "aws:RequestedRegion" + values = ["us-east-2"] + } + } +} + +resource "aws_iam_role_policy" "flex_apply_guardrails" { + name = "guardrails" + role = aws_iam_role.flex_terraform["apply"].name + policy = data.aws_iam_policy_document.flex_apply_guardrails.json +} + +# The most any role of the stack may do, whatever its own policies say. The control plane passes +# the runner role to the instances it launches and creates the Spot service-linked role; nothing +# of the stack writes other IAM, assumes roles outside the stack or touches the Terraform states. +data "aws_iam_policy_document" "flex_boundary" { + statement { + sid = "Everything" + actions = ["*"] + resources = ["*"] + } + statement { + sid = "NoIamWrites" + effect = "Deny" + # PassRole and CreateServiceLinkedRole are limited below. + actions = [ + "iam:Add*", "iam:Attach*", "iam:Change*", "iam:CreateA*", "iam:CreateG*", "iam:CreateI*", + "iam:CreateL*", "iam:CreateO*", "iam:CreateP*", "iam:CreateR*", "iam:CreateSA*", + "iam:CreateServiceS*", "iam:CreateU*", "iam:CreateV*", "iam:Deactivate*", "iam:Delete*", + "iam:Detach*", "iam:Enable*", "iam:Put*", "iam:Remove*", "iam:Reset*", "iam:Set*", "iam:Tag*", + "iam:Untag*", "iam:Update*", "iam:Upload*", + ] + resources = ["*"] + } + statement { + sid = "PassOnlyStackRoles" + effect = "Deny" + actions = ["iam:PassRole"] + not_resources = ["arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*"] + } + statement { + sid = "OnlyServiceLinkedRoles" + effect = "Deny" + actions = ["iam:CreateServiceLinkedRole"] + not_resources = ["arn:aws:iam::${local.account_id}:role/aws-service-role/*"] + } + statement { + sid = "AssumeOnlyStackRoles" + effect = "Deny" + actions = ["sts:AssumeRole"] + not_resources = ["arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*"] + } + statement { + sid = "NoTerraformState" + effect = "Deny" + actions = ["s3:*"] + resources = local.tfstate_bucket + } +} + +resource "aws_iam_policy" "flex_boundary" { + name = "${local.flex_stack}-boundary" + description = "Permissions boundary of every IAM role the runs-on-flex stack creates" + policy = data.aws_iam_policy_document.flex_boundary.json +} diff --git a/terraform/runs-on-flex/README.rst b/terraform/runs-on-flex/README.rst new file mode 100644 index 0000000..7152b73 --- /dev/null +++ b/terraform/runs-on-flex/README.rst @@ -0,0 +1,129 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +RunsOn Flex — the runners from RunsOn's Terraform module +======================================================== + +This module deploys the same RunsOn runners as `terraform/runs-on <../runs-on/README.rst>`_, from +RunsOn's `Terraform module <https://github.com/runs-on/terraform-aws-runs-on>`_ (Flex, v3.5.1) +instead of its CloudFormation template, into a VPC this module creates with a public subnet in +each of the three us-east-2 zones. Both stacks run side by side until the jobs of +``apache/airflow`` have moved here; the CloudFormation stack is then removed in a separate change. + +Why this module +--------------- + +The CloudFormation template places runners in the first two zones of the region, use2-az1 and +use2-az2 for this account, and has no parameter to change that. On 2026-10-30 those two zones +had the worst Spot capacity in the region for the runners' instance families: + +================================= ========== ========== ========== +Spot placement score (1-10) use2-az1 use2-az2 use2-az3 +================================= ========== ========== ========== +x64, 8-16 vCPU (m5-m8 a/i) 1 1 9 +x64, 4-8 vCPU 2 1 8 +arm64, 8-64 vCPU (m6g-m8g) 1 1 3 +================================= ========== ========== ========== + +From 2026-10-25 to 2026-10-30 AWS reclaimed 911 of 16,145 Spot runners (5.6%), at the median +7 minutes after launch, and 30% of all runners ran on-demand. A third zone gives the Spot +allocator the pools where capacity is. + +Files +----- + +``network.tf`` + VPC (``vpc_cidr``, default ``10.3.0.0/16``), internet gateway, one public subnet per zone ID + and an S3 gateway endpoint. No NAT: runners get public IPs, as on the CloudFormation stack. +``main.tf`` + The ``runs-on/runs-on/aws//modules/flex`` module with the CloudFormation stack's settings, the + license-key lookup and the permissions boundary of the stack's roles. +``variables.tf`` + All variables have defaults: ``stack_name`` (``runs-on-flex``), ``runs_on_environment`` + (``flex``), ``github_organization``, ``alert_email``, ``enable_admin_routes`` (``false``) and + ``vpc_cidr``. +``state.tf`` + S3 backend ``airflow-ci-tfstate`` / ``tf-state/runs-on-flex.tfstate``. + +How it differs from the CloudFormation stack +-------------------------------------------- + +- **Environment.** The stack takes only jobs whose label carries ``env=flex``; the CloudFormation + stack keeps the default ``production`` and ignores them. That is what lets both run at once. +- **Who creates the resources.** The CloudFormation stack is applied by a role that can only + resubmit RunsOn's published template; CloudFormation's service role does the work. Here the + ``runs-on-terraform-flex-apply`` role creates everything itself, with AdministratorAccess and + the denies in `terraform/runs-on-access/runs-on-flex.tf <../runs-on-access/runs-on-flex.tf>`_: + IAM writes only on ``runs-on-flex-*`` roles and only with the ``runs-on-flex-boundary`` + permissions boundary, only the two AWS-managed policies the module attaches, no S3 but the + stack's cache bucket and this state, no role assumption, and us-east-2 only. Anything merged to + ``main`` here and approved in the ``runs-on-prod`` environment can therefore change any non-IAM + resource in us-east-2, not just this stack: review pull requests to this module as such. +- **Upgrades** bump the module ``version`` in ``main.tf`` (with the lock file), not a template + URL. Before a bump, check the new release's IAM role names and managed-policy attachments + against ``flex_iam_resources`` and ``flex_managed_policies`` in ``runs-on-access``. + +Setup +----- + +1. **Access (account admin, once).** Apply `terraform/runs-on-access <../runs-on-access/README.rst>`_: + it adds the ``runs-on-terraform-flex-plan`` and ``-apply`` roles and the + ``runs-on-flex-boundary`` policy. +2. **License key (manual, once).** The same RunsOn license as the CloudFormation stack, under + this stack's name. The module receives only the parameter's ARN, so the key never reaches + Terraform state: + + .. code-block:: bash + + aws ssm put-parameter --region us-east-2 \ + --name /runs-on-flex/license-key \ + --type SecureString \ + --description "RunsOn license key for the runs-on-flex stack (GitHub organization apache)" \ + --value 'RUNS-ON-LICENSE-KEY' + +3. **Create the stack.** Merge the pull request that adds this module; the + ``Terraform (RunsOn Flex)`` workflow plans on ``main`` and applies once a reviewer approves + ``runs-on-prod``. Set ``enable_admin_routes = true`` in that first change. +4. **GitHub App (with ASF Infra).** Open the ``entry_point`` output → **Register app**. This + creates a second private GitHub App for this stack; ASF Infra installs it on + ``apache/airflow`` (and ``.github-private``, for the presets). Then set + ``enable_admin_routes`` back to ``false`` in a follow-up change. +5. **Move the jobs.** In ``apache/airflow``, add ``/env=flex`` to the RunsOn label templates in + ``dev/breeze/src/airflow_breeze/global_constants.py``. All jobs of a run must move together: + the stash actions keep image tars in the cache bucket of the stack that ran the job, so a + job on the other stack would not find them. Rolling back is removing ``/env=flex``. +6. **Retire the CloudFormation stack** once the runs have been green on this stack for a while: + a separate change in ``terraform/runs-on`` (remove ``prevent_destroy``, destroy), the old + GitHub App uninstalled by ASF Infra, and the ``runs-on-terraform-plan``, ``-apply`` and + ``runs-on-cloudformation`` roles removed from ``runs-on-access``. + +Locally, an account admin runs the same steps as the workflow:: + + cd terraform/runs-on-flex + terraform init + terraform plan -out=runs-on-flex.tfplan + terraform apply runs-on-flex.tfplan + terraform output -raw getting_started + +Not carried over yet +-------------------- + +- ``RUNS_ON_SPOT_FORCE_ZONE_SPREAD`` (RunsOn v3.5.0, beta) would spread Spot runners across the + zones instead of letting the allocator pick; it can be set through ``extra_env_vars`` once the + three zones have run for a while. +- ``prevent_destroy`` cannot be set on a module; a ``terraform destroy`` of this module is + stopped only by review. diff --git a/terraform/runs-on-flex/main.tf b/terraform/runs-on-flex/main.tf new file mode 100644 index 0000000..7ed8ce4 --- /dev/null +++ b/terraform/runs-on-flex/main.tf @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The license key is never in Terraform state: the Flex module accepts license_key as the ARN of +# an SSM SecureString parameter named /<stack name>/license-key, created by hand (README.rst). This +# lookup without decryption only proves it exists before the stack is planned. +data "aws_ssm_parameter" "license_key" { + name = "/${var.stack_name}/license-key" + with_decryption = false +} + +data "aws_caller_identity" "current" {} + +module "runs_on" { + source = "runs-on/runs-on/aws//modules/flex" + version = "v3.5.1" + + stack_name = var.stack_name + environment = var.runs_on_environment + github_organization = var.github_organization + license_key = data.aws_ssm_parameter.license_key.arn + email = var.alert_email + + vpc_id = aws_vpc.runners.id + public_subnet_ids = [for zone_id in local.zone_ids : aws_subnet.public[zone_id].id] + private_mode = "false" + + # Created in terraform/runs-on-access; the apply role cannot create a stack role without it. + permission_boundary_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/${var.stack_name}-boundary" + + # The values of the CloudFormation stack (terraform/runs-on/main.tf); everything else keeps the + # module default. + ssh_allowed = false + ssm_allowed = true + enable_admin_routes = var.enable_admin_routes + enable_ecr = false + runner_max_runtime = 720 + cache_expiration_days = 10 + app_size = "small" + # With hundreds of runners two interruptions in 15 minutes are routine, so the default 2/15/30 + # keeps Spot off (terraform/runs-on/main.tf). + spot_circuit_breaker = "30/15/10" + enable_cost_reports = "daily" + app_budget_daily_usd = 10 # alert only, not a spending cap +} diff --git a/terraform/runs-on-flex/network.tf b/terraform/runs-on-flex/network.tf new file mode 100644 index 0000000..0ddc73f --- /dev/null +++ b/terraform/runs-on-flex/network.tf @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Runners get public IPs in public subnets (no NAT cost, Docker Hub pulls per runner IP), as on the +# CloudFormation stack, but in all three zones of the region: that stack is fixed to the first two, +# and on 2026-10-30 their Spot placement score for the runners' families was 1 against 9 in +# use2-az3 (README, "Why this module"). Zones are pinned by ID because zone names map to different +# zones in every account, and Spot placement scores are reported by ID. +locals { + zone_ids = ["use2-az1", "use2-az2", "use2-az3"] +} + +resource "aws_vpc" "runners" { + cidr_block = var.vpc_cidr + enable_dns_support = true + enable_dns_hostnames = true + + tags = { + Name = "${var.stack_name}-vpc" + } +} + +resource "aws_internet_gateway" "runners" { + vpc_id = aws_vpc.runners.id + + tags = { + Name = "${var.stack_name}-igw" + } +} + +resource "aws_subnet" "public" { + for_each = toset(local.zone_ids) + + vpc_id = aws_vpc.runners.id + availability_zone_id = each.key + cidr_block = cidrsubnet(var.vpc_cidr, 4, index(local.zone_ids, each.key)) + map_public_ip_on_launch = true + + tags = { + Name = "${var.stack_name}-public-${each.key}" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.runners.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.runners.id + } + + tags = { + Name = "${var.stack_name}-public" + } +} + +resource "aws_route_table_association" "public" { + for_each = aws_subnet.public + + subnet_id = each.value.id + route_table_id = aws_route_table.public.id +} + +# Free, and keeps the stash payloads in the cache bucket (multi-gigabyte image tars) off the +# internet gateway. +resource "aws_vpc_endpoint" "s3" { + vpc_id = aws_vpc.runners.id + service_name = "com.amazonaws.us-east-2.s3" + vpc_endpoint_type = "Gateway" + route_table_ids = [aws_route_table.public.id] + + tags = { + Name = "${var.stack_name}-s3" + } +} diff --git a/terraform/runs-on-flex/outputs.tf b/terraform/runs-on-flex/outputs.tf new file mode 100644 index 0000000..60dcd5d --- /dev/null +++ b/terraform/runs-on-flex/outputs.tf @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +output "entry_point" { + description = "RunsOn web entry point: open it once after creation to register the GitHub App." + value = module.runs_on.ingress.url +} + +output "cache_bucket" { + description = "S3 bucket the runners can use under the cache/ prefix (stash payloads)." + value = module.runs_on.platform.cache.bucket_name +} + +output "subnets" { + description = "Public subnet of the runners in each zone, by zone ID." + value = { for zone_id, subnet in aws_subnet.public : zone_id => subnet.id } +} + +output "getting_started" { + description = "RunsOn's post-apply instructions, including the env= label jobs need for this stack." + value = module.runs_on.stack.getting_started +} diff --git a/terraform/runs-on-flex/provider.tf b/terraform/runs-on-flex/provider.tf new file mode 100644 index 0000000..0902b26 --- /dev/null +++ b/terraform/runs-on-flex/provider.tf @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Credentials come from the environment (AWS_PROFILE locally, the OIDC role in the +# Terraform (RunsOn Flex) workflow) — do not hardcode a profile here. +provider "aws" { + # Also hard-coded in the backend (state.tf), the zone IDs (network.tf) and the README commands; + # change all together. + region = "us-east-2" + + default_tags { + tags = { + Project = "airflow-ci" + ManagedBy = "terraform" + Repo = "apache/airflow-ci-infra" + } + } +} diff --git a/terraform/runs-on-flex/state.tf b/terraform/runs-on-flex/state.tf new file mode 100644 index 0000000..ab9c02d --- /dev/null +++ b/terraform/runs-on-flex/state.tf @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +terraform { + backend "s3" { + region = "us-east-2" + bucket = "airflow-ci-tfstate" + # Own key, next to the CloudFormation stack's tf-state/runs-on.tfstate and the EKS states. + key = "tf-state/runs-on-flex.tfstate" + encrypt = true + use_lockfile = true + } +} diff --git a/terraform/runs-on-flex/variables.tf b/terraform/runs-on-flex/variables.tf new file mode 100644 index 0000000..9a07768 --- /dev/null +++ b/terraform/runs-on-flex/variables.tf @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "stack_name" { + description = <<-EOT + RunsOn stack name. Every resource the module creates is named after it, the license-key SSM + parameter is /<stack_name>/license-key, and terraform/runs-on-access scopes the apply role's + IAM rights to it, so changing it re-creates the stack and needs an access change. + EOT + type = string + default = "runs-on-flex" +} + +variable "runs_on_environment" { + description = <<-EOT + RunsOn environment: the stack only takes jobs whose label carries env=<this value>. It differs + from the CloudFormation stack's "production" so both stacks can serve apache/airflow at once and + jobs move by label. + EOT + type = string + default = "flex" +} + +variable "github_organization" { + description = "GitHub organization the runners serve (case-sensitive)." + type = string + default = "apache" +} + +variable "alert_email" { + description = "Recipient of RunsOn cost reports and alerts; AWS sends an SNS confirmation email on creation." + type = string + default = "[email protected]" +} + +variable "enable_admin_routes" { + description = <<-EOT + Expose the /setup admin routes on the public entry point. Set to true only while registering + (or re-registering) the GitHub App, then back to false. + EOT + type = bool + default = false +} + +variable "vpc_cidr" { + description = <<-EOT + CIDR of the runners' VPC; must not overlap existing VPCs in the account (10.0.0.0/16, + 10.1.0.0/16 and the CloudFormation stack's 10.2.0.0/16 are taken). + EOT + type = string + default = "10.3.0.0/16" +} diff --git a/terraform/runs-on-flex/version.tf b/terraform/runs-on-flex/version.tf new file mode 100644 index 0000000..a2b7aec --- /dev/null +++ b/terraform/runs-on-flex/version.tf @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + # The RunsOn module needs >= 6.45. + version = "~> 6.45" + } + } + # S3-native state locking (use_lockfile) is GA from 1.11 + required_version = ">= 1.11.0" +}
