This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
The following commit(s) were added to refs/heads/main by this push:
new a28d1e1 Run the RunsOn runners in all three us-east-2 zones from
RunsOn's Terraform module (#82)
a28d1e1 is described below
commit a28d1e14ad37c1f5b5ff15c4d10b62435a98c8e0
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Oct 31 12:20:41 2026 +0100
Run the RunsOn runners in all three us-east-2 zones from RunsOn's Terraform
module (#82)
* Run the RunsOn runners in all three us-east-2 zones from RunsOn's
Terraform module
The CloudFormation template only uses the first two zones, use2-az1 and
use2-az2, whose Spot placement score for the runners' families is 1 against 9
in use2-az3; 5.6% of the Spot runners were reclaimed from 2026-10-25 to
2026-10-30. The template has no parameter for the zones, so a second stack from
the Flex module runs beside it until the jobs move with an env=flex label.
Generated-by: Claude Opus 5
* Lock the providers of the RunsOn Flex stack
Generated-by: Claude Opus 5
* Keep the runs-on-flex roles in us-east-2 and open the setup routes for
the first apply
The permissions boundary is the ceiling of every role the apply role
creates, and those roles can
be passed to Lambda, ECS or EC2, so the boundary now carries the region
deny the apply role has.
A job that uses the runner instance profile for AWS calls outside us-east-2
is now denied.
The first apply opens the /setup routes so ASF Infra can register the
stack's GitHub App; a
follow-up change closes them again.
Generated-by: Claude Opus 5.5
* Send the runs-on-flex alerts to a personal address for now
AWS confirms an SNS email subscription only after someone clicks the link
in the confirmation
email, and the moderated [email protected] holds that email back.
The alerts go to
[email protected] until there is a way to confirm the subscription for a
mailing list.
Generated-by: Claude Opus 5.5
---------
Co-authored-by: Shahar Epstein <[email protected]>
---
.github/workflows/terraform-runs-on-flex.yml | 116 ++++++++++++
terraform/runs-on-access/runs-on-flex.tf | 253 +++++++++++++++++++++++++++
terraform/runs-on-flex/.terraform.lock.hcl | 74 ++++++++
terraform/runs-on-flex/README.rst | 129 ++++++++++++++
terraform/runs-on-flex/main.tf | 59 +++++++
terraform/runs-on-flex/network.tf | 89 ++++++++++
terraform/runs-on-flex/outputs.tf | 36 ++++
terraform/runs-on-flex/provider.tf | 32 ++++
terraform/runs-on-flex/state.tf | 27 +++
terraform/runs-on-flex/variables.tf | 68 +++++++
terraform/runs-on-flex/version.tf | 28 +++
11 files changed, 911 insertions(+)
diff --git a/.github/workflows/terraform-runs-on-flex.yml
b/.github/workflows/terraform-runs-on-flex.yml
new file mode 100644
index 0000000..9981f49
--- /dev/null
+++ b/.github/workflows/terraform-runs-on-flex.yml
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+# Plans terraform/runs-on-flex on pull requests and on main; applies the plan
made on main once a
+# required reviewer approves the runs-on-prod environment. Same flow as
terraform-runs-on.yml; the
+# AWS roles live in terraform/runs-on-access and are applied by hand.
+name: Terraform (RunsOn Flex)
+
+on: # yamllint disable-line rule:truthy
+ pull_request:
+ paths:
+ - "terraform/runs-on-flex/*.tf"
+ - "terraform/runs-on-flex/.terraform.lock.hcl"
+ - ".github/workflows/terraform-runs-on-flex.yml"
+ # Pull requests only: a plan proves the plan role still admits pull
requests after an account
+ # admin applied the changed trust policies by hand. On main the change
needs no apply.
+ - "terraform/runs-on-access/*.tf"
+ push:
+ branches: [main]
+ paths:
+ - "terraform/runs-on-flex/*.tf"
+ - "terraform/runs-on-flex/.terraform.lock.hcl"
+ - ".github/workflows/terraform-runs-on-flex.yml"
+
+permissions: {}
+
+concurrency:
+ group: terraform-runs-on-flex-${{ github.ref }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+ AWS_REGION: us-east-2
+ TF_IN_AUTOMATION: "true"
+ TF_INPUT: "false"
+
+jobs:
+ plan:
+ # Fork pull requests get no OIDC token, so they cannot plan.
+ if: >-
+ github.event_name == 'push' ||
+ github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: ubuntu-slim
+ permissions:
+ contents: read
+ id-token: write
+ defaults:
+ run:
+ working-directory: terraform/runs-on-flex
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ persist-credentials: false
+ - uses:
aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd
# v6.3.0
+ with:
+ role-to-assume:
arn:aws:iam::827901512104:role/runs-on-terraform-flex-plan
+ aws-region: ${{ env.AWS_REGION }}
+ - uses:
hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
+ with:
+ terraform_version: 1.16.2
+ terraform_wrapper: false
+ - run: terraform init -lockfile=readonly
+ # The plan role cannot write the lock file; apply takes the lock and
refuses a stale plan.
+ - run: terraform plan -lock=false -out=runs-on-flex.tfplan
+ - if: github.event_name == 'push'
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
# v7.0.2
+ with:
+ name: runs-on-flex-tfplan
+ path: terraform/runs-on-flex/runs-on-flex.tfplan
+ retention-days: 7
+
+ apply:
+ needs: plan
+ if: github.event_name == 'push'
+ # Not ubuntu-slim: its 15-minute job limit is shorter than creating the
stack may take, and a
+ # killed apply leaves the state lock held.
+ runs-on: ubuntu-24.04
+ # Required reviewers and the main-only branch policy come from .asf.yaml.
+ environment: runs-on-prod
+ permissions:
+ contents: read
+ id-token: write
+ defaults:
+ run:
+ working-directory: terraform/runs-on-flex
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ persist-credentials: false
+ - uses:
aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd
# v6.3.0
+ with:
+ role-to-assume:
arn:aws:iam::827901512104:role/runs-on-terraform-flex-apply
+ aws-region: ${{ env.AWS_REGION }}
+ - uses:
hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
+ with:
+ terraform_version: 1.16.2
+ terraform_wrapper: false
+ - run: terraform init -lockfile=readonly
+ - uses:
actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ name: runs-on-flex-tfplan
+ path: terraform/runs-on-flex
+ - run: terraform apply runs-on-flex.tfplan
diff --git a/terraform/runs-on-access/runs-on-flex.tf
b/terraform/runs-on-access/runs-on-flex.tf
new file mode 100644
index 0000000..06a98b6
--- /dev/null
+++ b/terraform/runs-on-access/runs-on-flex.tf
@@ -0,0 +1,253 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Roles of the Terraform (RunsOn Flex) workflow, which applies
terraform/runs-on-flex: RunsOn's
+# Terraform module instead of its CloudFormation template. No CloudFormation
service role sits in
+# between, so the apply role itself creates the infrastructure. It gets what
the CloudFormation
+# service role in main.tf has (AdministratorAccess with denies) and three
limits more: it acts
+# only in us-east-2 (IAM and Budgets are global), it reaches no S3 bucket but
the stack's cache
+# buckets and its own state, and every role it creates carries the
runs-on-flex-boundary
+# permissions boundary, so no role of the stack can be given more than the
boundary allows.
+locals {
+ flex_stack = "runs-on-flex"
+ flex_state = "arn:aws:s3:::airflow-ci-tfstate/tf-state/runs-on-flex.tfstate"
+
+ # Every IAM role and instance profile of the module (v3.5.1) is named
<stack>-<purpose>. AWS
+ # creates the service-linked roles on first use of ECS and Spot.
+ flex_iam_resources = [
+ "arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*",
+ "arn:aws:iam::${local.account_id}:instance-profile/${local.flex_stack}-*",
+ "arn:aws:iam::${local.account_id}:role/aws-service-role/*",
+ ]
+ # The only AWS-managed policies the module attaches (runner SSM access, ECS
task execution).
+ flex_managed_policies = [
+ "arn:aws:iam::aws:policy/AmazonSSMManagedEC2InstanceDefaultPolicy",
+ "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
+ ]
+ iam_write_actions = [
+ "iam:Add*", "iam:Attach*", "iam:Change*", "iam:Create*",
"iam:Deactivate*", "iam:Delete*",
+ "iam:Detach*", "iam:Enable*", "iam:PassRole", "iam:Put*", "iam:Remove*",
"iam:Reset*",
+ "iam:Set*", "iam:Tag*", "iam:Untag*", "iam:Update*", "iam:Upload*",
+ ]
+ tfstate_bucket = ["arn:aws:s3:::airflow-ci-tfstate",
"arn:aws:s3:::airflow-ci-tfstate/*"]
+}
+
+# Named outside the stack's runs-on-flex-* prefix: the apply role may write
IAM only under that
+# prefix, and must not be able to remove its own guardrails.
+resource "aws_iam_role" "flex_terraform" {
+ for_each = local.github_subjects
+
+ name = "runs-on-terraform-flex-${each.key}"
+ description = "terraform ${each.key} of terraform/runs-on-flex from
apache/airflow-ci-infra GitHub Actions"
+ assume_role_policy = data.aws_iam_policy_document.github_trust[each.key].json
+}
+
+# Refreshing the module's ~150 resources reads across a dozen services, so
plans get
+# ReadOnlyAccess. Plans also run on pull requests, so the denies keep secrets
out of reach:
+# ReadOnlyAccess can read every S3 object (the other Terraform states
included) and SSM
+# parameters, which the aws/ssm key would decrypt.
+resource "aws_iam_role_policy_attachment" "flex_plan_read_only" {
+ role = aws_iam_role.flex_terraform["plan"].name
+ policy_arn = "arn:aws:iam::aws:policy/ReadOnlyAccess"
+}
+
+data "aws_iam_policy_document" "flex_plan_guardrails" {
+ statement {
+ sid = "OnlyThisStateObject"
+ effect = "Deny"
+ actions = ["s3:GetObject*"]
+ not_resources = [local.flex_state]
+ }
+ statement {
+ sid = "NoSsmDecrypt"
+ effect = "Deny"
+ actions = ["kms:Decrypt"]
+ resources = ["*"]
+ condition {
+ test = "StringLike"
+ variable = "kms:ViaService"
+ values = ["ssm.*.amazonaws.com"]
+ }
+ }
+ statement {
+ sid = "NoSecretValues"
+ effect = "Deny"
+ actions = ["secretsmanager:GetSecretValue",
"secretsmanager:BatchGetSecretValue"]
+ resources = ["*"]
+ }
+}
+
+resource "aws_iam_role_policy" "flex_plan_guardrails" {
+ name = "guardrails"
+ role = aws_iam_role.flex_terraform["plan"].name
+ policy = data.aws_iam_policy_document.flex_plan_guardrails.json
+}
+
+# RunsOn documents only "admin rights" for deploying the stack, and the
module's resource set
+# changes between releases, so AdministratorAccess as for the CloudFormation
service role.
+resource "aws_iam_role_policy_attachment" "flex_apply" {
+ role = aws_iam_role.flex_terraform["apply"].name
+ policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess"
+}
+
+data "aws_iam_policy_document" "flex_apply_guardrails" {
+ statement {
+ sid = "OnlyTheStackIam"
+ effect = "Deny"
+ actions = local.iam_write_actions
+ not_resources = local.flex_iam_resources
+ }
+ # A stack role without the boundary could be given any permission and then
be passed to a Lambda
+ # function or an ECS task, which would lift every deny of this policy.
+ statement {
+ sid = "StackRolesKeepTheBoundary"
+ effect = "Deny"
+ actions = ["iam:CreateRole", "iam:PutRolePermissionsBoundary",
"iam:PutRolePolicy", "iam:AttachRolePolicy"]
+ resources = ["arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*"]
+ condition {
+ test = "StringNotEquals"
+ variable = "iam:PermissionsBoundary"
+ values = [aws_iam_policy.flex_boundary.arn]
+ }
+ }
+ statement {
+ sid = "NoBoundaryRemoval"
+ effect = "Deny"
+ actions = ["iam:DeleteRolePermissionsBoundary"]
+ resources = ["*"]
+ }
+ statement {
+ sid = "OnlyTheModuleManagedPolicies"
+ effect = "Deny"
+ actions = ["iam:AttachRolePolicy"]
+ resources = ["*"]
+ condition {
+ test = "ArnNotEquals"
+ variable = "iam:PolicyARN"
+ values = local.flex_managed_policies
+ }
+ }
+ statement {
+ sid = "NoRoleHopping"
+ effect = "Deny"
+ actions = ["sts:AssumeRole", "sts:AssumeRoleWithWebIdentity",
"sts:AssumeRoleWithSAML"]
+ resources = ["*"]
+ }
+ # Only the stack's cache bucket (bucket_prefix "<stack>-cache-") and this
state and its lock file.
+ statement {
+ sid = "OnlyTheStackBuckets"
+ effect = "Deny"
+ actions = ["s3:*"]
+ not_resources = [
+ "arn:aws:s3:::${local.flex_stack}-cache-*",
+ local.flex_state,
+ "${local.flex_state}.tflock",
+ "arn:aws:s3:::airflow-ci-tfstate",
+ ]
+ }
+ statement {
+ sid = "NoStateBucketChanges"
+ effect = "Deny"
+ actions = ["s3:Put*", "s3:Delete*"]
+ resources = ["arn:aws:s3:::airflow-ci-tfstate"]
+ }
+ statement {
+ sid = "OnlyTheRunnersRegion"
+ effect = "Deny"
+ not_actions = ["iam:*", "budgets:*", "sts:*"]
+ resources = ["*"]
+ condition {
+ test = "StringNotEquals"
+ variable = "aws:RequestedRegion"
+ values = ["us-east-2"]
+ }
+ }
+}
+
+resource "aws_iam_role_policy" "flex_apply_guardrails" {
+ name = "guardrails"
+ role = aws_iam_role.flex_terraform["apply"].name
+ policy = data.aws_iam_policy_document.flex_apply_guardrails.json
+}
+
+# The most any role of the stack may do, whatever its own policies say. The
control plane passes
+# the runner role to the instances it launches and creates the Spot
service-linked role; nothing
+# of the stack writes other IAM, assumes roles outside the stack or touches
the Terraform states.
+data "aws_iam_policy_document" "flex_boundary" {
+ statement {
+ sid = "Everything"
+ actions = ["*"]
+ resources = ["*"]
+ }
+ statement {
+ sid = "NoIamWrites"
+ effect = "Deny"
+ # PassRole and CreateServiceLinkedRole are limited below.
+ actions = [
+ "iam:Add*", "iam:Attach*", "iam:Change*", "iam:CreateA*",
"iam:CreateG*", "iam:CreateI*",
+ "iam:CreateL*", "iam:CreateO*", "iam:CreateP*", "iam:CreateR*",
"iam:CreateSA*",
+ "iam:CreateServiceS*", "iam:CreateU*", "iam:CreateV*",
"iam:Deactivate*", "iam:Delete*",
+ "iam:Detach*", "iam:Enable*", "iam:Put*", "iam:Remove*", "iam:Reset*",
"iam:Set*", "iam:Tag*",
+ "iam:Untag*", "iam:Update*", "iam:Upload*",
+ ]
+ resources = ["*"]
+ }
+ statement {
+ sid = "PassOnlyStackRoles"
+ effect = "Deny"
+ actions = ["iam:PassRole"]
+ not_resources =
["arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*"]
+ }
+ statement {
+ sid = "OnlyServiceLinkedRoles"
+ effect = "Deny"
+ actions = ["iam:CreateServiceLinkedRole"]
+ not_resources =
["arn:aws:iam::${local.account_id}:role/aws-service-role/*"]
+ }
+ statement {
+ sid = "AssumeOnlyStackRoles"
+ effect = "Deny"
+ actions = ["sts:AssumeRole"]
+ not_resources =
["arn:aws:iam::${local.account_id}:role/${local.flex_stack}-*"]
+ }
+ # The apply role's own OnlyTheRunnersRegion deny does not reach the roles it
passes to Lambda,
+ # ECS or EC2, so the boundary repeats it. Exempted: IAM, STS and Budgets as
on the apply role,
+ # and the global endpoints the module's roles call (Cost Explorer, Pricing,
public ECR).
+ statement {
+ sid = "OnlyTheRunnersRegion"
+ effect = "Deny"
+ not_actions = ["iam:*", "sts:*", "budgets:*", "ce:*", "pricing:*",
"ecr-public:*"]
+ resources = ["*"]
+ condition {
+ test = "StringNotEquals"
+ variable = "aws:RequestedRegion"
+ values = ["us-east-2"]
+ }
+ }
+ statement {
+ sid = "NoTerraformState"
+ effect = "Deny"
+ actions = ["s3:*"]
+ resources = local.tfstate_bucket
+ }
+}
+
+resource "aws_iam_policy" "flex_boundary" {
+ name = "${local.flex_stack}-boundary"
+ description = "Permissions boundary of every IAM role the runs-on-flex stack
creates"
+ policy = data.aws_iam_policy_document.flex_boundary.json
+}
diff --git a/terraform/runs-on-flex/.terraform.lock.hcl
b/terraform/runs-on-flex/.terraform.lock.hcl
new file mode 100644
index 0000000..6990ac1
--- /dev/null
+++ b/terraform/runs-on-flex/.terraform.lock.hcl
@@ -0,0 +1,74 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+ version = "6.71.0"
+ constraints = ">= 6.0.0, >= 6.33.0, >= 6.39.0, >= 6.45.0, ~> 6.45"
+ hashes = [
+ "h1:aLOQts7nCLMR0kb/jCdP56wlQQoGqhKLctKgtQjDopc=",
+ "h1:sZCupscga0Grk22+04FNY1RYDeOcVbCTd6ff8+hsF94=",
+ "h1:tmOIsPrIHwR6en567EnkR0nqIr9yD0XQcadgSsPpG2g=",
+ "zh:09aa14408c5ab26915fb2a469b5eb67c06393a2054d78592ba4c32f7a882e918",
+ "zh:1d1138128dc4866f6c16b1ef3b871b3fd9a3dc648e9056e06e210d1817aa052b",
+ "zh:2c17a79821a0e06fbf1834a12e92227db29c2001fef7ef0b347af2e6dad83f14",
+ "zh:55616730a485c9c7283546a84f1ac5f8341152c5572338a54fc7af5f69ebfb92",
+ "zh:7e6461ec2976b5961a1d4f3c08c68fae0645d7f5e0a404f7647baa3b88d337d8",
+ "zh:7ec935166b2251d075fea980012f5267078785399585e8b443dca484fc57df0f",
+ "zh:885b3290cd48738cbf9ee8a6d47a4452889843f8f53b8cf67fc8c9b16e553721",
+ "zh:8930fb33c6c965f7cc82230cb7fbb0cfff9bb30becdfce5e0c2242512d849894",
+ "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+ "zh:a702f9a4dc7340182a8daf5b6ee562f14657fe14752f7d54dc799cf2fd91a868",
+ "zh:b624cb320d787fbfa6dc94be5a1fd52683d1ebd11b42646262b221a694e6ffb8",
+ "zh:bcbaded68b553afeea93604ec22c254b98f3951862bdeb68ae89c179c7dda117",
+ "zh:cb57bbcaf0fa3a70ea29b60d183d3a0dd020f041b18da5d1725d2acd17fa7d59",
+ "zh:d4ca164b1b6039557d8f5f5f911c4aaad1a187e9b3758f6885c95f8954f7de58",
+ "zh:eb0c705337cef7b96086e8b84702ba965eeeace7a4aa3235b2c8aad5f50f2fca",
+ "zh:f2787f40ce873d8c1308c92adec9b6adbd223506fe9d79bba447729e422f9146",
+ ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+ version = "3.9.1"
+ constraints = ">= 3.7.0"
+ hashes = [
+ "h1:PYbnOqRuGn4c0/Ae1f7yOS/0zvmXNHFJRZjuF4KECnM=",
+ "h1:PlW+UZ4EElQF3NQwf41KQwavFujab3Czc51zu9dyVM8=",
+ "h1:g40qr7yDmIpaur4SsK5BcOda3HSo1RJ6zHVMqN4EJ+0=",
+ "zh:05f4734c1f0be840b711b3eff259ebc5fca436784c728955b1678078466f48d7",
+ "zh:0b91bf19371d012434eba1deeb6aab77158def9b39601dcbd94450b3974a2a26",
+ "zh:0ee6eacd47ec00183d55d726a4b6c4ce951a199f944bf22f1aa58392ebdfa7a2",
+ "zh:19388a4074b76a89a43a6c8328d7ae8ee2e7de3d346af51e80d3e6d3d12925f1",
+ "zh:23e74d48c5e2ac2e823fd527f49fee9db37d32a1990c9e3bf126ead697b843eb",
+ "zh:3cabf7fbd096c520064aae3aba61aba670af83ab91291a71fa1b1332929c2b7f",
+ "zh:5c0a3b8af0be60be4eca12ddee385cfa8babc1ec8e98cdf9de2f2274c73eabfa",
+ "zh:60b4f8a8ef18f52bf8e19215229dae408bee732825964092db7c989fd2de4097",
+ "zh:7359015acfedcbd6366f2329c854cf8d3c8ca5cd0faa89d2d37db358d6eba6c5",
+ "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+ "zh:7b38758402f0e13a1071162da28994023cd2ac676e54af350c9ffd8dfa73fa7b",
+ "zh:7c7fbb8895eb75bb4de1f933e98553bd99c8d048c89a925ddba490aa5a67f7dc",
+ "zh:8c2b8c6a7ccdec16b73e2fb9f3700ea097f58c592571e4c5de60c93d2301732c",
+ ]
+}
+
+provider "registry.terraform.io/hashicorp/time" {
+ version = "0.14.2"
+ constraints = ">= 0.9.0"
+ hashes = [
+ "h1:eQRXh8mZFlUJfzYXKdaYRHRMhiS2cFyCfgP1mjkrtuI=",
+ "h1:gnP2hptiFIHSHUFBvAFKhE/Yh5u5yVEx+P7XSB58A/E=",
+ "h1:mwxQFMxGnM+Xm3F76yeHSAfPr3RZq/+XGT1vjgwiRWQ=",
+ "zh:0aa1028d91041f4dceba193e3707dac57358d0063d97e20700e554758b67baca",
+ "zh:32bee9f2b2678e2a0789ad86e716d09ca1d5450180b3cd8033ee7a251bfd352e",
+ "zh:3aded9ef4dc6f4aec202a50c68a08b40013d325f9947f10168ebc8bee54109fc",
+ "zh:4d924637f3115ffa4ffc7f16d3f366bc472594f7447d400adb9def7ac92e3fc8",
+ "zh:5c35008e1363deafaa440ab43519409866dd7ec72aabeb2317bc16cd82756784",
+ "zh:6b30d97c9827501d7010fe49c8889d7a6cc8b45b77cdb1668c65af7f28a27d73",
+ "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+ "zh:851e0f0e9c4de395e42220de51e7fb20d0e845e629f3cf37056da19e597304e7",
+ "zh:85622b4779b3ba7424780f7f8efcfa585227e245201cdb68303f2118b388e971",
+ "zh:9039153e3d45147183804188a1ce36c3811db9a4dca80f36fa382408d5f50b72",
+ "zh:a3d385413dc258a53fe8f4ded5d1fcd1eba715c65a015d74e70911bab2919207",
+ "zh:b9310a2327f7c8ad2aa3266f89c4d026b4bf18b09f7c4257c0ecd71a32f32db3",
+ "zh:cee7f2143da0c494115da94984bf3630e3e5855e27dae720a7bc29bb6acd9be2",
+ ]
+}
diff --git a/terraform/runs-on-flex/README.rst
b/terraform/runs-on-flex/README.rst
new file mode 100644
index 0000000..16ce262
--- /dev/null
+++ b/terraform/runs-on-flex/README.rst
@@ -0,0 +1,129 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+RunsOn Flex — the runners from RunsOn's Terraform module
+========================================================
+
+This module deploys the same RunsOn runners as `terraform/runs-on
<../runs-on/README.rst>`_, from
+RunsOn's `Terraform module
<https://github.com/runs-on/terraform-aws-runs-on>`_ (Flex, v3.5.1)
+instead of its CloudFormation template, into a VPC this module creates with a
public subnet in
+each of the three us-east-2 zones. Both stacks run side by side until the jobs
of
+``apache/airflow`` have moved here; the CloudFormation stack is then removed
in a separate change.
+
+Why this module
+---------------
+
+The CloudFormation template places runners in the first two zones of the
region, use2-az1 and
+use2-az2 for this account, and has no parameter to change that. On 2026-10-30
those two zones
+had the worst Spot capacity in the region for the runners' instance families:
+
+================================= ========== ========== ==========
+Spot placement score (1-10) use2-az1 use2-az2 use2-az3
+================================= ========== ========== ==========
+x64, 8-16 vCPU (m5-m8 a/i) 1 1 9
+x64, 4-8 vCPU 2 1 8
+arm64, 8-64 vCPU (m6g-m8g) 1 1 3
+================================= ========== ========== ==========
+
+From 2026-10-25 to 2026-10-30 AWS reclaimed 911 of 16,145 Spot runners (5.6%),
at the median
+7 minutes after launch, and 30% of all runners ran on-demand. A third zone
gives the Spot
+allocator the pools where capacity is.
+
+Files
+-----
+
+``network.tf``
+ VPC (``vpc_cidr``, default ``10.3.0.0/16``), internet gateway, one public
subnet per zone ID
+ and an S3 gateway endpoint. No NAT: runners get public IPs, as on the
CloudFormation stack.
+``main.tf``
+ The ``runs-on/runs-on/aws//modules/flex`` module with the CloudFormation
stack's settings, the
+ license-key lookup and the permissions boundary of the stack's roles.
+``variables.tf``
+ All variables have defaults: ``stack_name`` (``runs-on-flex``),
``runs_on_environment``
+ (``flex``), ``github_organization``, ``alert_email``,
``enable_admin_routes``
+ (``true`` until the GitHub App is registered) and ``vpc_cidr``.
+``state.tf``
+ S3 backend ``airflow-ci-tfstate`` / ``tf-state/runs-on-flex.tfstate``.
+
+How it differs from the CloudFormation stack
+--------------------------------------------
+
+- **Environment.** The stack takes only jobs whose label carries ``env=flex``;
the CloudFormation
+ stack keeps the default ``production`` and ignores them. That is what lets
both run at once.
+- **Who creates the resources.** The CloudFormation stack is applied by a role
that can only
+ resubmit RunsOn's published template; CloudFormation's service role does the
work. Here the
+ ``runs-on-terraform-flex-apply`` role creates everything itself, with
AdministratorAccess and
+ the denies in `terraform/runs-on-access/runs-on-flex.tf
<../runs-on-access/runs-on-flex.tf>`_:
+ IAM writes only on ``runs-on-flex-*`` roles and only with the
``runs-on-flex-boundary``
+ permissions boundary, only the two AWS-managed policies the module attaches,
no S3 but the
+ stack's cache bucket and this state, no role assumption, and us-east-2 only.
Anything merged to
+ ``main`` here and approved in the ``runs-on-prod`` environment can therefore
change any non-IAM
+ resource in us-east-2, not just this stack: review pull requests to this
module as such.
+- **Upgrades** bump the module ``version`` in ``main.tf`` (with the lock
file), not a template
+ URL. Before a bump, check the new release's IAM role names and
managed-policy attachments
+ against ``flex_iam_resources`` and ``flex_managed_policies`` in
``runs-on-access``.
+
+Setup
+-----
+
+1. **Access (account admin, once).** Apply `terraform/runs-on-access
<../runs-on-access/README.rst>`_:
+ it adds the ``runs-on-terraform-flex-plan`` and ``-apply`` roles and the
+ ``runs-on-flex-boundary`` policy.
+2. **License key (manual, once).** The same RunsOn license as the
CloudFormation stack, under
+ this stack's name. The module receives only the parameter's ARN, so the key
never reaches
+ Terraform state:
+
+ .. code-block:: bash
+
+ aws ssm put-parameter --region us-east-2 \
+ --name /runs-on-flex/license-key \
+ --type SecureString \
+ --description "RunsOn license key for the runs-on-flex stack
(GitHub organization apache)" \
+ --value 'RUNS-ON-LICENSE-KEY'
+
+3. **Create the stack.** Merge the pull request that adds this module; the
+ ``Terraform (RunsOn Flex)`` workflow plans on ``main`` and applies once a
reviewer approves
+ ``runs-on-prod``. Set ``enable_admin_routes = true`` in that first change.
+4. **GitHub App (with ASF Infra).** Open the ``entry_point`` output →
**Register app**. This
+ creates a second private GitHub App for this stack; ASF Infra installs it on
+ ``apache/airflow`` (and ``.github-private``, for the presets). Then set
+ ``enable_admin_routes`` back to ``false`` in a follow-up change.
+5. **Move the jobs.** In ``apache/airflow``, add ``/env=flex`` to the RunsOn
label templates in
+ ``dev/breeze/src/airflow_breeze/global_constants.py``. All jobs of a run
must move together:
+ the stash actions keep image tars in the cache bucket of the stack that ran
the job, so a
+ job on the other stack would not find them. Rolling back is removing
``/env=flex``.
+6. **Retire the CloudFormation stack** once the runs have been green on this
stack for a while:
+ a separate change in ``terraform/runs-on`` (remove ``prevent_destroy``,
destroy), the old
+ GitHub App uninstalled by ASF Infra, and the ``runs-on-terraform-plan``,
``-apply`` and
+ ``runs-on-cloudformation`` roles removed from ``runs-on-access``.
+
+Locally, an account admin runs the same steps as the workflow::
+
+ cd terraform/runs-on-flex
+ terraform init
+ terraform plan -out=runs-on-flex.tfplan
+ terraform apply runs-on-flex.tfplan
+ terraform output -raw getting_started
+
+Not carried over yet
+--------------------
+
+- ``RUNS_ON_SPOT_FORCE_ZONE_SPREAD`` (RunsOn v3.5.0, beta) would spread Spot
runners across the
+ zones instead of letting the allocator pick; it can be set through
``extra_env_vars`` once the
+ three zones have run for a while.
+- ``prevent_destroy`` cannot be set on a module; a ``terraform destroy`` of
this module is
+ stopped only by review.
diff --git a/terraform/runs-on-flex/main.tf b/terraform/runs-on-flex/main.tf
new file mode 100644
index 0000000..7ed8ce4
--- /dev/null
+++ b/terraform/runs-on-flex/main.tf
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# The license key is never in Terraform state: the Flex module accepts
license_key as the ARN of
+# an SSM SecureString parameter named /<stack name>/license-key, created by
hand (README.rst). This
+# lookup without decryption only proves it exists before the stack is planned.
+data "aws_ssm_parameter" "license_key" {
+ name = "/${var.stack_name}/license-key"
+ with_decryption = false
+}
+
+data "aws_caller_identity" "current" {}
+
+module "runs_on" {
+ source = "runs-on/runs-on/aws//modules/flex"
+ version = "v3.5.1"
+
+ stack_name = var.stack_name
+ environment = var.runs_on_environment
+ github_organization = var.github_organization
+ license_key = data.aws_ssm_parameter.license_key.arn
+ email = var.alert_email
+
+ vpc_id = aws_vpc.runners.id
+ public_subnet_ids = [for zone_id in local.zone_ids :
aws_subnet.public[zone_id].id]
+ private_mode = "false"
+
+ # Created in terraform/runs-on-access; the apply role cannot create a stack
role without it.
+ permission_boundary_arn =
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/${var.stack_name}-boundary"
+
+ # The values of the CloudFormation stack (terraform/runs-on/main.tf);
everything else keeps the
+ # module default.
+ ssh_allowed = false
+ ssm_allowed = true
+ enable_admin_routes = var.enable_admin_routes
+ enable_ecr = false
+ runner_max_runtime = 720
+ cache_expiration_days = 10
+ app_size = "small"
+ # With hundreds of runners two interruptions in 15 minutes are routine, so
the default 2/15/30
+ # keeps Spot off (terraform/runs-on/main.tf).
+ spot_circuit_breaker = "30/15/10"
+ enable_cost_reports = "daily"
+ app_budget_daily_usd = 10 # alert only, not a spending cap
+}
diff --git a/terraform/runs-on-flex/network.tf
b/terraform/runs-on-flex/network.tf
new file mode 100644
index 0000000..0ddc73f
--- /dev/null
+++ b/terraform/runs-on-flex/network.tf
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Runners get public IPs in public subnets (no NAT cost, Docker Hub pulls per
runner IP), as on the
+# CloudFormation stack, but in all three zones of the region: that stack is
fixed to the first two,
+# and on 2026-10-30 their Spot placement score for the runners' families was 1
against 9 in
+# use2-az3 (README, "Why this module"). Zones are pinned by ID because zone
names map to different
+# zones in every account, and Spot placement scores are reported by ID.
+locals {
+ zone_ids = ["use2-az1", "use2-az2", "use2-az3"]
+}
+
+resource "aws_vpc" "runners" {
+ cidr_block = var.vpc_cidr
+ enable_dns_support = true
+ enable_dns_hostnames = true
+
+ tags = {
+ Name = "${var.stack_name}-vpc"
+ }
+}
+
+resource "aws_internet_gateway" "runners" {
+ vpc_id = aws_vpc.runners.id
+
+ tags = {
+ Name = "${var.stack_name}-igw"
+ }
+}
+
+resource "aws_subnet" "public" {
+ for_each = toset(local.zone_ids)
+
+ vpc_id = aws_vpc.runners.id
+ availability_zone_id = each.key
+ cidr_block = cidrsubnet(var.vpc_cidr, 4, index(local.zone_ids,
each.key))
+ map_public_ip_on_launch = true
+
+ tags = {
+ Name = "${var.stack_name}-public-${each.key}"
+ }
+}
+
+resource "aws_route_table" "public" {
+ vpc_id = aws_vpc.runners.id
+
+ route {
+ cidr_block = "0.0.0.0/0"
+ gateway_id = aws_internet_gateway.runners.id
+ }
+
+ tags = {
+ Name = "${var.stack_name}-public"
+ }
+}
+
+resource "aws_route_table_association" "public" {
+ for_each = aws_subnet.public
+
+ subnet_id = each.value.id
+ route_table_id = aws_route_table.public.id
+}
+
+# Free, and keeps the stash payloads in the cache bucket (multi-gigabyte image
tars) off the
+# internet gateway.
+resource "aws_vpc_endpoint" "s3" {
+ vpc_id = aws_vpc.runners.id
+ service_name = "com.amazonaws.us-east-2.s3"
+ vpc_endpoint_type = "Gateway"
+ route_table_ids = [aws_route_table.public.id]
+
+ tags = {
+ Name = "${var.stack_name}-s3"
+ }
+}
diff --git a/terraform/runs-on-flex/outputs.tf
b/terraform/runs-on-flex/outputs.tf
new file mode 100644
index 0000000..60dcd5d
--- /dev/null
+++ b/terraform/runs-on-flex/outputs.tf
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+output "entry_point" {
+ description = "RunsOn web entry point: open it once after creation to
register the GitHub App."
+ value = module.runs_on.ingress.url
+}
+
+output "cache_bucket" {
+ description = "S3 bucket the runners can use under the cache/ prefix (stash
payloads)."
+ value = module.runs_on.platform.cache.bucket_name
+}
+
+output "subnets" {
+ description = "Public subnet of the runners in each zone, by zone ID."
+ value = { for zone_id, subnet in aws_subnet.public : zone_id =>
subnet.id }
+}
+
+output "getting_started" {
+ description = "RunsOn's post-apply instructions, including the env= label
jobs need for this stack."
+ value = module.runs_on.stack.getting_started
+}
diff --git a/terraform/runs-on-flex/provider.tf
b/terraform/runs-on-flex/provider.tf
new file mode 100644
index 0000000..0902b26
--- /dev/null
+++ b/terraform/runs-on-flex/provider.tf
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Credentials come from the environment (AWS_PROFILE locally, the OIDC role in
the
+# Terraform (RunsOn Flex) workflow) — do not hardcode a profile here.
+provider "aws" {
+ # Also hard-coded in the backend (state.tf), the zone IDs (network.tf) and
the README commands;
+ # change all together.
+ region = "us-east-2"
+
+ default_tags {
+ tags = {
+ Project = "airflow-ci"
+ ManagedBy = "terraform"
+ Repo = "apache/airflow-ci-infra"
+ }
+ }
+}
diff --git a/terraform/runs-on-flex/state.tf b/terraform/runs-on-flex/state.tf
new file mode 100644
index 0000000..ab9c02d
--- /dev/null
+++ b/terraform/runs-on-flex/state.tf
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+terraform {
+ backend "s3" {
+ region = "us-east-2"
+ bucket = "airflow-ci-tfstate"
+ # Own key, next to the CloudFormation stack's tf-state/runs-on.tfstate and
the EKS states.
+ key = "tf-state/runs-on-flex.tfstate"
+ encrypt = true
+ use_lockfile = true
+ }
+}
diff --git a/terraform/runs-on-flex/variables.tf
b/terraform/runs-on-flex/variables.tf
new file mode 100644
index 0000000..5e1ea65
--- /dev/null
+++ b/terraform/runs-on-flex/variables.tf
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+variable "stack_name" {
+ description = <<-EOT
+ RunsOn stack name. Every resource the module creates is named after it,
the license-key SSM
+ parameter is /<stack_name>/license-key, and terraform/runs-on-access
scopes the apply role's
+ IAM rights to it, so changing it re-creates the stack and needs an access
change.
+ EOT
+ type = string
+ default = "runs-on-flex"
+}
+
+variable "runs_on_environment" {
+ description = <<-EOT
+ RunsOn environment: the stack only takes jobs whose label carries
env=<this value>. It differs
+ from the CloudFormation stack's "production" so both stacks can serve
apache/airflow at once and
+ jobs move by label.
+ EOT
+ type = string
+ default = "flex"
+}
+
+variable "github_organization" {
+ description = "GitHub organization the runners serve (case-sensitive)."
+ type = string
+ default = "apache"
+}
+
+variable "alert_email" {
+ description = "Recipient of RunsOn cost reports and alerts; AWS sends an SNS
confirmation email on creation."
+ type = string
+ # A personal address until we find a way to confirm the SNS subscription for
a mailing list:
+ # the confirmation email needs a click, and the moderated
[email protected] holds it.
+ default = "[email protected]"
+}
+
+variable "enable_admin_routes" {
+ description = <<-EOT
+ Expose the /setup admin routes on the public entry point. Set to true only
while registering
+ (or re-registering) the GitHub App, then back to false.
+ EOT
+ type = bool
+ default = true
+}
+
+variable "vpc_cidr" {
+ description = <<-EOT
+ CIDR of the runners' VPC; must not overlap existing VPCs in the account
(10.0.0.0/16,
+ 10.1.0.0/16 and the CloudFormation stack's 10.2.0.0/16 are taken).
+ EOT
+ type = string
+ default = "10.3.0.0/16"
+}
diff --git a/terraform/runs-on-flex/version.tf
b/terraform/runs-on-flex/version.tf
new file mode 100644
index 0000000..a2b7aec
--- /dev/null
+++ b/terraform/runs-on-flex/version.tf
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ # The RunsOn module needs >= 6.45.
+ version = "~> 6.45"
+ }
+ }
+ # S3-native state locking (use_lockfile) is GA from 1.11
+ required_version = ">= 1.11.0"
+}
