This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch remove-legacy-runner-code in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit 710fe7f5a184ffe7ebf6f7ae35ca71bacfb740a4 Author: Shahar Epstein <[email protected]> AuthorDate: Fri Nov 6 17:16:08 2026 +0200 Remove the code of the retired self-hosted runners Airflow's jobs run on RunsOn or GitHub-hosted runners, and the AWS resources this code built and deployed (the scale-out Lambda, the runner AMIs and their SSM parameters) were deleted on 2026-11-03. Keeping it means Dependabot bumps, license checks and reviews for code nothing can run any more, and it suggests to readers that the 2021 runners still exist. Generated-by: Claude Opus 5.5 --- .apache-magpie-overrides/repo-health-config.md | 3 - .github/dependabot.yml | 4 +- .gitignore | 5 - .pre-commit-config.yaml | 5 +- arm-docker-ami/packer/files/docker-permissions.sh | 18 - .../packer/files/install-dependencies.sh | 22 - arm-docker-ami/packer/linux-ami.pkr.hcl | 96 --- arm-docker-ami/packer/vars/variables.pkrvars.hcl | 22 - cloud-init.yml | 20 - .../packer/files/actions-runner-ec2-reporting.sh | 22 - .../packer/files/actions.runner-supervisor.service | 30 - .../packer/files/actions.runner.service | 39 - github-runner-ami/packer/files/cleanup.sh | 30 - .../packer/files/cloudwatch-metrics-github-runners | 18 - github-runner-ami/packer/files/configure_kernel.sh | 22 - .../packer/files/create-hostedtools-cache.sh | 21 - github-runner-ami/packer/files/docker.sh | 47 -- github-runner-ami/packer/files/git.sh | 88 --- .../packer/files/install-dependencies.sh | 50 -- github-runner-ami/packer/files/install-files.sh | 29 - .../packer/files/install-github-cli.sh | 28 - github-runner-ami/packer/files/install-nodejs.sh | 25 - .../packer/files/install-subversion.sh | 22 - github-runner-ami/packer/files/mounts_setup.sh | 19 - github-runner-ami/packer/files/regctl.sh | 29 - github-runner-ami/packer/files/rules.v4 | 28 - github-runner-ami/packer/files/runner | 18 - .../packer/files/runner-cleanup-workdir.sh | 44 -- .../packer/files/runner-supervisor.py | 853 --------------------- github-runner-ami/packer/files/runner_bootstrap.sh | 45 -- .../packer/files/source-list-additions.sh | 23 - .../packer/files/stop-runner-if-no-job.sh | 46 -- github-runner-ami/packer/files/timber.key | 36 - github-runner-ami/packer/files/vector.toml | 105 --- github-runner-ami/packer/ubuntu2004.pkr.hcl | 166 ---- .../packer/vars/variables.pkrvars.hcl | 24 - lambdas/scale_out_runner/.chalice/config.json | 20 - .../scale_out_runner/.chalice/deployed/prod.json | 28 - lambdas/scale_out_runner/.chalice/prod_iam.json | 33 - lambdas/scale_out_runner/.gitignore | 20 - lambdas/scale_out_runner/app.py | 210 ----- lambdas/scale_out_runner/requirements.txt | 19 - requirements-test.txt | 1 - requirements.txt | 27 - scripts/list_committers | 102 --- scripts/store-agent-creds.py | 206 ----- tests/conftest.py | 16 - tests/lambdas/scale_out_runner/conftest.py | 35 - tests/lambdas/scale_out_runner/test_app.py | 61 -- 49 files changed, 3 insertions(+), 2877 deletions(-) diff --git a/.apache-magpie-overrides/repo-health-config.md b/.apache-magpie-overrides/repo-health-config.md index 9b97730..7a2775c 100644 --- a/.apache-magpie-overrides/repo-health-config.md +++ b/.apache-magpie-overrides/repo-health-config.md @@ -117,9 +117,6 @@ repo_health: # Source paths to audit for license headers (relative to upstream repo root). source_paths: - ".github/" - - "arm-docker-ami/" - - "github-runner-ami/" - - "lambdas/" - "scripts/" - "terraform/" - "tests/" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e0bd437..fa91eb6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -40,9 +40,7 @@ updates: patterns: ["*"] - package-ecosystem: pip - directories: - - / - - /lambdas/scale_out_runner + directory: / schedule: interval: weekly cooldown: diff --git a/.gitignore b/.gitignore index 994f4d7..17d19fb 100644 --- a/.gitignore +++ b/.gitignore @@ -57,12 +57,7 @@ override.tf.json .idea *.iml -# Session manager binaries -session-manager-plugin.* - # End of https://www.toptal.com/developers/gitignore/api/terraform -github-runner-ami/packer/session-manager-plugin.deb -github-runner-ami/packer/session-manager-plugin.rpm # Apache Magpie personal layer /.apache-magpie-local/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3020db8..e2e5778 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,9 +31,8 @@ repos: - id: forbid-tabs - id: insert-license name: Add license - # LICENSE is the license itself, lock files are generated, and cloud-init.yml must start with - # #cloud-config. - exclude: ^\.github/.*$|^license-templates/|\.json$|\.md$|\.rst$|^LICENSE$|\.terraform\.lock\.hcl$|^cloud-init\.yml$ + # LICENSE is the license itself and lock files are generated. + exclude: ^\.github/.*$|^license-templates/|\.json$|\.md$|\.rst$|^LICENSE$|\.terraform\.lock\.hcl$ args: - --comment-style - "|#|" diff --git a/arm-docker-ami/packer/files/docker-permissions.sh b/arm-docker-ami/packer/files/docker-permissions.sh deleted file mode 100644 index cf1fb13..0000000 --- a/arm-docker-ami/packer/files/docker-permissions.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -sudo usermod -a -G docker ec2-user diff --git a/arm-docker-ami/packer/files/install-dependencies.sh b/arm-docker-ami/packer/files/install-dependencies.sh deleted file mode 100644 index 3486545..0000000 --- a/arm-docker-ami/packer/files/install-dependencies.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -yum update -yum install docker --assumeyes diff --git a/arm-docker-ami/packer/linux-ami.pkr.hcl b/arm-docker-ami/packer/linux-ami.pkr.hcl deleted file mode 100644 index 64c6ecb..0000000 --- a/arm-docker-ami/packer/linux-ami.pkr.hcl +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -variable "vpc_id" { - type = string -} -variable "ami_name" { - type = string -} -variable "aws_regions" { - type = list(string) -} -variable "packer_role_arn" { - type = string -} -variable "session_manager_instance_profile_name" { - type = string -} - -source "amazon-ebs" "docker-runner" { - region = var.aws_regions[0] - ami_name = "${var.ami_name}-v2" - ami_regions = var.aws_regions - tag { - key = "Name" - value = "arm-docker-ami" - } - snapshot_tag { - key = "Name" - value = "arm-docker-ami-root" - } - encrypt_boot = false - instance_type = "m6g.large" - communicator = "ssh" - ssh_username = "ec2-user" - ssh_interface = "session_manager" - iam_instance_profile = var.session_manager_instance_profile_name - subnet_filter { - # Just pick a random subnet in the VPC -- we only have the three defaults so this is fine! - random = true - } - vpc_id = var.vpc_id - source_ami_filter { - filters = { - virtualization-type = "hvm" - architecture= "arm64", - name = "amzn2-ami-kernel-5.10-hvm-*" - root-device-type = "ebs" - } - owners = ["amazon"] - most_recent = true - } -} - -build { - sources = [ - "source.amazon-ebs.docker-runner" - ] - - provisioner "shell" { - inline = [ - "echo Connected via SSM at '${build.User}@${build.Host}:${build.Port}'" - ] - } - - # Since we connect as a non-root user, we have to "stage" the files to a writable folder, which we then move - # in to place with the approriate permissions via install-files.sh provisioner step - provisioner "shell" { - inline = [ - "mkdir -p /tmp/etc-systemd-system /tmp/usr-local-sbin /tmp/usr-local-bin /tmp/etc-sudoers.d /tmp/etc-iptables /tmp/etc-cron.d" - ] - } - provisioner "shell" { - scripts = [ - "./files/install-dependencies.sh", - "./files/docker-permissions.sh", - ] - execute_command = "chmod +x '{{ .Path }}'; sudo sh -c '{{ .Vars }} {{ .Path }}'" - environment_vars = [ - ] - } -} diff --git a/arm-docker-ami/packer/vars/variables.pkrvars.hcl b/arm-docker-ami/packer/vars/variables.pkrvars.hcl deleted file mode 100644 index f01a76e..0000000 --- a/arm-docker-ami/packer/vars/variables.pkrvars.hcl +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -vpc_id = "vpc-d73487bd" -ami_name = "arm-docker-ami" -aws_regions = ["eu-central-1", "us-east-2"] -packer_role_arn = "arn:aws:iam::827901512104:role/packer-role" -session_manager_instance_profile_name = "packer_ssm_instance_profile" diff --git a/cloud-init.yml b/cloud-init.yml deleted file mode 100644 index 459708d..0000000 --- a/cloud-init.yml +++ /dev/null @@ -1,20 +0,0 @@ -#cloud-config -# 2022-01-21 19:31 BMT - -mounts: - - [tmpfs, /var/lib/docker, tmpfs, "defaults,noatime,size=85%"] - - [tmpfs, /home/runner/actions-runner/_work, tmpfs, "defaults,noatime"] - - -runcmd: - - - - bash - - -c - - | - set -eu -o pipefail - echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment - echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/default/vector - #aws logs create-log-stream --log-group-name GitHubRunners --log-stream-name $(hostname) || true - #systemctl enable --now vector - systemctl enable --now actions.runner - sysctl -w net.netfilter.nf_conntrack_tcp_be_liberal=1 diff --git a/github-runner-ami/packer/files/actions-runner-ec2-reporting.sh b/github-runner-ami/packer/files/actions-runner-ec2-reporting.sh deleted file mode 100644 index d79b0c9..0000000 --- a/github-runner-ami/packer/files/actions-runner-ec2-reporting.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if pgrep -c Runner.Worker >/dev/null; then - # Only report metric when we're doing something -- no point paying to submit zeros - aws cloudwatch put-metric-data --metric-name jobs-running --value "$(pgrep -c Runner.Worker)" --namespace github.actions -fi diff --git a/github-runner-ami/packer/files/actions.runner-supervisor.service b/github-runner-ami/packer/files/actions.runner-supervisor.service deleted file mode 100644 index baffc90..0000000 --- a/github-runner-ami/packer/files/actions.runner-supervisor.service +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[Unit] -Description=Fetch credentials and supervise GitHub Actions Runner -After=network.target -Before=actions.runner.service - -[Service] -Type=notify -ExecStart=/opt/runner-supervisor/bin/python /opt/runner-supervisor/bin/runner-supervisor -# We need to run as root to have the ability to open the netlink connector socket -User=root -WorkingDirectory=/home/runner/actions-runner -Restart=always -EnvironmentFile=/etc/environment diff --git a/github-runner-ami/packer/files/actions.runner.service b/github-runner-ami/packer/files/actions.runner.service deleted file mode 100644 index 67a1d44..0000000 --- a/github-runner-ami/packer/files/actions.runner.service +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[Unit] -Description=GitHub Actions Runner -After=network.target actions.runner-supervisor.service -Requires=actions.runner-supervisor.service -BindsTo=actions.runner-supervisor.service - -[Service] -ExecStartPre=!/usr/local/sbin/runner-cleanup-workdir.sh -ExecStart=/home/runner/actions-runner/run.sh --once --startuptype service --labels $RUNNER_LABELS -ExecStop=/usr/local/bin/stop-runner-if-no-job.sh $MAINPID -EnvironmentFile=/etc/environment -Environment=GITHUB_ACTIONS_RUNNER_CHANNEL_TIMEOUT=300 -Environment=RUNNER_LABELS=airflow-runner,vm-runner -User=runner -WorkingDirectory=/home/runner/actions-runner -KillMode=mixed -KillSignal=SIGTERM -TimeoutStopSec=30min -Restart=on-success - -[Install] -WantedBy=multi-user.target diff --git a/github-runner-ami/packer/files/cleanup.sh b/github-runner-ami/packer/files/cleanup.sh deleted file mode 100644 index 4b6443a..0000000 --- a/github-runner-ami/packer/files/cleanup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -rm ~ubuntu/.ssh/authorized_keys - -journalctl --rotate -sleep 2 -journalctl --vacuum-time=1s - -rm -rf /var/lib/cloud/ /var/log/cloud-init*.log - -rm -rf /var/cache/* /var/lib/apt/* diff --git a/github-runner-ami/packer/files/cloudwatch-metrics-github-runners b/github-runner-ami/packer/files/cloudwatch-metrics-github-runners deleted file mode 100644 index 0380246..0000000 --- a/github-runner-ami/packer/files/cloudwatch-metrics-github-runners +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -*/1 * * * * nobody /usr/local/sbin/actions-runner-ec2-reporting.sh diff --git a/github-runner-ami/packer/files/configure_kernel.sh b/github-runner-ami/packer/files/configure_kernel.sh deleted file mode 100644 index 5fb2855..0000000 --- a/github-runner-ami/packer/files/configure_kernel.sh +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# Needed to run multiple kind instances -cat >>/etc/sysctl.conf <<EOF -fs.inotify.max_user_watches = 524288 -fs.inotify.max_user_instances = 512 -net.netfilter.nf_conntrack_tcp_be_liberal = 1 -EOF diff --git a/github-runner-ami/packer/files/create-hostedtools-cache.sh b/github-runner-ami/packer/files/create-hostedtools-cache.sh deleted file mode 100644 index 89a9f74..0000000 --- a/github-runner-ami/packer/files/create-hostedtools-cache.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eu -o pipefail -echo "Creating hosted tools cache:" -install --owner runner --directory /opt/hostedtoolcache diff --git a/github-runner-ami/packer/files/docker.sh b/github-runner-ami/packer/files/docker.sh deleted file mode 100644 index 73e4be9..0000000 --- a/github-runner-ami/packer/files/docker.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -exu -o pipefail - -for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do sudo apt-get remove $pkg --assume-yes || true; done - -sudo apt-get update -sudo apt-get install ca-certificates curl gnupg - -sudo install -m 0755 -d /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg -sudo chmod a+r /etc/apt/keyrings/docker.gpg - -echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - -sudo apt-get update - -# Same version as MIN_VERSIONS in Breeze -DOCKER_VERSION_STRING="5:24.0.7-1~ubuntu.20.04~focal" -DOCKER_COMPOSE_VERSION_STRING="2.20.2-1~ubuntu.20.04~focal" -DOCKER_BUILDX_VERSION_STRING="0.11.2-1~ubuntu.20.04~focal" -sudo apt-get install \ - "docker-ce=${DOCKER_VERSION_STRING}" \ - "docker-ce-cli=${DOCKER_VERSION_STRING}" \ - containerd.io \ - "docker-buildx-plugin=${DOCKER_BUILDX_VERSION_STRING}" \ - "docker-compose-plugin=${DOCKER_COMPOSE_VERSION_STRING}" --assume-yes --allow-downgrades diff --git a/github-runner-ami/packer/files/git.sh b/github-runner-ami/packer/files/git.sh deleted file mode 100644 index fc79f0a..0000000 --- a/github-runner-ami/packer/files/git.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -GIT_REPO="ppa:git-core/ppa" -GIT_LFS_REPO="https://packagecloud.io/install/repositories/github/git-lfs"; - -function download_with_retries() { -# Due to restrictions of bash functions, positional arguments are used here. -# In case if you using latest argument NAME, you should also set value to all previous parameters. -# Example: download_with_retries $ANDROID_SDK_URL "." "android_sdk.zip" - local URL="$1" - local DEST="${2:-.}" - local NAME="${3:-${URL##*/}}" - local COMPRESSED="$4" - - if [[ $COMPRESSED == "compressed" ]]; then - local COMMAND="curl $URL -4 -sL --compressed -o '$DEST/$NAME' -w '%{http_code}'" - else - local COMMAND="curl $URL -4 -sL -o '$DEST/$NAME' -w '%{http_code}'" - fi - - echo "Downloading '$URL' to '${DEST}/${NAME}'..." - retries=20 - interval=30 - while [ $retries -gt 0 ]; do - ((retries--)) - # Temporary disable exit on error to retry on non-zero exit code - set +e - http_code=$(eval $COMMAND) - exit_code=$? - if [ $http_code -eq 200 ] && [ $exit_code -eq 0 ]; then - echo "Download completed" - return 0 - else - echo "Error — Either HTTP response code for '$URL' is wrong - '$http_code' or exit code is not 0 - '$exit_code'. Waiting $interval seconds before the next attempt, $retries attempts left" - sleep $interval - fi - # Enable exit on error back - set -e - done - - echo "Could not download $URL" - return 1 -} - -## Install git -add-apt-repository $GIT_REPO -y -apt-get update -apt-get install --upgrade git -y -git --version - -# Install git-lfs -curl -s $GIT_LFS_REPO/script.deb.sh | bash -apt-get install -y git-lfs=2.13.3 - -# Install git-ftp -apt-get install git-ftp -y - -# Remove source repo's -add-apt-repository --remove $GIT_REPO -rm /etc/apt/sources.list.d/github_git-lfs.list - -#Install hub -tmp_hub="/tmp/hub" -mkdir -p "$tmp_hub" -url=$(curl --location -s https://api.github.com/repos/github/hub/releases/latest | jq -r '.assets[].browser_download_url | select(contains("hub-linux-amd64"))') -download_with_retries "$url" "$tmp_hub" -tar xzf "$tmp_hub"/hub-linux-amd64-*.tgz --strip-components 1 -C "$tmp_hub" -mv "$tmp_hub"/bin/hub /usr/local/bin - -# Add well-known SSH host keys to known_hosts -ssh-keyscan -t rsa github.com >> /etc/ssh/ssh_known_hosts -ssh-keyscan -t rsa ssh.dev.azure.com >> /etc/ssh/ssh_known_hosts diff --git a/github-runner-ami/packer/files/install-dependencies.sh b/github-runner-ami/packer/files/install-dependencies.sh deleted file mode 100644 index 9d24d99..0000000 --- a/github-runner-ami/packer/files/install-dependencies.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -debconf-set-selections <<EOF -iptables-persistent iptables-persistent/autosave_done boolean true -iptables-persistent iptables-persistent/autosave_v4 boolean false -iptables-persistent iptables-persistent/autosave_v6 boolean false -EOF - -export DEBIAN_FRONTEND=noninteractive - -apt-get update -apt-get install -yq --no-install-recommends -o Dpkg::Options::="--force-confold" \ - awscli \ - build-essential \ - autossh \ - git \ - haveged \ - iptables-persistent \ - jq \ - parallel \ - python3-dev \ - python3-venv \ - python3-wheel \ - yarn \ - vector='0.15.*' - - -# Re-enabled in clout-init once AWS_DEFAULT_REGION env var is set -systemctl disable vector - -# validate the vector config file we have already installed -sudo -u vector vector validate --no-environment diff --git a/github-runner-ami/packer/files/install-files.sh b/github-runner-ami/packer/files/install-files.sh deleted file mode 100644 index 9c726f6..0000000 --- a/github-runner-ami/packer/files/install-files.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eu -o pipefail - -mkdir /etc/iptables/ /etc/vector - -install --owner root --mode=0644 --target-directory "/etc/systemd/system/" "/tmp/etc-systemd-system/"* -install --owner root --mode=0755 --target-directory "/usr/local/sbin" "/tmp/usr-local-sbin/"* -install --owner root --mode=0755 --target-directory "/usr/local/bin" "/tmp/usr-local-bin/"* -install --owner root --mode=0644 --target-directory "/etc/iptables" "/tmp/etc-iptables/"* -install --owner root --mode=0644 --target-directory "/etc/cron.d" "/tmp/etc-cron.d/"* -install --owner root --mode=0644 --target-directory "/etc/sudoers.d" "/tmp/etc-sudoers.d/"* -install --owner root --mode=0644 --target-directory "/etc/vector/" "/tmp/etc-vector/"* diff --git a/github-runner-ami/packer/files/install-github-cli.sh b/github-runner-ami/packer/files/install-github-cli.sh deleted file mode 100644 index a09714e..0000000 --- a/github-runner-ami/packer/files/install-github-cli.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -e - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -uo pipefail - -# https://github.com/actions/virtual-environments/blob/f2fdcef0e020770b1b6cc58bda3b4a01f0286f5e/images/linux/scripts/installers/github-cli.sh -url=$(curl -s https://api.github.com/repos/cli/cli/releases/latest | jq -r '.assets[].browser_download_url|select(contains("linux") and contains("amd64") and contains(".deb"))') - -cd /tmp -curl -fsL --remote-name "$url" -apt install /tmp/gh_*_linux_amd64.deb -rm /tmp/gh_*_linux_amd64.deb diff --git a/github-runner-ami/packer/files/install-nodejs.sh b/github-runner-ami/packer/files/install-nodejs.sh deleted file mode 100644 index d2a4eed..0000000 --- a/github-runner-ami/packer/files/install-nodejs.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# https://github.com/actions/virtual-environments/blob/525f79f479cca77aef4e0a680548b65534c64a18/images/linux/scripts/installers/nodejs.sh -curl -sL https://raw.githubusercontent.com/mklement0/n-install/stable/bin/n-install | bash -s -- -ny - -~/n/bin/n lts -npm install -g grunt gulp n parcel-bundler typescript newman -npm install -g --save-dev webpack webpack-cli -npm install -g npm -rm -rf ~/n diff --git a/github-runner-ami/packer/files/install-subversion.sh b/github-runner-ami/packer/files/install-subversion.sh deleted file mode 100644 index 9a974f3..0000000 --- a/github-runner-ami/packer/files/install-subversion.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -e - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -uo pipefail - -apt install subversion -y diff --git a/github-runner-ami/packer/files/mounts_setup.sh b/github-runner-ami/packer/files/mounts_setup.sh deleted file mode 100644 index 03f449b..0000000 --- a/github-runner-ami/packer/files/mounts_setup.sh +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sudo mount -t tmpfs -o size=85% tmpfs /var/lib/docker -sudo mount -t tmpfs -o tmpfs /home/runner/actions-runner/_work diff --git a/github-runner-ami/packer/files/regctl.sh b/github-runner-ami/packer/files/regctl.sh deleted file mode 100644 index 1a8e712..0000000 --- a/github-runner-ami/packer/files/regctl.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -set -exu -o pipefail - -architecture=$(uname -m) -if [[ ${architecture} == "x86_64" ]] ; then - # Well. Docker compose got it right, but regctl didn't ¯\_(ツ)_/¯ - architecture="amd64" -fi -# Hard-code regctl version -regctl_version="v0.4.3" -regctl_binary="regctl-$(uname -s)-${architecture}" -curl -L "https://github.com/regclient/regclient/releases/download/${regctl_version}/${regctl_binary}"; -o "/usr/local/bin/regctl" -chmod a+x "/usr/local/bin/regctl" diff --git a/github-runner-ami/packer/files/rules.v4 b/github-runner-ami/packer/files/rules.v4 deleted file mode 100644 index 6d19ac1..0000000 --- a/github-runner-ami/packer/files/rules.v4 +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Generated by iptables-save v1.8.4 on Thu Jan 14 13:59:27 2021 -*filter -:INPUT ACCEPT [833:75929] -:FORWARD DROP [0:0] -:OUTPUT ACCEPT [794:143141] -:DOCKER-USER - [0:0] --A FORWARD -j DOCKER-USER -# Dis-allow any docker container to access the metadata service --A DOCKER-USER -d 169.254.169.254/32 -j REJECT --reject-with icmp-port-unreachable --A DOCKER-USER -j RETURN -COMMIT diff --git a/github-runner-ami/packer/files/runner b/github-runner-ami/packer/files/runner deleted file mode 100644 index 1d79760..0000000 --- a/github-runner-ami/packer/files/runner +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -runner ALL=(ALL) NOPASSWD:/usr/sbin/swapoff -a, /usr/bin/rm -f /swapfile, /usr/bin/apt clean diff --git a/github-runner-ami/packer/files/runner-cleanup-workdir.sh b/github-runner-ami/packer/files/runner-cleanup-workdir.sh deleted file mode 100644 index 23141d7..0000000 --- a/github-runner-ami/packer/files/runner-cleanup-workdir.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eu -o pipefail -echo "Left-over containers:" -docker ps -a -docker ps -qa | xargs --verbose --no-run-if-empty docker rm -fv - -echo "Log in to a paid docker user to get unlimited docker pulls" -aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \ - jq .Parameter.Value -r | \ - sudo -u runner docker login --username airflowcirunners --password-stdin - -if [[ -d ~runner/actions-runner/_work/airflow/airflow ]]; then - cd ~runner/actions-runner/_work/airflow/airflow - - chown --changes -R runner: . - if [[ -e .git ]]; then - sudo -u runner bash -c " - git reset --hard && \ - git submodule deinit --all -f && \ - git submodule foreach git clean -fxd && \ - git clean -fxd \ - " - fi -fi - -# Remove left over mssql data dirs -find . -maxdepth 1 -name 'tmp-mssql-volume-*' -type d -printf 'Deleting %f\n' -exec sudo rm -r {} + diff --git a/github-runner-ami/packer/files/runner-supervisor.py b/github-runner-ami/packer/files/runner-supervisor.py deleted file mode 100755 index fe6e8e5..0000000 --- a/github-runner-ami/packer/files/runner-supervisor.py +++ /dev/null @@ -1,853 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Obtain credentials for Actions runner and co-operate with auto scaling group - -The purpose of this script is to enable the self-hosted runners to operate in -an AutoScaling environment (without needing admin permissions on the GitHub -repo to create and delete runners.) - -The order of operations is: - -1. Obtain credentials - - We have pre-created a number of credentials and stored them in Amazon KMS. - This script makes use of dynamodb to obtain an exclusive lock on a set of - credentials. - - We need the "locking" as if you use credentials that are already - in use the new runner process will wait (but never error) until they are not - in use. - -2. Complete the ASG lifecycle action so the instance is marked as InService - - This might not be strictly necessary, we don't want the instance to be "in - service" until the runner has started. - -3. Emit metric saying whether instance is running a job or not - - This is used to drive the scale-in CloudWatch alarm - -4. Monitor for the runner starting jobs, and protecting the instance from Scale-In when it is - - Since we are running in an autoscaling group we can't dictate which instance - AWS choses to terminate, so we instead have to set scale-in protection when a job is running. - - The way we watch for jobs being executed is using the Netlink Process - Connector, which is a datagram socket that a (root) process can open to the - kernel, to receive push events for whenever a process starts or stops. - - There are more events than that send, and to limit it to the only ones we - care about we use a BPF filter to drop everything else. - - Since it is a datagram socket it is possible we might miss a notification, so - we also periodically check if the process is still alive - -5. Watch for ASG instance state changing to Terminating:Wait - - When the ASG wants to terminate the instance, we have it configured to put - the instance in to a "requested" state -- this is to avoid a race condition - where the instance _isn't_ running a job (so isn't protected from scale in), - gets set to Terminating, but before AWS shuts down the machine the runner - process picks up and starts a Job, which leads to the job failing with "The - self-hosted runner: Airflow Runner $N lost communication with the server". - - When we notice being in this state, we _gracefully_ shut down the runner - (letting it complete any job it might have), stop it from restarting, and - then allow the termination lifecycle to continue - -""" - -import ctypes -import datetime -import enum -import errno -import json -import logging -import os -import random -import selectors -import shutil -import signal -import socket -from subprocess import check_call -from typing import Callable, List, Tuple, Union - -import boto3 -import click -import psutil -from python_dynamodb_lock.python_dynamodb_lock import DynamoDBLockClient, DynamoDBLockError -from tenacity import before_sleep_log, retry, stop_after_delay, wait_exponential - -logging.basicConfig(level=logging.INFO) -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - -logging.getLogger('python_dynamodb_lock').setLevel(logging.WARNING) - - -TABLE_NAME = os.getenv('COUNTER_TABLE', 'GithubRunnerQueue') - - [email protected]() [email protected]('--repo', default='apache/airflow') [email protected]('--user', default='runner') [email protected]( - '--output-folder', - help="Folder to write credentials to. Default of ~runner/actions-runner", - default='~runner/actions-runner', -) -def main(repo, output_folder, user): - global INSTANCE_ID - # Notify the ASG LifeCycle hook that we are now In Service and ready to - # process requests/safe to be shut down - - # Fetch current instance ID from where cloutinit writes it to - if not INSTANCE_ID: - with open('/var/lib/cloud/data/instance-id') as fh: - INSTANCE_ID = fh.readline().strip() - - log.info("Starting on %s...", INSTANCE_ID) - - output_folder = os.path.expanduser(output_folder) - - short_time = datetime.timedelta(microseconds=1) - - dynamodb = boto3.resource('dynamodb') - client = DynamoDBLockClient( - dynamodb, - table_name='GitHubRunnerLocks', - expiry_period=datetime.timedelta(0, 300), - heartbeat_period=datetime.timedelta(seconds=10), - ) - - # Just keep trying until we get some credentials. - while True: - # Have each runner try to get a credential in a random order. - possibles = get_possible_credentials(repo) - random.shuffle(possibles) - - log.info("Trying to get a set of credentials in this order: %r", possibles) - - notify = get_sd_notify_func() - - for index in possibles: - try: - lock = client.acquire_lock( - f'{repo}/{index}', - retry_period=short_time, - retry_timeout=short_time, - raise_context_exception=True, - ) - except DynamoDBLockError as e: - log.info("Could not lock %s (%s)", index, e) - continue - - with lock: - log.info("Obtained lock on %s", index) - write_credentials_to_files(repo, index, output_folder, user) - merge_in_settings(repo, output_folder) - notify(f"STATUS=Obtained lock on {index}") - - if get_lifecycle_state() == "Pending:Wait": - complete_asg_lifecycle_hook() - - notify("READY=1") - log.info("Watching for Runner.Worker processes") - ProcessWatcher().run() - - client.close() - - exit() - - -def get_sd_notify_func() -> Callable[[str], None]: - # http://www.freedesktop.org/software/systemd/man/sd_notify.html - addr = os.getenv('NOTIFY_SOCKET') - if not addr: - return lambda status: None - - sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - if addr[0] == '@': - addr = '\0' + addr[1:] - sock.connect(addr) - - def notify(status: str): - sock.sendall(status.encode('utf-8')) - - return notify - - -def write_credentials_to_files( - repo: str, index: str, out_folder: str = '~runner/actions-runner', user: str = 'runner' -): - param_path = os.path.join('/runners/', repo, index) - - resp = boto3.client("ssm").get_parameters_by_path(Path=param_path, Recursive=False, WithDecryption=True) - - param_to_file = { - 'config': '.runner', - 'credentials': '.credentials', - 'rsaparams': '.credentials_rsaparams', - } - - for param in resp['Parameters']: - # "/runners/apache/airflow/config" -> "config" - name = os.path.basename(param['Name']) - filename = param_to_file.get(name, None) - if filename is None: - log.info("Unknown Parameter from SSM: %r", param['Name']) - continue - log.info("Writing %r to %r", param['Name'], filename) - with open(os.path.join(out_folder, filename), "w") as fh: - fh.write(param['Value']) - shutil.chown(fh.name, user) - os.chmod(fh.name, 0o600) - del param_to_file[name] - if param_to_file: - raise RuntimeError(f"Missing expected params: {list(param_to_file.keys())}") - - -def merge_in_settings(repo: str, out_folder: str) -> None: - client = boto3.client('ssm') - - param_path = os.path.join('/runners/', repo, 'configOverlay') - log.info("Loading config overlay from %s", param_path) - - try: - - resp = client.get_parameter(Name=param_path, WithDecryption=True) - except client.exceptions.ParameterNotFound: - log.debug("Failed to load config overlay", exc_info=True) - return - - try: - overlay = json.loads(resp['Parameter']['Value']) - except ValueError: - log.debug("Failed to parse config overlay", exc_info=True) - return - - with open(os.path.join(out_folder, ".runner"), "r+") as fh: - settings = json.load(fh) - - for key, val in overlay.items(): - settings[key] = val - - fh.seek(0, os.SEEK_SET) - os.ftruncate(fh.fileno(), 0) - json.dump(settings, fh, indent=2) - - -def get_possible_credentials(repo: str) -> List[str]: - client = boto3.client("ssm") - paginator = client.get_paginator("describe_parameters") - - path = os.path.join('/runners/', repo, '') - baked_path = os.path.join(path, 'runnersList') - - # Pre-compute the list, to avoid making lots of requests and getting throttled by SSM API in case of - # thundering herd - try: - log.info("Using pre-computed credentials indexes from %s", baked_path) - resp = client.get_parameter(Name=baked_path) - return resp['Parameter']['Value'].split(',') - except client.exceptions.ParameterNotFound: - pass - - log.info("Looking at %s for possible credentials", path) - - pages = paginator.paginate( - ParameterFilters=[{"Key": "Path", "Option": "Recursive", "Values": [path]}], - PaginationConfig={ - "PageSize": 50, - }, - ) - - seen = set() - - for i, page in enumerate(pages): - log.info("Page %d", i) - for param in page['Parameters']: - name = param['Name'] - log.info("%s", name) - - # '/runners/x/1/config' -> '1/config', - # '/runners/x/y/1/config' -> 'y/1/config', - local_name = name[len(path) :] - - try: - # '1/config' -> '1' - index, _ = local_name.split('/') - except ValueError: - # Ignore any 'x/y' when we asked for 'x'. There should only be an index and a filename - log.debug("Ignoring nested path %s", name) - continue - - try: - # Check it's a number, but keep variable as string - int(index) - except ValueError: - log.debug("Ignoring non-numeric index %s", name) - continue - - index = os.path.basename(os.path.dirname(name)) - seen.add(index) - - if not seen: - raise RuntimeError(f'No credentials found in SSM ParameterStore for {repo!r}') - - try: - resp = client.put_parameter( - Name=baked_path, Type='StringList', Value=','.join(list(seen)), Overwrite=False - ) - log.info("Stored pre-computed credentials indexes at %s", baked_path) - except client.exceptions.ParameterAlreadyExists: - # Race, we lost, never mind! - pass - - return list(seen) - - -OWN_ASG = None -INSTANCE_ID = None - - -def get_lifecycle_state() -> str: - global INSTANCE_ID, OWN_ASG - - if not INSTANCE_ID: - with open('/var/lib/cloud/data/instance-id') as fh: - INSTANCE_ID = fh.readline().strip() - - asg_client = boto3.client('autoscaling') - - try: - instances = asg_client.describe_auto_scaling_instances( - InstanceIds=[INSTANCE_ID], - )['AutoScalingInstances'] - except asg_client.exceptions.ClientError: - return "UNKNOWN" - - if len(instances) != 1: - return "UNKNOWN" - - details = instances[0] - - if not OWN_ASG: - OWN_ASG = details['AutoScalingGroupName'] - - return details['LifecycleState'] - - -def complete_asg_lifecycle_hook(hook_name='WaitForInstanceReportReady', retry=False): - global OWN_ASG, INSTANCE_ID - # Notify the ASG LifeCycle hook that we are now InService and ready to - # process requests/safe to be shut down - - asg_client = boto3.client('autoscaling') - - try: - asg_client.complete_lifecycle_action( - AutoScalingGroupName=OWN_ASG, - InstanceId=INSTANCE_ID, - LifecycleHookName=hook_name, - LifecycleActionResult='CONTINUE', - ) - log.info("LifeCycle hook %s set to CONTINUE instance=%s", hook_name, INSTANCE_ID) - except asg_client.exceptions.ClientError as e: - # If the script fails for whatever reason and we re-run, the lifecycle hook may have already be - # completed, so this would fail. That is not an error - - # We don't want the stacktrace here, just the message - log.warning("Failed to complete lifecycle hook %s: %s", hook_name, str(e)) - pass - - -# Constants and types from -# https://github.com/torvalds/linux/blob/fcadab740480e0e0e9fa9bd272acd409884d431a/include/uapi/linux/cn_proc.h -class NlMsgFlag(enum.IntEnum): - NoOp = 1 - Error = 2 - Done = 3 - Overrun = 4 - - -class NLMsgHdr(ctypes.Structure): - """Netlink Message Header""" - - _fields_ = [ - ("len", ctypes.c_uint32), - ("type", ctypes.c_uint16), - ("flags", ctypes.c_uint16), - ("seq", ctypes.c_uint32), - ("pid", ctypes.c_uint32), - ] - - -class ProcConnectorOp(enum.IntEnum): - MCAST_LISTEN = 1 - MCAST_IGNORE = 2 - - -class cn_msg(ctypes.Structure): - """Linux kernel Connector message""" - - CN_IDX_PROC = 1 - CN_VAL_PROC = 1 - - _fields_ = [ - ("cb_id_idx", ctypes.c_uint32), - ("cb_id_val", ctypes.c_uint32), - ("seq", ctypes.c_uint32), - ("ack", ctypes.c_uint32), - ("len", ctypes.c_uint16), - ("flags", ctypes.c_uint16), - ] - - def __init__(self, header, data, **kwargs): - super().__init__(**kwargs) - self.header = header - self.len = ctypes.sizeof(data) - self.data = data - self.header.len = ctypes.sizeof(header) + ctypes.sizeof(self) + self.len - - def to_bytes(self): - return bytes(self.header) + bytes(self) + bytes(self.data) # type: ignore - - -class ProcEventWhat(enum.IntFlag): - NONE = 0x0 - FORK = 0x1 - EXEC = 0x2 - UID = 0x4 - GID = 0x40 - SID = 0x80 - PTRACE = 0x0000010 - COMM = 0x0000020 - COREDUMP = 0x40000000 - EXIT = 0x80000000 - - -class proc_event(ctypes.Structure): - """Base proc_event field""" - - _fields_ = [ - ("what", ctypes.c_uint32), - ("cpu", ctypes.c_uint32), - ("timestamp", ctypes.c_uint64), # Number of nano seconds since system boot - ] - - @classmethod - def from_netlink_packet( - cls, data - ) -> Tuple["proc_event", Union[None, "exec_proc_event", "exit_proc_event"]]: - """ - Parse the netlink packet in to a - """ - # Netlink message header (struct nlmsghdr) - header = NLMsgHdr.from_buffer_copy(data) - data = data[ctypes.sizeof(header) :] - - # We already checked/filtered on header.type == NlMsgFlag.Done - - # Connector message header (struct cn_msg) - connector_msg = cn_msg.from_buffer_copy(data) - - # Ignore messages from other Netlink connector types: done via BPF - - data = data[ctypes.sizeof(connector_msg) :] - - event = proc_event.from_buffer_copy(data) - data = data[ctypes.sizeof(event) :] - event.what = ProcEventWhat(event.what) - - if event.what == ProcEventWhat.EXEC: - return event, exec_proc_event.from_buffer_copy(data) - elif event.what == ProcEventWhat.EXIT: - return event, exit_proc_event.from_buffer_copy(data) - return event, None - - -class exec_proc_event(ctypes.Structure): - _fields_ = [ - ("pid", ctypes.c_int32), - ("tid", ctypes.c_int32), - ] - - -class exit_proc_event(ctypes.Structure): - _fields_ = [ - ("pid", ctypes.c_int32), - ("tid", ctypes.c_int32), - ("exit_code", ctypes.c_int32), - ("signal", ctypes.c_int32), - ] - - -class ProcessWatcher: - interesting_processes = {} - - protected = None - in_termating_lifecycle = False - - def run(self): - # Create a signal pipe that we can poll on - sig_read, sig_write = socket.socketpair() - - sel = selectors.DefaultSelector() - - def sig_handler(signal, frame): - # no-op - ... - - sig_read.setblocking(False) - sig_write.setblocking(False) - sel.register(sig_read, selectors.EVENT_READ, None) - - proc_socket = self.open_proc_connector_socket() - proc_socket.setblocking(False) - - signal.signal(signal.SIGINT, sig_handler) - signal.signal(signal.SIGALRM, sig_handler) - signal.setitimer(signal.ITIMER_REAL, 30, 30.0) - signal.set_wakeup_fd(sig_write.fileno(), warn_on_full_buffer=False) - - sel.register(proc_socket, selectors.EVENT_READ, self.handle_proc_event) - - self.pgrep() - - try: - while True: - for key, mask in sel.select(): - - if key.fileobj == sig_read: - sig = signal.Signals(key.fileobj.recv(1)[0]) # type: ignore - if sig == signal.SIGALRM: - self.check_still_alive() - continue - else: - log.info(f"Got {sig.name}, exiting") - return - callback = key.data - callback(key.fileobj, mask) - finally: - # Disable the timers for any cleanup code to run - signal.setitimer(signal.ITIMER_REAL, 0) - signal.set_wakeup_fd(-1) - - def pgrep(self): - """Check for any interesting processes we might have missed.""" - listener_found = False - - for proc in psutil.process_iter(['name', 'cmdline']): - try: - if proc.name() == "Runner.Worker" and proc.pid not in self.interesting_processes: - log.info( - "Found existing interesting processes, protecting from scale in %d: %s", - proc.pid, - proc.cmdline(), - ) - self.interesting_processes[proc.pid] = proc - self.protect_from_scale_in(protect=True) - self.dynamodb_atomic_decrement() - if proc.name() == "Runner.Listener": - listener_found = True - except psutil.NoSuchProcess: - # Process went away before we could - pass - - if not listener_found: - if self.in_termating_lifecycle: - log.info("Runner.Listener process not found - OkayToTerminate instance") - complete_asg_lifecycle_hook('OkayToTerminate') - else: - # Unprotect ourselves if somehow the runner is no longer working - self.protect_from_scale_in(protect=False) - - def check_still_alive(self): - # Check ASG status - if not self.in_termating_lifecycle: - state = get_lifecycle_state() - if state == 'Terminating:Wait': - self.in_termating_lifecycle = True - self.gracefully_terminate_runner() - elif state == 'Pending:Wait': - complete_asg_lifecycle_hook() - - # proc_connector is un-reliable (UDP) so periodically check if the processes are still alive - if not self.interesting_processes: - self.pgrep() - return - - # list() is used to prevent "Dict changed size during iteration" during loop below - pids = list(self.interesting_processes.keys()) - log.info("Checking processes %r are still alive", pids) - - for pid in pids: - proc = self.interesting_processes[pid] - if not proc.is_running() or proc.status() == psutil.STATUS_ZOMBIE: - log.info("Proc %d dead but we didn't notice!", pid) - del self.interesting_processes[pid] - - if not self.interesting_processes: - log.info("No interesting processes left, unprotecting from scale in") - self.protect_from_scale_in(protect=False) - elif not self.protected: - # If we didn't manage to protect last time, try again - self.protect_from_scale_in() - - def gracefully_terminate_runner(self): - check_call(['systemctl', 'stop', 'actions.runner', '--no-block']) - - def protect_from_scale_in(self, protect: bool = True): - """Set (or unset) ProtectedFromScaleIn on our instance""" - if not OWN_ASG: - # Not part of an ASG - return - - if self.in_termating_lifecycle: - log.info("Not trying to SetInstanceProtection, we are already in the terminating lifecycle step") - return - - asg_client = boto3.client('autoscaling') - try: - self._protect_from_scale_in(asg_client, protect) - self.protected = protect - except asg_client.exceptions.ClientError as e: - # This can happen if this the runner picks up a job "too quick", and the ASG still has the state - # as Pending:Proceed, so we can't yet set it as protected - log.warning("Failed to set scale in protection: %s", str(e)) - - @retry( - wait=wait_exponential(multiplier=1, max=10), - stop=stop_after_delay(30), - before_sleep=before_sleep_log(log, logging.INFO), - reraise=True, - ) - def _protect_from_scale_in(self, asg_client, protect): - asg_client.set_instance_protection( - AutoScalingGroupName=OWN_ASG, - InstanceIds=[INSTANCE_ID], - ProtectedFromScaleIn=protect, - ) - - def dynamodb_atomic_decrement(self): - dynamodb = boto3.client('dynamodb') - try: - resp = dynamodb.update_item( - TableName=TABLE_NAME, - Key={'id': {'S': 'queued_jobs'}}, - ExpressionAttributeValues={':delta': {'N': '-1'}, ':limit': {'N': '0'}}, - UpdateExpression='ADD queued :delta', - # Make sure it never goes below zero! - ConditionExpression='queued > :limit', - ReturnValues='UPDATED_NEW', - ) - - log.info("Updated DynamoDB queue length: %s", resp['Attributes']['queued']['N']) - except dynamodb.exceptions.ConditionalCheckFailedException: - log.warning("%s.queued was already 0, we won't decrease it any further!", TABLE_NAME) - - def handle_proc_event(self, sock, mask): - try: - data, (nlpid, nlgrps) = sock.recvfrom(1024) - except OSError as e: - if e.errno == errno.ENOBUFS: - return - raise - if nlpid != 0: - # Ignore messages from non-root processes - return - - event, detail = proc_event.from_netlink_packet(data) - if event.what == ProcEventWhat.EXEC: - try: - proc = psutil.Process(detail.pid) - - with proc.oneshot(): - if proc.name() == "Runner.Worker": - log.info( - "Found new interesting processes, protecting from scale in %d: %s", - detail.pid, - proc.cmdline(), - ) - self.interesting_processes[detail.pid] = proc - self.protect_from_scale_in(protect=True) - self.dynamodb_atomic_decrement() - - except psutil.NoSuchProcess: - # We lost the race, process has already exited. If it was that short lived it wasn't that - # interesting anyway - pass - elif event.what == ProcEventWhat.EXIT: - if detail.pid in self.interesting_processes: - log.info("Interesting process %d exited", detail.pid) - del self.interesting_processes[detail.pid] - - if not self.interesting_processes: - log.info("Watching no processes, disabling termination protection") - self.protect_from_scale_in(protect=False) - elif self.in_termating_lifecycle: - try: - proc = psutil.Process(detail.pid) - if proc.name() == "Runner.Listener": - log.info("Runner.Listener process %d exited - OkayToTerminate instance", detail.pid) - complete_asg_lifecycle_hook('OkayToTerminate') - except psutil.NoSuchProcess: - # We lost the race, process has already exited. If it was that short lived it wasn't that - # interesting anyway - pass - - def open_proc_connector_socket(self) -> socket.socket: - """Open and set up a socket connected to the kernel's Proc Connector event stream - - This uses the Netlink family of socket, the Connector message type and the proc_event connector to get - send a (UDP) message whenever a process starts or exits. - - We use this mechansim to get notified when processes start or stop, so we can watch for the - "Runner.Worker" and enable/disable termination protection. - """ - - class bpf_insn(ctypes.Structure): - """The BPF instruction data structure""" - - _fields_ = [ - ("code", ctypes.c_ushort), - ("jt", ctypes.c_ubyte), - ("jf", ctypes.c_ubyte), - ("k", ctypes.c_uint32), - ] - - class bpf_program(ctypes.Structure): - """Structure for BIOCSETF""" - - _fields_ = [("bf_len", ctypes.c_uint), ("bf_insns", ctypes.POINTER(bpf_insn))] - - def __init__(self, program): - self.bf_len = len(program) - bpf_insn_array = bpf_insn * self.bf_len - self.bf_insns = bpf_insn_array() - - # Fill the pointer - for i, insn in enumerate(program): - self.bf_insns[i] = insn - - def bpf_jump(code, k, jt, jf) -> bpf_insn: - """ - :param code: BPF instruction op codes - :param k: argument - :param jt: jump offset if true - :param jf: jump offset if false - """ - return bpf_insn(code, jt, jf, k) - - def bpf_stmt(code, k): - return bpf_jump(code, k, 0, 0) - - def packet_filter_prog(): - """ - A Berkley Packet Filter program to filter down the "firehose" of info we receive over the netlink - socket. - - The Proc Connector doesn't provide any easy way to filter out the firehose of package events, and - while we could ignore the things we don't care about in Python, it's more efficient to never - receive those packets. "Luckily" there is the BPF, or Berkley Packet Filter, which can operate on - any socket. This BPF program was taken from - https://web.archive.org/web/20130601175512/https://netsplit.com/2011/02/09/the-proc-connector-and-socket-filters/ - """ - # A subset of Berkeley Packet Filter constants and macros, as defined in linux/filter.h. - - # Instruction classes - BPF_LD = 0x00 - BPF_JMP = 0x05 - BPF_RET = 0x06 - - # ld/ldx fields - BPF_W = 0x00 - BPF_H = 0x08 - BPF_ABS = 0x20 - - # alu/jmp fields - BPF_JEQ = 0x10 - BPF_K = 0x00 - - return bpf_program( - [ - # Load 16-bit ("half"-word) nlmsg.type field - bpf_stmt(BPF_LD | BPF_H | BPF_ABS, NLMsgHdr.type.offset), - bpf_jump(BPF_JMP | BPF_JEQ | BPF_K, socket.htons(NlMsgFlag.Done), 1, 0), - # Not NlMsgFlag.Done, return whole packet - bpf_stmt(BPF_RET | BPF_K, 0xFFFFFFFF), - # - # Load 32-bit (word) cb_id_idx field - bpf_stmt(BPF_LD | BPF_W | BPF_ABS, ctypes.sizeof(NLMsgHdr) + cn_msg.cb_id_idx.offset), - bpf_jump(BPF_JMP | BPF_JEQ | BPF_K, socket.htonl(cn_msg.CN_IDX_PROC), 1, 0), - # If not CN_IDX_PROC, return whole packet - bpf_stmt(BPF_RET | BPF_K, 0xFFFFFFFF), - # - # Load cb_id_val field - bpf_stmt(BPF_LD | BPF_W | BPF_ABS, ctypes.sizeof(NLMsgHdr) + cn_msg.cb_id_val.offset), - bpf_jump(BPF_JMP | BPF_JEQ | BPF_K, socket.htonl(cn_msg.CN_VAL_PROC), 1, 0), - # If not CN_VAL_PROC, return whole packet - bpf_stmt(BPF_RET | BPF_K, 0xFFFFFFFF), - # - # If not ProcEventWhat.EXEC or ProcEventWhat.EXIT, event, filter out the packet - bpf_stmt( - BPF_LD | BPF_W | BPF_ABS, - ctypes.sizeof(NLMsgHdr) + ctypes.sizeof(cn_msg) + proc_event.what.offset, - ), - bpf_jump(BPF_JMP | BPF_JEQ | BPF_K, socket.htonl(ProcEventWhat.EXEC), 2, 0), - bpf_jump(BPF_JMP | BPF_JEQ | BPF_K, socket.htonl(ProcEventWhat.EXIT), 1, 0), - bpf_stmt(BPF_RET | BPF_K, 0x0), - # Return everything - bpf_stmt(BPF_RET | BPF_K, 0xFFFFFFFF), - ] - ) - - # Create Netlink socket - - # Missing from most/all pythons - NETLINK_CONNECTOR = getattr(socket, "NETLINK_CONNECTOR", 11) - SO_ATTACH_FILTER = getattr(socket, "SO_ATTACH_FILTER", 26) - - sock = socket.socket(socket.AF_NETLINK, socket.SOCK_DGRAM, NETLINK_CONNECTOR) - - filter_prog = packet_filter_prog() - sock.setsockopt(socket.SOL_SOCKET, SO_ATTACH_FILTER, bytes(filter_prog)) # type: ignore - - sock.bind((os.getpid(), cn_msg.CN_IDX_PROC)) - - # Send PROC_CN_MCAST_LISTEN to start receiving messages - msg = cn_msg( - header=NLMsgHdr(type=NlMsgFlag.Done, pid=os.getpid()), - cb_id_idx=cn_msg.CN_IDX_PROC, - cb_id_val=cn_msg.CN_VAL_PROC, - seq=0, - ack=0, - data=ctypes.c_uint32(ProcConnectorOp.MCAST_LISTEN), - ) - - data = msg.to_bytes() - if sock.send(data) != len(data): - raise RuntimeError("Failed to send PROC_CN_MCAST_LISTEN") - - return sock - - -if __name__ == "__main__": - main() diff --git a/github-runner-ami/packer/files/runner_bootstrap.sh b/github-runner-ami/packer/files/runner_bootstrap.sh deleted file mode 100644 index 132d870..0000000 --- a/github-runner-ami/packer/files/runner_bootstrap.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -exu -o pipefail - -# Validate params -: "${RUNNER_VERSION?}" - -# Set an env var (that is visible in runners) that will let us know we are on a self-hosted runner -echo 'AIRFLOW_SELF_HOSTED_RUNNER="[\"self-hosted\"]"' >> /etc/environment - -useradd --create-home runner -G docker - -install --owner runner --directory ~runner/actions-runner - -cd ~runner/actions-runner -curl -L "https://github.com/ashb/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz"; | tar -zx - -python3 -mvenv /opt/runner-supervisor -/opt/runner-supervisor/bin/pip install -U pip python-dynamodb-lock-whatnick==0.9.3 click==7.1.2 psutil 'tenacity~=6.0' - -install --owner root --mode 0755 /tmp/runner-supervisor /opt/runner-supervisor/bin/runner-supervisor - -systemctl enable iptables.service -systemctl enable vector.service - -# We don't enable actions.runner.service here, but instead in the user-data -# script, as otherwise it would happen to early, before we have had a chance to -# drop the AWS_DEFAULT_REGION in to /etc/environment diff --git a/github-runner-ami/packer/files/source-list-additions.sh b/github-runner-ami/packer/files/source-list-additions.sh deleted file mode 100644 index 6f8b0af..0000000 --- a/github-runner-ami/packer/files/source-list-additions.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -apt-key adv --recv-key 0x1646B01B86E50310 "1646B01B86E50310" -echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list -apt-key add /tmp/timber.key -echo "deb https://repositories.timber.io/public/vector/deb/ubuntu focal main" > /etc/apt/sources.list.d/timber.list diff --git a/github-runner-ami/packer/files/stop-runner-if-no-job.sh b/github-runner-ami/packer/files/stop-runner-if-no-job.sh deleted file mode 100644 index 9d988de..0000000 --- a/github-runner-ami/packer/files/stop-runner-if-no-job.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -u - -MAINPID="${MAINPID:-${1:-}}" - -if [[ -z "$MAINPID" ]]; then - echo "No MAINPID, assuming it already crashed!" - exit 0 -fi - -if pgrep --ns $MAINPID -a Runner.Worker > /dev/null; then - echo "Waiting for current job to finish" - while pgrep --ns $MAINPID -a Runner.Worker; do - # Job running -- just wait for it to exit - sleep 10 - done - -else - # If there were _no_ Workers running, ask the main process to stop. If there - # were Workers running, then Runner.Listener would stop automatically because - # of the `--once` - pkill --ns $MAINPID Runner.Listener || true -fi - - # Wait for it to shut down -echo "Waiting for main Runner.Listener $MAINPID process to stop" -while pgrep --ns $MAINPID -a Runner.Listener; do - sleep 5 -done diff --git a/github-runner-ami/packer/files/timber.key b/github-runner-ami/packer/files/timber.key deleted file mode 100644 index f8b8df7..0000000 --- a/github-runner-ami/packer/files/timber.key +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v2 - -mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ -jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d -2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy -Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL -Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz -629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj -a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE -EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor -xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X -muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE -RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA -gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm -0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g -3vq0NvX0k2CLi/W9mTiSdHXFChI15A== -=k36w ------END PGP PUBLIC KEY BLOCK----- diff --git a/github-runner-ami/packer/files/vector.toml b/github-runner-ami/packer/files/vector.toml deleted file mode 100644 index f9f3308..0000000 --- a/github-runner-ami/packer/files/vector.toml +++ /dev/null @@ -1,105 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -data_dir = "/var/lib/vector" - -[api] - enabled = true - -# Input data. Change me to a valid input source. -[sources.logs] - type = "journald" - include_units = ["actions.runner.service", "actions.runner-supervisor.service"] - -[transforms.without_systemd_fields] - type = "remove_fields" - inputs = ["logs"] - fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP", - "_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID", - "_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT", - "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "SYSLOG_TIMESTAMP", - "PRIORITY", "_EXE", "source_type"] - -[sources.runner-logs] - type = "file" - include = ["/home/runner/actions-runner/_diag/*.log"] - - [sources.runner-logs.multiline] - start_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}' - mode = "halt_before" - condition_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}' - timeout_ms = 250 - -[transforms.grok-runner-logs] - type = "remap" - inputs=["runner-logs"] - source = ''' - structured, err = parse_grok(.message, "(?m)\\[%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{NOTSPACE:logger}\\] %{GREEDYDATA:message}") - - if err != null { - .err = err - } else { - . = merge(., structured) - } - ''' -[transforms.filter-runner-logs] - type = "filter" - inputs = ['grok-runner-logs'] - condition = ''' - if .logger == "JobServerQueue" { - !match!(.message, r'Try to append \d+ batches web console lines for record') - } else if .logger == "HostContext" { - !starts_with!(.message, "Well known directory") - } else if .logger == "JobDispatcher" { - !starts_with!(.message, "Successfully renew job request") - } - } else { - true - } - ''' - -[sources.job-logs] - type = "file" - include = ["/home/runner/actions-runner/_diag/pages/*.log"] - -[transforms.grok-job-logs] - type = "remap" - inputs = ["job-logs"] - drop_on_abort = true - source = ''' - structured, err = parse_grok(.message, "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:message}") - - if err == null { - . = merge(., structured) - .type = "job-output" - - if length(.message) == 0 ?? true { - abort # Don't send empty messages to cloud watch! - } - } - ''' - -# Output data -[sinks.cloudwatch] -inputs = ["without_systemd_fields", "filter-runner-logs", "grok-job-logs"] -type = "aws_cloudwatch_logs" -encoding = "json" -create_missing_group = false -create_missing_stream = true -group_name = "GitHubRunners" -stream_name = "{{ host }}" -region = "${AWS_DEFAULT_REGION}" diff --git a/github-runner-ami/packer/ubuntu2004.pkr.hcl b/github-runner-ami/packer/ubuntu2004.pkr.hcl deleted file mode 100644 index 0f56d47..0000000 --- a/github-runner-ami/packer/ubuntu2004.pkr.hcl +++ /dev/null @@ -1,166 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -variable "vpc_id" { - type = string -} -variable "ami_name" { - type = string -} -variable "aws_regions" { - type = list(string) -} -variable "packer_role_arn" { - type = string -} -variable "runner_version" { - type = string -} -variable "runner_labels" { - type = string -} -variable "session_manager_instance_profile_name" { - type = string -} - -source "amazon-ebs" "runner_builder" { - #access_key = "" - #secret_key = "" - region = var.aws_regions[0] - ami_name = "${var.ami_name}-${var.runner_version}-v20" - ami_regions = var.aws_regions - tag { - key = "Name" - value = "github-runner-ami" - } - snapshot_tag { - key = "Name" - value = "github-runner-ami-root" - } - encrypt_boot = false - instance_type = "t3.micro" - communicator = "ssh" - ssh_username = "ubuntu" - ssh_interface = "session_manager" - iam_instance_profile = var.session_manager_instance_profile_name - subnet_filter { - # Just pick a random subnet in the VPC -- we only have the three defaults so this is fine! - random = true - } - vpc_id = var.vpc_id - source_ami_filter { - filters = { - virtualization-type = "hvm" - name = "ubuntu/images/*buntu-focal-20.04-amd64-server-*" - root-device-type = "ebs" - } - owners = ["099720109477"] - most_recent = true - } - launch_block_device_mappings { - device_name = "/dev/sda1" - volume_size = 16 - volume_type = "gp3" - delete_on_termination = true - } -} - -build { - sources = [ - "source.amazon-ebs.runner_builder" - ] - - provisioner "shell" { - inline = [ - "echo Connected via SSM at '${build.User}@${build.Host}:${build.Port}'" - ] - } - - # Since we connect as a non-root user, we have to "stage" the files to a writable folder, which we then move - # in to place with the approriate permissions via install-files.sh provisioner step - provisioner "shell" { - inline = [ - "mkdir -p /tmp/etc-systemd-system /tmp/usr-local-sbin /tmp/usr-local-bin /tmp/etc-sudoers.d /tmp/etc-iptables /tmp/etc-cron.d /tmp/etc-vector /tmp/etc-systemd-system-vector.service.d" - ] - } - provisioner "file" { - destination = "/tmp/etc-systemd-system/actions.runner.service" - source = "./files/actions.runner.service" - } - provisioner "file" { - destination = "/tmp/etc-systemd-system/actions.runner-supervisor.service" - source = "./files/actions.runner-supervisor.service" - } - provisioner "file" { - destination = "/tmp/usr-local-sbin/runner-cleanup-workdir.sh" - source = "./files/runner-cleanup-workdir.sh" - } - provisioner "file" { - destination = "/tmp/usr-local-bin/stop-runner-if-no-job.sh" - source = "./files/stop-runner-if-no-job.sh" - } - provisioner "file" { - destination = "/tmp/etc-sudoers.d/runner" - source = "./files/runner" - } - provisioner "file" { - destination = "/tmp/etc-iptables/rules.v4" - source = "./files/rules.v4" - } - provisioner "file" { - destination = "/tmp/usr-local-sbin/actions-runner-ec2-reporting.sh" - source = "./files/actions-runner-ec2-reporting.sh" - } - provisioner "file" { - destination = "/tmp/etc-cron.d/cloudwatch-metrics-github-runners" - source = "./files/cloudwatch-metrics-github-runners" - } - provisioner "file" { - destination = "/tmp/timber.key" - source = "./files/timber.key" - } - provisioner "file" { - destination = "/tmp/runner-supervisor" - source = "./files/runner-supervisor.py" - } - provisioner "file" { - destination = "/tmp/etc-vector/vector.toml" - source = "./files/vector.toml" - } - provisioner "shell" { - scripts = [ - "./files/install-files.sh", - "./files/source-list-additions.sh", - "./files/install-dependencies.sh", - "./files/install-nodejs.sh", - "./files/install-github-cli.sh", - "./files/install-subversion.sh", - "./files/docker.sh", - "./files/configure_kernel.sh", - "./files/git.sh", - "./files/runner_bootstrap.sh", - "./files/create-hostedtools-cache.sh", - "./files/regctl.sh", - "./files/cleanup.sh", - ] - execute_command = "chmod +x '{{ .Path }}'; sudo sh -c '{{ .Vars }} {{ .Path }}'" - environment_vars = [ - "RUNNER_VERSION=${var.runner_version}", - "RUNNER_LABELS=${var.runner_labels}", - ] - } -} diff --git a/github-runner-ami/packer/vars/variables.pkrvars.hcl b/github-runner-ami/packer/vars/variables.pkrvars.hcl deleted file mode 100644 index c9ad301..0000000 --- a/github-runner-ami/packer/vars/variables.pkrvars.hcl +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -vpc_id = "vpc-d73487bd" -ami_name = "airflow-runner-ami" -aws_regions = ["eu-central-1", "us-east-2"] -packer_role_arn = "arn:aws:iam::827901512104:role/packer-role" -runner_version = "2.309.0-airflow11" -session_manager_instance_profile_name = "packer_ssm_instance_profile" -runner_labels = "airflow-runner,vm-runner" diff --git a/lambdas/scale_out_runner/.chalice/config.json b/lambdas/scale_out_runner/.chalice/config.json deleted file mode 100644 index 669f935..0000000 --- a/lambdas/scale_out_runner/.chalice/config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "version": "2.0", - "app_name": "scale_out_runner", - "stages": { - "dev": { - "api_gateway_stage": "api" - }, - "prod": { - "api_gateway_stage": "api", - "autogen_policy": false, - "iam_policy_file": "prod_iam.json", - "environment_variables": { - "GH_WEBHOOK_TOKEN_ENCRYPTED": "AQICAHg1MGVq8MAqYXSkkgy6iL19KSI14nJw8DelmHFuRpAfvQEh/mniWicOD0N1aVuHhp+VAAAAfDB6BgkqhkiG9w0BBwagbTBrAgEAMGYGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMVqkAu70BaR2rDQ4qAgEQgDlIAxDrBlblL6XNKkJIs+zrNEMcGpS68JnoVh2s1oKEzbBXzYS16/9ZrW9CwUV4ULAd2EsGb90iz74=", - "ASG_NAME": "RunnerASG", - "ASG_REGION_NAME": "us-east-2" - } - } - }, - "automatic_layer": true -} diff --git a/lambdas/scale_out_runner/.chalice/deployed/prod.json b/lambdas/scale_out_runner/.chalice/deployed/prod.json deleted file mode 100644 index 7f1a411..0000000 --- a/lambdas/scale_out_runner/.chalice/deployed/prod.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "resources": [ - { - "name": "managed-layer", - "resource_type": "lambda_layer", - "layer_version_arn": "arn:aws:lambda:eu-central-1:827901512104:layer:scale_out_runner-prod-managed-layer:35" - }, - { - "name": "api_handler_role", - "resource_type": "iam_role", - "role_arn": "arn:aws:iam::827901512104:role/scale_out_runner-prod-api_handler", - "role_name": "scale_out_runner-prod-api_handler" - }, - { - "name": "api_handler", - "resource_type": "lambda_function", - "lambda_arn": "arn:aws:lambda:eu-central-1:827901512104:function:scale_out_runner-prod" - }, - { - "name": "rest_api", - "resource_type": "rest_api", - "rest_api_id": "2onm92olq7", - "rest_api_url": "https://2onm92olq7.execute-api.eu-central-1.amazonaws.com/api/"; - } - ], - "schema_version": "2.0", - "backend": "api" -} diff --git a/lambdas/scale_out_runner/.chalice/prod_iam.json b/lambdas/scale_out_runner/.chalice/prod_iam.json deleted file mode 100644 index 539bac3..0000000 --- a/lambdas/scale_out_runner/.chalice/prod_iam.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "logs:CreateLogStream", - "kms:Decrypt", - "autoscaling:SetDesiredCapacity", - "ssm:GetParameter", - "logs:CreateLogGroup", - "logs:PutLogEvents", - "dynamodb:UpdateItem" - ], - "Resource": [ - "arn:aws:ssm:*:827901512104:parameter/runners/*/configOverlay", - "arn:aws:autoscaling:*:827901512104:autoScalingGroup:*:autoScalingGroupName/AshbRunnerASG", - "arn:aws:kms:*:827901512104:key/48a58710-7ac6-4f88-995f-758a6a450faa", - "arn:aws:dynamodb:*:827901512104:table/GithubRunnerQueue", - "arn:*:logs:*:*:*" - ] - }, - { - "Sid": "VisualEditor1", - "Effect": "Allow", - "Action": [ - "autoscaling:DescribeAutoScalingGroups" - ], - "Resource": "*" - } - ] -} diff --git a/lambdas/scale_out_runner/.gitignore b/lambdas/scale_out_runner/.gitignore deleted file mode 100644 index 9e4df26..0000000 --- a/lambdas/scale_out_runner/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -.chalice/deployments/ -.chalice/venv/ -__pycache__/ diff --git a/lambdas/scale_out_runner/app.py b/lambdas/scale_out_runner/app.py deleted file mode 100644 index a02b341..0000000 --- a/lambdas/scale_out_runner/app.py +++ /dev/null @@ -1,210 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import codecs -import hmac -import json -import logging -import os -from typing import cast - -import boto3 -from chalice import BadRequestError, Chalice, ForbiddenError -from chalice.app import Request - -app = Chalice(app_name='scale_out_runner') -app.log.setLevel(logging.INFO) - -ASG_GROUP_NAME = os.getenv('ASG_NAME', 'AshbRunnerASG') -ASG_REGION_NAME = os.getenv('ASG_REGION_NAME', None) -TABLE_NAME = os.getenv('COUNTER_TABLE', 'GithubRunnerQueue') -_commiters = set() -GH_WEBHOOK_TOKEN = None - -REPOS = os.getenv('REPOS') -if REPOS: - REPO_CONFIGURATION = json.loads(REPOS) -else: - REPO_CONFIGURATION = { - # <repo>: [list-of-branches-to-use-self-hosted-on] - 'apache/airflow': {'main', 'master'}, - } -del REPOS - - [email protected]('/', methods=['POST']) -def index(): - validate_gh_sig(app.current_request) - - if app.current_request.headers.get('X-GitHub-Event', None) != "check_run": - # Ignore things about installs/permissions etc - return {'ignored': 'not about check_runs'} - - body = app.current_request.json_body - - repo = body['repository']['full_name'] - - sender = body['sender']['login'] - - # Other repos configured with this app, but we don't do anything with them - # yet. - if repo not in REPO_CONFIGURATION: - app.log.info("Ignoring event for %r", repo) - return {'ignored': 'Other repo'} - - interested_branches = REPO_CONFIGURATION[repo] - - branch = body['check_run']['check_suite']['head_branch'] - - use_self_hosted = sender in commiters() or branch in interested_branches - payload = {'sender': sender, 'use_self_hosted': use_self_hosted} - - if body['action'] == 'completed' and body['check_run']['conclusion'] == 'cancelled': - if use_self_hosted: - # The only time we get a "cancelled" job is when it wasn't yet running. - queue_length = increment_dynamodb_counter(-1) - # Don't scale in the ASG -- let the CloudWatch alarm do that. - payload['new_queue'] = queue_length - else: - payload = {'ignored': 'unknown sender'} - - elif body['action'] != 'created': - payload = {'ignored': "action is not 'created'"} - - elif body['check_run']['status'] != 'queued': - # Skipped runs are "created", but are instantly completed. Ignore anything that is not queued - payload = {'ignored': "check_run.status is not 'queued'"} - else: - if use_self_hosted: - # Increment counter in DynamoDB - queue_length = increment_dynamodb_counter() - payload.update(**scale_asg_if_needed(queue_length)) - app.log.info( - "delivery=%s branch=%s: %r", - app.current_request.headers.get('X-GitHub-Delivery', None), - branch, - payload, - ) - return payload - - -def commiters(ssm_repo_name: str = os.getenv('SSM_REPO_NAME', 'apache/airflow')): - global _commiters - - if not _commiters: - client = boto3.client('ssm') - param_path = os.path.join('/runners/', ssm_repo_name, 'configOverlay') - app.log.info("Loading config overlay from %s", param_path) - - try: - - resp = client.get_parameter(Name=param_path, WithDecryption=True) - except client.exceptions.ParameterNotFound: - app.log.debug("Failed to load config overlay", exc_info=True) - return set() - - try: - overlay = json.loads(resp['Parameter']['Value']) - except ValueError: - app.log.debug("Failed to parse config overlay", exc_info=True) - return set() - - _commiters = set(overlay['pullRequestSecurity']['allowedAuthors']) - - return _commiters - - -def validate_gh_sig(request: Request): - sig = request.headers.get('X-Hub-Signature-256', None) - if not sig or not sig.startswith('sha256='): - raise BadRequestError('X-Hub-Signature-256 not of expected format') - - sig = sig[len('sha256=') :] - calculated_sig = sign_request_body(request) - - app.log.debug('Checksum verification - expected %s got %s', calculated_sig, sig) - - if not hmac.compare_digest(sig, calculated_sig): - raise ForbiddenError('Spoofed request') - - -def sign_request_body(request: Request) -> str: - global GH_WEBHOOK_TOKEN - if GH_WEBHOOK_TOKEN is None: - if 'GH_WEBHOOK_TOKEN' in os.environ: - # Local dev support: - GH_WEBHOOK_TOKEN = os.environ['GH_WEBHOOK_TOKEN'].encode('utf-8') - else: - encrypted = os.environb[b'GH_WEBHOOK_TOKEN_ENCRYPTED'] - - kms = boto3.client('kms') - response = kms.decrypt(CiphertextBlob=codecs.decode(encrypted, 'base64')) - GH_WEBHOOK_TOKEN = response['Plaintext'] - body = cast(bytes, request.raw_body) - return hmac.new(GH_WEBHOOK_TOKEN, body, digestmod='SHA256').hexdigest() # type: ignore - - -def increment_dynamodb_counter(delta: int = 1) -> int: - dynamodb = boto3.client('dynamodb') - args = dict( - TableName=TABLE_NAME, - Key={'id': {'S': 'queued_jobs'}}, - ExpressionAttributeValues={':delta': {'N': str(delta)}}, - UpdateExpression='ADD queued :delta', - ReturnValues='UPDATED_NEW', - ) - - if delta < 0: - # Make sure it never goes below zero! - args['ExpressionAttributeValues'][':limit'] = {'N': str(-delta)} - args['ConditionExpression'] = 'queued >= :limit' - - resp = dynamodb.update_item(**args) - return int(resp['Attributes']['queued']['N']) - - -def scale_asg_if_needed(num_queued_jobs: int) -> dict: - asg = boto3.client('autoscaling', region_name=ASG_REGION_NAME) - - resp = asg.describe_auto_scaling_groups( - AutoScalingGroupNames=[ASG_GROUP_NAME], - ) - - asg_info = resp['AutoScalingGroups'][0] - - current = asg_info['DesiredCapacity'] - max_size = asg_info['MaxSize'] - - busy = 0 - for instance in asg_info['Instances']: - if instance['LifecycleState'] == 'InService' and instance['ProtectedFromScaleIn']: - busy += 1 - app.log.info("Busy instances: %d, num_queued_jobs: %d, current_size: %d", busy, num_queued_jobs, current) - - new_size = num_queued_jobs + busy - if new_size > current: - if new_size <= max_size or current < max_size: - try: - new_size = min(new_size, max_size) - asg.set_desired_capacity(AutoScalingGroupName=ASG_GROUP_NAME, DesiredCapacity=new_size) - return {'new_capcity': new_size} - except asg.exceptions.ScalingActivityInProgressFault as e: - return {'error': str(e)} - else: - return {'capacity_at_max': True} - else: - return {'idle_instances': True} diff --git a/lambdas/scale_out_runner/requirements.txt b/lambdas/scale_out_runner/requirements.txt deleted file mode 100644 index c5402f0..0000000 --- a/lambdas/scale_out_runner/requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -boto3 -chalice diff --git a/requirements-test.txt b/requirements-test.txt index 5ca36b4..6023b51 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -16,4 +16,3 @@ # under the License. pytest~=9.1 -moto diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a36cc06..0000000 --- a/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -boto3 -click~=8.5 -chalice -pygithub -pytest~=9.1 -python-dynamodb-lock-whatnick~=0.9.3 -psutil -rich-click -requests -tenacity~=9.2 diff --git a/scripts/list_committers b/scripts/list_committers deleted file mode 100755 index d0b0d09..0000000 --- a/scripts/list_committers +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import rich_click as click -from github import Github - -from rich.console import Console - -console = Console(color_system="standard", width=200) - - [email protected](short_help='List committer logins - used to sync list of committers in CI configuration') [email protected]( - '--github-token', - envvar='GITHUB_TOKEN', - help="You can generate the token with readOrg permissions: " - "https://github.com/settings/tokens/new?description=Read%20Org&scopes=read:org";, -) -def main(github_token): - gh = Github(github_token) - org = gh.get_organization('apache') - committers = org.get_team_by_slug('airflow-committers') - committer_usernames = sorted(f'"{c.login}"' for c in committers.get_members()) - - click.echo("Take the below list and:") - click.echo(" - update the `/runners/apache/airflow/configOverlay` parameter in AWS SSM ParameterStore") - click.echo(" - restart the self-hosted runners") - click.echo( - " - Inform the new committer, that it's time to open PR to update list of committers in " - "dev/breeze/src/airflow_breeze/global_constants.py (COMMITTERS variable)" - ) - click.echo(',\n'.join(committer_usernames)) - click.echo(""" - -Add the following explanation: - -As a new committer, adding yourself to committer list gives you automated -acccess to our self-hosted runners for your PRs. Once you merge the changes, -all your PRs by default will run using our self-hosted infrastructure, that is -sponsored currently by AWS and Astronomer and involves running the builds on -AWS spot instances that are started on-demand. - -Those instances are way bigger than the Public Runners available for all ASF -projects. They have 8 cores and 64 GB memory and all the builds happen -in-memory, and our builds are optimized to use parallelism and memory so that -often what you will get is 3-4 times speed-up vs the Public instances. - -We currently have up to 35 such instances running at a time, so there is a -possibility - if there are many committers pushing their changes in a short -time that they will get into a queue so it's not always faster to get the -feedback. Currently ASF has plenty (900) public runners, so the delays we -experienced in the past for Public Runners is a distant pass so they should run -relatively quickly as well, but this might change depending on other projects, -because the runners are shared between all ASF projects. - -At any of your PRs you can apply the label "use public runners" before -triggering the build and your build will switch to using Public Runners. - -Most PRs are "selective" - they will only run a subset of tests - the subeset -is calculated by "Selective checks" job (you can open the job output and find -out in details how the decisions are made - we have a simple rule engine that -determines the set of tests to run. You can apply "full tests needed" label to -PR before it starts - this way a complete set of tests will be executed - -including all Python and DB versions. - -As a committer, you can also apply the labels to other's PRs and retrigger the -build if you are in doubt if the selective checks decision was good-enough -(looking at the content of the PR). - -We also have "canary" builds that run a complete set of tests in "main". They -are run every night and after merging PRs to main. Usually 3-5 canary builds -complete during the day when there are a number of PRs merged - we cancel -running build when new PR gets merged before canary build succeeds and we start -a new one (saves a lot of build time). The canary builds are designed to signal -that some problems passed through the net and need to be fixed in order to turn -"main" into "green" so watch out for those - we sometimes revert culprit PRs or -simply implement fix PRs to do so. If you merged a PR and "canary" build failed, -you should receieve notification about it (but it could be one of the previous -commits that is the root cause so you need to verify that when diagnosing it -by looking at past builds and other PRs. - -""") - - -if __name__ == "__main__": - main() diff --git a/scripts/store-agent-creds.py b/scripts/store-agent-creds.py deleted file mode 100755 index 7f00dfa..0000000 --- a/scripts/store-agent-creds.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import json -import os -import platform -import subprocess -import tempfile -from typing import Optional, Tuple - -import boto3 -import click -import requests -from botocore.exceptions import NoCredentialsError - - [email protected]() [email protected]( - "--runner-version", - default="2.275.1", - help="Runner version to register with", - metavar="VER", -) [email protected]("--repo", default="apache/airflow") [email protected]("--store-as", default="apache/airflow") [email protected]("--runnergroup") [email protected]("--token", help="GitHub runner registration token", required=False) [email protected]("--index", type=int, required=False) -def main( - token, runner_version, store_as: Optional[str], repo, runnergroup: Optional[str], index: Optional[int] -): - check_aws_config() - dir = make_runner_dir(runner_version) - - if not token: - token = click.prompt("GitHub runner registration token") - - if store_as is None: - store_as = repo - - if index is None: - index = get_next_index(store_as) - click.echo(f"Registering as runner {index}") - - register_runner(dir.name, token, repo, runnergroup, store_as, index) - - -def check_aws_config(): - click.echo("Checking AWS account credentials") - try: - whoami = boto3.client("sts").get_caller_identity() - except NoCredentialsError: - click.echo("No AWS credentials found -- maybe you need to set AWS_PROFILE?", err=True) - exit(1) - - if whoami["Account"] != "827901512104": - click.echo("Wrong AWS account in use -- maybe you need to set AWS_PROFILE?", err=True) - exit(1) - - -def make_runner_dir(version): - """Extract the runner tar to a temporary directory""" - dir = tempfile.TemporaryDirectory() - - tar = _get_runner_tar(version) - - subprocess.check_call( - ["tar", "-xzf", tar], - cwd=dir.name, - ) - - return dir - - -def get_next_index(repo: str) -> int: - """Find the next available index to store the runner credentials in AWS SSM ParameterStore""" - paginator = boto3.client("ssm").get_paginator("describe_parameters") - - path = os.path.join('/runners/', repo, '') - - pages = paginator.paginate(ParameterFilters=[{"Key": "Path", "Option": "Recursive", "Values": [path]}]) - - seen = set() - - for page in pages: - for param in page['Parameters']: - name = param['Name'] - - # '/runners/1/config' -> '1' - index = os.path.basename(os.path.dirname(name)) - seen.add(int(index)) - - if not seen: - return 1 - - # Fill in any gaps too. - for n in range(1, max(seen) + 2): - if n not in seen: - return n - - -def register_runner(dir: str, token: str, repo: str, runnergroup: Optional[str], store_as: str, index: int): - os.chdir(dir) - - cmd = [ - "./config.sh", - "--unattended", - "--url", - f"https://github.com/{repo}";, - "--token", - token, - "--name", - f"Airflow Runner {index}", - ] - - if runnergroup: - cmd += ['--runnergroup', runnergroup] - - res = subprocess.call(cmd) - - if res != 0: - exit(res) - _put_runner_creds(store_as, index) - - -def _put_runner_creds(repo: str, index: int): - client = boto3.client("ssm") - - with open(".runner", encoding='utf-8-sig') as fh: - # We want to adjust the config before storing it! - config = json.load(fh) - config["pullRequestSecurity"] = {} - - client.put_parameter( - Name=f"/runners/{repo}/{index}/config", - Type="String", - Value=json.dumps(config, indent=2), - ) - - with open(".credentials", encoding='utf-8-sig') as fh: - client.put_parameter(Name=f"/runners/{repo}/{index}/credentials", Type="String", Value=fh.read()) - - with open(".credentials_rsaparams", encoding='utf-8-sig') as fh: - client.put_parameter(Name=f"/runners/{repo}/{index}/rsaparams", Type="SecureString", Value=fh.read()) - - -def _get_system_arch() -> Tuple[str, str]: - uname = platform.uname() - if uname.system == "Linux": - system = "linux" - elif uname.system == "Darwin": - system = "osx" - else: - raise RuntimeError("Un-supported platform") - - if uname.machine == "x86_64": - arch = "x64" - else: - raise RuntimeError("Un-supported architecture") - - return system, arch - - -def _get_runner_tar(version) -> str: - system, arch = _get_system_arch() - - cache = os.path.abspath(".cache") - - try: - os.mkdir(cache) - except FileExistsError: - pass - - fname = f"actions-runner-{system}-{arch}-{version}.tar.gz" - local_file = os.path.join(cache, fname) - - if os.path.exists(local_file): - return local_file - - url = f"https://github.com/actions/runner/releases/download/v{version}/{fname}"; - click.echo(f"Getting {url}") - resp = requests.get(url, stream=True) - resp.raise_for_status() - with open(local_file, "wb") as fh, click.progressbar(length=int(resp.headers["content-length"])) as bar: - for chunk in resp.iter_content(chunk_size=40960): - fh.write(chunk) - bar.update(len(chunk)) - return local_file - - -if __name__ == "__main__": - main() diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 13a8339..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/tests/lambdas/scale_out_runner/conftest.py b/tests/lambdas/scale_out_runner/conftest.py deleted file mode 100644 index 378e130..0000000 --- a/tests/lambdas/scale_out_runner/conftest.py +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -import sys - -import pytest -from chalice.test import Client - -path = os.path.dirname(__file__) -idx = path.rfind('/tests/') -path = path[:idx] + path[idx + 6 :] -sys.path.append(path) - - [email protected] -def client(request): - app = getattr(request.module, "app") - - with Client(app) as client: - yield client diff --git a/tests/lambdas/scale_out_runner/test_app.py b/tests/lambdas/scale_out_runner/test_app.py deleted file mode 100644 index 52f8e99..0000000 --- a/tests/lambdas/scale_out_runner/test_app.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import json - -import pytest -from app import app # noqa - - [email protected](autouse=True) -def no_requests(monkeypatch): - monkeypatch.setenv("GH_WEBHOOK_TOKEN", "abc") - - -def test_no_auth(client): - response = client.http.post('/', body=json.dumps({'hello': 'world'})) - assert response.status_code == 400 - - [email protected]( - "sig", - [ - "md5=", - # Valid, but not prefixed - "160156e060356c9444613b224fc5613a0a25315b7898fd5d8c7656bd8a6654af", - ], -) -def test_bad_auth(sig, client): - response = client.http.post( - '/', - headers={ - 'X-Hub-Signature-256': sig, - }, - body=json.dumps({'hello': 'world'}), - ) - assert response.status_code == 400 - - -def test_auth(client): - response = client.http.post( - '/', - headers={ - 'X-Hub-Signature-256': 'sha256=160156e060356c9444613b224fc5613a0a25315b7898fd5d8c7656bd8a6654af' - }, - body=json.dumps({'hello': 'world'}), - ) - assert response.status_code == 200
