This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch switch-to-ami
in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit 4eb8f645f306e31c0b76ca9e1761dafabd0a50c7
Author: Ash Berlin-Taylor <ash_git...@firemirror.com>
AuthorDate: Mon May 17 09:29:23 2021 +0100
Switch to using the built AMI instead of cloud-init
We have packer configued to build an AMI that has everything
pre-installed, so all we have left to do in the cloud init is drop the
env var so that we know which region we are in (without having to query
it every time) and start the runner and log shipper services.
---
cloud-init.yml | 318 +--------------------------------------------------------
1 file changed, 4 insertions(+), 314 deletions(-)
diff --git a/cloud-init.yml b/cloud-init.yml
index 05d71d5..183a6f4 100644
--- a/cloud-init.yml
+++ b/cloud-init.yml
@@ -1,329 +1,19 @@
#cloud-config
-# 2021-03-20 14:00 CET
+# 2021-05-17 09:21+01:00
mounts:
- [tmpfs, /tmp, tmpfs, "defaults,noatime,size=10%"]
- [tmpfs, /var/lib/docker, tmpfs, "defaults,noatime,size=66%"]
- [tmpfs, /home/runner/actions-runner/_work, tmpfs, "defaults,noatime"]
-users:
- - default
- - name: runner
-
-packages:
- - awscli
- - build-essential
- - docker.io
- - git
- - iptables-persistent
- - jq
- - parallel
- - python3-dev
- - python3-venv
- - python3-wheel
- - yarn
- - vector
runcmd:
-
- bash
- -c
- #
https://github.com/actions/virtual-environments/blob/525f79f479cca77aef4e0a680548b65534c64a18/images/linux/scripts/installers/docker-compose.sh
- - |
- set -eu -o pipefail
- URL=$(curl -s
https://api.github.com/repos/docker/compose/releases/latest | jq -r
'.assets[].browser_download_url |
select(endswith("docker-compose-Linux-x86_64"))')
- curl -L $URL -o /usr/local/bin/docker-compose
- chmod +x /usr/local/bin/docker-compose
- -
- - bash
- - -c
- |
set -eu -o pipefail
echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >> /etc/environment
- # Set an env var (that is visible in runners) that will let us know we
are on a self-hosted runner
- echo 'AIRFLOW_SELF_HOSTED_RUNNER="[\"self-hosted\"]"' >> /etc/environment
- set -a
- . /etc/environment
- set +a
- echo "ASG_GROUP_NAME=$(aws ec2 describe-tags --filter
Name=resource-id,Values=$(cloud-init query instance_id)
Name=key,Values=aws:autoscaling:groupName \
- | jq -r '@sh "\(.Tags[0].Value)"')" >> /etc/environment
- - [systemctl, daemon-reload]
- -
- - bash
- - -c
- - |
- set -eu -o pipefail
- python3 -mvenv /opt/runner-supervisor
- /opt/runner-supervisor/bin/pip install -U pip
python-dynamodb-lock-whatnick==0.9.3 click==7.1.2 psutil 'tenacity~=6.0'
- -
- - bash
- - -c
- - |
- set -eu -o pipefail
-
- usermod -G docker -a runner
-
- mkdir -p ~runner/actions-runner
- find ~runner -exec chown runner: {} +
- cd ~runner/actions-runner
-
- RUNNER_VERSION="$0"
-
- curl -L
"https://github.com/ashb/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz";
| tar -zx
-
- set -a
- . /etc/environment
- set +a
-
- aws s3 cp s3://airflow-ci-assets/runner-supervisor.py
/opt/runner-supervisor/bin/runner-supervisor
- chmod 755 /opt/runner-supervisor/bin/runner-supervisor
-
- - 2.277.1-airflow3
- - [systemctl, enable, --now, iptables.service]
- # Restart docker after applying the user firewall -- else some rules/chains
might be list!
- - [systemctl, restart, docker.service]
- - [systemctl, enable, --now, vector.service]
- - [systemctl, enable, --now, actions.runner.service]
-
-write_files:
- - path: /etc/systemd/system/actions.runner.service
- content: |
- [Unit]
- Description=GitHub Actions Runner
- After=network.target actions.runner-supervisor.service
- Requires=actions.runner-supervisor.service
- BindsTo=actions.runner-supervisor.service
-
- [Service]
- ExecStartPre=!/usr/local/sbin/runner-cleanup-workdir.sh
- ExecStart=/home/runner/actions-runner/run.sh --once --startuptype service
- ExecStop=/usr/local/bin/stop-runner-if-no-job.sh $MAINPID
- EnvironmentFile=/etc/environment
- Environment=GITHUB_ACTIONS_RUNNER_CHANNEL_TIMEOUT=300
- User=runner
- WorkingDirectory=/home/runner/actions-runner
- KillMode=mixed
- KillSignal=SIGTERM
- TimeoutStopSec=5min
- Restart=on-success
-
- [Install]
- WantedBy=multi-user.target
-
- # Don't put this in ~runner, as these get written before the user is added,
and this messes up creating the home dir
- - path: /usr/local/sbin/runner-cleanup-workdir.sh
- content: |
- #!/bin/bash
- set -eu -o pipefail
- echo "Left-over containers:"
- docker ps -a
- docker ps -qa | xargs --verbose --no-run-if-empty docker rm -fv
-
- echo "Log in to a paid docker user to get unlimited docker pulls"
- aws ssm get-parameter --with-decryption --name
/runners/apache/airflow/dockerPassword | \
- jq .Parameter.Value -r | \
- sudo -u runner docker login --username airflowcirunners
--password-stdin
-
- if [[ -d ~runner/actions-runner/_work/airflow/airflow ]]; then
- cd ~runner/actions-runner/_work/airflow/airflow
-
- chown --changes -R runner: .
- if [[ -e .git ]]; then
- sudo -u runner bash -c "
- git reset --hard && \
- git submodule deinit --all -f && \
- git submodule foreach git clean -fxd && \
- git clean -fxd \
- "
- fi
- fi
- owner: root:root
- permissions: '0775'
- - path: /usr/local/bin/stop-runner-if-no-job.sh
- content: |
- #!/bin/bash
- set -eu -o pipefail
-
- MAINPID="${MAINPID:-${1:-}}"
-
- if [[ -z "$MAINPID" ]]; then
- echo "No MAINPID, assuming it already crashed!"
- exit 0
- fi
-
- if pgrep --ns $MAINPID -a Runner.Worker > /dev/null; then
- echo "Waiting for current job to finish"
- while pgrep --ns $MAINPID -a Runner.Worker; do
- # Job running -- just wait for it to exit
- sleep 10
- done
- fi
-
- # Request shutdown if it's still alive -- because we are in "stop" state
it should not restart
- if pkill --ns $MAINPID Runner.Listener; then
- # Wait for it to shut down
- echo "Waiting for main Runner.Listener process to stop"
- while pgrep --ns $MAINPID -a Runner.Listener; do
- sleep 5
- done
- fi
- owner: root:root
- permissions: '0775'
- - path: /etc/sudoers.d/runner
- owner: root:root
- permissions: '0440'
- content: |
- runner ALL=(ALL) NOPASSWD:/usr/sbin/swapoff -a, /usr/bin/rm -f
/swapfile, /usr/bin/apt clean
- - path: /etc/iptables/rules.v4
- content: |
- # Generated by iptables-save v1.8.4 on Thu Jan 14 13:59:27 2021
- *filter
- :INPUT ACCEPT [833:75929]
- :FORWARD DROP [0:0]
- :OUTPUT ACCEPT [794:143141]
- :DOCKER-USER - [0:0]
- -A FORWARD -j DOCKER-USER
- # Dis-allow any docker container to access the metadata service
- -A DOCKER-USER -d 169.254.169.254/32 -j REJECT --reject-with
icmp-port-unreachable
- -A DOCKER-USER -j RETURN
- COMMIT
-
- - path: /usr/local/sbin/actions-runner-ec2-reporting
- permissions: '0775'
- content: |
- #!/bin/bash
- set -eu -o pipefail
- if pgrep -c Runner.Worker >/dev/null; then
- # Only report metric when we're doing something -- no point paying
to submit zeros
- aws cloudwatch put-metric-data --metric-name jobs-running --value
"$(pgrep -c Runner.Worker)" --namespace github.actions
- fi
- - path: /etc/cron.d/cloudwatch-metrics-github-runners
- content: |
- */1 * * * * nobody /usr/local/sbin/actions-runner-ec2-reporting
-
- - path: /etc/systemd/system/actions.runner-supervisor.service
- content: |
- [Unit]
- Description=Fetch credentials and supervise GitHub Actions Runner
- After=network.target
- Before=actions.runner.service
-
- [Service]
- Type=notify
- ExecStart=/opt/runner-supervisor/bin/python
/opt/runner-supervisor/bin/runner-supervisor
- # We need to run as root to have the ability to open the netlink
connector socket
- User=root
- WorkingDirectory=/home/runner/actions-runner
- Restart=always
- EnvironmentFile=/etc/environment
- - path: /etc/vector/vector.toml
- content: |
- data_dir = "/var/lib/vector"
-
- [api]
- enabled = true
-
- # Input data. Change me to a valid input source.
- [sources.logs]
- type = "journald"
- include_units = ["actions.runner.service",
"actions.runner-supervisor.service"]
-
- [transforms.without_systemd_fields]
- type = "remove_fields"
- inputs = ["logs"]
- fields = ["_CAP_EFFECTIVE", "_SYSTEMD_SLICE", "_SYSTEMD_CGROUP",
- "_SYSTEMD_INVOCATION_ID", "_SELINUX_CONTEXT", "_COMM", "_BOOT_ID",
- "_MACHINE_ID", "_STREAM_ID", "_PID", "_GID", "_UID","_TRANSPORT",
- "__MONOTONIC_TIMESTAMP", "SYSLOG_IDENTIFIER", "PRIORITY",
- "source_type"]
-
- [sources.runner-logs]
- type = "file"
- include = ["/home/runner/actions-runner/_diag/*.log"]
-
- [sources.runner-logs.multiline]
- start_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
- mode = "halt_before"
- condition_pattern = '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
- timeout_ms = 250
-
- [transforms.grok-runner-logs]
- type = "remap"
- inputs=["runner-logs"]
- source = '''
- structured, err = parse_grok(.message,
"(?m)\\[%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{NOTSPACE:logger}\\]
%{GREEDYDATA:message}")
-
- if err != null {
- .err = err
- } else {
- . = merge(., structured)
- }
- '''
- [transforms.filter-runner-logs]
- type = "filter"
- inputs = ['grok-runner-logs']
- condition.type = "remap"
- condition.source = '''
- if .logger == "JobServerQueue" {
- !match!(.message, r'Try to append \d+ batches web console lines
for record')
- } else if .logger == "HostContext" {
- !starts_with!(.message, "Well known directory")
- } else {
- true
- }
- '''
-
- [sources.job-logs]
- type = "file"
- include = ["/home/runner/actions-runner/_diag/pages/*.log"]
-
- [transforms.grok-job-logs]
- type = "remap"
- inputs = ["job-logs"]
- source = '''
- structured, err = parse_grok(.message,
"%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:message}")
-
- if err == null {
- . = merge(., structured)
- .message = strip_ansi_escape_codes!(.message)
- .type = "job-output"
- }
- '''
-
- # Output data
- [sinks.cloudwatch]
- inputs = ["without_systemd_fields", "filter-runner-logs",
"grok-job-logs"]
- type = "aws_cloudwatch_logs"
- encoding = "json"
- create_missing_group = false
- create_missing_stream = true
- group_name = "GitHubRunners"
- stream_name = "{{ host }}"
- region = "eu-central-1"
-apt:
- sources:
- yarn:
- source: "deb https://dl.yarnpkg.com/debian/ stable main"
- keyid: "1646B01B86E50310"
- timber:
- source: "deb https://repositories.timber.io/public/vector/deb/ubuntu
focal main"
- key: |
- -----BEGIN PGP PUBLIC KEY BLOCK-----
- Version: GnuPG v2
-
- mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ
- jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d
- 2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy
- Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL
- Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz
- 629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj
- a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE
- EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor
- xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X
- muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE
- RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA
- gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm
- 0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g
- 3vq0NvX0k2CLi/W9mTiSdHXFChI15A==
- =k36w
- -----END PGP PUBLIC KEY BLOCK-----
+ echo "AWS_DEFAULT_REGION=$(cloud-init query region)" >>
/etc/default/vector
+ systemctl enable --now vector
+ systemctl enable --now actions.runner
