This is an automated email from the ASF dual-hosted git repository. fgerlits pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 4356e902f41ea492ff88e72b0f4d2d69182a02e8 Author: Gabor Gyimesi <[email protected]> AuthorDate: Wed Jun 14 18:03:24 2023 +0200 MINIFICPP-2130 Custom cache eviction strategy for GitHub Actions Signed-off-by: Ferenc Gerlits <[email protected]> This closes #1591 --- .github/workflows/ci.yml | 66 +++++++++--- .github/workflows/clear-actions-cache.yml | 18 ++++ github_scripts/github_actions_cache_cleanup.py | 117 +++++++++++++++++++++ .../github_actions_cache_cleanup_tests.py | 80 ++++++++++++++ github_scripts/requirements.txt | 1 + run_flake8.sh | 2 +- 6 files changed, 268 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95fb35f49..9480bf81c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,8 @@ jobs: steps: - id: checkout uses: actions/checkout@v3 - - id: cache - uses: actions/cache@v3 + - name: cache restore + uses: actions/cache/restore@v3 with: path: ${{ env.CCACHE_DIR }} key: macos-xcode-ccache-${{github.ref}}-${{github.sha}} @@ -43,6 +43,12 @@ jobs: export CPPFLAGS="-I/usr/local/opt/flex/include" # CPPFLAGS are not recognized by cmake, so we have to force them to CFLAGS and CXXFLAGS to have flex 2.6 working ./bootstrap.sh -e -t && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DCMAKE_C_FLAGS="${CPPFLAGS} ${CFLAGS}" -DCMAKE_CXX_FLAGS="${CPPFLAGS} ${CXXFLAGS}" -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_SQL=ON -DUSE_REAL_ODBC_TEST_DRIVER=ON -DENABLE_AZURE=ON -DENABLE_GCP=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES=OFF -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON .. && cmake --build . --parallel 4 + - name: cache save + uses: actions/cache/save@v3 + if: always() + with: + path: ${{ env.CCACHE_DIR }} + key: macos-xcode-ccache-${{github.ref}}-${{github.sha}} - name: test id: test run: | @@ -74,10 +80,8 @@ jobs: run: git config --system core.longpaths true - id: checkout uses: actions/checkout@v3 - - name: Run sccache-cache - uses: mozilla-actions/[email protected] - - name: sccache cache - uses: actions/cache@v3 + - name: sccache cache restore + uses: actions/cache/restore@v3 with: path: ~/AppData/Local/Mozilla/sccache/cache key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }} @@ -85,6 +89,8 @@ jobs: ${{ runner.os }}-sccache-${{ github.ref }} ${{ runner.os }}-sccache-refs/heads/main ${{ runner.os }}-sccache + - name: Run sccache-cache + uses: mozilla-actions/[email protected] - name: Install ninja-build tool uses: seanmiddleditch/gha-setup-ninja@v3 - name: Set up Python @@ -107,6 +113,12 @@ jobs: win_build_vs.bat ..\b /64 /CI /S /A /PDH /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA sccache --show-stats shell: cmd + - name: sccache cache save + uses: actions/cache/save@v3 + if: always() + with: + path: ~/AppData/Local/Mozilla/sccache/cache + key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }} - name: test run: cd ..\b && ctest --timeout 300 --parallel %NUMBER_OF_PROCESSORS% -C Release --output-on-failure shell: cmd @@ -120,8 +132,8 @@ jobs: steps: - id: checkout uses: actions/checkout@v3 - - id: cache - uses: actions/cache@v3 + - name: cache restore + uses: actions/cache/restore@v3 with: path: ~/.ccache key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}} @@ -144,6 +156,12 @@ jobs: -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON \ -DENABLE_LUA_SCRIPTING=OFF -DENABLE_PYTHON_SCRIPTING=OFF -DENABLE_MQTT=OFF -DENABLE_ELASTICSEARCH=OFF -DENABLE_KUBERNETES=OFF -DENABLE_OPC=OFF .. make -j$(nproc) VERBOSE=1 + - name: cache save + uses: actions/cache/save@v3 + if: always() + with: + path: ~/.ccache + key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}} - name: test id: test run: | @@ -171,8 +189,8 @@ jobs: steps: - id: checkout uses: actions/checkout@v3 - - id: cache - uses: actions/cache@v3 + - name: cache restore + uses: actions/cache/restore@v3 with: path: ~/.ccache key: ubuntu-20.04-all-clang-ccache-${{github.ref}}-${{github.sha}} @@ -204,6 +222,12 @@ jobs: -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. cmake --build . --parallel $(nproc) + - id: cache_save + uses: actions/cache/save@v3 + if: always() + with: + path: ~/.ccache + key: ubuntu-20.04-all-clang-ccache-${{github.ref}}-${{github.sha}} - name: test id: test run: | @@ -254,8 +278,8 @@ jobs: steps: - id: checkout uses: actions/checkout@v3 - - id: cache - uses: actions/cache@v3 + - name: cache restore + uses: actions/cache/restore@v3 with: path: ~/.ccache key: centos-ccache-${{github.ref}}-${{github.sha}} @@ -276,6 +300,12 @@ jobs: -DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \ -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \ -DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test + - name: cache save + uses: actions/cache/save@v3 + if: always() + with: + path: ~/.ccache + key: centos-ccache-${{github.ref}}-${{github.sha}} - id: test run: | # Set core file size limit to 1GiB @@ -309,8 +339,8 @@ jobs: steps: - id: checkout uses: actions/checkout@v3 - - id: cache - uses: actions/cache@v3 + - name: cache restore + uses: actions/cache/restore@v3 with: path: ~/.ccache key: docker-ccache-${{github.ref}}-${{github.sha}} @@ -324,6 +354,12 @@ jobs: cd build cmake ${DOCKER_CMAKE_FLAGS} .. make docker + - name: cache save + uses: actions/cache/save@v3 + if: always() + with: + path: ~/.ccache + key: docker-ccache-${{github.ref}}-${{github.sha}} - name: Save docker image run: cd build && docker save -o minifi_docker.tar apacheminificpp:$(grep CMAKE_PROJECT_VERSION:STATIC CMakeCache.txt | cut -d "=" -f2) - name: Upload artifact @@ -331,7 +367,7 @@ jobs: with: name: minifi_docker path: build/minifi_docker.tar - docker_tests_q1: + docker_tests: name: "Docker integration tests" needs: docker_build runs-on: ubuntu-20.04 diff --git a/.github/workflows/clear-actions-cache.yml b/.github/workflows/clear-actions-cache.yml new file mode 100644 index 000000000..65eb3efbf --- /dev/null +++ b/.github/workflows/clear-actions-cache.yml @@ -0,0 +1,18 @@ +name: "MiNiFi-CPP Github Actions Cache Eviction" +on: + schedule: + - cron: '*/30 * * * *' + workflow_dispatch: +jobs: + ubuntu_20_04: + name: "ubuntu-20.04" + runs-on: ubuntu-20.04 + steps: + - id: checkout + uses: actions/checkout@v3 + - name: clear cache + run: | + python3 -m venv github_env + source github_env/bin/activate + pip install -r github_scripts/requirements.txt + python3 github_scripts/github_actions_cache_cleanup.py -t ${{github.token}} -r ${{github.repository}} diff --git a/github_scripts/github_actions_cache_cleanup.py b/github_scripts/github_actions_cache_cleanup.py new file mode 100755 index 000000000..0f4d67970 --- /dev/null +++ b/github_scripts/github_actions_cache_cleanup.py @@ -0,0 +1,117 @@ +#!/bin/python3 + +import requests +import re +import argparse +import logging +from typing import Dict, List + + +logging.basicConfig(level=logging.INFO) + + +class GithubRequestSender: + def __init__(self, token: str, repository: str): + self.headers = { + 'Accept': 'application/vnd.github+json', + 'Authorization': 'Bearer ' + token, + 'X-GitHub-Api-Version': '2022-11-28' + } + self.repository = repository + + def _send_get_request(self, url: str): + response = requests.get(url, headers=self.headers) + response.raise_for_status() + + return response.json() + + def _send_delete_request(self, url: str, params: Dict[str, str]): + response = requests.delete(url, headers=self.headers, params=params) + response.raise_for_status() + + def list_open_pull_requests(self): + url = "https://api.github.com/repos/"; + self.repository + "/pulls?state=open" + return self._send_get_request(url) + + def list_caches(self): + url = "https://api.github.com/repos/"; + self.repository + "/actions/caches" + return self._send_get_request(url) + + def delete_cache(self, key: str): + url = "https://api.github.com/repos/"; + self.repository + "/actions/caches" + self._send_delete_request(url, params={"key": key}) + + +class CacheEntry: + def __init__(self, id: int, key: str): + self.id = id + self.key = key + + def __str__(self) -> str: + return "{id: " + str(self.id) + ", key: " + self.key + "}" + + def __repr__(self): + return str(self) + + +class GithubActionsCacheCleaner: + def __init__(self, token: str, repository: str): + self.github_request_sender = GithubRequestSender(token, repository) + + def _list_open_pr_ids(self) -> List[str]: + open_tickets = [] + for request in self.github_request_sender.list_open_pull_requests(): + open_tickets.append(request['number']) + return open_tickets + + def _get_cache_entries(self) -> List[CacheEntry]: + entries = [] + json_result = self.github_request_sender.list_caches() + for json_entry in json_result["actions_caches"]: + entries.append(CacheEntry(int(json_entry["id"]), json_entry["key"])) + return entries + + def _is_pr_already_closed(self, entry, open_tickets): + match = re.search(r'refs/pull/([\d]+)/merge', entry.key) + return match and match[1] not in open_tickets + + def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cache_map: Dict[str, CacheEntry], removable_entries: List[str]): + cache_mapping_key = "-".join(entry.key.split("-")[0:-1]) + if cache_mapping_key not in latest_branch_cache_map: + latest_branch_cache_map[cache_mapping_key] = entry + else: + if latest_branch_cache_map[cache_mapping_key].id < entry.id: + removable_entries.append(latest_branch_cache_map[cache_mapping_key].key) + latest_branch_cache_map[cache_mapping_key] = entry + else: + removable_entries.append(entry.key) + + def _get_removable_cache_entries(self) -> List[str]: + removable_entries = [] + latest_branch_cache_map = dict() + open_prs = self._list_open_pr_ids() + for entry in self._get_cache_entries(): + if self._is_pr_already_closed(entry, open_prs): + removable_entries.append(entry.key) + continue + + self._remove_non_latest_branch_caches(entry, latest_branch_cache_map, removable_entries) + + return removable_entries + + def _remove_cache_entries(self, entries_to_remove: List[str]): + for key in entries_to_remove: + logging.info('Removing cache entry: %s', key) + self.github_request_sender.delete_cache(key) + + def remove_obsolete_cache_entries(self): + self._remove_cache_entries(self._get_removable_cache_entries()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog='Github Actions Cache Cleaner', description='Cleans up obsolete cache entries of github actions') + parser.add_argument('-r', '--repository', help='The name of the repository in <owner>/<repository> format', required=True) + parser.add_argument('-t', '--token', help='Github token for API access', required=True) + args = parser.parse_args() + cache_cleaner = GithubActionsCacheCleaner(args.token, args.repository) + cache_cleaner.remove_obsolete_cache_entries() diff --git a/github_scripts/github_actions_cache_cleanup_tests.py b/github_scripts/github_actions_cache_cleanup_tests.py new file mode 100755 index 000000000..6d33ececb --- /dev/null +++ b/github_scripts/github_actions_cache_cleanup_tests.py @@ -0,0 +1,80 @@ +#!/bin/python3 + +import unittest +from unittest.mock import MagicMock +from github_actions_cache_cleanup import GithubActionsCacheCleaner + + +class TestGithubActionsCacheCleaner(unittest.TestCase): + def create_mock_github_request_sender(self): + mock = MagicMock() + mock.list_open_pull_requests = MagicMock() + open_pull_requests = [ + { + "number": "227", + "title": "MINIFICPP-13712 TEST1", + }, + { + "number": "228", + "title": "MINIFICPP-9999 TEST2", + }, + { + "number": "229", + "title": "MINIFICPP-123 TEST3", + } + ] + mock.list_open_pull_requests.return_value = open_pull_requests + caches = { + "actions_caches": [ + { + "id": "999", + "key": "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + }, + { + "id": "11111", + "key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + }, + { + "id": "11112", + "key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + }, + { + "id": "12345", + "key": "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", + }, + { + "id": "22221", + "key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + }, + { + "id": "22222", + "key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-8f4d283f5bc894af8dfc295e5976a5f1b4664123", + }, + { + "id": "44444", + "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + }, + { + "id": "55555", + "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", + } + ] + } + mock.list_caches = MagicMock() + mock.list_caches.return_value = caches + mock.delete_cache = MagicMock() + return mock + + def test_cache_cleanup(self): + cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") + cleaner.github_request_sender = self.create_mock_github_request_sender() + cleaner.remove_obsolete_cache_entries() + self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), + {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + + +if __name__ == '__main__': + unittest.main() diff --git a/github_scripts/requirements.txt b/github_scripts/requirements.txt new file mode 100644 index 000000000..2c24336eb --- /dev/null +++ b/github_scripts/requirements.txt @@ -0,0 +1 @@ +requests==2.31.0 diff --git a/run_flake8.sh b/run_flake8.sh index b0577c428..e1a409f09 100755 --- a/run_flake8.sh +++ b/run_flake8.sh @@ -19,4 +19,4 @@ set -euo pipefail directory=${1:-.} -flake8 --exclude venv,thirdparty,build,cmake-build-* --builtins log,REL_SUCCESS,REL_FAILURE,raw_input --ignore E501,W503 --per-file-ignores="steps.py:F811" "${directory}" +flake8 --exclude venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,raw_input --ignore E501,W503 --per-file-ignores="steps.py:F811" "${directory}"
