This is an automated email from the ASF dual-hosted git repository. martinzink pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit a266a30475b76d3ceeecaa38f4711f331d0a3901 Author: Gabor Gyimesi <[email protected]> AuthorDate: Mon Jul 31 11:30:50 2023 +0200 MINIFICPP-2168 Fix Github Actions cache issues - Environment variable for docker cache only used after defined - Fix mistake of comparing int with string representation of PR id Closes #1613 Signed-off-by: Martin Zink <[email protected]> --- .github/workflows/ci.yml | 4 +- github_scripts/github_actions_cache_cleanup.py | 2 +- .../github_actions_cache_cleanup_tests.py | 111 +++++++++++++++++++-- 3 files changed, 103 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23ec4ca3a..8f95a7463 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ on: [push, pull_request, workflow_dispatch] env: DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ - -DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache + -DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON SCCACHE_GHA_ENABLE: true jobs: macos_xcode: @@ -352,7 +352,7 @@ jobs: if [ -d ~/.ccache ]; then mv ~/.ccache .; fi mkdir build cd build - cmake ${DOCKER_CMAKE_FLAGS} .. + cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. make docker - name: cache save uses: actions/cache/save@v3 diff --git a/github_scripts/github_actions_cache_cleanup.py b/github_scripts/github_actions_cache_cleanup.py index 0f4d67970..8e70e9538 100755 --- a/github_scripts/github_actions_cache_cleanup.py +++ b/github_scripts/github_actions_cache_cleanup.py @@ -73,7 +73,7 @@ class GithubActionsCacheCleaner: def _is_pr_already_closed(self, entry, open_tickets): match = re.search(r'refs/pull/([\d]+)/merge', entry.key) - return match and match[1] not in open_tickets + return match and int(match[1]) not in open_tickets def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cache_map: Dict[str, CacheEntry], removable_entries: List[str]): cache_mapping_key = "-".join(entry.key.split("-")[0:-1]) diff --git a/github_scripts/github_actions_cache_cleanup_tests.py b/github_scripts/github_actions_cache_cleanup_tests.py index 6d33ececb..857bbd921 100755 --- a/github_scripts/github_actions_cache_cleanup_tests.py +++ b/github_scripts/github_actions_cache_cleanup_tests.py @@ -11,15 +11,15 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): mock.list_open_pull_requests = MagicMock() open_pull_requests = [ { - "number": "227", + "number": 227, "title": "MINIFICPP-13712 TEST1", }, { - "number": "228", + "number": 228, "title": "MINIFICPP-9999 TEST2", }, { - "number": "229", + "number": 229, "title": "MINIFICPP-123 TEST3", } ] @@ -27,35 +27,35 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): caches = { "actions_caches": [ { - "id": "999", + "id": 999, "key": "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", }, { - "id": "11111", + "id": 11111, "key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", }, { - "id": "11112", + "id": 11112, "key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", }, { - "id": "12345", + "id": 12345, "key": "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", }, { - "id": "22221", + "id": 22221, "key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", }, { - "id": "22222", + "id": 22222, "key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-8f4d283f5bc894af8dfc295e5976a5f1b4664123", }, { - "id": "44444", + "id": 44444, "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", }, { - "id": "55555", + "id": 55555, "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", } ] @@ -65,6 +65,77 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): mock.delete_cache = MagicMock() return mock + def create_empty_open_pr_mock_github_request_sender(self): + mock = MagicMock() + mock.list_open_pull_requests = MagicMock() + mock.list_open_pull_requests.return_value = [] + caches = { + "actions_caches": [ + { + "id": 999, + "key": "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + }, + { + "id": 11111, + "key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + }, + { + "id": 11112, + "key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + }, + { + "id": 12345, + "key": "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", + }, + { + "id": 22221, + "key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + }, + { + "id": 22222, + "key": "macos-xcode-ccache-refs/heads/MINIFICPP-9999-8f4d283f5bc894af8dfc295e5976a5f1b4664123", + }, + { + "id": 44444, + "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + }, + { + "id": 55555, + "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", + } + ] + } + mock.list_caches = MagicMock() + mock.list_caches.return_value = caches + mock.delete_cache = MagicMock() + return mock + + def create_empty_caches_mock_github_request_sender(self): + mock = MagicMock() + mock.list_open_pull_requests = MagicMock() + open_pull_requests = [ + { + "number": 227, + "title": "MINIFICPP-13712 TEST1", + }, + { + "number": 228, + "title": "MINIFICPP-9999 TEST2", + }, + { + "number": 229, + "title": "MINIFICPP-123 TEST3", + } + ] + mock.list_open_pull_requests.return_value = open_pull_requests + mock.list_caches = MagicMock() + caches = { + "actions_caches": [] + } + mock.list_caches.return_value = caches + mock.delete_cache = MagicMock() + return mock + def test_cache_cleanup(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") cleaner.github_request_sender = self.create_mock_github_request_sender() @@ -75,6 +146,24 @@ class TestGithubActionsCacheCleaner(unittest.TestCase): "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + def test_cache_cleanup_with_zero_open_prs(self): + cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") + cleaner.github_request_sender = self.create_empty_open_pr_mock_github_request_sender() + cleaner.remove_obsolete_cache_entries() + self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), + {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", + "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + + def test_cache_cleanup_with_zero_action_caches(self): + cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") + cleaner.github_request_sender = self.create_empty_caches_mock_github_request_sender() + cleaner.remove_obsolete_cache_entries() + self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), set()) + if __name__ == '__main__': unittest.main()
