This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch collection-manager-s3-simultaneous-update in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f2bedc9ddf8d0dc69bcc750b1f7b3891c7008177 Author: Eamon Ford <[email protected]> AuthorDate: Fri Jan 8 14:34:44 2021 -0800 SDAP-302: Fix bug where the Collection Manager would crash if the Collections Config was updated while Collection Manager was actively scanning S3 directories --- collection_manager/collection_manager/services/S3Observer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/collection_manager/collection_manager/services/S3Observer.py b/collection_manager/collection_manager/services/S3Observer.py index 6d87d91..f8efe4b 100644 --- a/collection_manager/collection_manager/services/S3Observer.py +++ b/collection_manager/collection_manager/services/S3Observer.py @@ -75,7 +75,11 @@ class S3Observer: new_cache = {} watch_index = {} - for watch in self._watches: + # We need to iterate on a copy of self._watches rather than on the original set itself + # because it is very possible that the original set could get updated while we are in the + # middle of scanning S3, which will cause an exception. + watches_copy = self._watches.copy() + for watch in watches_copy: new_cache_for_watch = await self._get_s3_files(watch.path) new_index = {file: watch for file in new_cache_for_watch}
