This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch s3-support in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 180ea1e5bc7b5c4d53f6b11be99931e3b8c066db Author: Eamon Ford <[email protected]> AuthorDate: Thu Oct 1 18:20:22 2020 -0700 Make some public methods private, for clarity --- .../services/history_manager/IngestionHistory.py | 46 +++++++++++----------- .../history_manager/test_FileIngestionHistory.py | 6 +-- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/collection_manager/collection_manager/services/history_manager/IngestionHistory.py b/collection_manager/collection_manager/services/history_manager/IngestionHistory.py index ef73ccb..b71c32f 100644 --- a/collection_manager/collection_manager/services/history_manager/IngestionHistory.py +++ b/collection_manager/collection_manager/services/history_manager/IngestionHistory.py @@ -55,27 +55,6 @@ class IngestionHistory(ABC): await self._save_latest_timestamp() - def latest_ingested_mtime(self) -> Optional[datetime]: - """ - Return the modified time of the most recently modified file that was ingested. - :return: A datetime or None - """ - if self._latest_ingested_file_update: - return datetime.fromtimestamp(self._latest_ingested_file_update) - else: - return None - - async def already_ingested(self, file_path: str) -> bool: - """ - Return a boolean indicating whether the specified file has already been ingested, based on its signature. - :param file_path: The full path of a file to search for in the history. - :return: A boolean indicating whether this file has already been ingested or not - """ - file_path = file_path.strip() - file_name = os.path.basename(file_path) - signature = self._signature_fun(file_path) - return signature == await self._get_signature(file_name) - async def get_granule_status(self, file_path: str, date_from: datetime = None, @@ -94,13 +73,34 @@ class IngestionHistory(ABC): should fall in order to be "desired". :return: A GranuleStatus enum. """ - if self._in_time_range(file_path, date_from=self.latest_ingested_mtime()): + if self._in_time_range(file_path, date_from=self._latest_ingested_mtime()): return GranuleStatus.DESIRED_FORWARD_PROCESSING - elif self._in_time_range(file_path, date_from, date_to) and not await self.already_ingested(file_path): + elif self._in_time_range(file_path, date_from, date_to) and not await self._already_ingested(file_path): return GranuleStatus.DESIRED_HISTORICAL else: return GranuleStatus.UNDESIRED + def _latest_ingested_mtime(self) -> Optional[datetime]: + """ + Return the modified time of the most recently modified file that was ingested. + :return: A datetime or None + """ + if self._latest_ingested_file_update: + return datetime.fromtimestamp(self._latest_ingested_file_update) + else: + return None + + async def _already_ingested(self, file_path: str) -> bool: + """ + Return a boolean indicating whether the specified file has already been ingested, based on its signature. + :param file_path: The full path of a file to search for in the history. + :return: A boolean indicating whether this file has already been ingested or not + """ + file_path = file_path.strip() + file_name = os.path.basename(file_path) + signature = self._signature_fun(file_path) + return signature == await self._get_signature(file_name) + @abstractmethod async def _save_latest_timestamp(self): pass diff --git a/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py b/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py index 07ab0e1..8bd939e 100644 --- a/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py +++ b/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py @@ -36,7 +36,7 @@ class TestFileIngestionHistory(unittest.TestCase): # history_manager with this file current_file_path = pathlib.Path(__file__) await ingestion_history.push(str(current_file_path)) - self.assertTrue(await ingestion_history.already_ingested(str(current_file_path))) + self.assertTrue(await ingestion_history._already_ingested(str(current_file_path))) del ingestion_history @@ -47,7 +47,7 @@ class TestFileIngestionHistory(unittest.TestCase): # history_manager with this file current_file_path = pathlib.Path(__file__) await ingestion_history.push(str(current_file_path)) - self.assertTrue(await ingestion_history.already_ingested(str(current_file_path))) + self.assertTrue(await ingestion_history._already_ingested(str(current_file_path))) del ingestion_history @@ -57,7 +57,7 @@ class TestFileIngestionHistory(unittest.TestCase): ingestion_history = FileIngestionHistory(history_dir, DATASET_ID, md5sum_from_filepath) # history_manager with this file current_file_path = pathlib.Path(__file__) - self.assertFalse(await ingestion_history.already_ingested(str(current_file_path))) + self.assertFalse(await ingestion_history._already_ingested(str(current_file_path))) if __name__ == '__main__':
