This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch less_solr_commit in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 07d887c66333c869eae43593696af3707ff7a036 Author: Thomas Loubrieu <[email protected]> AuthorDate: Thu Dec 9 14:46:30 2021 -0500 remove the auto commit from the solr collection --- .../granule_ingester/writers/ElasticsearchStore.py | 5 ++++- .../granule_ingester/writers/SolrStore.py | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/granule_ingester/granule_ingester/writers/ElasticsearchStore.py b/granule_ingester/granule_ingester/writers/ElasticsearchStore.py index b18b566..dfed039 100644 --- a/granule_ingester/granule_ingester/writers/ElasticsearchStore.py +++ b/granule_ingester/granule_ingester/writers/ElasticsearchStore.py @@ -48,7 +48,10 @@ class ElasticsearchStore(MetadataStore): async def save_metadata(self, nexus_tile: NexusTile) -> None: es_doc = self.build_es_doc(nexus_tile) await self.save_document(es_doc) - + + def commit(self): + pass + @run_in_executor def save_document(self, doc: dict): try: diff --git a/granule_ingester/granule_ingester/writers/SolrStore.py b/granule_ingester/granule_ingester/writers/SolrStore.py index 02a568d..fad8a84 100644 --- a/granule_ingester/granule_ingester/writers/SolrStore.py +++ b/granule_ingester/granule_ingester/writers/SolrStore.py @@ -88,7 +88,7 @@ class SolrStore(MetadataStore): self._set_solr_status(zk) return pysolr.SolrCloud(zk, self._collection) elif self._solr_url: - return pysolr.Solr(f'{self._solr_url}/solr/{self._collection}', always_commit=True) + return pysolr.Solr(f'{self._solr_url}/solr/{self._collection}') else: raise RuntimeError("You must provide either solr_host or zookeeper_host.") @@ -111,20 +111,21 @@ class SolrStore(MetadataStore): logger.debug(f'solr_doc: {solr_doc}') await self._save_document(solr_doc) + def commit(self): + try: + self._solr.commit() + except pysolr.SolrError as e: + logger.exception(f'Lost connection to Solr, and cannot commit tiles. cause: {e}. creating SolrLostConnectionError') + raise SolrLostConnectionError(f'Lost connection to Solr, and cannot commit tiles. cause: {e}') + @run_in_executor def _save_document(self, doc: dict): try: self._solr.add([doc]) except pysolr.SolrError as e: - logger.exception(f'Lost connection to Solr, and cannot save tiles while adding doc. cause: {e}. creating SolrLostConnectionError') - raise SolrLostConnectionError(f'Lost connection to Solr, and cannot save tiles. cause: {e}') + logger.exception(f'Lost connection to Solr, and cannot add tile. cause: {e}. creating SolrLostConnectionError') + raise SolrLostConnectionError(f'Lost connection to Solr, and cannot add tiles. cause: {e}') - def _commit(self): - try: - self._solr.commit() - except pysolr.SolrError as e: - logger.exception(f'Lost connection to Solr, and cannot save tiles while commiting. cause: {e}. creating SolrLostConnectionError') - raise SolrLostConnectionError(f'Lost connection to Solr, and cannot save tiles. cause: {e}') def _build_solr_doc(self, tile: NexusTile) -> Dict: summary: TileSummary = tile.summary
