This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch solr_stable_359 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f206e91162fe008a8891f802f835b984eea0d526 Author: Thomas Loubrieu <[email protected]> AuthorDate: Sun Nov 28 12:15:18 2021 -0500 upgrade solr connection status control for latest solr versions --- .../granule_ingester/writers/SolrStore.py | 37 +++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/granule_ingester/granule_ingester/writers/SolrStore.py b/granule_ingester/granule_ingester/writers/SolrStore.py index def70c5..5c5f088 100644 --- a/granule_ingester/granule_ingester/writers/SolrStore.py +++ b/granule_ingester/granule_ingester/writers/SolrStore.py @@ -50,13 +50,42 @@ class SolrStore(MetadataStore): self.log.setLevel(logging.DEBUG) self._solr = None + def _get_collections(self, zk, parent_nodes): + """ + try to get list of collection from zookeper, on a list of candidate nodes, + return the first successful request result + """ + + try: + logger.debug("getting solr configuration from zookeeper, node '%s'", parent_nodes[0]) + return parent_nodes[0], zk.zk.get_children(parent_nodes[0]) + except NoNodeError: + logger.debug("solr configuration not found in node '%s'", parent_nodes[0]) + if len(parent_nodes)>1: + return self._get_collections(zk, parent_nodes[1:]) + else: + raise + + def _set_solr_status(self, zk): + """ because of something not working right between zookeeper and solr + we need to manually update the solr status on zookeeper + see https://github.com/django-haystack/pysolr/issues/189 + """ + collections = {} + parent_node, zk_collections = self._get_collections(zk, + ['collections', + 'solr/collections'] + # with bitnami/solr 0.3.3 helm chart deployment + ) + + for c in zk_collections: + collections.update(json.loads(zk.zk.get(f"{parent_node}/{c}/state.json")[0].decode("utf-8"))) + zk.collections = collections + def _get_connection(self) -> pysolr.Solr: if self._zk_url: zk = pysolr.ZooKeeper(f"{self._zk_url}") - collections = {} - for c in zk.zk.get_children("collections"): - collections.update(json.loads(zk.zk.get("collections/{}/state.json".format(c))[0].decode("ascii"))) - zk.collections = collections + self._set_solr_status(zk) return pysolr.SolrCloud(zk, self._collection, always_commit=True) elif self._solr_url: return pysolr.Solr(f'{self._solr_url}/solr/{self._collection}', always_commit=True)
