RKuttruff commented on code in PR #275:
URL:
https://github.com/apache/incubator-sdap-nexus/pull/275#discussion_r1323772668
##########
analysis/webservice/algorithms/doms/ResultsStorage.py:
##########
@@ -297,18 +297,34 @@ def __retrieveData(self, id, trim_data=False, page_num=1,
page_size=1000):
return data
def __enrichPrimaryDataWithMatches(self, id, dataMap, trim_data=False):
- cql = "SELECT * FROM doms_data where execution_id = %s and is_primary
= false"
- rows = self._session.execute(cql, (id,))
-
- for row in rows:
- entry = self.__rowToDataEntry(row, trim_data=trim_data)
- if row.primary_value_id in dataMap:
- if not "matches" in dataMap[row.primary_value_id]:
- dataMap[row.primary_value_id]["matches"] = []
- dataMap[row.primary_value_id]["matches"].append(entry)
+ cql = f"SELECT * FROM doms_data where execution_id = {str(id)} and
is_primary = false and primary_value_id = ?"
+ statement = self._session.prepare(cql)
+
+ primary_ids = list(dataMap.keys())
+
+ logger.info(f'Getting secondary data for {len(primary_ids)} primaries
of {str(id)}')
+
+ for (success, rows) in execute_concurrent_with_args(
+ self._session, statement, [(i,) for i in primary_ids],
concurrency=50, results_generator=True
+ ):
+ for row in rows:
+ entry = self.__rowToDataEntry(row, trim_data=trim_data)
+ if row.primary_value_id in dataMap:
+ if not "matches" in dataMap[row.primary_value_id]:
+ dataMap[row.primary_value_id]["matches"] = []
+ dataMap[row.primary_value_id]["matches"].append(entry)
+
+ # rows = self._session.execute(cql, (id,))
+ #
+ # for row in rows:
+ # entry = self.__rowToDataEntry(row, trim_data=trim_data)
+ # if row.primary_value_id in dataMap:
+ # if not "matches" in dataMap[row.primary_value_id]:
+ # dataMap[row.primary_value_id]["matches"] = []
+ # dataMap[row.primary_value_id]["matches"].append(entry)
def __retrievePrimaryData(self, id, trim_data=False, page_num=2,
page_size=10):
- cql = "SELECT * FROM doms_data where execution_id = %s and is_primary
= true limit %s"
+ cql = "SELECT * FROM doms_data_temp where execution_id = %s and
is_primary = true limit %s"
Review Comment:
Yes I did, thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscr...@sdap.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
