hongzhi-gao commented on code in PR #18638:
URL: https://github.com/apache/iotdb/pull/18638#discussion_r4013521824
##########
iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py:
##########
@@ -148,16 +150,74 @@ def close(self):
self.__client = None
def next(self):
- if not self.has_cached_data_frame:
- self.construct_one_data_frame()
- if self.has_cached_data_frame:
- return True
- if self.__empty_resultSet:
- return False
- if self.__more_data and self.fetch_results():
- self.construct_one_data_frame()
- return True
- return False
+ return bool(self.__row_buffer) or self._fill_row_buffer()
+
+ def _pop_row(self):
+ if not self.next():
+ return None
+ return self.__row_buffer.popleft()
+
+ def _pop_rows(self, max_rows=None):
+ rows = []
+ while max_rows is None or len(rows) < max_rows:
+ if not self.__row_buffer and not self._fill_row_buffer():
+ break
+ remaining = (
+ len(self.__row_buffer)
+ if max_rows is None
+ else min(len(self.__row_buffer), max_rows - len(rows))
+ )
+ rows.extend(self.__row_buffer.popleft() for _ in range(remaining))
+ return rows
+
+ def _fill_row_buffer(self):
Review Comment:
This first consumes already-returned TSBlocks and only issues another RPC
fetch when the local row buffer is empty, preserving the existing paged-fetch
behavior.
##########
iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py:
##########
@@ -148,16 +150,74 @@ def close(self):
self.__client = None
def next(self):
- if not self.has_cached_data_frame:
- self.construct_one_data_frame()
- if self.has_cached_data_frame:
- return True
- if self.__empty_resultSet:
- return False
- if self.__more_data and self.fetch_results():
- self.construct_one_data_frame()
- return True
- return False
+ return bool(self.__row_buffer) or self._fill_row_buffer()
+
+ def _pop_row(self):
+ if not self.next():
+ return None
+ return self.__row_buffer.popleft()
+
+ def _pop_rows(self, max_rows=None):
+ rows = []
+ while max_rows is None or len(rows) < max_rows:
+ if not self.__row_buffer and not self._fill_row_buffer():
+ break
+ remaining = (
+ len(self.__row_buffer)
+ if max_rows is None
+ else min(len(self.__row_buffer), max_rows - len(rows))
+ )
+ rows.extend(self.__row_buffer.popleft() for _ in range(remaining))
+ return rows
+
+ def _fill_row_buffer(self):
+ while not self.__row_buffer:
+ while self.__query_result is not None and
self.__query_result_index < len(
+ self.__query_result
+ ):
+ block = self.__query_result[self.__query_result_index]
+ self.__query_result[self.__query_result_index] = None
+ self.__query_result_index += 1
+ self.__row_buffer.extend(self._deserialize_rows(block))
+ if self.__row_buffer:
+ return True
+ if self.__empty_resultSet or not self.__more_data:
+ return False
+ if not self.fetch_results():
+ return False
+ return True
+
+ def _deserialize_rows(self, serialized_block):
+ time_array, column_arrays, null_indicators, row_count = deserialize(
+ memoryview(serialized_block)
+ )
+ columns = []
+ if not self.ignore_timestamp:
+ columns.append(time_array.tolist())
+ for location in self.__column_index_2_tsblock_column_index_list:
+ if location < 0:
+ continue
+ columns.append(
+ self._expand_column(
+ column_arrays[location],
+ null_indicators[location],
+ row_count,
+ self.__data_type_for_tsblock_column[location],
+ )
+ )
+ return list(zip(*columns)) if columns else [tuple() for _ in
range(row_count)]
Review Comment:
The RPC block is column-oriented. Transposing the decoded columns with `zip`
produces tuple rows directly and avoids constructing an intermediate DataFrame
for row iteration.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]