hongzhi-gao commented on code in PR #18638:
URL: https://github.com/apache/iotdb/pull/18638#discussion_r4013521824


##########
iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py:
##########
@@ -148,16 +150,74 @@ def close(self):
             self.__client = None
 
     def next(self):
-        if not self.has_cached_data_frame:
-            self.construct_one_data_frame()
-        if self.has_cached_data_frame:
-            return True
-        if self.__empty_resultSet:
-            return False
-        if self.__more_data and self.fetch_results():
-            self.construct_one_data_frame()
-            return True
-        return False
+        return bool(self.__row_buffer) or self._fill_row_buffer()
+
+    def _pop_row(self):
+        if not self.next():
+            return None
+        return self.__row_buffer.popleft()
+
+    def _pop_rows(self, max_rows=None):
+        rows = []
+        while max_rows is None or len(rows) < max_rows:
+            if not self.__row_buffer and not self._fill_row_buffer():
+                break
+            remaining = (
+                len(self.__row_buffer)
+                if max_rows is None
+                else min(len(self.__row_buffer), max_rows - len(rows))
+            )
+            rows.extend(self.__row_buffer.popleft() for _ in range(remaining))
+        return rows
+
+    def _fill_row_buffer(self):

Review Comment:
   Example: for 50,000 rows with `fetch_size=10,000`, this method drains the 
current RPC page into the row buffer first. It calls `fetch_results()` for the 
next page only after that buffer is empty.



##########
iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py:
##########
@@ -148,16 +150,74 @@ def close(self):
             self.__client = None
 
     def next(self):
-        if not self.has_cached_data_frame:
-            self.construct_one_data_frame()
-        if self.has_cached_data_frame:
-            return True
-        if self.__empty_resultSet:
-            return False
-        if self.__more_data and self.fetch_results():
-            self.construct_one_data_frame()
-            return True
-        return False
+        return bool(self.__row_buffer) or self._fill_row_buffer()
+
+    def _pop_row(self):
+        if not self.next():
+            return None
+        return self.__row_buffer.popleft()
+
+    def _pop_rows(self, max_rows=None):
+        rows = []
+        while max_rows is None or len(rows) < max_rows:
+            if not self.__row_buffer and not self._fill_row_buffer():
+                break
+            remaining = (
+                len(self.__row_buffer)
+                if max_rows is None
+                else min(len(self.__row_buffer), max_rows - len(rows))
+            )
+            rows.extend(self.__row_buffer.popleft() for _ in range(remaining))
+        return rows
+
+    def _fill_row_buffer(self):
+        while not self.__row_buffer:
+            while self.__query_result is not None and 
self.__query_result_index < len(
+                self.__query_result
+            ):
+                block = self.__query_result[self.__query_result_index]
+                self.__query_result[self.__query_result_index] = None
+                self.__query_result_index += 1
+                self.__row_buffer.extend(self._deserialize_rows(block))
+                if self.__row_buffer:
+                    return True
+            if self.__empty_resultSet or not self.__more_data:
+                return False
+            if not self.fetch_results():
+                return False
+        return True
+
+    def _deserialize_rows(self, serialized_block):
+        time_array, column_arrays, null_indicators, row_count = deserialize(
+            memoryview(serialized_block)
+        )
+        columns = []
+        if not self.ignore_timestamp:
+            columns.append(time_array.tolist())
+        for location in self.__column_index_2_tsblock_column_index_list:
+            if location < 0:
+                continue
+            columns.append(
+                self._expand_column(
+                    column_arrays[location],
+                    null_indicators[location],
+                    row_count,
+                    self.__data_type_for_tsblock_column[location],
+                )
+            )
+        return list(zip(*columns)) if columns else [tuple() for _ in 
range(row_count)]

Review Comment:
   A TSBlock is column-oriented. For example, `Time=[1, 2]` and `s1=[10, 20]` 
become row tuples `[(1, 10), (2, 20)]` through `zip(*columns)`, without 
creating an intermediate DataFrame.



##########
iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py:
##########
@@ -148,18 +151,78 @@ def close(self):
             self.__client = None
 
     def next(self):
-        if not self.has_cached_data_frame:
-            self.construct_one_data_frame()
-        if self.has_cached_data_frame:
-            return True
-        if self.__empty_resultSet:
-            return False
-        if self.__more_data and self.fetch_results():
-            self.construct_one_data_frame()
-            return True
-        return False
+        return bool(self.__row_buffer) or self._fill_row_buffer()
+
+    def _pop_row(self):
+        if not self.next():
+            return None
+        return self.__row_buffer.popleft()
+
+    def _pop_rows(self, max_rows=None):
+        rows = []
+        while max_rows is None or len(rows) < max_rows:
+            if not self.__row_buffer and not self._fill_row_buffer():
+                break
+            remaining = (
+                len(self.__row_buffer)
+                if max_rows is None
+                else min(len(self.__row_buffer), max_rows - len(rows))
+            )
+            rows.extend(self.__row_buffer.popleft() for _ in range(remaining))
+        return rows
+
+    def _fill_row_buffer(self):
+        while not self.__row_buffer:
+            while self.__query_result is not None and 
self.__query_result_index < len(
+                self.__query_result
+            ):
+                block = self.__query_result[self.__query_result_index]
+                self.__query_result[self.__query_result_index] = None
+                self.__query_result_index += 1
+                self.__row_buffer.extend(self._deserialize_rows(block))
+                if self.__row_buffer:
+                    return True
+            if self.__empty_resultSet or not self.__more_data:
+                return False
+            if not self.fetch_results():
+                return False
+        return True
+
+    def _deserialize_rows(self, serialized_block):
+        time_array, column_arrays, null_indicators, row_count = deserialize(
+            memoryview(serialized_block)
+        )
+        columns = []
+        if not self.ignore_timestamp:
+            columns.append(time_array.tolist())
+        for location in self.__column_index_2_tsblock_column_index_list:
+            if location < 0:
+                continue
+            columns.append(
+                self._expand_column(
+                    column_arrays[location],
+                    null_indicators[location],
+                    row_count,
+                    self.__data_type_for_tsblock_column[location],
+                )
+            )
+        return list(zip(*columns)) if columns else [tuple() for _ in 
range(row_count)]
+
+    @staticmethod
+    def _expand_column(values, nulls, row_count, data_type):
+        source = values.tolist() if hasattr(values, "tolist") else list(values)
+        if nulls is None:
+            return source
+        if data_type == TSDataType.BOOLEAN and len(source) == row_count:

Review Comment:
   For example, Boolean `values=[True, False, True]` with `nulls=[False, True, 
False]` must become `[True, None, True]`. Boolean blocks keep positional 
values, unlike other nullable columns that omit null entries.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to