hongzhi-gao commented on code in PR #18638:
URL: https://github.com/apache/iotdb/pull/18638#discussion_r4013465896
##########
iotdb-client/client-py/iotdb/utils/Field.py:
##########
@@ -16,12 +16,18 @@
# under the License.
#
-# for package
-from iotdb.utils.IoTDBConstants import TSDataType
-from iotdb.tsfile.utils.date_utils import parse_int_to_date
-from iotdb.utils.rpc_utils import convert_to_timestamp, isoformat
import numpy as np
-import pandas as pd
+from iotdb.tsfile.utils.date_utils import parse_int_to_date
+from iotdb.utils.IoTDBConstants import TSDataType
+
+
+def _is_missing(value):
+ if value is None:
+ return True
+ value_type = type(value)
Review Comment:
`pd.NA` cannot be referenced without importing pandas. Checking the sentinel
type metadata preserves the existing null behavior while keeping pandas out of
the normal row-iteration import path.
##########
iotdb-client/client-py/iotdb/utils/SessionDataSet.py:
##########
@@ -16,16 +16,17 @@
# under the License.
#
import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
from iotdb.utils.Field import Field
+from iotdb.utils.iotdb_rpc_dataset import IoTDBRpcDataSet
# for package
from iotdb.utils.IoTDBConstants import TSDataType
-from iotdb.utils.iotdb_rpc_dataset import IoTDBRpcDataSet
from iotdb.utils.RowRecord import RowRecord
-import pandas as pd
+if TYPE_CHECKING:
+ import pandas as pd
Review Comment:
Pandas is needed here only for static type checking. The runtime annotations
are quoted, so importing and iterating a normal Session result does not load
pandas.
##########
iotdb-client/client-py/iotdb/utils/rpc_utils.py:
##########
@@ -78,6 +74,10 @@ def
verify_success_with_redirection_for_multi_devices(status: TSStatus, devices:
def convert_to_timestamp(time: int, precision: str, timezone: str):
+ import pandas as pd
+ from pandas._libs import OutOfBoundsDatetime
+ from tzlocal import get_localzone_name
Review Comment:
These imports are intentionally local: timestamp conversion keeps returning
the same pandas type, while queries that do not convert timestamps avoid
loading pandas and tzlocal.
##########
iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py:
##########
@@ -148,18 +151,78 @@ def close(self):
self.__client = None
def next(self):
- if not self.has_cached_data_frame:
- self.construct_one_data_frame()
- if self.has_cached_data_frame:
- return True
- if self.__empty_resultSet:
- return False
- if self.__more_data and self.fetch_results():
- self.construct_one_data_frame()
- return True
- return False
+ return bool(self.__row_buffer) or self._fill_row_buffer()
+
+ def _pop_row(self):
+ if not self.next():
+ return None
+ return self.__row_buffer.popleft()
+
+ def _pop_rows(self, max_rows=None):
+ rows = []
+ while max_rows is None or len(rows) < max_rows:
+ if not self.__row_buffer and not self._fill_row_buffer():
+ break
+ remaining = (
+ len(self.__row_buffer)
+ if max_rows is None
+ else min(len(self.__row_buffer), max_rows - len(rows))
+ )
+ rows.extend(self.__row_buffer.popleft() for _ in range(remaining))
+ return rows
+
+ def _fill_row_buffer(self):
+ while not self.__row_buffer:
+ while self.__query_result is not None and
self.__query_result_index < len(
+ self.__query_result
+ ):
+ block = self.__query_result[self.__query_result_index]
+ self.__query_result[self.__query_result_index] = None
+ self.__query_result_index += 1
+ self.__row_buffer.extend(self._deserialize_rows(block))
+ if self.__row_buffer:
+ return True
+ if self.__empty_resultSet or not self.__more_data:
+ return False
+ if not self.fetch_results():
+ return False
+ return True
+
+ def _deserialize_rows(self, serialized_block):
+ time_array, column_arrays, null_indicators, row_count = deserialize(
+ memoryview(serialized_block)
+ )
+ columns = []
+ if not self.ignore_timestamp:
+ columns.append(time_array.tolist())
+ for location in self.__column_index_2_tsblock_column_index_list:
+ if location < 0:
+ continue
+ columns.append(
+ self._expand_column(
+ column_arrays[location],
+ null_indicators[location],
+ row_count,
+ self.__data_type_for_tsblock_column[location],
+ )
+ )
+ return list(zip(*columns)) if columns else [tuple() for _ in
range(row_count)]
+
+ @staticmethod
+ def _expand_column(values, nulls, row_count, data_type):
+ source = values.tolist() if hasattr(values, "tolist") else list(values)
+ if nulls is None:
+ return source
+ if data_type == TSDataType.BOOLEAN and len(source) == row_count:
Review Comment:
Boolean TSBlocks retain one encoded value per row when a null bitmap is
present, while other nullable columns omit null entries. Boolean expansion
therefore has to preserve positional indexing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]