This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch query_v3_py
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/query_v3_py by this push:
new 7e04bc48b63 update
7e04bc48b63 is described below
commit 7e04bc48b63b7ebe62d7a60474ecd86ce425b326
Author: HTHou <[email protected]>
AuthorDate: Mon Mar 24 23:44:23 2025 +0800
update
---
.../client-py/iotdb/tsfile/utils/tsblock_serde.py | 44 ++++++++++++----------
.../client-py/iotdb/utils/iotdb_rpc_dataset.py | 21 +++--------
2 files changed, 30 insertions(+), 35 deletions(-)
diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py
b/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py
index dee944387f3..881113f62e3 100644
--- a/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py
+++ b/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py
@@ -272,16 +272,16 @@ def deserialize(buffer):
column_encodings, buffer = read_column_encoding(buffer, value_column_count
+ 1)
time_column_values, _, buffer = read_column(
- column_encodings[0], buffer, TSDataType.INT64, position_count
+ column_encodings[0], buffer, 2, position_count
)
- column_values = [None] * value_column_count
- null_indicators = [None] * value_column_count
+ column_values = []
+ null_indicators = []
for i in range(value_column_count):
column_value, null_indicator, buffer = read_column(
column_encodings[i + 1], buffer, data_types[i], position_count
)
- column_values[i] = column_value
- null_indicators[i] = null_indicator
+ column_values.append(column_value)
+ null_indicators.append(null_indicator)
return time_column_values, column_values, null_indicators, position_count
@@ -290,8 +290,9 @@ def deserialize(buffer):
def read_int_from_buffer(buffer):
- res, buffer = read_from_buffer(buffer, 4)
- return int.from_bytes(res, "big"), buffer
+ res = np.frombuffer(buffer, dtype=">i4", count=1)
+ buffer = buffer[4:]
+ return res[0], buffer
def read_byte_from_buffer(buffer):
@@ -299,9 +300,8 @@ def read_byte_from_buffer(buffer):
def read_from_buffer(buffer, size):
- mv = memoryview(buffer)
- res = mv[:size]
- new_buffer = mv[size:]
+ res = buffer[:size]
+ new_buffer = buffer[size:]
return res, new_buffer
@@ -310,7 +310,7 @@ def read_from_buffer(buffer, size):
def read_column_types(buffer, value_column_count):
data_types = np.frombuffer(buffer, dtype=np.uint8,
count=value_column_count)
- new_buffer = memoryview(buffer)[value_column_count:]
+ new_buffer = buffer[value_column_count:]
if not np.all(np.isin(data_types, [0, 1, 2, 3, 4, 5])):
raise Exception("Invalid data type encountered: " + str(data_types))
return data_types, new_buffer
@@ -348,7 +348,7 @@ def get_data_type_byte_from_str(value):
def read_column_encoding(buffer, size):
encodings = np.frombuffer(buffer, dtype=np.uint8, count=size)
- new_buffer = memoryview(buffer)[size:]
+ new_buffer = buffer[size:]
return encodings, new_buffer
@@ -356,9 +356,9 @@ def read_column_encoding(buffer, size):
def deserialize_null_indicators(buffer, size):
- may_have_null = np.frombuffer(buffer, dtype=np.uint8, count=1)
- buffer = memoryview(buffer)[1:]
- if may_have_null[0] != 0:
+ may_have_null = buffer[0]
+ buffer = buffer[1:]
+ if may_have_null != 0:
return deserialize_from_boolean_array(buffer, size)
return None, buffer
@@ -378,11 +378,15 @@ def read_int64_column(buffer, data_type, position_count):
else:
size = null_indicators.count(False)
- if data_type == 2 or data_type == 4:
- values, buffer = read_from_buffer(buffer, size * 8)
- return values, null_indicators, buffer
+ if data_type == 2:
+ dtype = ">i8"
+ elif data_type == 4:
+ dtype = ">f8"
else:
- raise Exception("Invalid data type: " + data_type)
+ raise Exception("Invalid data type: " + str(data_type))
+ values = np.frombuffer(buffer, dtype, count=size)
+ buffer = buffer[size * 8 :]
+ return values, null_indicators, buffer
# Serialized data layout:
@@ -407,7 +411,7 @@ def read_int32_column(buffer, data_type, position_count):
else:
raise Exception("Invalid data type: " + str(data_type))
values = np.frombuffer(buffer, dtype, count=size)
- buffer = memoryview(buffer)[size * 4:]
+ buffer = buffer[size * 4 :]
return values, null_indicators, buffer
diff --git a/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py
b/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py
index 5055593574c..1933667b77b 100644
--- a/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py
+++ b/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py
@@ -163,21 +163,18 @@ class IoTDBRpcDataSet(object):
has_pd_series.append(False)
total_length = 0
while self.__query_result_index < len(self.__query_result):
- time_column_values, column_values, null_indicators, _ =
deserialize(
- self.__query_result[self.__query_result_index]
+ time_column_values, column_values, null_indicators, current_length
= (
+
deserialize(memoryview(self.__query_result[self.__query_result_index]))
)
self.__query_result[self.__query_result_index] = None
self.__query_result_index += 1
- time_array = np.frombuffer(
- time_column_values, np.dtype(np.longlong).newbyteorder(">")
- )
+ time_array = time_column_values
if time_array.dtype.byteorder == ">":
time_array = time_array.byteswap().view(
time_array.dtype.newbyteorder("<")
)
if self.ignore_timestamp is None or self.ignore_timestamp is False:
result[0].append(time_array)
- current_length = len(time_array)
total_length += current_length
for i, location in enumerate(
self.__column_index_2_tsblock_column_index_list
@@ -189,9 +186,7 @@ class IoTDBRpcDataSet(object):
value_buffer_len = len(value_buffer)
# DOUBLE
if data_type == 4:
- data_array = np.frombuffer(
- value_buffer, np.dtype(np.double).newbyteorder(">")
- )
+ data_array = value_buffer
# FLOAT
elif data_type == 3:
data_array = value_buffer
@@ -200,14 +195,10 @@ class IoTDBRpcDataSet(object):
data_array = np.array(value_buffer).astype("bool")
# INT32, DATE
elif data_type == 1 or data_type == 9:
- data_array = np.frombuffer(
- value_buffer, np.dtype(np.int32).newbyteorder(">")
- )
+ data_array = value_buffer
# INT64, TIMESTAMP
elif data_type == 2 or data_type == 8:
- data_array = np.frombuffer(
- value_buffer, np.dtype(np.int64).newbyteorder(">")
- )
+ data_array = value_buffer
# TEXT, STRING, BLOB
elif data_type == 5 or data_type == 11 or data_type == 10:
index = 0
