This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 2fd218422b6 Optimize the python client performance for Python 3.10 or
later (#11198)
2fd218422b6 is described below
commit 2fd218422b68307bb13e15d42ccf6d6fea7f0919
Author: Haonan <[email protected]>
AuthorDate: Wed Sep 27 10:00:04 2023 +0800
Optimize the python client performance for Python 3.10 or later (#11198)
---
iotdb-client/client-py/iotdb/Session.py | 57 +++++++---------------
.../iotdb/tsfile/utils/ReadWriteIOUtils.py | 12 ++---
.../client-py/iotdb/utils/IoTDBConstants.py | 33 ++-----------
iotdb-client/client-py/iotdb/utils/NumpyTablet.py | 8 +--
iotdb-client/client-py/iotdb/utils/Tablet.py | 14 +++---
5 files changed, 39 insertions(+), 85 deletions(-)
diff --git a/iotdb-client/client-py/iotdb/Session.py
b/iotdb-client/client-py/iotdb/Session.py
index f41c9b9efbe..af4ead9dd52 100644
--- a/iotdb-client/client-py/iotdb/Session.py
+++ b/iotdb-client/client-py/iotdb/Session.py
@@ -59,8 +59,6 @@ from .thrift.rpc.ttypes import (
)
from .utils.IoTDBConnectionException import IoTDBConnectionException
-from .utils.IoTDBConstants import TSDataType
-
logger = logging.getLogger("IoTDB")
@@ -176,9 +174,9 @@ class Session(object):
raise IoTDBConnectionException(e) from None
if self.__enable_rpc_compression:
- client = Client(TCompactProtocol.TCompactProtocol(transport))
+ client =
Client(TCompactProtocol.TCompactProtocolAccelerated(transport))
else:
- client = Client(TBinaryProtocol.TBinaryProtocol(transport))
+ client =
Client(TBinaryProtocol.TBinaryProtocolAccelerated(transport))
open_req = TSOpenSessionReq(
client_protocol=self.protocol_version,
@@ -310,9 +308,6 @@ class Session(object):
:param attributes: Dictionary, attribute map for time series
:param alias: String, measurement alias for time series
"""
- data_type = data_type.value
- encoding = encoding.value
- compressor = compressor.value
request = TSCreateTimeseriesReq(
self.__session_id,
ts_path,
@@ -349,9 +344,6 @@ class Session(object):
:param encoding_lst: List of TSEncoding, encodings for time series
:param compressor_lst: List of Compressor, compressing types for time
series
"""
- data_type_lst = [data_type.value for data_type in data_type_lst]
- encoding_lst = [encoding.value for encoding in encoding_lst]
- compressor_lst = [compressor.value for compressor in compressor_lst]
request = TSCreateAlignedTimeseriesReq(
self.__session_id,
@@ -399,9 +391,6 @@ class Session(object):
:param attributes_lst: List of attribute Dictionary, attribute maps
for time series
:param alias_lst: List of alias, measurement alias for time series
"""
- data_type_lst = [data_type.value for data_type in data_type_lst]
- encoding_lst = [encoding.value for encoding in encoding_lst]
- compressor_lst = [compressor.value for compressor in compressor_lst]
request = TSCreateMultiTimeseriesReq(
self.__session_id,
@@ -936,10 +925,7 @@ class Session(object):
request.measurementsList.append(tablet_lst[i].get_measurements())
request.valuesList.append(tablet_lst[i].get_binary_values())
request.sizeList.append(tablet_lst[i].get_row_number())
- data_type_values = [
- data_type.value for data_type in
tablet_lst[i].get_data_types()
- ]
- request.typesList.append(data_type_values)
+ request.typesList.append(tablet_lst[i].get_data_types())
for client, request in request_group.items():
try:
Session.verify_success_with_redirection_for_multi_devices(
@@ -1030,10 +1016,7 @@ class Session(object):
request.measurementsList.append(tablet_lst[i].get_measurements())
request.valuesList.append(tablet_lst[i].get_binary_values())
request.sizeList.append(tablet_lst[i].get_row_number())
- data_type_values = [
- data_type.value for data_type in
tablet_lst[i].get_data_types()
- ]
- request.typesList.append(data_type_values)
+ request.typesList.append(tablet_lst[i].get_data_types())
for client, request in request_group.items():
try:
Session.verify_success_with_redirection_for_multi_devices(
@@ -1287,14 +1270,13 @@ class Session(object):
raise IoTDBConnectionException(self.connection_error_msg())
from None
def gen_insert_tablet_req(self, tablet, is_aligned=False):
- data_type_values = [data_type.value for data_type in
tablet.get_data_types()]
return TSInsertTabletReq(
self.__session_id,
tablet.get_device_id(),
tablet.get_measurements(),
tablet.get_binary_values(),
tablet.get_binary_timestamps(),
- data_type_values,
+ tablet.get_data_types(),
tablet.get_row_number(),
is_aligned,
)
@@ -1307,14 +1289,11 @@ class Session(object):
type_lst = []
size_lst = []
for tablet in tablet_lst:
- data_type_values = [
- data_type.value for data_type in tablet.get_data_types()
- ]
device_id_lst.append(tablet.get_device_id())
measurements_lst.append(tablet.get_measurements())
values_lst.append(tablet.get_binary_values())
timestamps_lst.append(tablet.get_binary_timestamps())
- type_lst.append(data_type_values)
+ type_lst.append(tablet.get_data_types())
size_lst.append(tablet.get_row_number())
return TSInsertTabletsReq(
self.__session_id,
@@ -1423,36 +1402,34 @@ class Session(object):
def value_to_bytes(data_types, values):
format_str_list = [">"]
values_tobe_packed = []
- for i in range(len(data_types)):
- value = values[i]
- data_type_value = data_types[i].value
+ for data_type, value in zip(data_types, values):
# BOOLEAN
- if data_type_value == 0:
+ if data_type == 0:
format_str_list.append("c?")
values_tobe_packed.append(b"\x00")
values_tobe_packed.append(value)
# INT32
- elif data_type_value == 1:
+ elif data_type == 1:
format_str_list.append("ci")
values_tobe_packed.append(b"\x01")
values_tobe_packed.append(value)
# INT64
- elif data_type_value == 2:
+ elif data_type == 2:
format_str_list.append("cq")
values_tobe_packed.append(b"\x02")
values_tobe_packed.append(value)
# FLOAT
- elif data_type_value == 3:
+ elif data_type == 3:
format_str_list.append("cf")
values_tobe_packed.append(b"\x03")
values_tobe_packed.append(value)
# DOUBLE
- elif data_type_value == 4:
+ elif data_type == 4:
format_str_list.append("cd")
values_tobe_packed.append(b"\x04")
values_tobe_packed.append(value)
# TEXT
- elif data_type_value == 5:
+ elif data_type == 5:
if isinstance(value, str):
value_bytes = bytes(value, "utf-8")
else:
@@ -1464,7 +1441,7 @@ class Session(object):
values_tobe_packed.append(len(value_bytes))
values_tobe_packed.append(value_bytes)
else:
- raise RuntimeError("Unsupported data type:" +
str(data_types[i]))
+ raise RuntimeError("Unsupported data type:" + str(data_type))
format_str = "".join(format_str_list)
return struct.pack(format_str, *values_tobe_packed)
@@ -1899,9 +1876,9 @@ class Session(object):
template_name,
is_aligned,
measurements_path,
- list(map(lambda x: x.value, data_types)),
- list(map(lambda x: x.value, encodings)),
- list(map(lambda x: x.value, compressors)),
+ data_types,
+ encodings,
+ compressors,
)
try:
return
Session.verify_success(self.__client.appendSchemaTemplate(request))
diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/ReadWriteIOUtils.py
b/iotdb-client/client-py/iotdb/tsfile/utils/ReadWriteIOUtils.py
index 6101906ced9..f6a5c02f235 100644
--- a/iotdb-client/client-py/iotdb/tsfile/utils/ReadWriteIOUtils.py
+++ b/iotdb-client/client-py/iotdb/tsfile/utils/ReadWriteIOUtils.py
@@ -39,14 +39,14 @@ class ReadWriteUtils:
cls.write_bool(value, format_str_list, values_tobe_packed)
elif isinstance(value, str):
cls.write_str(value, format_str_list, values_tobe_packed)
- elif isinstance(value, int):
- cls.write_int(value, format_str_list, values_tobe_packed)
elif isinstance(value, TSDataType):
- cls.write_byte(value.value, format_str_list, values_tobe_packed)
+ cls.write_byte(bytes([int(value)]), format_str_list,
values_tobe_packed)
elif isinstance(value, TSEncoding):
- cls.write_byte(value.value, format_str_list, values_tobe_packed)
+ cls.write_byte(bytes([int(value)]), format_str_list,
values_tobe_packed)
elif isinstance(value, Compressor):
- cls.write_byte(value.value, format_str_list, values_tobe_packed)
+ cls.write_byte(bytes([int(value)]), format_str_list,
values_tobe_packed)
+ elif isinstance(value, int):
+ cls.write_int(value, format_str_list, values_tobe_packed)
@classmethod
def write_str(cls, s: str, format_str_list, values_tobe_packed):
@@ -73,5 +73,5 @@ class ReadWriteUtils:
@classmethod
def write_byte(cls, b, format_str_list, values_tobe_packed):
- format_str_list.append("b")
+ format_str_list.append("c")
values_tobe_packed.append(b)
diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
index fac308b7556..9b671663ff4 100644
--- a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
+++ b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
@@ -15,13 +15,12 @@
# specific language governing permissions and limitations
# under the License.
#
-
-from enum import Enum, unique
+from enum import unique, IntEnum
import numpy as np
@unique
-class TSDataType(Enum):
+class TSDataType(IntEnum):
BOOLEAN = 0
INT32 = 1
INT64 = 2
@@ -29,14 +28,6 @@ class TSDataType(Enum):
DOUBLE = 4
TEXT = 5
- # this method is implemented to avoid the issue reported by:
- # https://bugs.python.org/issue30545
- def __eq__(self, other) -> bool:
- return self.value == other.value
-
- def __hash__(self):
- return self.value
-
def np_dtype(self):
return {
TSDataType.BOOLEAN: np.dtype(">?"),
@@ -49,7 +40,7 @@ class TSDataType(Enum):
@unique
-class TSEncoding(Enum):
+class TSEncoding(IntEnum):
PLAIN = 0
DICTIONARY = 1
RLE = 2
@@ -64,17 +55,9 @@ class TSEncoding(Enum):
SPRINTZ = 12
RLBE = 13
- # this method is implemented to avoid the issue reported by:
- # https://bugs.python.org/issue30545
- def __eq__(self, other) -> bool:
- return self.value == other.value
-
- def __hash__(self):
- return self.value
-
@unique
-class Compressor(Enum):
+class Compressor(IntEnum):
UNCOMPRESSED = 0
SNAPPY = 1
GZIP = 2
@@ -85,11 +68,3 @@ class Compressor(Enum):
LZ4 = 7
ZSTD = 8
LZMA2 = 9
-
- # this method is implemented to avoid the issue reported by:
- # https://bugs.python.org/issue30545
- def __eq__(self, other) -> bool:
- return self.value == other.value
-
- def __hash__(self):
- return self.value
diff --git a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
index dbb0d354dbd..4577f7f880c 100644
--- a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
+++ b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
@@ -101,8 +101,9 @@ class NumpyTablet(object):
def get_binary_values(self):
bs_len = 0
bs_list = []
- for i, value in enumerate(self.__values):
- if self.__data_types[i] == TSDataType.TEXT:
+ for data_type, value in zip(self.__data_types, self.__values):
+ # TEXT
+ if data_type == 5:
format_str_list = [">"]
values_tobe_packed = []
for str_list in value:
@@ -118,6 +119,7 @@ class NumpyTablet(object):
values_tobe_packed.append(value_bytes)
format_str = "".join(format_str_list)
bs = struct.pack(format_str, *values_tobe_packed)
+ # Non-TEXT
else:
bs = value.tobytes()
bs_list.append(bs)
@@ -145,7 +147,7 @@ class NumpyTablet(object):
_l = len(bs)
ret[offset : offset + _l] = bs
offset += _l
- return ret
+ return bytes(ret)
def mark_none_value(self, column, row):
if self.bitmaps is None:
diff --git a/iotdb-client/client-py/iotdb/utils/Tablet.py
b/iotdb-client/client-py/iotdb/utils/Tablet.py
index 61a3e1d9263..508361dc857 100644
--- a/iotdb-client/client-py/iotdb/utils/Tablet.py
+++ b/iotdb-client/client-py/iotdb/utils/Tablet.py
@@ -18,7 +18,6 @@
import struct
-from iotdb.utils.IoTDBConstants import TSDataType
from iotdb.utils.BitMap import BitMap
@@ -95,7 +94,8 @@ class Tablet(object):
for i in range(self.__column_number):
bitmap = None
bitmaps.append(bitmap)
- if self.__data_types[i] == TSDataType.BOOLEAN:
+ data_type_value = self.__data_types[i]
+ if data_type_value == 0:
format_str_list.append(str(self.__row_number))
format_str_list.append("?")
for j in range(self.__row_number):
@@ -106,7 +106,7 @@ class Tablet(object):
self.__mark_none_value(bitmaps, i, j)
has_none = True
- elif self.__data_types[i] == TSDataType.INT32:
+ elif data_type_value == 1:
format_str_list.append(str(self.__row_number))
format_str_list.append("i")
for j in range(self.__row_number):
@@ -117,7 +117,7 @@ class Tablet(object):
self.__mark_none_value(bitmaps, i, j)
has_none = True
- elif self.__data_types[i] == TSDataType.INT64:
+ elif data_type_value == 2:
format_str_list.append(str(self.__row_number))
format_str_list.append("q")
for j in range(self.__row_number):
@@ -128,7 +128,7 @@ class Tablet(object):
self.__mark_none_value(bitmaps, i, j)
has_none = True
- elif self.__data_types[i] == TSDataType.FLOAT:
+ elif data_type_value == 3:
format_str_list.append(str(self.__row_number))
format_str_list.append("f")
for j in range(self.__row_number):
@@ -139,7 +139,7 @@ class Tablet(object):
self.__mark_none_value(bitmaps, i, j)
has_none = True
- elif self.__data_types[i] == TSDataType.DOUBLE:
+ elif data_type_value == 4:
format_str_list.append(str(self.__row_number))
format_str_list.append("d")
for j in range(self.__row_number):
@@ -150,7 +150,7 @@ class Tablet(object):
self.__mark_none_value(bitmaps, i, j)
has_none = True
- elif self.__data_types[i] == TSDataType.TEXT:
+ elif data_type_value == 5:
for j in range(self.__row_number):
if self.__values[j][i] is not None:
if isinstance(self.__values[j][i], str):
