This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 72175750334 [Py-Client] support new data types (#13494)
72175750334 is described below
commit 721757503349a991466231d0b8886e6c92c1a33a
Author: Haonan <[email protected]>
AuthorDate: Fri Sep 13 10:08:42 2024 +0800
[Py-Client] support new data types (#13494)
* [Py-Client] support new data types
* add test
* recover useless change
* more change
---
iotdb-client/client-py/SessionExample.py | 53 ++++++-
iotdb-client/client-py/iotdb/Session.py | 33 ++++-
.../client-py/iotdb/tsfile/utils/DateUtils.py | 41 ++++++
iotdb-client/client-py/iotdb/utils/Field.py | 34 ++++-
.../client-py/iotdb/utils/IoTDBConstants.py | 7 +-
.../client-py/iotdb/utils/IoTDBRpcDataSet.py | 85 +++++++++--
iotdb-client/client-py/iotdb/utils/NumpyTablet.py | 30 +++-
iotdb-client/client-py/iotdb/utils/RowRecord.py | 8 +-
.../client-py/iotdb/utils/SessionDataSet.py | 24 ++--
iotdb-client/client-py/iotdb/utils/Tablet.py | 47 +++---
.../tests/integration/test_new_data_types.py | 157 +++++++++++++++++++++
11 files changed, 459 insertions(+), 60 deletions(-)
diff --git a/iotdb-client/client-py/SessionExample.py
b/iotdb-client/client-py/SessionExample.py
index c489fe7d03d..ca610de9a0c 100644
--- a/iotdb-client/client-py/SessionExample.py
+++ b/iotdb-client/client-py/SessionExample.py
@@ -18,7 +18,7 @@
# Uncomment the following line to use apache-iotdb module installed by pip3
import numpy as np
-
+from datetime import date
from iotdb.Session import Session
from iotdb.utils.BitMap import BitMap
from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor
@@ -360,6 +360,57 @@ with session.execute_last_data_query(
while session_data_set.has_next():
print(session_data_set.next())
+# insert tablet with new data types
+measurements_new_type = ["s_01", "s_02", "s_03", "s_04"]
+data_types_new_type = [
+ TSDataType.DATE,
+ TSDataType.TIMESTAMP,
+ TSDataType.BLOB,
+ TSDataType.STRING,
+]
+values_new_type = [
+ [date(2024, 1, 1), 1, b"\x12\x34", "test01"],
+ [date(2024, 1, 2), 2, b"\x12\x34", "test02"],
+ [date(2024, 1, 3), 3, b"\x12\x34", "test03"],
+ [date(2024, 1, 4), 4, b"\x12\x34", "test04"],
+]
+timestamps_new_type = [1, 2, 3, 4]
+tablet_new_type = Tablet(
+ "root.sg_test_01.d_04",
+ measurements_new_type,
+ data_types_new_type,
+ values_new_type,
+ timestamps_new_type,
+)
+session.insert_tablet(tablet_new_type)
+np_values_new_type = [
+ np.array([date(2024, 2, 4), date(2024, 3, 4), date(2024, 4, 4), date(2024,
5, 4)]),
+ np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()),
+ np.array([b"\x12\x34", b"\x12\x34", b"\x12\x34", b"\x12\x34"]),
+ np.array(["test01", "test02", "test03", "test04"]),
+]
+np_timestamps_new_type = np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype())
+np_tablet_new_type = NumpyTablet(
+ "root.sg_test_01.d_04",
+ measurements_new_type,
+ data_types_new_type,
+ np_values_new_type,
+ np_timestamps_new_type,
+)
+session.insert_tablet(np_tablet_new_type)
+with session.execute_query_statement(
+ "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+) as dataset:
+ print(dataset.get_column_names())
+ while dataset.has_next():
+ print(dataset.next())
+
+with session.execute_query_statement(
+ "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+) as dataset:
+ df = dataset.todf()
+ print(df.to_string())
+
# delete database
session.delete_storage_group("root.sg_test_01")
diff --git a/iotdb-client/client-py/iotdb/Session.py
b/iotdb-client/client-py/iotdb/Session.py
index 098faf0d520..12545e6f7a2 100644
--- a/iotdb-client/client-py/iotdb/Session.py
+++ b/iotdb-client/client-py/iotdb/Session.py
@@ -58,6 +58,7 @@ from .thrift.rpc.ttypes import (
TSLastDataQueryReq,
TSInsertStringRecordsOfOneDeviceReq,
)
+from .tsfile.utils.DateUtils import parse_date_to_int
from .utils.IoTDBConnectionException import IoTDBConnectionException
logger = logging.getLogger("IoTDB")
@@ -118,7 +119,7 @@ class Session(object):
fetch_size=DEFAULT_FETCH_SIZE,
zone_id=DEFAULT_ZONE_ID,
enable_redirection=True,
- sql_dialect=None,
+ sql_dialect=SQL_DIALECT,
database=None,
):
if node_urls is None:
@@ -1520,6 +1521,36 @@ class Session(object):
values_tobe_packed.append(b"\x05")
values_tobe_packed.append(len(value_bytes))
values_tobe_packed.append(value_bytes)
+ # TIMESTAMP
+ elif data_type == 8:
+ format_str_list.append("cq")
+ values_tobe_packed.append(b"\x08")
+ values_tobe_packed.append(value)
+ # DATE
+ elif data_type == 9:
+ format_str_list.append("ci")
+ values_tobe_packed.append(b"\x09")
+ values_tobe_packed.append(parse_date_to_int(value))
+ # BLOB
+ elif data_type == 10:
+ format_str_list.append("ci")
+ format_str_list.append(str(len(value)))
+ format_str_list.append("s")
+ values_tobe_packed.append(b"\x0a")
+ values_tobe_packed.append(len(value))
+ values_tobe_packed.append(value)
+ # STRING
+ elif data_type == 11:
+ if isinstance(value, str):
+ value_bytes = bytes(value, "utf-8")
+ else:
+ value_bytes = value
+ format_str_list.append("ci")
+ format_str_list.append(str(len(value_bytes)))
+ format_str_list.append("s")
+ values_tobe_packed.append(b"\x0b")
+ values_tobe_packed.append(len(value_bytes))
+ values_tobe_packed.append(value_bytes)
else:
raise RuntimeError("Unsupported data type:" + str(data_type))
format_str = "".join(format_str_list)
diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/DateUtils.py
b/iotdb-client/client-py/iotdb/tsfile/utils/DateUtils.py
new file mode 100644
index 00000000000..4cafedea618
--- /dev/null
+++ b/iotdb-client/client-py/iotdb/tsfile/utils/DateUtils.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from datetime import date
+
+
+class DateTimeParseException(Exception):
+ pass
+
+
+def parse_int_to_date(date_int: int) -> date:
+ try:
+ year = date_int // 10000
+ month = (date_int // 100) % 100
+ day = date_int % 100
+ return date(year, month, day)
+ except ValueError as e:
+ raise DateTimeParseException("Invalid date format.") from e
+
+
+def parse_date_to_int(local_date: date) -> int:
+ if local_date is None:
+ raise DateTimeParseException("Date expression is none or empty.")
+ if local_date.year < 1000:
+ raise DateTimeParseException("Year must be between 1000 and 9999.")
+ return local_date.year * 10000 + local_date.month * 100 + local_date.day
diff --git a/iotdb-client/client-py/iotdb/utils/Field.py
b/iotdb-client/client-py/iotdb/utils/Field.py
index 7b4b05eab72..fdc1396c8fb 100644
--- a/iotdb-client/client-py/iotdb/utils/Field.py
+++ b/iotdb-client/client-py/iotdb/utils/Field.py
@@ -17,7 +17,8 @@
#
# for package
-from .IoTDBConstants import TSDataType
+from iotdb.utils.IoTDBConstants import TSDataType
+from iotdb.tsfile.utils.DateUtils import parse_int_to_date
import numpy as np
import pandas as pd
@@ -36,7 +37,10 @@ class Field(object):
if output.get_data_type() is not None:
if output.get_data_type() == TSDataType.BOOLEAN:
output.set_bool_value(field.get_bool_value())
- elif output.get_data_type() == TSDataType.INT32:
+ elif (
+ output.get_data_type() == TSDataType.INT32
+ or output.get_data_type() == TSDataType.DATE
+ ):
output.set_int_value(field.get_int_value())
elif (
output.get_data_type() == TSDataType.INT64
@@ -50,6 +54,7 @@ class Field(object):
elif (
output.get_data_type() == TSDataType.TEXT
or output.get_data_type() == TSDataType.STRING
+ or output.get_data_type() == TSDataType.BLOB
):
output.set_binary_value(field.get_binary_value())
else:
@@ -86,6 +91,7 @@ class Field(object):
raise Exception("Null Field Exception!")
if (
self.__data_type != TSDataType.INT32
+ and self.__data_type != TSDataType.DATE
or self.value is None
or self.value is pd.NA
):
@@ -151,11 +157,26 @@ class Field(object):
return None
return self.value
+ def get_date_value(self):
+ if self.__data_type is None:
+ raise Exception("Null Field Exception!")
+ if (
+ self.__data_type != TSDataType.DATE
+ or self.value is None
+ or self.value is pd.NA
+ ):
+ return None
+ return parse_int_to_date(self.value)
+
def get_string_value(self):
if self.__data_type is None or self.value is None or self.value is
pd.NA:
return "None"
+ # TEXT, STRING
elif self.__data_type == 5 or self.__data_type == 11:
return self.value.decode("utf-8")
+ # BLOB
+ elif self.__data_type == 10:
+ return str(hex(int.from_bytes(self.value, byteorder="big")))
else:
return str(self.get_object_value(self.__data_type))
@@ -172,13 +193,18 @@ class Field(object):
return bool(self.value)
elif data_type == 1:
return np.int32(self.value)
- elif data_type == 2:
+ elif data_type == 2 or data_type == 8:
return np.int64(self.value)
elif data_type == 3:
return np.float32(self.value)
elif data_type == 4:
return np.float64(self.value)
- return self.value
+ elif data_type == 9:
+ return parse_int_to_date(self.value)
+ elif data_type == 5 or data_type == 10 or data_type == 11:
+ return self.value
+ else:
+ raise RuntimeError("Unsupported data type:" + str(data_type))
@staticmethod
def get_field(value, data_type):
diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
index daae37d23ba..4b9082b5353 100644
--- a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
+++ b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
#
+from datetime import date
from enum import unique, IntEnum
import numpy as np
@@ -39,9 +40,11 @@ class TSDataType(IntEnum):
TSDataType.DOUBLE: np.dtype(">f8"),
TSDataType.INT32: np.dtype(">i4"),
TSDataType.INT64: np.dtype(">i8"),
- TSDataType.TEXT: np.dtype("str"),
+ TSDataType.TEXT: str,
TSDataType.TIMESTAMP: np.dtype(">i8"),
- TSDataType.STRING: np.dtype("str"),
+ TSDataType.DATE: date,
+ TSDataType.BLOB: bytes,
+ TSDataType.STRING: str,
}[self]
diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py
b/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py
index 2aa885125d8..73d4f317de6 100644
--- a/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py
+++ b/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py
@@ -24,6 +24,7 @@ import numpy as np
import pandas as pd
from thrift.transport import TTransport
from iotdb.thrift.rpc.IClientRPCService import TSFetchResultsReq,
TSCloseOperationReq
+from iotdb.tsfile.utils.DateUtils import parse_int_to_date
from iotdb.utils.IoTDBConstants import TSDataType
logger = logging.getLogger("IoTDB")
@@ -168,25 +169,31 @@ class IoTDBRpcDataSet(object):
data_type = self.column_type_deduplicated_list[location]
value_buffer = self.__query_data_set.valueList[location]
value_buffer_len = len(value_buffer)
+ # DOUBLE
if data_type == 4:
data_array = np.frombuffer(
value_buffer, np.dtype(np.double).newbyteorder(">")
)
+ # FLOAT
elif data_type == 3:
data_array = np.frombuffer(
value_buffer, np.dtype(np.float32).newbyteorder(">")
)
+ # BOOLEAN
elif data_type == 0:
data_array = np.frombuffer(value_buffer, np.dtype("?"))
- elif data_type == 1:
+ # INT32, DATE
+ elif data_type == 1 or data_type == 9:
data_array = np.frombuffer(
value_buffer, np.dtype(np.int32).newbyteorder(">")
)
+ # INT64, TIMESTAMP
elif data_type == 2 or data_type == 8:
data_array = np.frombuffer(
value_buffer, np.dtype(np.int64).newbyteorder(">")
)
- elif data_type == 5 or data_type == 11:
+ # TEXT, STRING, BLOB
+ elif data_type == 5 or data_type == 11 or data_type == 10:
j = 0
offset = 0
data_array = []
@@ -208,10 +215,15 @@ class IoTDBRpcDataSet(object):
data_array = data_array.byteswap().view(
data_array.dtype.newbyteorder("<")
)
- # self.__query_data_set.valueList[location] = None
if len(data_array) < total_length:
- # INT32 or INT64 or boolean
- if data_type == 0 or data_type == 1 or data_type == 2:
+ # INT32, INT64, BOOLEAN, TIMESTAMP, DATE
+ if (
+ data_type == 0
+ or data_type == 1
+ or data_type == 2
+ or data_type == 8
+ or data_type == 9
+ ):
tmp_array = np.full(total_length, np.nan, np.float32)
else:
tmp_array = np.full(total_length, None, dtype=object)
@@ -223,10 +235,13 @@ class IoTDBRpcDataSet(object):
bit_mask = bit_mask[:total_length]
tmp_array[bit_mask] = data_array
- if data_type == 1:
+ # INT32, DATE
+ if data_type == 1 or data_type == 9:
tmp_array = pd.Series(tmp_array, dtype="Int32")
+ # INT64, TIMESTAMP
elif data_type == 2 or data_type == 8:
tmp_array = pd.Series(tmp_array, dtype="Int64")
+ # BOOLEAN
elif data_type == 0:
tmp_array = pd.Series(tmp_array, dtype="boolean")
data_array = tmp_array
@@ -251,7 +266,7 @@ class IoTDBRpcDataSet(object):
return True
return False
- def resultset_to_pandas(self):
+ def result_set_to_pandas(self):
result = {}
for column_name in self.__column_name_list:
result[column_name] = []
@@ -283,24 +298,30 @@ class IoTDBRpcDataSet(object):
data_type = self.column_type_deduplicated_list[location]
value_buffer = self.__query_data_set.valueList[location]
value_buffer_len = len(value_buffer)
+ # DOUBLE
if data_type == 4:
data_array = np.frombuffer(
value_buffer, np.dtype(np.double).newbyteorder(">")
)
+ # FLOAT
elif data_type == 3:
data_array = np.frombuffer(
value_buffer, np.dtype(np.float32).newbyteorder(">")
)
+ # BOOLEAN
elif data_type == 0:
data_array = np.frombuffer(value_buffer, np.dtype("?"))
+ # INT32
elif data_type == 1:
data_array = np.frombuffer(
value_buffer, np.dtype(np.int32).newbyteorder(">")
)
+ # INT64, TIMESTAMP
elif data_type == 2 or data_type == 8:
data_array = np.frombuffer(
value_buffer, np.dtype(np.int64).newbyteorder(">")
)
+ # TEXT, STRING
elif data_type == 5 or data_type == 11:
j = 0
offset = 0
@@ -317,7 +338,30 @@ class IoTDBRpcDataSet(object):
data_array.append(value)
j += 1
offset += length
- data_array = np.array(data_array, dtype=object)
+ data_array = pd.Series(data_array).astype(str)
+ # BLOB
+ elif data_type == 10:
+ j = 0
+ offset = 0
+ data_array = []
+ while offset < value_buffer_len:
+ length = int.from_bytes(
+ value_buffer[offset : offset + 4],
+ byteorder="big",
+ signed=False,
+ )
+ offset += 4
+ value = value_buffer[offset : offset + length]
+ data_array.append(value)
+ j += 1
+ offset += length
+ data_array = pd.Series(data_array)
+ # DATE
+ elif data_type == 9:
+ data_array = np.frombuffer(
+ value_buffer, np.dtype(np.int32).newbyteorder(">")
+ )
+ data_array = pd.Series(data_array).apply(parse_int_to_date)
else:
raise RuntimeError("unsupported data type
{}.".format(data_type))
if data_array.dtype.byteorder == ">":
@@ -327,13 +371,26 @@ class IoTDBRpcDataSet(object):
self.__query_data_set.valueList[location] = None
tmp_array = []
if len(data_array) < total_length:
- if data_type == 1 or data_type == 2 or data_type == 8:
- tmp_array = np.full(total_length, np.nan, np.float32)
+ # BOOLEAN, INT32, INT64, TIMESTAMP
+ if (
+ data_type == 0
+ or data_type == 1
+ or data_type == 2
+ or data_type == 8
+ ):
+ tmp_array = np.full(total_length, np.nan,
dtype=np.float32)
+ # FLOAT, DOUBLE
elif data_type == 3 or data_type == 4:
- tmp_array = np.full(total_length, np.nan,
data_array.dtype)
- elif data_type == 0:
- tmp_array = np.full(total_length, np.nan, np.float32)
- elif data_type == 5 or data_type == 11 or data_type == 10:
+ tmp_array = np.full(
+ total_length, np.nan, dtype=data_array.dtype
+ )
+ # TEXT, STRING, BLOB, DATE
+ elif (
+ data_type == 5
+ or data_type == 11
+ or data_type == 10
+ or data_type == 9
+ ):
tmp_array = np.full(total_length, None,
dtype=data_array.dtype)
bitmap_buffer = self.__query_data_set.bitmapList[location]
diff --git a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
index f00b61de519..4ce623c9db9 100644
--- a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
+++ b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py
@@ -18,8 +18,11 @@
import struct
+import numpy as np
from numpy import ndarray
from typing import List
+
+from iotdb.tsfile.utils.DateUtils import parse_date_to_int
from iotdb.utils.IoTDBConstants import TSDataType
from iotdb.utils.BitMap import BitMap
from iotdb.utils.Tablet import ColumnType
@@ -49,7 +52,7 @@ class NumpyTablet(object):
2, id:1, attr:1, 2.0
3, id:2, attr:2, 3.0
Notice: The tablet will be sorted at the initialization by timestamps
- :param insert_target_name: Str, DeviceId if using tree-view interfaces
or TableName when using table-view interfaces.
+ :param insert_target_name: Str, DeviceId if using tree model or
TableName when using table model.
:param column_names: Str List, names of columns
:param data_types: TSDataType List, specify value types for columns
:param values: ndarray List, one ndarray contains the value of one
column
@@ -128,12 +131,22 @@ class NumpyTablet(object):
bs_len = 0
bs_list = []
for data_type, value in zip(self.__data_types, self.__values):
+ # BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TIMESTAMP
+ if (
+ data_type == 0
+ or data_type == 1
+ or data_type == 2
+ or data_type == 3
+ or data_type == 4
+ or data_type == 8
+ ):
+ bs = value.tobytes()
# TEXT, STRING, BLOB
- if data_type == 5 or data_type == 11 or data_type == 10:
+ elif data_type == 5 or data_type == 11 or data_type == 10:
format_str_list = [">"]
values_tobe_packed = []
for str_list in value:
- # Fot TEXT, it's same as the original solution
+ # For TEXT, it's same as the original solution
if isinstance(str_list, str):
value_bytes = bytes(str_list, "utf-8")
else:
@@ -145,11 +158,18 @@ class NumpyTablet(object):
values_tobe_packed.append(value_bytes)
format_str = "".join(format_str_list)
bs = struct.pack(format_str, *values_tobe_packed)
- # Non-TEXT
+ # DATE
+ elif data_type == 9:
+ bs = (
+ np.vectorize(parse_date_to_int)(value)
+ .astype(np.dtype(">i4"))
+ .tobytes()
+ )
else:
- bs = value.tobytes()
+ raise RuntimeError("Unsupported data type:" + str(data_type))
bs_list.append(bs)
bs_len += len(bs)
+
if self.bitmaps is not None:
format_str_list = [">"]
values_tobe_packed = []
diff --git a/iotdb-client/client-py/iotdb/utils/RowRecord.py
b/iotdb-client/client-py/iotdb/utils/RowRecord.py
index 16a88f1edf9..91bcebf948b 100644
--- a/iotdb-client/client-py/iotdb/utils/RowRecord.py
+++ b/iotdb-client/client-py/iotdb/utils/RowRecord.py
@@ -17,15 +17,11 @@
#
# for package
-from .Field import Field
-
-# for debug
-# from IoTDBConstants import TSDataType
-# from Field import Field
+from iotdb.utils.Field import Field
class RowRecord(object):
- def __init__(self, timestamp, field_list=None):
+ def __init__(self, timestamp, field_list: list = None):
self.__timestamp = timestamp
self.__field_list = field_list
diff --git a/iotdb-client/client-py/iotdb/utils/SessionDataSet.py
b/iotdb-client/client-py/iotdb/utils/SessionDataSet.py
index c69aca3ebbe..f4d63035efc 100644
--- a/iotdb-client/client-py/iotdb/utils/SessionDataSet.py
+++ b/iotdb-client/client-py/iotdb/utils/SessionDataSet.py
@@ -125,30 +125,32 @@ class SessionDataSet(object):
self.iotdb_rpc_data_set.close()
def todf(self) -> pd.DataFrame:
- return resultset_to_pandas(self)
+ return result_set_to_pandas(self)
-def resultset_to_pandas(result_set: SessionDataSet) -> pd.DataFrame:
+def result_set_to_pandas(result_set: SessionDataSet) -> pd.DataFrame:
"""
Transforms a SessionDataSet from IoTDB to a Pandas Data Frame
Each Field from IoTDB is a column in Pandas
:param result_set:
:return:
"""
- return result_set.iotdb_rpc_data_set.resultset_to_pandas()
+ return result_set.iotdb_rpc_data_set.result_set_to_pandas()
def get_typed_point(field: Field, none_value=None):
choices = {
# In Case of Boolean, cast to 0 / 1
- TSDataType.BOOLEAN: lambda field: 1 if field.get_bool_value() else 0,
- TSDataType.TEXT: lambda field: field.get_string_value(),
- TSDataType.FLOAT: lambda field: field.get_float_value(),
- TSDataType.INT32: lambda field: field.get_int_value(),
- TSDataType.DOUBLE: lambda field: field.get_double_value(),
- TSDataType.INT64: lambda field: field.get_long_value(),
- TSDataType.TIMESTAMP: lambda field: field.get_long_value(),
- TSDataType.STRING: lambda field: field.get_string_value(),
+ TSDataType.BOOLEAN: lambda f: 1 if f.get_bool_value() else 0,
+ TSDataType.TEXT: lambda f: f.get_string_value(),
+ TSDataType.FLOAT: lambda f: f.get_float_value(),
+ TSDataType.INT32: lambda f: f.get_int_value(),
+ TSDataType.DOUBLE: lambda f: f.get_double_value(),
+ TSDataType.INT64: lambda f: f.get_long_value(),
+ TSDataType.TIMESTAMP: lambda f: f.get_long_value(),
+ TSDataType.STRING: lambda f: f.get_string_value(),
+ TSDataType.DATE: lambda f: f.get_date_value(),
+ TSDataType.BLOB: lambda f: f.get_binary_value(),
}
result_next_type: TSDataType = field.get_data_type()
diff --git a/iotdb-client/client-py/iotdb/utils/Tablet.py
b/iotdb-client/client-py/iotdb/utils/Tablet.py
index 0f086530347..fbae5fe2293 100644
--- a/iotdb-client/client-py/iotdb/utils/Tablet.py
+++ b/iotdb-client/client-py/iotdb/utils/Tablet.py
@@ -18,7 +18,9 @@
import struct
from enum import unique, IntEnum
-from typing import List
+from typing import List, Union
+
+from iotdb.tsfile.utils.DateUtils import parse_date_to_int
from iotdb.utils.BitMap import BitMap
from iotdb.utils.IoTDBConstants import TSDataType
@@ -59,7 +61,7 @@ class Tablet(object):
2, id:1, attr:1, 2.0
3, id:2, attr:2, 3.0
Notice: The tablet will be sorted at the initialization by timestamps
- :param insert_target_name: Str, DeviceId if using tree-view interfaces
or TableName when using table-view interfaces.
+ :param insert_target_name: Str, DeviceId if using tree model or
TableName when using table model.
:param column_names: Str List, names of columns
:param data_types: TSDataType List, specify value types for columns
:param values: 2-D List, the values of each row should be the outer
list element
@@ -126,13 +128,14 @@ class Tablet(object):
def get_binary_values(self):
format_str_list = [">"]
values_tobe_packed = []
- bitmaps = []
+ bitmaps: List[Union[BitMap, None]] = []
has_none = False
for i in range(self.__column_number):
bitmap = None
bitmaps.append(bitmap)
- data_type_value = self.__data_types[i]
- if data_type_value == 0:
+ data_type = self.__data_types[i]
+ # BOOLEAN
+ if data_type == 0:
format_str_list.append(str(self.__row_number))
format_str_list.append("?")
for j in range(self.__row_number):
@@ -142,8 +145,8 @@ class Tablet(object):
values_tobe_packed.append(False)
self.__mark_none_value(bitmaps, i, j)
has_none = True
-
- elif data_type_value == 1:
+ # INT32
+ elif data_type == 1:
format_str_list.append(str(self.__row_number))
format_str_list.append("i")
for j in range(self.__row_number):
@@ -153,8 +156,8 @@ class Tablet(object):
values_tobe_packed.append(0)
self.__mark_none_value(bitmaps, i, j)
has_none = True
-
- elif data_type_value == 2:
+ # INT64 or TIMESTAMP
+ elif data_type == 2 or data_type == 8:
format_str_list.append(str(self.__row_number))
format_str_list.append("q")
for j in range(self.__row_number):
@@ -164,8 +167,8 @@ class Tablet(object):
values_tobe_packed.append(0)
self.__mark_none_value(bitmaps, i, j)
has_none = True
-
- elif data_type_value == 3:
+ # FLOAT
+ elif data_type == 3:
format_str_list.append(str(self.__row_number))
format_str_list.append("f")
for j in range(self.__row_number):
@@ -175,8 +178,8 @@ class Tablet(object):
values_tobe_packed.append(0)
self.__mark_none_value(bitmaps, i, j)
has_none = True
-
- elif data_type_value == 4:
+ # DOUBLE
+ elif data_type == 4:
format_str_list.append(str(self.__row_number))
format_str_list.append("d")
for j in range(self.__row_number):
@@ -186,8 +189,8 @@ class Tablet(object):
values_tobe_packed.append(0)
self.__mark_none_value(bitmaps, i, j)
has_none = True
-
- elif data_type_value == 5 or data_type_value == 11:
+ # TEXT, STRING, BLOB
+ elif data_type == 5 or data_type == 11 or data_type == 10:
for j in range(self.__row_number):
if self.__values[j][i] is not None:
if isinstance(self.__values[j][i], str):
@@ -208,7 +211,19 @@ class Tablet(object):
values_tobe_packed.append(value_bytes)
self.__mark_none_value(bitmaps, i, j)
has_none = True
-
+ # DATE
+ elif data_type == 9:
+ format_str_list.append(str(self.__row_number))
+ format_str_list.append("i")
+ for j in range(self.__row_number):
+ if self.__values[j][i] is not None:
+ values_tobe_packed.append(
+ parse_date_to_int(self.__values[j][i])
+ )
+ else:
+ values_tobe_packed.append(0)
+ self.__mark_none_value(bitmaps, i, j)
+ has_none = True
else:
raise RuntimeError("Unsupported data type:" +
str(self.__data_types[i]))
diff --git a/iotdb-client/client-py/tests/integration/test_new_data_types.py
b/iotdb-client/client-py/tests/integration/test_new_data_types.py
new file mode 100644
index 00000000000..3452d6c1b7b
--- /dev/null
+++ b/iotdb-client/client-py/tests/integration/test_new_data_types.py
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from datetime import date
+
+import numpy as np
+
+from iotdb.Session import Session
+from iotdb.SessionPool import PoolConfig, create_session_pool
+from iotdb.utils.IoTDBConstants import TSDataType
+from iotdb.utils.NumpyTablet import NumpyTablet
+from iotdb.utils.Tablet import Tablet
+from iotdb.IoTDBContainer import IoTDBContainer
+
+
+def test_session():
+ session_test()
+
+
+def test_session_pool():
+ session_test(True)
+
+
+def session_test(use_session_pool=False):
+ with IoTDBContainer("iotdb:dev") as db:
+ db: IoTDBContainer
+
+ if use_session_pool:
+ pool_config = PoolConfig(
+ db.get_container_host_ip(),
+ db.get_exposed_port(6667),
+ "root",
+ "root",
+ None,
+ 1024,
+ "Asia/Shanghai",
+ 3,
+ )
+ session_pool = create_session_pool(pool_config, 1, 3000)
+ session = session_pool.get_session()
+ else:
+ session = Session(db.get_container_host_ip(),
db.get_exposed_port(6667))
+ session.open(False)
+
+ if not session.is_open():
+ print("can't open session")
+ exit(1)
+
+ device_id = "root.sg_test_01.d_04"
+ measurements_new_type = ["s_01", "s_02", "s_03", "s_04"]
+ data_types_new_type = [
+ TSDataType.DATE,
+ TSDataType.TIMESTAMP,
+ TSDataType.BLOB,
+ TSDataType.STRING,
+ ]
+ values_new_type = [
+ [date(2024, 1, 1), 1, b"\x12\x34", "test01"],
+ [date(2024, 1, 2), 2, b"\x12\x34", "test02"],
+ [date(2024, 1, 3), 3, b"\x12\x34", "test03"],
+ [date(2024, 1, 4), 4, b"\x12\x34", "test04"],
+ ]
+ timestamps_new_type = [1, 2, 3, 4]
+ tablet_new_type = Tablet(
+ device_id,
+ measurements_new_type,
+ data_types_new_type,
+ values_new_type,
+ timestamps_new_type,
+ )
+ session.insert_tablet(tablet_new_type)
+ np_values_new_type = [
+ np.array(
+ [date(2024, 1, 5), date(2024, 1, 6), date(2024, 1, 7),
date(2024, 1, 8)]
+ ),
+ np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()),
+ np.array([b"\x12\x34", b"\x12\x34", b"\x12\x34", b"\x12\x34"]),
+ np.array(["test05", "test06", "test07", "test08"]),
+ ]
+ np_timestamps_new_type = np.array([5, 6, 7, 8],
TSDataType.INT64.np_dtype())
+ np_tablet_new_type = NumpyTablet(
+ device_id,
+ measurements_new_type,
+ data_types_new_type,
+ np_values_new_type,
+ np_timestamps_new_type,
+ )
+ session.insert_tablet(np_tablet_new_type)
+ session.insert_records(
+ [device_id, device_id],
+ [9, 10],
+ [measurements_new_type, measurements_new_type],
+ [data_types_new_type, data_types_new_type],
+ [
+ [date(2024, 1, 9), 9, b"\x12\x34", "test09"],
+ [date(2024, 1, 10), 10, b"\x12\x34", "test010"],
+ ],
+ )
+
+ with session.execute_query_statement(
+ "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+ ) as dataset:
+ print(dataset.get_column_names())
+ while dataset.has_next():
+ print(dataset.next())
+
+ with session.execute_query_statement(
+ "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+ ) as dataset:
+ df = dataset.todf()
+ print(df.to_string())
+
+ with session.execute_query_statement(
+ "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+ ) as dataset:
+ cnt = 0
+ while dataset.has_next():
+ row_record = dataset.next()
+ timestamp = row_record.get_timestamp()
+ assert row_record.get_fields()[0].get_date_value() == date(
+ 2024, 1, timestamp
+ )
+ assert (
+
row_record.get_fields()[1].get_object_value(TSDataType.TIMESTAMP)
+ == timestamp
+ )
+ assert row_record.get_fields()[2].get_binary_value() ==
b"\x12\x34"
+ assert row_record.get_fields()[3].get_string_value() ==
"test0" + str(
+ timestamp
+ )
+ cnt += 1
+ assert cnt == 10
+
+ with session.execute_query_statement(
+ "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+ ) as dataset:
+ df = dataset.todf()
+ rows, columns = df.shape
+ assert rows == 10
+ assert columns == 5
+
+ # close session connection.
+ session.close()
