This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch numpy_dtype_13 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d549facce2ce76496b37e280ae52738375aab296 Author: Haonan <[email protected]> AuthorDate: Fri Apr 8 14:18:52 2022 +0800 [IOTDB-2838] Check and auto correct endian type for NumpyTablet (#5448) --- client-py/README.md | 19 +-- client-py/SessionExample.py | 14 +- client-py/iotdb/utils/IoTDBConstants.py | 11 ++ client-py/iotdb/utils/NumpyTablet.py | 15 ++- client-py/tests/test_numpy_tablet.py | 147 +++++++++++++++++++++ .../UserGuide/API/Programming-Python-Native-API.md | 19 +-- .../UserGuide/API/Programming-Python-Native-API.md | 18 +-- 7 files changed, 206 insertions(+), 37 deletions(-) diff --git a/client-py/README.md b/client-py/README.md index 6c0acd62e7..41c0a113b8 100644 --- a/client-py/README.md +++ b/client-py/README.md @@ -182,8 +182,9 @@ Comparing with Tablet, Numpy Tablet is using [numpy.ndarray](https://numpy.org/d With less memory footprint and time cost of serialization, the insert performance will be better. **Notice** -1. time and numerical value columns in Tablet is ndarray -2. ndarray should be big-endian, see the example below +1. time and value columns in Tablet are ndarray. +2. recommended to use the specific dtypes to each ndarray, see the example below +(if not, the default dtypes are also ok). ```python data_types_ = [ @@ -195,14 +196,14 @@ data_types_ = [ TSDataType.TEXT, ] np_values_ = [ - np.array([False, True, False, True], np.dtype(">?")), - np.array([10, 100, 100, 0], np.dtype(">i4")), - np.array([11, 11111, 1, 0], np.dtype(">i8")), - np.array([1.1, 1.25, 188.1, 0], np.dtype(">f4")), - np.array([10011.1, 101.0, 688.25, 6.25], np.dtype(">f8")), - np.array(["test01", "test02", "test03", "test04"]), + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), ] -np_timestamps_ = np.array([1, 2, 3, 4], np.dtype(">i8")) +np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) np_tablet_ = NumpyTablet( "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ ) diff --git a/client-py/SessionExample.py b/client-py/SessionExample.py index 75897a44fb..bbc9669527 100644 --- a/client-py/SessionExample.py +++ b/client-py/SessionExample.py @@ -185,14 +185,14 @@ session.insert_tablet(tablet_) # insert one numpy tablet into the database. np_values_ = [ - np.array([False, True, False, True], np.dtype(">?")), - np.array([10, 100, 100, 0], np.dtype(">i4")), - np.array([11, 11111, 1, 0], np.dtype(">i8")), - np.array([1.1, 1.25, 188.1, 0], np.dtype(">f4")), - np.array([10011.1, 101.0, 688.25, 6.25], np.dtype(">f8")), - np.array(["test01", "test02", "test03", "test04"]), + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), ] -np_timestamps_ = np.array([1, 2, 3, 4], np.dtype(">i8")) +np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) np_tablet_ = NumpyTablet( "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ ) diff --git a/client-py/iotdb/utils/IoTDBConstants.py b/client-py/iotdb/utils/IoTDBConstants.py index 50344d1480..b01cd50783 100644 --- a/client-py/iotdb/utils/IoTDBConstants.py +++ b/client-py/iotdb/utils/IoTDBConstants.py @@ -17,6 +17,7 @@ # from enum import Enum, unique +import numpy as np @unique @@ -36,6 +37,16 @@ class TSDataType(Enum): def __hash__(self): return self.value + def np_dtype(self): + return { + TSDataType.BOOLEAN: np.dtype(">?"), + TSDataType.FLOAT: np.dtype(">f4"), + TSDataType.DOUBLE: np.dtype(">f8"), + TSDataType.INT32: np.dtype(">i4"), + TSDataType.INT64: np.dtype(">i8"), + TSDataType.TEXT: np.dtype("str"), + }[self] + @unique class TSEncoding(Enum): diff --git a/client-py/iotdb/utils/NumpyTablet.py b/client-py/iotdb/utils/NumpyTablet.py index 8dd8457459..b81a172a40 100644 --- a/client-py/iotdb/utils/NumpyTablet.py +++ b/client-py/iotdb/utils/NumpyTablet.py @@ -17,7 +17,6 @@ # import struct - from iotdb.utils.IoTDBConstants import TSDataType from iotdb.utils.BitMap import BitMap @@ -41,15 +40,25 @@ class NumpyTablet(object): """ if len(values) > 0 and len(values[0]) != len(timestamps): raise RuntimeError( - "Input error! len(timestamps) does not equal to len(values)!" + "Input error! len(timestamps) does not equal to len(values[0])!" + ) + if len(values) != len(data_types): + raise RuntimeError( + "Input error! len(values) does not equal to len(data_types)!" ) - if not NumpyTablet.check_sorted(timestamps): + if not self.check_sorted(timestamps): index = timestamps.argsort() timestamps = timestamps[index] for i in range(len(values)): values[i] = values[i][index] + if timestamps.dtype != TSDataType.INT64.np_dtype(): + timestamps = timestamps.astype(TSDataType.INT64.np_dtype()) + for i in range(len(values)): + if values[i].dtype != data_types[i].np_dtype(): + values[i] = values[i].astype(data_types[i].np_dtype()) + self.__values = values self.__timestamps = timestamps self.__device_id = device_id diff --git a/client-py/tests/test_numpy_tablet.py b/client-py/tests/test_numpy_tablet.py new file mode 100644 index 0000000000..b984193975 --- /dev/null +++ b/client-py/tests/test_numpy_tablet.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import numpy as np +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.NumpyTablet import NumpyTablet +from iotdb.utils.Tablet import Tablet + + +def test_numpy_tablet_serialization(): + + measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + ] + values_ = [ + [False, 10, 11, 1.1, 10011.1, "test01"], + [True, 100, 11111, 1.25, 101.0, "test02"], + [False, 100, 1, 188.1, 688.25, "test03"], + [True, 0, 0, 0, 6.25, "test04"], + ] + timestamps_ = [16, 17, 18, 19] + tablet_ = Tablet( + "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ + ) + np_values_ = [ + np.array([False, True, False, True], np.dtype(">?")), + np.array([10, 100, 100, 0], np.dtype(">i4")), + np.array([11, 11111, 1, 0], np.dtype(">i8")), + np.array([1.1, 1.25, 188.1, 0], np.dtype(">f4")), + np.array([10011.1, 101.0, 688.25, 6.25], np.dtype(">f8")), + np.array(["test01", "test02", "test03", "test04"]), + ] + np_timestamps_ = np.array([16, 17, 18, 19], np.dtype(">i8")) + np_tablet_ = NumpyTablet( + "root.sg_test_01.d_01", measurements_, data_types_, np_values_, np_timestamps_ + ) + assert tablet_.get_binary_timestamps() == np_tablet_.get_binary_timestamps() + assert tablet_.get_binary_values() == np_tablet_.get_binary_values() + + +def test_sort_numpy_tablet(): + + measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + ] + values_ = [ + [True, 10000, 11111, 8.999, 776, "test05"], + [True, 1000, 1111, 0, 6.25, "test06"], + [False, 100, 111, 188.1, 688.25, "test07"], + [False, 10, 11, 1.25, 101.0, "test08"], + [False, 0, 1, 1.1, 10011.1, "test09"], + ] + timestamps_ = [5, 6, 7, 8, 9] + tablet_ = Tablet( + "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ + ) + np_values_unsorted = [ + np.array([False, False, False, True, True], np.dtype(">?")), + np.array([0, 10, 100, 1000, 10000], np.dtype(">i4")), + np.array([1, 11, 111, 1111, 11111], np.dtype(">i8")), + np.array([1.1, 1.25, 188.1, 0, 8.999], np.dtype(">f4")), + np.array([10011.1, 101.0, 688.25, 6.25, 776], np.dtype(">f8")), + np.array(["test09", "test08", "test07", "test06", "test05"]), + ] + np_timestamps_unsorted = np.array([9, 8, 7, 6, 5], np.dtype(">i8")) + np_tablet_ = NumpyTablet( + "root.sg_test_01.d_01", + measurements_, + data_types_, + np_values_unsorted, + np_timestamps_unsorted, + ) + assert tablet_.get_binary_timestamps() == np_tablet_.get_binary_timestamps() + assert tablet_.get_binary_values() == np_tablet_.get_binary_values() + + +def test_numpy_tablet_auto_correct_datatype(): + + measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] + data_types_ = [ + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + ] + values_ = [ + [True, 10000, 11111, 8.999, 776, "test05"], + [True, 1000, 1111, 0, 6.25, "test06"], + [False, 100, 111, 188.1, 688.25, "test07"], + [False, 10, 11, 1.25, 101.0, "test08"], + [False, 0, 1, 1.1, 10011.1, "test09"], + ] + timestamps_ = [5, 6, 7, 8, 9] + tablet_ = Tablet( + "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_ + ) + np_values_unsorted = [ + np.array([False, False, False, True, True]), + np.array([0, 10, 100, 1000, 10000]), + np.array([1, 11, 111, 1111, 11111]), + np.array([1.1, 1.25, 188.1, 0, 8.999]), + np.array([10011.1, 101.0, 688.25, 6.25, 776]), + np.array(["test09", "test08", "test07", "test06", "test05"]), + ] + np_timestamps_unsorted = np.array([9, 8, 7, 6, 5]) + # numpy.dtype of int and float should be little endian by default + assert np_timestamps_unsorted.dtype != np.dtype(">i8") + for i in range(1, 4): + assert np_values_unsorted[i].dtype != data_types_[i].np_dtype() + np_tablet_ = NumpyTablet( + "root.sg_test_01.d_01", + measurements_, + data_types_, + np_values_unsorted, + np_timestamps_unsorted, + ) + assert tablet_.get_binary_timestamps() == np_tablet_.get_binary_timestamps() + assert tablet_.get_binary_values() == np_tablet_.get_binary_values() diff --git a/docs/UserGuide/API/Programming-Python-Native-API.md b/docs/UserGuide/API/Programming-Python-Native-API.md index c167af2ea4..fc5e80c161 100644 --- a/docs/UserGuide/API/Programming-Python-Native-API.md +++ b/docs/UserGuide/API/Programming-Python-Native-API.md @@ -163,7 +163,8 @@ With less memory footprint and time cost of serialization, the insert performanc **Notice** 1. time and numerical value columns in Tablet is ndarray -2. ndarray should be big-endian, see the example below +2. recommended to use the specific dtypes to each ndarray, see the example below + (if not, the default dtypes are also ok). ```python data_types_ = [ @@ -175,14 +176,14 @@ data_types_ = [ TSDataType.TEXT, ] np_values_ = [ - np.array([False, True, False, True], np.dtype(">?")), - np.array([10, 100, 100, 0], np.dtype(">i4")), - np.array([11, 11111, 1, 0], np.dtype(">i8")), - np.array([1.1, 1.25, 188.1, 0], np.dtype(">f4")), - np.array([10011.1, 101.0, 688.25, 6.25], np.dtype(">f8")), - np.array(["test01", "test02", "test03", "test04"]), + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), ] -np_timestamps_ = np.array([1, 2, 3, 4], np.dtype(">i8")) +np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) np_tablet_ = NumpyTablet( "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ ) @@ -206,7 +207,7 @@ session.insert_record(device_id, timestamp, measurements_, data_types_, values_) ```python session.insert_records( device_ids_, time_list_, measurements_list_, data_type_list_, values_list_ - ) +) ``` * Insert multiple Records that belong to the same device. diff --git a/docs/zh/UserGuide/API/Programming-Python-Native-API.md b/docs/zh/UserGuide/API/Programming-Python-Native-API.md index f32c8d4aa5..d15bb830e2 100644 --- a/docs/zh/UserGuide/API/Programming-Python-Native-API.md +++ b/docs/zh/UserGuide/API/Programming-Python-Native-API.md @@ -162,8 +162,8 @@ session.insert_tablet(tablet_) 内存占用和序列化耗时会降低很多，写入效率也会有很大提升。 **注意** -1. Tablet 中的每一列值记录为一个 ndarray -2. ndarray 需要为大端类型的数据类型，具体可参考下面的例子 +1. Tablet 中的每一列时间戳和值记录为一个 ndarray +2. ndarray 推荐使用如下面例子中的特定的 dtype，如果不使用，不会影响正确性。 ```python data_types_ = [ @@ -175,14 +175,14 @@ data_types_ = [ TSDataType.TEXT, ] np_values_ = [ - np.array([False, True, False, True], np.dtype(">?")), - np.array([10, 100, 100, 0], np.dtype(">i4")), - np.array([11, 11111, 1, 0], np.dtype(">i8")), - np.array([1.1, 1.25, 188.1, 0], np.dtype(">f4")), - np.array([10011.1, 101.0, 688.25, 6.25], np.dtype(">f8")), - np.array(["test01", "test02", "test03", "test04"]), + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), ] -np_timestamps_ = np.array([1, 2, 3, 4], np.dtype(">i8")) +np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) np_tablet_ = NumpyTablet( "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ )
