This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch numpy_tablet_none in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b502d6a65b0d94d7b53c980d7ad16d4a948da7f7 Author: HTHou <[email protected]> AuthorDate: Thu Dec 22 15:07:42 2022 +0800 finish --- client-py/README.md | 37 +++++++++++++++++++- client-py/SessionExample.py | 12 +++++-- client-py/iotdb/utils/BitMap.py | 9 +++++ client-py/iotdb/utils/NumpyTablet.py | 6 ++-- client-py/tests/test_numpy_tablet.py | 19 +++++++---- client-py/tests/test_session.py | 26 +++++++++++++++ .../UserGuide/API/Programming-Python-Native-API.md | 39 ++++++++++++++++++++-- .../UserGuide/API/Programming-Python-Native-API.md | 39 ++++++++++++++++++++-- 8 files changed, 170 insertions(+), 17 deletions(-) diff --git a/client-py/README.md b/client-py/README.md index 68c1697acf..dcb6bdcca1 100644 --- a/client-py/README.md +++ b/client-py/README.md @@ -172,6 +172,18 @@ tablet_ = Tablet( device_id, measurements_, data_types_, values_, timestamps_ ) session.insert_tablet(tablet_) + +values_ = [ + [None, 10, 11, 1.1, 10011.1, "test01"], + [True, None, 11111, 1.25, 101.0, "test02"], + [False, 100, None, 188.1, 688.25, "test03"], + [True, 0, 0, 0, None, None], +] +timestamps_ = [16, 17, 18, 19] +tablet_ = Tablet( + device_id, measurements_, data_types_, values_, timestamps_ +) +session.insert_tablet(tablet_) ``` * Numpy Tablet @@ -203,9 +215,32 @@ np_values_ = [ ] np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) np_tablet_ = NumpyTablet( - "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ + device_id, measurements_, data_types_, np_values_, np_timestamps_ ) session.insert_tablet(np_tablet_) + +# insert one numpy tablet with none into the database. +np_values_ = [ + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), +] +np_timestamps_ = np.array([98, 99, 100, 101], TSDataType.INT64.np_dtype()) +np_bitmaps_ = [] +for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) +np_bitmaps_[0].mark(0) +np_bitmaps_[1].mark(1) +np_bitmaps_[2].mark(2) +np_bitmaps_[4].mark(3) +np_bitmaps_[5].mark(3) +np_tablet_with_none = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_, np_bitmaps_ +) +session.insert_tablet(np_tablet_with_none) ``` * Insert multiple Tablets diff --git a/client-py/SessionExample.py b/client-py/SessionExample.py index 372a4ea457..e1cb1ef4c2 100644 --- a/client-py/SessionExample.py +++ b/client-py/SessionExample.py @@ -23,6 +23,7 @@ from iotdb.Session import Session from iotdb.template.InternalNode import InternalNode from iotdb.template.MeasurementNode import MeasurementNode from iotdb.template.Template import Template +from iotdb.utils.BitMap import BitMap from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor from iotdb.utils.Tablet import Tablet from iotdb.utils.NumpyTablet import NumpyTablet @@ -229,10 +230,17 @@ np_values_ = [ np.array(["test01", "test02", "test03", "test04"]), ] np_timestamps_ = np.array([98, 99, 100, 101], TSDataType.INT64.np_dtype()) +np_bitmaps_ = [] +for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) +np_bitmaps_[0].mark(0) +np_bitmaps_[1].mark(1) +np_bitmaps_[2].mark(2) +np_bitmaps_[4].mark(3) +np_bitmaps_[5].mark(3) np_tablet_with_none = NumpyTablet( - "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ + "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_, np_bitmaps_ ) -np_tablet_with_none.mark_none_value(0,0) session.insert_tablet(np_tablet_with_none) diff --git a/client-py/iotdb/utils/BitMap.py b/client-py/iotdb/utils/BitMap.py index 7b171f6f81..621bf6c7df 100644 --- a/client-py/iotdb/utils/BitMap.py +++ b/client-py/iotdb/utils/BitMap.py @@ -28,3 +28,12 @@ class BitMap(object): def mark(self, position): self.bits[position // 8] |= BitMap.BIT_UTIL[position % 8] + + def is_all_unmarked(self): + for i in range(self.__size // 8): + if self.bits[i] != 0: + return False + for i in range(self.__size % 8): + if (self.bits[self.__size // 8] & BitMap.BIT_UTIL[i]) != 0: + return False + return True diff --git a/client-py/iotdb/utils/NumpyTablet.py b/client-py/iotdb/utils/NumpyTablet.py index 20dc958943..b217841f74 100644 --- a/client-py/iotdb/utils/NumpyTablet.py +++ b/client-py/iotdb/utils/NumpyTablet.py @@ -22,7 +22,7 @@ from iotdb.utils.BitMap import BitMap class NumpyTablet(object): - def __init__(self, device_id, measurements, data_types, values, timestamps): + def __init__(self, device_id, measurements, data_types, values, timestamps, bitmaps=None): """ creating a numpy tablet for insertion for example, considering device: root.sg1.d1 @@ -38,7 +38,6 @@ class NumpyTablet(object): :param values: List of numpy array, the values of each column should be the inner numpy array :param timestamps: Numpy array, the timestamps """ - self.bitmaps = None if len(values) > 0 and len(values[0]) != len(timestamps): raise RuntimeError( "Input error! len(timestamps) does not equal to len(values[0])!" @@ -67,6 +66,7 @@ class NumpyTablet(object): self.__data_types = data_types self.__row_number = len(timestamps) self.__column_number = len(measurements) + self.bitmaps = bitmaps @staticmethod def check_sorted(timestamps): @@ -122,7 +122,7 @@ class NumpyTablet(object): values_tobe_packed = [] for i in range(self.__column_number): format_str_list.append("?") - if self.bitmaps[i] is None: + if self.bitmaps[i] is None or self.bitmaps[i].is_all_unmarked(): values_tobe_packed.append(False) else: values_tobe_packed.append(True) diff --git a/client-py/tests/test_numpy_tablet.py b/client-py/tests/test_numpy_tablet.py index 74fe5bfe73..ddcb17b070 100644 --- a/client-py/tests/test_numpy_tablet.py +++ b/client-py/tests/test_numpy_tablet.py @@ -17,6 +17,8 @@ # import numpy as np + +from iotdb.utils.BitMap import BitMap from iotdb.utils.IoTDBConstants import TSDataType from iotdb.utils.NumpyTablet import NumpyTablet from iotdb.utils.Tablet import Tablet @@ -59,7 +61,7 @@ def test_numpy_tablet_serialization(): assert tablet_.get_binary_values() == np_tablet_.get_binary_values() -def test_numpy_tablet_serialization2(): +def test_numpy_tablet_with_none_serialization(): measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"] data_types_ = [ @@ -89,14 +91,17 @@ def test_numpy_tablet_serialization2(): np.array(["test01", "test02", "test03", ""]), ] np_timestamps_ = np.array([16, 17, 18, 19], np.dtype(">i8")) + np_bitmaps_ = [] + for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) + np_bitmaps_[0].mark(0) + np_bitmaps_[1].mark(1) + np_bitmaps_[2].mark(2) + np_bitmaps_[4].mark(3) + np_bitmaps_[5].mark(3) np_tablet_ = NumpyTablet( - "root.sg_test_01.d_01", measurements_, data_types_, np_values_, np_timestamps_ + "root.sg_test_01.d_01", measurements_, data_types_, np_values_, np_timestamps_, np_bitmaps_ ) - np_tablet_.mark_none_value(0, 0) - np_tablet_.mark_none_value(1, 1) - np_tablet_.mark_none_value(2, 2) - np_tablet_.mark_none_value(4, 3) - np_tablet_.mark_none_value(5, 3) assert tablet_.get_binary_timestamps() == np_tablet_.get_binary_timestamps() assert tablet_.get_binary_values() == np_tablet_.get_binary_values() diff --git a/client-py/tests/test_session.py b/client-py/tests/test_session.py index e7ae79fcb9..92bc87d012 100644 --- a/client-py/tests/test_session.py +++ b/client-py/tests/test_session.py @@ -20,6 +20,7 @@ import numpy as np from iotdb.Session import Session +from iotdb.utils.BitMap import BitMap from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor from iotdb.utils.NumpyTablet import NumpyTablet from iotdb.utils.Tablet import Tablet @@ -290,6 +291,31 @@ def test_session(): test_fail() print_message("insert tablet with empty cells failed") + # insert one numpy tablet with empty cells into the database. + np_values_ = [ + np.array([False, True, False, True], np.dtype(">?")), + np.array([10, 0, 100, 0], np.dtype(">i4")), + np.array([11, 11111, 0, 0], np.dtype(">i8")), + np.array([1.1, 1.25, 188.1, 0], np.dtype(">f4")), + np.array([10011.1, 101.0, 688.25, 0], np.dtype(">f8")), + np.array(["test01", "test02", "test03", ""]), + ] + np_timestamps_ = np.array([30, 31, 32, 33], np.dtype(">i8")) + np_bitmaps_ = [] + for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) + np_bitmaps_[0].mark(0) + np_bitmaps_[1].mark(1) + np_bitmaps_[2].mark(2) + np_bitmaps_[4].mark(3) + np_bitmaps_[5].mark(3) + np_tablet_ = NumpyTablet( + "root.sg_test_01.d_01", measurements_, data_types_, np_values_, np_timestamps_, np_bitmaps_ + ) + if session.insert_tablet(np_tablet_) < 0: + test_fail() + print_message("insert numpy tablet with empty cells failed") + # insert records of one device time_list = [1, 2, 3] measurements_list = [ diff --git a/docs/UserGuide/API/Programming-Python-Native-API.md b/docs/UserGuide/API/Programming-Python-Native-API.md index 30ea146e4b..c1cd8995ad 100644 --- a/docs/UserGuide/API/Programming-Python-Native-API.md +++ b/docs/UserGuide/API/Programming-Python-Native-API.md @@ -153,6 +153,18 @@ tablet_ = Tablet( device_id, measurements_, data_types_, values_, timestamps_ ) session.insert_tablet(tablet_) + +values_ = [ + [None, 10, 11, 1.1, 10011.1, "test01"], + [True, None, 11111, 1.25, 101.0, "test02"], + [False, 100, None, 188.1, 688.25, "test03"], + [True, 0, 0, 0, None, None], +] +timestamps_ = [16, 17, 18, 19] +tablet_ = Tablet( + device_id, measurements_, data_types_, values_, timestamps_ +) +session.insert_tablet(tablet_) ``` * Numpy Tablet @@ -180,13 +192,36 @@ np_values_ = [ np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), - np.array(["test01", "test02", "test03", "test04"]), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), ] np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) np_tablet_ = NumpyTablet( - "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ + device_id, measurements_, data_types_, np_values_, np_timestamps_ ) session.insert_tablet(np_tablet_) + +# insert one numpy tablet with None into the database. +np_values_ = [ + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), +] +np_timestamps_ = np.array([98, 99, 100, 101], TSDataType.INT64.np_dtype()) +np_bitmaps_ = [] +for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) +np_bitmaps_[0].mark(0) +np_bitmaps_[1].mark(1) +np_bitmaps_[2].mark(2) +np_bitmaps_[4].mark(3) +np_bitmaps_[5].mark(3) +np_tablet_with_none = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_, np_bitmaps_ +) +session.insert_tablet(np_tablet_with_none) ``` * Insert multiple Tablets diff --git a/docs/zh/UserGuide/API/Programming-Python-Native-API.md b/docs/zh/UserGuide/API/Programming-Python-Native-API.md index 115488b729..3b213d8b79 100644 --- a/docs/zh/UserGuide/API/Programming-Python-Native-API.md +++ b/docs/zh/UserGuide/API/Programming-Python-Native-API.md @@ -153,6 +153,18 @@ tablet_ = Tablet( device_id, measurements_, data_types_, values_, timestamps_ ) session.insert_tablet(tablet_) + +values_ = [ + [None, 10, 11, 1.1, 10011.1, "test01"], + [True, None, 11111, 1.25, 101.0, "test02"], + [False, 100, None, 188.1, 688.25, "test03"], + [True, 0, 0, 0, None, None], +] +timestamps_ = [16, 17, 18, 19] +tablet_ = Tablet( + device_id, measurements_, data_types_, values_, timestamps_ +) +session.insert_tablet(tablet_) ``` * Numpy Tablet @@ -179,13 +191,36 @@ np_values_ = [ np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), - np.array(["test01", "test02", "test03", "test04"]), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), ] np_timestamps_ = np.array([1, 2, 3, 4], TSDataType.INT64.np_dtype()) np_tablet_ = NumpyTablet( - "root.sg_test_01.d_02", measurements_, data_types_, np_values_, np_timestamps_ + device_id, measurements_, data_types_, np_values_, np_timestamps_ ) session.insert_tablet(np_tablet_) + +# insert one numpy tablet with None into the database. +np_values_ = [ + np.array([False, True, False, True], TSDataType.BOOLEAN.np_dtype()), + np.array([10, 100, 100, 0], TSDataType.INT32.np_dtype()), + np.array([11, 11111, 1, 0], TSDataType.INT64.np_dtype()), + np.array([1.1, 1.25, 188.1, 0], TSDataType.FLOAT.np_dtype()), + np.array([10011.1, 101.0, 688.25, 6.25], TSDataType.DOUBLE.np_dtype()), + np.array(["test01", "test02", "test03", "test04"], TSDataType.TEXT.np_dtype()), +] +np_timestamps_ = np.array([98, 99, 100, 101], TSDataType.INT64.np_dtype()) +np_bitmaps_ = [] +for i in range(len(measurements_)): + np_bitmaps_.append(BitMap(len(np_timestamps_))) +np_bitmaps_[0].mark(0) +np_bitmaps_[1].mark(1) +np_bitmaps_[2].mark(2) +np_bitmaps_[4].mark(3) +np_bitmaps_[5].mark(3) +np_tablet_with_none = NumpyTablet( + device_id, measurements_, data_types_, np_values_, np_timestamps_, np_bitmaps_ +) +session.insert_tablet(np_tablet_with_none) ``` * 插入多个 Tablet
