This is an automated email from the ASF dual-hosted git repository. colinlee pushed a commit to branch colin_fix_config in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 4b716d569b4712053f9a9ccd995e379366144b65 Author: colin <[email protected]> AuthorDate: Wed Apr 23 17:27:51 2025 +0800 add config show. --- .gitignore | 1 + python/tests/test_write_and_read.py | 20 ++++++++++++++++++-- python/tsfile/tsfile_cpp.pxd | 4 ---- python/tsfile/tsfile_py_cpp.pyx | 20 ++++++++------------ python/tsfile/tsfile_table_writer.py | 4 ++-- python/tsfile/tsfile_writer.pyx | 2 +- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index b9070546..4fa45012 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ python/data python/venv/* python/tests/__pycache__/* python/tests/*.tsfile +python/tsfile/include cpp/cmake-build-debug-mingw/ cpp/third_party/googletest-release-1.12.1.zip diff --git a/python/tests/test_write_and_read.py b/python/tests/test_write_and_read.py index 0b666d15..ea84d898 100644 --- a/python/tests/test_write_and_read.py +++ b/python/tests/test_write_and_read.py @@ -26,6 +26,7 @@ from tsfile import Tablet, RowRecord, Field from tsfile import TimeseriesSchema from tsfile import TsFileTableWriter from tsfile import TsFileWriter, TsFileReader, ColumnCategory +from tsfile import Compressor def test_row_record_write_and_read(): @@ -217,11 +218,12 @@ def test_tsfile_config(): from tsfile import get_tsfile_config, set_tsfile_config config = get_tsfile_config() + assert config["chunk_group_size_threshold_"] == 0 table = TableSchema("tEst_Table", [ColumnSchema("Device", TSDataType.STRING, ColumnCategory.TAG), ColumnSchema("vAlue", TSDataType.DOUBLE, ColumnCategory.FIELD)]) - + os.remove("test1.tsfile") with TsFileTableWriter("test1.tsfile", table) as writer: tablet = Tablet(["device", "VALUE"], [TSDataType.STRING, TSDataType.DOUBLE]) for i in range(100): @@ -232,8 +234,10 @@ def test_tsfile_config(): writer.write_table(tablet) config_normal = get_tsfile_config() + assert config_normal["chunk_group_size_threshold_"] == 128 * 1024 * 1024 - with TsFileTableWriter("test2.tsfile", table, 100 * 100) as writer: + os.remove("test1.tsfile") + with TsFileTableWriter("test1.tsfile", table, 100 * 100) as writer: tablet = Tablet(["device", "VALUE"], [TSDataType.STRING, TSDataType.DOUBLE]) for i in range(100): tablet.add_timestamp(i, i) @@ -241,6 +245,18 @@ def test_tsfile_config(): tablet.add_value_by_name("valuE", i, i * 1.1) writer.write_table(tablet) + config_modified = get_tsfile_config() + assert config_normal != config_modified + assert config_modified["chunk_group_size_threshold_"] == 100 * 100 + set_tsfile_config({'chunk_group_size_threshold_': 100 * 20}) + assert get_tsfile_config()["chunk_group_size_threshold_"] == 100 * 20 + with pytest.raises(TypeError): + set_tsfile_config({"time_compress_type_": TSDataType.DOUBLE}) + with pytest.raises(TypeError): + set_tsfile_config({'chunk_group_size_threshold_': -1 *100 * 20}) + + + diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd index c726c220..e4782672 100644 --- a/python/tsfile/tsfile_cpp.pxd +++ b/python/tsfile/tsfile_cpp.pxd @@ -17,11 +17,7 @@ # #cython: language_level=3 -<<<<<<< HEAD from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t, uint8_t -======= -from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t ->>>>>>> 0db9e5c1 (add memory threshold in py writer.) ctypedef int32_t ErrorCode diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx index a554bab9..55446ee7 100644 --- a/python/tsfile/tsfile_py_cpp.pyx +++ b/python/tsfile/tsfile_py_cpp.pyx @@ -384,22 +384,22 @@ cpdef void set_tsfile_config(dict new_config): _check_double(new_config["tsfile_index_bloom_filter_error_percent_"]) g_config_value_.tsfile_index_bloom_filter_error_percent_ = new_config["tsfile_index_bloom_filter_error_percent_"] if "time_encoding_type_" in new_config: - if new_config["time_encoding_type_"] not in TSEncodingPy: - raise ValueError(f"Unsupported TSEncoding: {new_config['time_encoding_type_']}") + if not isinstance(new_config["time_encoding_type_"], TSEncodingPy): + raise TypeError(f"Unsupported TSEncoding: {new_config['time_encoding_type_']}") set_global_time_encoding(<uint8_t>(new_config["time_encoding_type_"].value)) if "time_data_type_" in new_config: - if new_config["time_data_type_"] not in TSDataTypePy: - raise ValueError(f"Unsupported TSDataType: {new_config['time_data_type_']}") + if not isinstance(new_config["time_data_type_"], TSDataTypePy): + raise TypeError(f"Unsupported TSDataType: {new_config['time_data_type_']}") set_global_time_data_type(<uint8_t>(new_config["time_data_type_"].value)) if "time_compress_type_" in new_config: - if new_config["time_compress_type_"] not in CompressorPy: - raise ValueError(f"Unsupported Compressor: {new_config['time_compress_type_']}") + if not isinstance(new_config["time_compress_type_"], CompressorPy): + raise TypeError(f"Unsupported Compressor: {new_config['time_compress_type_']}") set_global_time_compression(<uint8_t>(new_config["time_compress_type_"].value)) if "chunk_group_size_threshold_" in new_config: - _check_int32(new_config["chunk_group_size_threshold_"]) + _check_uint32(new_config["chunk_group_size_threshold_"]) g_config_value_.chunk_group_size_threshold_ = new_config["chunk_group_size_threshold_"] if "record_count_for_next_mem_check_" in new_config: - _check_int32(new_config["record_count_for_next_mem_check_"]) + _check_uint32(new_config["record_count_for_next_mem_check_"]) g_config_value_.record_count_for_next_mem_check_ = new_config["record_count_for_next_mem_check_"] if "encrypt_flag_" in new_config: _check_bool(new_config["encrypt_flag_"]) @@ -409,10 +409,6 @@ cdef _check_uint32(value): if not isinstance(value, int) or value < 0 or value > 0xFFFFFFFF: raise TypeError(f"Expected uint32, got {type(value)}") -cdef _check_int32(value): - if not isinstance(value, int) or value < -0x80000000 or value > 0x7FFFFFFF: - raise TypeError(f"Expected int32, got {type(value)}") - cdef _check_double(value): if not isinstance(value, (int, float)): raise TypeError(f"Expected float, got {type(value)}") diff --git a/python/tsfile/tsfile_table_writer.py b/python/tsfile/tsfile_table_writer.py index c7ab9492..f5568dae 100644 --- a/python/tsfile/tsfile_table_writer.py +++ b/python/tsfile/tsfile_table_writer.py @@ -31,12 +31,12 @@ class TsFileTableWriter: according to that schema, and serialize this data into a TsFile. """ - def __init__(self, path: str, table_schema: TableSchema): + def __init__(self, path: str, table_schema: TableSchema, memory_threshold = 128 * 1024 * 1024): """ :param path: The path of tsfile, will create if it doesn't exist. :param table_schema: describes the schema of the tables they want to write. """ - self.writer = TsFileWriter(path) + self.writer = TsFileWriter(path, memory_threshold) self.writer.register_table(table_schema) self.exclusive_table_name_ = table_schema.get_table_name() diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx index 4d1f53ed..8db496cd 100644 --- a/python/tsfile/tsfile_writer.pyx +++ b/python/tsfile/tsfile_writer.pyx @@ -29,7 +29,7 @@ from tsfile.tablet import Tablet as TabletPy cdef class TsFileWriterPy: cdef TsFileWriter writer - def __init__(self, pathname, memory_threshold = 128 * 1024 * 1024): + def __init__(self, pathname:str, memory_threshold:int): self.writer = tsfile_writer_new_c(pathname, memory_threshold) def register_timeseries(self, device_name : str, timeseries_schema : TimeseriesSchemaPy):
