This is an automated email from the ASF dual-hosted git repository. colinlee pushed a commit to branch rc/2.2.0_bak in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 141f6af80a300cd7d47f91eb51d19c080be5a64e Author: Colin Lee <[email protected]> AuthorDate: Wed Dec 17 09:36:30 2025 +0800 tmp code (#666) fix TEXT null. tmp code. --- cpp/CMakeLists.txt | 19 +- cpp/src/reader/aligned_chunk_reader.cc | 113 ++++---- cpp/src/reader/result_set.h | 87 +++++++ cpp/src/reader/tsfile_series_scan_iterator.cc | 27 +- cpp/src/utils/db_utils.h | 59 ----- .../reader/tree_view/tsfile_reader_tree_test.cc | 86 ------- cpp/test/writer/tsfile_writer_test.cc | 27 +- python/tests/resources/README.md | 285 +++++++++++++++++++++ python/tests/resources/simple_table_t1.tsfile | Bin 0 -> 2965 bytes python/tests/resources/simple_table_t2.tsfile | Bin 0 -> 2677 bytes python/tests/resources/simple_tree.tsfile | Bin 0 -> 842 bytes python/tests/test_load_tsfile_from_iotdb.py | 111 ++++++++ python/tsfile/constants.py | 2 +- python/tsfile/tsfile_reader.pyx | 1 + 14 files changed, 593 insertions(+), 224 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3128c996..b4b8c39b 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -108,7 +108,7 @@ if (NOT WIN32) endif () endif () -option(BUILD_TEST "Build tests" OFF) +option(BUILD_TEST "Build tests" ON) message("cmake using: BUILD_TEST=${BUILD_TEST}") option(ENABLE_ANTLR4 "Enable ANTLR4 runtime" ON) @@ -117,15 +117,32 @@ message("cmake using: ENABLE_ANTLR4=${ENABLE_ANTLR4}") option(ENABLE_SNAPPY "Enable Google Snappy compression" ON) message("cmake using: ENABLE_SNAPPY=${ENABLE_SNAPPY}") +if (ENABLE_SNAPPY) + add_definitions(-DENABLE_SNAPPY) +endif() + option(ENABLE_LZ4 "Enable LZ4 compression" ON) message("cmake using: ENABLE_LZ4=${ENABLE_LZ4}") +if (ENABLE_LZ4) + add_definitions(-DENABLE_LZ4) +endif() + option(ENABLE_LZOKAY "Enable LZOKAY compression" ON) message("cmake using: ENABLE_LZOKAY=${ENABLE_LZOKAY}") +if (ENABLE_LZOKAY) + add_definitions(-DENABLE_LZOKAY) +endif() + option(ENABLE_ZLIB "Enable Zlib compression" ON) message("cmake using: ENABLE_ZLIB=${ENABLE_ZLIB}") +if (ENABLE_ZLIB) + add_definitions(-DENABLE_ZLIB) + add_definitions(-DENABLE_GZIP) +endif() + # All libs will be stored here, including libtsfile, compress-encoding lib. set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) diff --git a/cpp/src/reader/aligned_chunk_reader.cc b/cpp/src/reader/aligned_chunk_reader.cc index fc944ca7..b39db564 100644 --- a/cpp/src/reader/aligned_chunk_reader.cc +++ b/cpp/src/reader/aligned_chunk_reader.cc @@ -25,8 +25,8 @@ using namespace common; namespace storage { -int AlignedChunkReader::init(ReadFile *read_file, String m_name, - TSDataType data_type, Filter *time_filter) { +int AlignedChunkReader::init(ReadFile* read_file, String m_name, + TSDataType data_type, Filter* time_filter) { read_file_ = read_file; measurement_name_.shallow_copy_from(m_name); time_decoder_ = DecoderFactory::alloc_time_decoder(); @@ -50,7 +50,7 @@ void AlignedChunkReader::reset() { cur_time_page_header_.reset(); cur_value_page_header_.reset(); - char *file_data_buf = time_in_stream_.get_wrapped_buf(); + char* file_data_buf = time_in_stream_.get_wrapped_buf(); if (file_data_buf != nullptr) { mem_free(file_data_buf); } @@ -87,7 +87,7 @@ void AlignedChunkReader::destroy() { CompressorFactory::free(value_compressor_); value_compressor_ = nullptr; } - char *buf = time_in_stream_.get_wrapped_buf(); + char* buf = time_in_stream_.get_wrapped_buf(); if (buf != nullptr) { mem_free(buf); time_in_stream_.clear_wrapped_buf(); @@ -102,8 +102,8 @@ void AlignedChunkReader::destroy() { chunk_header_.~ChunkHeader(); } -int AlignedChunkReader::load_by_aligned_meta(ChunkMeta *time_chunk_meta, - ChunkMeta *value_chunk_meta) { +int AlignedChunkReader::load_by_aligned_meta(ChunkMeta* time_chunk_meta, + ChunkMeta* value_chunk_meta) { int ret = E_OK; time_chunk_meta_ = time_chunk_meta; value_chunk_meta_ = value_chunk_meta; @@ -116,8 +116,8 @@ int AlignedChunkReader::load_by_aligned_meta(ChunkMeta *time_chunk_meta, file_data_time_buf_size_ = 1024; file_data_value_buf_size_ = 1024; int32_t ret_read_len = 0; - char *time_file_data_buf = - (char *)mem_alloc(file_data_time_buf_size_, MOD_CHUNK_READER); + char* time_file_data_buf = + (char*)mem_alloc(file_data_time_buf_size_, MOD_CHUNK_READER); if (IS_NULL(time_file_data_buf)) { return E_OOM; } @@ -140,8 +140,8 @@ int AlignedChunkReader::load_by_aligned_meta(ChunkMeta *time_chunk_meta, } /* ================ deserialize value_chunk_header ================*/ ret_read_len = 0; - char *value_file_data_buf = - (char *)mem_alloc(file_data_value_buf_size_, MOD_CHUNK_READER); + char* value_file_data_buf = + (char*)mem_alloc(file_data_value_buf_size_, MOD_CHUNK_READER); if (IS_NULL(value_file_data_buf)) { return E_OOM; } @@ -182,7 +182,7 @@ int AlignedChunkReader::load_by_aligned_meta(ChunkMeta *time_chunk_meta, } int AlignedChunkReader::alloc_compressor_and_decoder( - storage::Decoder *&decoder, storage::Compressor *&compressor, + storage::Decoder*& decoder, storage::Compressor*& compressor, TSEncoding encoding, TSDataType data_type, CompressionType compression) { if (decoder != nullptr) { decoder->reset(); @@ -204,10 +204,10 @@ int AlignedChunkReader::alloc_compressor_and_decoder( return E_OK; } -int AlignedChunkReader::get_next_page(TsBlock *ret_tsblock, - Filter *oneshoot_filter, PageArena &pa) { +int AlignedChunkReader::get_next_page(TsBlock* ret_tsblock, + Filter* oneshoot_filter, PageArena& pa) { int ret = E_OK; - Filter *filter = + Filter* filter = (oneshoot_filter != nullptr ? oneshoot_filter : time_filter_); if (prev_time_page_not_finish() && prev_value_page_not_finish()) { ret = decode_time_value_buf_into_tsblock(ret_tsblock, oneshoot_filter, @@ -243,11 +243,11 @@ int AlignedChunkReader::get_next_page(TsBlock *ret_tsblock, return ret; } -int AlignedChunkReader::get_cur_page_header(ChunkMeta *&chunk_meta, - common::ByteStream &in_stream, - PageHeader &cur_page_header, - uint32_t &chunk_visit_offset, - ChunkHeader &chunk_header) { +int AlignedChunkReader::get_cur_page_header(ChunkMeta*& chunk_meta, + common::ByteStream& in_stream, + PageHeader& cur_page_header, + uint32_t& chunk_visit_offset, + ChunkHeader& chunk_header) { int ret = E_OK; bool retry = true; int cur_page_header_serialized_size = 0; @@ -263,7 +263,7 @@ int AlignedChunkReader::get_cur_page_header(ChunkMeta *&chunk_meta, if (deserialize_buf_not_enough(ret) && retry) { retry = false; retry_read_want_size += 1024; - int32_t &file_data_buf_size = + int32_t& file_data_buf_size = chunk_header.data_type_ == common::VECTOR ? file_data_time_buf_size_ : file_data_value_buf_size_; @@ -295,18 +295,18 @@ int AlignedChunkReader::get_cur_page_header(ChunkMeta *&chunk_meta, // reader at least @want_size bytes from file and wrap the buffer into // @in_stream_ int AlignedChunkReader::read_from_file_and_rewrap( - common::ByteStream &in_stream_, ChunkMeta *&chunk_meta, - uint32_t &chunk_visit_offset, int32_t &file_data_buf_size, int want_size, + common::ByteStream& in_stream_, ChunkMeta*& chunk_meta, + uint32_t& chunk_visit_offset, int32_t& file_data_buf_size, int want_size, bool may_shrink) { int ret = E_OK; const int DEFAULT_READ_SIZE = 4096; // may use page_size + page_header_size - char *file_data_buf = in_stream_.get_wrapped_buf(); + char* file_data_buf = in_stream_.get_wrapped_buf(); int offset = chunk_meta->offset_of_chunk_header_ + chunk_visit_offset; int read_size = (want_size < DEFAULT_READ_SIZE ? DEFAULT_READ_SIZE : want_size); if (file_data_buf_size < read_size || (may_shrink && read_size < file_data_buf_size / 10)) { - file_data_buf = (char *)mem_realloc(file_data_buf, read_size); + file_data_buf = (char*)mem_realloc(file_data_buf, read_size); if (IS_NULL(file_data_buf)) { return E_OOM; } @@ -326,7 +326,7 @@ int AlignedChunkReader::read_from_file_and_rewrap( return ret; } -bool AlignedChunkReader::cur_page_statisify_filter(Filter *filter) { +bool AlignedChunkReader::cur_page_statisify_filter(Filter* filter) { bool value_satisfy = filter == nullptr || cur_value_page_header_.statistic_ == nullptr || filter->satisfy(cur_value_page_header_.statistic_); @@ -364,8 +364,8 @@ int AlignedChunkReader::decode_cur_time_page_data() { } } - char *time_compressed_buf = nullptr; - char *time_uncompressed_buf = nullptr; + char* time_compressed_buf = nullptr; + char* time_uncompressed_buf = nullptr; uint32_t time_compressed_buf_size = 0; uint32_t time_uncompressed_buf_size = 0; @@ -376,7 +376,7 @@ int AlignedChunkReader::decode_cur_time_page_data() { #ifdef DEBUG_SE std::cout << "AlignedChunkReader::decode_cur_page_data,time_in_stream_." "get_wrapped_buf=" - << (void *)(time_in_stream_.get_wrapped_buf()) + << (void*)(time_in_stream_.get_wrapped_buf()) << ", time_in_stream_.read_pos=" << time_in_stream_.read_pos() << std::endl; #endif @@ -427,11 +427,11 @@ int AlignedChunkReader::decode_cur_value_page_data() { } } - char *value_compressed_buf = nullptr; - char *value_uncompressed_buf = nullptr; + char* value_compressed_buf = nullptr; + char* value_uncompressed_buf = nullptr; uint32_t value_compressed_buf_size = 0; uint32_t value_uncompressed_buf_size = 0; - char *value_buf = nullptr; + char* value_buf = nullptr; uint32_t value_buf_size = 0; // Step 2: do uncompress @@ -467,7 +467,7 @@ int AlignedChunkReader::decode_cur_value_page_data() { SerializationUtil::read_ui32(value_uncompressed_buf); value_uncompressed_buf_offset += sizeof(uint32_t); value_page_col_notnull_bitmap_.resize((value_page_data_num_ + 7) / 8); - for (unsigned char &i : value_page_col_notnull_bitmap_) { + for (unsigned char& i : value_page_col_notnull_bitmap_) { i = *(value_uncompressed_buf + value_uncompressed_buf_offset); value_uncompressed_buf_offset++; } @@ -486,7 +486,7 @@ int AlignedChunkReader::decode_cur_value_page_data() { } int AlignedChunkReader::decode_time_value_buf_into_tsblock( - TsBlock *&ret_tsblock, Filter *filter, common::PageArena *pa) { + TsBlock*& ret_tsblock, Filter* filter, common::PageArena* pa) { int ret = common::E_OK; ret = decode_tv_buf_into_tsblock_by_datatype(time_in_, value_in_, ret_tsblock, filter, pa); @@ -535,7 +535,7 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock( ret = E_OVERFLOW; \ break; \ } \ - row_appender.append(0, (char *)&time, sizeof(time)); \ + row_appender.append(0, (char*)&time, sizeof(time)); \ row_appender.append_null(1); \ continue; \ } \ @@ -552,15 +552,15 @@ int AlignedChunkReader::decode_time_value_buf_into_tsblock( } else { \ /*std::cout << "decoder: time=" << time << ", value=" << value \ * << std::endl;*/ \ - row_appender.append(0, (char *)&time, sizeof(time)); \ - row_appender.append(1, (char *)&value, sizeof(value)); \ + row_appender.append(0, (char*)&time, sizeof(time)); \ + row_appender.append(1, (char*)&value, sizeof(value)); \ } \ } \ } while (false) int AlignedChunkReader::i32_DECODE_TYPED_TV_INTO_TSBLOCK( - ByteStream &time_in, ByteStream &value_in, RowAppender &row_appender, - Filter *filter) { + ByteStream& time_in, ByteStream& value_in, RowAppender& row_appender, + Filter* filter) { int ret = E_OK; uint32_t mask = 1 << 7; int64_t time = 0; @@ -578,7 +578,7 @@ int AlignedChunkReader::i32_DECODE_TYPED_TV_INTO_TSBLOCK( ret = E_OVERFLOW; break; } - row_appender.append(0, (char *)&time, sizeof(time)); + row_appender.append(0, (char*)&time, sizeof(time)); row_appender.append_null(1); continue; } @@ -594,16 +594,16 @@ int AlignedChunkReader::i32_DECODE_TYPED_TV_INTO_TSBLOCK( } else { /*std::cout << "decoder: time=" << time << ", value=" << value * << std::endl;*/ - row_appender.append(0, (char *)&time, sizeof(time)); - row_appender.append(1, (char *)&value, sizeof(value)); + row_appender.append(0, (char*)&time, sizeof(time)); + row_appender.append(1, (char*)&value, sizeof(value)); } } return ret; } int AlignedChunkReader::decode_tv_buf_into_tsblock_by_datatype( - ByteStream &time_in, ByteStream &value_in, TsBlock *ret_tsblock, - Filter *filter, common::PageArena *pa) { + ByteStream& time_in, ByteStream& value_in, TsBlock* ret_tsblock, + Filter* filter, common::PageArena* pa) { int ret = E_OK; RowAppender row_appender(ret_tsblock); switch (value_chunk_header_.data_type_) { @@ -648,24 +648,37 @@ int AlignedChunkReader::decode_tv_buf_into_tsblock_by_datatype( } int AlignedChunkReader::STRING_DECODE_TYPED_TV_INTO_TSBLOCK( - ByteStream &time_in, ByteStream &value_in, RowAppender &row_appender, - PageArena &pa, Filter *filter) { + ByteStream& time_in, ByteStream& value_in, RowAppender& row_appender, + PageArena& pa, Filter* filter) { int ret = E_OK; int64_t time = 0; common::String value; - while (time_decoder_->has_remaining(time_in)) { - ASSERT(value_decoder_->has_remaining(value_in)); + uint32_t mask = 1 << 7; + while (time_decoder_->has_remaining(time_in) && + value_decoder_->has_remaining(value_in)) { + cur_value_index++; + bool should_read_data = true; + if (((value_page_col_notnull_bitmap_[cur_value_index / 8] & 0xFF) & + (mask >> (cur_value_index % 8))) == 0) { + should_read_data = false; + } if (UNLIKELY(!row_appender.add_row())) { ret = E_OVERFLOW; + cur_value_index--; break; } else if (RET_FAIL(time_decoder_->read_int64(time, time_in))) { - } else if (RET_FAIL(value_decoder_->read_String(value, pa, value_in))) { + } else if (should_read_data && + RET_FAIL(value_decoder_->read_String(value, pa, value_in))) { } else if (filter != nullptr && !filter->satisfy(time, value)) { row_appender.backoff_add_row(); continue; } else { - row_appender.append(0, (char *)&time, sizeof(time)); - row_appender.append(1, value.buf_, value.len_); + row_appender.append(0, (char*)&time, sizeof(time)); + if (!should_read_data) { + row_appender.append_null(1); + } else { + row_appender.append(1, value.buf_, value.len_); + } } } return ret; diff --git a/cpp/src/reader/result_set.h b/cpp/src/reader/result_set.h index da9891f2..228a9333 100644 --- a/cpp/src/reader/result_set.h +++ b/cpp/src/reader/result_set.h @@ -297,6 +297,93 @@ inline ResultSetIterator ResultSet::iterator() { return ResultSetIterator(this); } +static void print_table_result_set(storage::ResultSet* table_result_set) { + if (table_result_set == nullptr) { + std::cout << "TableResultSet is nullptr" << std::endl; + return; + } + + auto metadata = table_result_set->get_metadata(); + if (metadata == nullptr) { + std::cout << "Metadata is nullptr" << std::endl; + return; + } + + uint32_t column_count = metadata->get_column_count(); + if (column_count == 0) { + std::cout << "No columns in result set" << std::endl; + return; + } + + for (uint32_t i = 1; i <= column_count; i++) { + std::cout << metadata->get_column_name(i); + if (i < column_count) { + std::cout << "\t"; + } + } + std::cout << std::endl; + + bool has_next = false; + int row_count = 0; + while (IS_SUCC(table_result_set->next(has_next)) && has_next) { + for (uint32_t i = 1; i <= column_count; i++) { + if (table_result_set->is_null(i)) { + std::cout << "NULL"; + } else { + common::TSDataType col_type = metadata->get_column_type(i); + switch (col_type) { + case common::INT64: { + int64_t val = table_result_set->get_value<int64_t>(i); + std::cout << val; + break; + } + case common::INT32: { + int32_t val = table_result_set->get_value<int32_t>(i); + std::cout << val; + break; + } + case common::FLOAT: { + float val = table_result_set->get_value<float>(i); + std::cout << val; + break; + } + case common::DOUBLE: { + double val = table_result_set->get_value<double>(i); + std::cout << val; + break; + } + case common::BOOLEAN: { + bool val = table_result_set->get_value<bool>(i); + std::cout << (val ? "true" : "false"); + break; + } + case common::TEXT: + case common::STRING: { + common::String* str = + table_result_set->get_value<common::String*>(i); + if (str == nullptr) { + std::cout << "null"; + } else { + std::cout << std::string(str->buf_, str->len_); + } + break; + } + default: { + std::cout << "<UNKNOWN>"; + break; + } + } + } + if (i < column_count) { + std::cout << "\t"; + } + } + std::cout << std::endl; + row_count++; + } + std::cout << "Total rows: " << row_count << std::endl; +} + } // namespace storage #endif // READER_QUERY_DATA_SET_H diff --git a/cpp/src/reader/tsfile_series_scan_iterator.cc b/cpp/src/reader/tsfile_series_scan_iterator.cc index b1be6835..5e78574e 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.cc +++ b/cpp/src/reader/tsfile_series_scan_iterator.cc @@ -36,11 +36,11 @@ void TsFileSeriesScanIterator::destroy() { } } -int TsFileSeriesScanIterator::get_next(TsBlock *&ret_tsblock, bool alloc, - Filter *oneshoot_filter) { +int TsFileSeriesScanIterator::get_next(TsBlock*& ret_tsblock, bool alloc, + Filter* oneshoot_filter) { // TODO @filter int ret = E_OK; - Filter *filter = + Filter* filter = (oneshoot_filter != nullptr) ? oneshoot_filter : time_filter_; if (!chunk_reader_->has_more_data()) { while (true) { @@ -48,7 +48,7 @@ int TsFileSeriesScanIterator::get_next(TsBlock *&ret_tsblock, bool alloc, return E_NO_MORE_DATA; } else { if (!is_aligned_) { - ChunkMeta *cm = get_current_chunk_meta(); + ChunkMeta* cm = get_current_chunk_meta(); advance_to_next_chunk(); if (filter != nullptr && cm->statistic_ != nullptr && !filter->satisfy(cm->statistic_)) { @@ -59,8 +59,8 @@ int TsFileSeriesScanIterator::get_next(TsBlock *&ret_tsblock, bool alloc, } break; } else { - ChunkMeta *value_cm = value_chunk_meta_cursor_.get(); - ChunkMeta *time_cm = time_chunk_meta_cursor_.get(); + ChunkMeta* value_cm = value_chunk_meta_cursor_.get(); + ChunkMeta* time_cm = time_chunk_meta_cursor_.get(); advance_to_next_chunk(); if (filter != nullptr && value_cm->statistic_ != nullptr && !filter->satisfy(value_cm->statistic_)) { @@ -96,10 +96,10 @@ int TsFileSeriesScanIterator::init_chunk_reader() { int ret = E_OK; is_aligned_ = itimeseries_index_->get_data_type() == common::VECTOR; if (!is_aligned_) { - void *buf = common::mem_alloc(sizeof(ChunkReader), common::MOD_DEFAULT); + void* buf = common::mem_alloc(sizeof(ChunkReader), common::MOD_DEFAULT); chunk_reader_ = new (buf) ChunkReader; chunk_meta_cursor_ = itimeseries_index_->get_chunk_meta_list()->begin(); - ChunkMeta *cm = chunk_meta_cursor_.get(); + ChunkMeta* cm = chunk_meta_cursor_.get(); ASSERT(!chunk_reader_->has_more_data()); if (RET_FAIL(chunk_reader_->init( read_file_, itimeseries_index_->get_measurement_name(), @@ -109,16 +109,15 @@ int TsFileSeriesScanIterator::init_chunk_reader() { chunk_meta_cursor_++; } } else { - void *buf = + void* buf = common::mem_alloc(sizeof(AlignedChunkReader), common::MOD_DEFAULT); chunk_reader_ = new (buf) AlignedChunkReader; time_chunk_meta_cursor_ = itimeseries_index_->get_time_chunk_meta_list()->begin(); value_chunk_meta_cursor_ = itimeseries_index_->get_value_chunk_meta_list()->begin(); - ChunkMeta *time_cm = time_chunk_meta_cursor_.get(); - ChunkMeta *value_cm = value_chunk_meta_cursor_.get(); - chunk_reader_->has_more_data(); + ChunkMeta* time_cm = time_chunk_meta_cursor_.get(); + ChunkMeta* value_cm = value_chunk_meta_cursor_.get(); ASSERT(!chunk_reader_->has_more_data()); if (RET_FAIL(chunk_reader_->init( read_file_, itimeseries_index_->get_measurement_name(), @@ -134,8 +133,8 @@ int TsFileSeriesScanIterator::init_chunk_reader() { return ret; } -TsBlock *TsFileSeriesScanIterator::alloc_tsblock() { - ChunkHeader &ch = chunk_reader_->get_chunk_header(); +TsBlock* TsFileSeriesScanIterator::alloc_tsblock() { + ChunkHeader& ch = chunk_reader_->get_chunk_header(); // TODO config ColumnSchema time_cd("time", common::INT64, common::SNAPPY, diff --git a/cpp/src/utils/db_utils.h b/cpp/src/utils/db_utils.h index 90d0b969..8c0b89ce 100644 --- a/cpp/src/utils/db_utils.h +++ b/cpp/src/utils/db_utils.h @@ -315,65 +315,6 @@ FORCE_INLINE int64_t get_cur_timestamp() { } return timestamp; } - -#if 0 -struct DatabaseIdTTL -{ - NodeID db_nid_; - int64_t ttl_; - int16_t counter_; // suppose we at most support 64k timeseries. - DatabaseIdTTL() {} - DatabaseIdTTL(NodeID db_nid, int64_t ttl, int16_t counter) : db_nid_(db_nid), ttl_(ttl), counter_(counter) {} - DatabaseIdTTL(const DatabaseIdTTL &other) : db_nid_(other.db_nid_), ttl_(other.ttl_), counter_(other.counter_) {} - DatabaseIdTTL & operator = (const DatabaseIdTTL &other) - { - this->db_nid_ = other.db_nid_; - this->ttl_ = other.ttl_; - this->counter_ = other.counter_; - return *this; - } - bool operator == (const DatabaseIdTTL &other) - { - if (db_nid_ != other.db_nid_ || ttl_ != other.ttl_ || counter_ != other.counter_) { - return false; - } - return true; - } - friend std::ostream& operator << (std::ostream& out, DatabaseIdTTL& di) - { - - return out; - } -}; - -struct DeviceIDWithCounter -{ - NodeID device_nid_; - int16_t counter_; // suppose we at most support 64k timeseries. - DeviceIDWithCounter() {} - DeviceIDWithCounter(NodeID device_nid, int16_t counter) : device_nid_(device_nid), counter_(counter) {} - DeviceIDWithCounter(const DeviceIDWithCounter &other) : device_nid_(other.device_nid_), counter_(other.counter_) {} - DeviceIDWithCounter& operator = (const DeviceIDWithCounter &other) - { - this->device_nid_ = other.device_nid_; - this->counter_ = other.counter_; - return *this; - } - bool operator == (const DeviceID &other) - { - if (device_nid_ != other.device_nid_ || counter_ != other.counter_) { - return false; - } - return true; - } - friend std::ostream& operator << (std::ostream& out, DeviceID& di) - { - out << "(" << di.device_nid_ << ", " << di.counter_ << ") "; - return out; - } -}; -#endif - } // end namespace common #endif // UTILS_UTILS_H diff --git a/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc b/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc index a5c46102..ffcaa20f 100644 --- a/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc +++ b/cpp/test/reader/tree_view/tsfile_reader_tree_test.cc @@ -32,92 +32,6 @@ using namespace storage; using namespace common; -static void print_table_result_set(storage::TableResultSet* table_result_set) { - if (table_result_set == nullptr) { - std::cout << "TableResultSet is nullptr" << std::endl; - return; - } - - auto metadata = table_result_set->get_metadata(); - if (metadata == nullptr) { - std::cout << "Metadata is nullptr" << std::endl; - return; - } - - uint32_t column_count = metadata->get_column_count(); - if (column_count == 0) { - std::cout << "No columns in result set" << std::endl; - return; - } - - for (uint32_t i = 1; i <= column_count; i++) { - std::cout << metadata->get_column_name(i); - if (i < column_count) { - std::cout << "\t"; - } - } - std::cout << std::endl; - - bool has_next = false; - int row_count = 0; - while (IS_SUCC(table_result_set->next(has_next)) && has_next) { - for (uint32_t i = 1; i <= column_count; i++) { - if (table_result_set->is_null(i)) { - std::cout << "NULL"; - } else { - common::TSDataType col_type = metadata->get_column_type(i); - switch (col_type) { - case common::INT64: { - int64_t val = table_result_set->get_value<int64_t>(i); - std::cout << val; - break; - } - case common::INT32: { - int32_t val = table_result_set->get_value<int32_t>(i); - std::cout << val; - break; - } - case common::FLOAT: { - float val = table_result_set->get_value<float>(i); - std::cout << val; - break; - } - case common::DOUBLE: { - double val = table_result_set->get_value<double>(i); - std::cout << val; - break; - } - case common::BOOLEAN: { - bool val = table_result_set->get_value<bool>(i); - std::cout << (val ? "true" : "false"); - break; - } - case common::STRING: { - common::String* str = - table_result_set->get_value<common::String*>(i); - if (str == nullptr) { - std::cout << "null"; - } else { - std::cout << std::string(str->buf_, str->len_); - } - break; - } - default: { - std::cout << "<UNKNOWN>"; - break; - } - } - } - if (i < column_count) { - std::cout << "\t"; - } - } - std::cout << std::endl; - row_count++; - } - std::cout << "Total rows: " << row_count << std::endl; -} - class TsFileTreeReaderTest : public ::testing::Test { protected: void SetUp() override { diff --git a/cpp/test/writer/tsfile_writer_test.cc b/cpp/test/writer/tsfile_writer_test.cc index 791cf8e7..07d6cf78 100644 --- a/cpp/test/writer/tsfile_writer_test.cc +++ b/cpp/test/writer/tsfile_writer_test.cc @@ -320,19 +320,20 @@ TEST_F(TsFileWriterTest, WriteDiffrentTypeCombination) { char *literal = new char[std::strlen("literal") + 1]; std::strcpy(literal, "literal"); String literal_str(literal, std::strlen("literal")); - - for (size_t i = 0; i < schema_vecs.size(); ++i) { - TsRecord record(1622505600000 + i * 1000, device_path); - if (schema_vecs[i].data_type_ == TSDataType::INT32) { - record.add_point(schema_vecs[i].measurement_name_, (int32_t)i); - } else if (schema_vecs[i].data_type_ == TSDataType::FLOAT) { - record.add_point(schema_vecs[i].measurement_name_, 3.14); - } else if (schema_vecs[i].data_type_ == TSDataType::DOUBLE) { - record.add_point(schema_vecs[i].measurement_name_, 3.1415926); - } else if (schema_vecs[i].data_type_ == TSDataType::BOOLEAN) { - record.add_point(schema_vecs[i].measurement_name_, true); - } else if (schema_vecs[i].data_type_ == TSDataType::STRING) { - record.add_point(schema_vecs[i].measurement_name_, literal_str); + for (size_t l = 0; l < 100; ++l) { + TsRecord record(1622505600000 + 1 * 1000, device_path); + for (size_t i = 0; i < schema_vecs.size(); ++i) { + if (schema_vecs[i].data_type_ == TSDataType::INT32) { + record.add_point(schema_vecs[i].measurement_name_, (int32_t)i); + } else if (schema_vecs[i].data_type_ == TSDataType::FLOAT) { + record.add_point(schema_vecs[i].measurement_name_, 3.14); + } else if (schema_vecs[i].data_type_ == TSDataType::DOUBLE) { + record.add_point(schema_vecs[i].measurement_name_, 3.1415926); + } else if (schema_vecs[i].data_type_ == TSDataType::BOOLEAN) { + record.add_point(schema_vecs[i].measurement_name_, true); + } else if (schema_vecs[i].data_type_ == TSDataType::STRING) { + record.add_point(schema_vecs[i].measurement_name_, literal_str); + } } ASSERT_EQ(tsfile_writer_->write_record(record), E_OK); } diff --git a/python/tests/resources/README.md b/python/tests/resources/README.md new file mode 100644 index 00000000..ca80bb43 --- /dev/null +++ b/python/tests/resources/README.md @@ -0,0 +1,285 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +In `simple_tree.tsfile`: + +```sql + +CREATE TIMESERIES root.ln.wf01.wt01.status WITH DATATYPE=BOOLEAN, ENCODING=PLAIN; +CREATE TIMESERIES root.ln.wf01.wt01.temperature WITH DATATYPE=FLOAT, ENCODING=PLAIN; +IoTDB> show timeseries; ++-----------------------------+-----+--------+--------+--------+-----------+----+----------+--------+------------------+--------+ +| Timeseries|Alias|Database|DataType|Encoding|Compression|Tags|Attributes|Deadband|DeadbandParameters|ViewType| ++-----------------------------+-----+--------+--------+--------+-----------+----+----------+--------+------------------+--------+ +|root.ln.wf01.wt01.temperature| null| root.ln| FLOAT| RLE| LZ4|null| null| null| null| BASE| +| root.ln.wf01.wt01.status| null| root.ln| BOOLEAN| PLAIN| LZ4|null| null| null| null| BASE| ++-----------------------------+-----+--------+--------+--------+-----------+----+----------+--------+------------------+--------+ +``` + +```sql +IoTDB> select * from root.ln.wf01.wt01; ++-----------------------------+-----------------------------+------------------------+ +| Time|root.ln.wf01.wt01.temperature|root.ln.wf01.wt01.status| ++-----------------------------+-----------------------------+------------------------+ +|1970-01-01T08:00:00.100+08:00| null| true| +|1970-01-01T08:00:00.101+08:00| null| false| +|1970-01-01T08:00:00.102+08:00| null| true| +|1970-01-01T08:00:00.103+08:00| null| false| +|1970-01-01T08:00:00.104+08:00| null| true| +|1970-01-01T08:00:00.105+08:00| 20.0| false| +|1970-01-01T08:00:00.106+08:00| 20.1| true| +|1970-01-01T08:00:00.107+08:00| 20.2| false| +|1970-01-01T08:00:00.108+08:00| 20.3| true| +|1970-01-01T08:00:00.109+08:00| 20.4| false| +|1970-01-01T08:00:00.110+08:00| 20.5| true| +|1970-01-01T08:00:00.111+08:00| 20.6| false| +|1970-01-01T08:00:00.112+08:00| 20.7| true| +|1970-01-01T08:00:00.113+08:00| 20.8| false| +|1970-01-01T08:00:00.114+08:00| 20.9| true| +|1970-01-01T08:00:00.115+08:00| 21.0| false| +|1970-01-01T08:00:00.116+08:00| 21.1| true| +|1970-01-01T08:00:00.117+08:00| 21.2| false| +|1970-01-01T08:00:00.118+08:00| 21.3| true| +|1970-01-01T08:00:00.119+08:00| 21.4| false| +|1970-01-01T08:00:00.120+08:00| 21.5| true| +|1970-01-01T08:00:00.121+08:00| 21.6| false| +|1970-01-01T08:00:00.122+08:00| 21.7| true| +|1970-01-01T08:00:00.123+08:00| 21.8| false| +|1970-01-01T08:00:00.124+08:00| 21.9| true| +|1970-01-01T08:00:00.125+08:00| 22.0| false| +|1970-01-01T08:00:00.126+08:00| 22.1| true| +|1970-01-01T08:00:00.127+08:00| 22.2| false| +|1970-01-01T08:00:00.128+08:00| 22.3| true| +|1970-01-01T08:00:00.129+08:00| 22.4| false| +|1970-01-01T08:00:00.130+08:00| 22.5| true| +|1970-01-01T08:00:00.131+08:00| 22.6| false| +|1970-01-01T08:00:00.132+08:00| 22.7| true| +|1970-01-01T08:00:00.133+08:00| 22.8| false| +|1970-01-01T08:00:00.134+08:00| 22.9| true| +|1970-01-01T08:00:00.135+08:00| 23.0| false| +|1970-01-01T08:00:00.136+08:00| 23.1| true| +|1970-01-01T08:00:00.137+08:00| 23.2| false| +|1970-01-01T08:00:00.138+08:00| 23.3| true| +|1970-01-01T08:00:00.139+08:00| 23.4| false| +|1970-01-01T08:00:00.140+08:00| 23.5| true| +|1970-01-01T08:00:00.141+08:00| 23.6| false| +|1970-01-01T08:00:00.142+08:00| 23.7| true| +|1970-01-01T08:00:00.143+08:00| 23.8| false| +|1970-01-01T08:00:00.144+08:00| 23.9| true| +|1970-01-01T08:00:00.145+08:00| 24.0| false| +|1970-01-01T08:00:00.146+08:00| 24.1| true| +|1970-01-01T08:00:00.147+08:00| 24.2| false| +|1970-01-01T08:00:00.148+08:00| 24.3| true| +|1970-01-01T08:00:00.149+08:00| 24.4| false| +|1970-01-01T08:00:00.150+08:00| 24.5| true| +|1970-01-01T08:00:00.151+08:00| 24.6| false| +|1970-01-01T08:00:00.152+08:00| 24.7| true| +|1970-01-01T08:00:00.153+08:00| 24.8| false| +|1970-01-01T08:00:00.154+08:00| 24.9| true| +|1970-01-01T08:00:00.155+08:00| 25.0| false| +|1970-01-01T08:00:00.156+08:00| 25.1| true| +|1970-01-01T08:00:00.157+08:00| 25.2| false| +|1970-01-01T08:00:00.158+08:00| 25.3| true| +|1970-01-01T08:00:00.159+08:00| 25.4| false| +|1970-01-01T08:00:00.160+08:00| 25.5| true| +|1970-01-01T08:00:00.161+08:00| 25.6| false| +|1970-01-01T08:00:00.162+08:00| 25.7| true| +|1970-01-01T08:00:00.163+08:00| 25.8| false| +|1970-01-01T08:00:00.164+08:00| 25.9| true| +|1970-01-01T08:00:00.165+08:00| 26.0| false| +|1970-01-01T08:00:00.166+08:00| 26.1| true| +|1970-01-01T08:00:00.167+08:00| 26.2| false| +|1970-01-01T08:00:00.168+08:00| 26.3| true| +|1970-01-01T08:00:00.169+08:00| 26.4| false| +|1970-01-01T08:00:00.170+08:00| 26.5| true| +|1970-01-01T08:00:00.171+08:00| 26.6| false| +|1970-01-01T08:00:00.172+08:00| 26.7| true| +|1970-01-01T08:00:00.173+08:00| 26.8| false| +|1970-01-01T08:00:00.174+08:00| 26.9| true| +|1970-01-01T08:00:00.175+08:00| 27.0| false| +|1970-01-01T08:00:00.176+08:00| 27.1| true| +|1970-01-01T08:00:00.177+08:00| 27.2| false| +|1970-01-01T08:00:00.178+08:00| 27.3| true| +|1970-01-01T08:00:00.179+08:00| 27.4| false| +|1970-01-01T08:00:00.180+08:00| 27.5| true| +|1970-01-01T08:00:00.181+08:00| 27.6| false| +|1970-01-01T08:00:00.182+08:00| 27.7| true| +|1970-01-01T08:00:00.183+08:00| 27.8| false| +|1970-01-01T08:00:00.184+08:00| 27.9| true| +|1970-01-01T08:00:00.185+08:00| 28.0| false| +|1970-01-01T08:00:00.186+08:00| 28.1| true| +|1970-01-01T08:00:00.187+08:00| 28.2| false| +|1970-01-01T08:00:00.188+08:00| 28.3| true| +|1970-01-01T08:00:00.189+08:00| 28.4| false| +|1970-01-01T08:00:00.190+08:00| 28.5| true| +|1970-01-01T08:00:00.191+08:00| 28.6| false| +|1970-01-01T08:00:00.192+08:00| 28.7| true| +|1970-01-01T08:00:00.193+08:00| 28.8| false| +|1970-01-01T08:00:00.194+08:00| 28.9| true| +|1970-01-01T08:00:00.195+08:00| 29.0| false| +|1970-01-01T08:00:00.196+08:00| 29.1| true| +|1970-01-01T08:00:00.197+08:00| 29.2| false| +|1970-01-01T08:00:00.198+08:00| 29.3| true| +|1970-01-01T08:00:00.199+08:00| 29.4| false| +|1970-01-01T08:00:00.200+08:00| 29.5| null| +|1970-01-01T08:00:00.201+08:00| 29.6| null| +|1970-01-01T08:00:00.202+08:00| 29.7| null| +|1970-01-01T08:00:00.203+08:00| 29.8| null| +|1970-01-01T08:00:00.204+08:00| 29.9| null| ++-----------------------------+-----------------------------+------------------------+ +Total line number = 105 +``` + +In `simple_table_t1.tsfile` + +```sql +IoTDB:test> CREATE TABLE test ( +> s0 STRING TAG, +> s1 STRING TAG, +> s2 INT64 FIELD, +> s3 FLOAT FIELD, +> s4 STRING FIELD, +> s5 TIMESTAMP FIELD, +> s6 INT32 FIELD, +> s7 DOUBLE FIELD, +> s8 DATE FIELD, +> s9 TEXT FIELD +> ); + +IoTDB:test> select * from test; ++-----------------------------+--+--+----+----+---+-----------------------------+----+----+----------+------+ +| time|s0|s1| s2| s3| s4| s5| s6| s7| s8| s9| ++-----------------------------+--+--+----+----+---+-----------------------------+----+----+----------+------+ +|2025-10-10T22:20:20.000+08:00| a| b|1010| 2.0|v10|1970-01-01T08:00:01.010+08:00| 20|11.0|2024-10-20|text10| +|2025-10-10T22:20:21.000+08:00| a| b|1011| 2.1|v11|1970-01-01T08:00:01.011+08:00| 21|11.1|2024-10-21|text11| +|2025-10-10T22:20:22.000+08:00| a| b|1012| 2.2|v12|1970-01-01T08:00:01.012+08:00| 22|11.2|2024-10-22|text12| +|2025-10-10T22:20:23.000+08:00| a| b|1013| 2.3|v13|1970-01-01T08:00:01.013+08:00| 23|11.3|2024-10-23|text13| +|2025-10-10T22:20:24.000+08:00| a| b|1014| 2.4|v14|1970-01-01T08:00:01.014+08:00| 24|11.4|2024-10-24|text14| +|2025-10-10T22:20:25.000+08:00| a| b|1015| 2.5|v15|1970-01-01T08:00:01.015+08:00| 25|11.5|2024-10-25|text15| +|2025-10-10T22:20:26.000+08:00| a| b|null| 2.6|v16|1970-01-01T08:00:01.016+08:00|null|11.6|2024-10-26| null| +|2025-10-10T22:20:27.000+08:00| a| b|1017| 2.7|v17|1970-01-01T08:00:01.017+08:00| 27|11.7|2024-10-27|text17| +|2025-10-10T22:20:28.000+08:00| a| b|1018| 2.8|v18|1970-01-01T08:00:01.018+08:00| 28|11.8|2024-10-28|text18| +|2025-10-10T22:20:29.000+08:00| a| b|1019| 2.9|v19|1970-01-01T08:00:01.019+08:00| 29|11.9|2024-10-29|text19| +|2025-10-10T22:20:30.000+08:00| a| b|1020| 3.0|v20|1970-01-01T08:00:01.020+08:00| 30|12.0|2024-10-30|text20| +|2025-10-10T22:20:31.000+08:00| a| b|1021| 3.1|v21|1970-01-01T08:00:01.021+08:00| 31|12.1|2024-10-31|text21| +|2025-10-10T22:20:32.000+08:00| a| b|1022| 3.2|v22|1970-01-01T08:00:01.022+08:00| 32|12.2|2024-11-01|text22| +|2025-10-10T22:20:33.000+08:00| a| b|null| 3.3|v23|1970-01-01T08:00:01.023+08:00|null|12.3|2024-11-02| null| +|2025-10-10T22:20:34.000+08:00| a| b|1024| 3.4|v24|1970-01-01T08:00:01.024+08:00| 34|12.4|2024-11-03|text24| +|2025-10-10T22:20:35.000+08:00| a| b|1025| 3.5|v25|1970-01-01T08:00:01.025+08:00| 35|12.5|2024-11-04|text25| +|2025-10-10T22:20:36.000+08:00| a| b|1026| 3.6|v26|1970-01-01T08:00:01.026+08:00| 36|12.6|2024-11-05|text26| +|2025-10-10T22:20:37.000+08:00| a| b|1027| 3.7|v27|1970-01-01T08:00:01.027+08:00| 37|12.7|2024-11-06|text27| +|2025-10-10T22:20:38.000+08:00| a| b|1028| 3.8|v28|1970-01-01T08:00:01.028+08:00| 38|12.8|2024-11-07|text28| +|2025-10-10T22:20:39.000+08:00| a| b|null| 3.9|v29|1970-01-01T08:00:01.029+08:00|null|12.9|2024-11-08| null| +|2025-10-10T22:20:40.000+08:00| a| b|1030| 4.0|v30|1970-01-01T08:00:01.030+08:00| 40|13.0|2024-11-09|text30| +|2025-10-10T22:20:41.000+08:00| a| b|1031| 4.1|v31|1970-01-01T08:00:01.031+08:00| 41|13.1|2024-11-10|text31| +|2025-10-10T22:20:42.000+08:00| a| b|1032| 4.2|v32|1970-01-01T08:00:01.032+08:00| 42|13.2|2024-11-11|text32| +|2025-10-10T22:20:43.000+08:00| a| b|1033| 4.3|v33|1970-01-01T08:00:01.033+08:00| 43|13.3|2024-11-12|text33| +|2025-10-10T22:20:44.000+08:00| a| b|1034| 4.4|v34|1970-01-01T08:00:01.034+08:00| 44|13.4|2024-11-13|text34| +|2025-10-10T22:20:45.000+08:00| a| b|1035| 4.5|v35|1970-01-01T08:00:01.035+08:00| 45|13.5|2024-11-14|text35| +|2025-10-10T22:20:46.000+08:00| a| b|null| 4.6|v36|1970-01-01T08:00:01.036+08:00|null|13.6|2024-11-15| null| +|2025-10-10T22:20:47.000+08:00| a| b|1037| 4.7|v37|1970-01-01T08:00:01.037+08:00| 47|13.7|2024-11-16|text37| +|2025-10-10T22:20:48.000+08:00| a| b|1038| 4.8|v38|1970-01-01T08:00:01.038+08:00| 48|13.8|2024-11-17|text38| +|2025-10-10T22:20:49.000+08:00| a| b|1039| 4.9|v39|1970-01-01T08:00:01.039+08:00| 49|13.9|2024-11-18|text39| +|2025-10-10T22:21:20.000+08:00| b| d|1070| 8.0|v70|1970-01-01T08:00:01.070+08:00| 80|17.0|2024-12-19|text70| +|2025-10-10T22:21:21.000+08:00| b| d|1071| 8.1|v71|1970-01-01T08:00:01.071+08:00| 81|17.1|2024-12-20|text71| +|2025-10-10T22:21:22.000+08:00| b| d|1072| 8.2|v72|1970-01-01T08:00:01.072+08:00| 82|17.2|2024-12-21|text72| +|2025-10-10T22:21:23.000+08:00| b| d|1073| 8.3|v73|1970-01-01T08:00:01.073+08:00| 83|17.3|2024-12-22|text73| +|2025-10-10T22:21:24.000+08:00| b| d|1074| 8.4|v74|1970-01-01T08:00:01.074+08:00| 84|17.4|2024-12-23|text74| +|2025-10-10T22:21:25.000+08:00| b| d|1075| 8.5|v75|1970-01-01T08:00:01.075+08:00| 85|17.5|2024-12-24|text75| +|2025-10-10T22:21:26.000+08:00| b| d|null| 8.6|v76|1970-01-01T08:00:01.076+08:00|null|17.6|2024-12-25| null| +|2025-10-10T22:21:27.000+08:00| b| d|1077| 8.7|v77|1970-01-01T08:00:01.077+08:00| 87|17.7|2024-12-26|text77| +|2025-10-10T22:21:28.000+08:00| b| d|1078| 8.8|v78|1970-01-01T08:00:01.078+08:00| 88|17.8|2024-12-27|text78| +|2025-10-10T22:21:29.000+08:00| b| d|1079| 8.9|v79|1970-01-01T08:00:01.079+08:00| 89|17.9|2024-12-28|text79| +|2025-10-10T22:21:30.000+08:00| b| d|1080| 9.0|v80|1970-01-01T08:00:01.080+08:00| 90|18.0|2024-12-29|text80| +|2025-10-10T22:21:31.000+08:00| b| d|1081| 9.1|v81|1970-01-01T08:00:01.081+08:00| 91|18.1|2024-12-30|text81| +|2025-10-10T22:21:32.000+08:00| b| d|1082| 9.2|v82|1970-01-01T08:00:01.082+08:00| 92|18.2|2024-12-31|text82| +|2025-10-10T22:21:33.000+08:00| b| d|null| 9.3|v83|1970-01-01T08:00:01.083+08:00|null|18.3|2025-01-01| null| +|2025-10-10T22:21:34.000+08:00| b| d|1084| 9.4|v84|1970-01-01T08:00:01.084+08:00| 94|18.4|2025-01-02|text84| +|2025-10-10T22:21:35.000+08:00| b| d|1085| 9.5|v85|1970-01-01T08:00:01.085+08:00| 95|18.5|2025-01-03|text85| +|2025-10-10T22:21:36.000+08:00| b| d|1086| 9.6|v86|1970-01-01T08:00:01.086+08:00| 96|18.6|2025-01-04|text86| +|2025-10-10T22:21:37.000+08:00| b| d|1087| 9.7|v87|1970-01-01T08:00:01.087+08:00| 97|18.7|2025-01-05|text87| +|2025-10-10T22:21:38.000+08:00| b| d|1088| 9.8|v88|1970-01-01T08:00:01.088+08:00| 98|18.8|2025-01-06|text88| +|2025-10-10T22:21:39.000+08:00| b| d|null| 9.9|v89|1970-01-01T08:00:01.089+08:00|null|18.9|2025-01-07| null| +|2025-10-10T22:21:40.000+08:00| b| d|1090|10.0|v90|1970-01-01T08:00:01.090+08:00| 100|19.0|2025-01-08|text90| +|2025-10-10T22:21:41.000+08:00| b| d|1091|10.1|v91|1970-01-01T08:00:01.091+08:00| 101|19.1|2025-01-09|text91| +|2025-10-10T22:21:42.000+08:00| b| d|1092|10.2|v92|1970-01-01T08:00:01.092+08:00| 102|19.2|2025-01-10|text92| +|2025-10-10T22:21:43.000+08:00| b| d|1093|10.3|v93|1970-01-01T08:00:01.093+08:00| 103|19.3|2025-01-11|text93| +|2025-10-10T22:21:44.000+08:00| b| d|1094|10.4|v94|1970-01-01T08:00:01.094+08:00| 104|19.4|2025-01-12|text94| +|2025-10-10T22:21:45.000+08:00| b| d|1095|10.5|v95|1970-01-01T08:00:01.095+08:00| 105|19.5|2025-01-13|text95| +|2025-10-10T22:21:46.000+08:00| b| d|null|10.6|v96|1970-01-01T08:00:01.096+08:00|null|19.6|2025-01-14| null| +|2025-10-10T22:21:47.000+08:00| b| d|1097|10.7|v97|1970-01-01T08:00:01.097+08:00| 107|19.7|2025-01-15|text97| +|2025-10-10T22:21:48.000+08:00| b| d|1098|10.8|v98|1970-01-01T08:00:01.098+08:00| 108|19.8|2025-01-16|text98| +|2025-10-10T22:21:49.000+08:00| b| d|1099|10.9|v99|1970-01-01T08:00:01.099+08:00| 109|19.9|2025-01-17|text99| ++-----------------------------+--+--+----+----+---+-----------------------------+----+----+----------+------+ +Total line number = 60 +``` + +In `simple_table_t2.tsfile` + +```sql + +IoTDB:test> select * from test; ++-----------------------------+--+--+----+---+---+-----------------------------+----+----+----------+------+ +| time|s0|s1| s2| s3| s4| s5| s6| s7| s8| s9| ++-----------------------------+--+--+----+---+---+-----------------------------+----+----+----------+------+ +|2025-10-10T22:20:10.000+08:00| a| a|1000|1.0| v0|1970-01-01T08:00:01.000+08:00| 10|10.0|2024-10-10| text0| +|2025-10-10T22:20:11.000+08:00| a| a|1001|1.1| v1|1970-01-01T08:00:01.001+08:00| 11|10.1|2024-10-11| text1| +|2025-10-10T22:20:12.000+08:00| a| a|1002|1.2| v2|1970-01-01T08:00:01.002+08:00| 12|10.2|2024-10-12| text2| +|2025-10-10T22:20:13.000+08:00| a| a|1003|1.3| v3|1970-01-01T08:00:01.003+08:00| 13|10.3|2024-10-13| text3| +|2025-10-10T22:20:14.000+08:00| a| a|1004|1.4| v4|1970-01-01T08:00:01.004+08:00| 14|10.4|2024-10-14| text4| +|2025-10-10T22:20:15.000+08:00| a| a|1005|1.5| v5|1970-01-01T08:00:01.005+08:00| 15|10.5|2024-10-15| text5| +|2025-10-10T22:20:16.000+08:00| a| a|null|1.6| v6|1970-01-01T08:00:01.006+08:00|null|10.6|2024-10-16| null| +|2025-10-10T22:20:17.000+08:00| a| a|1007|1.7| v7|1970-01-01T08:00:01.007+08:00| 17|10.7|2024-10-17| text7| +|2025-10-10T22:20:18.000+08:00| a| a|1008|1.8| v8|1970-01-01T08:00:01.008+08:00| 18|10.8|2024-10-18| text8| +|2025-10-10T22:20:19.000+08:00| a| a|1009|1.9| v9|1970-01-01T08:00:01.009+08:00| 19|10.9|2024-10-19| text9| +|2025-10-10T22:20:50.000+08:00| b| c|1040|5.0|v40|1970-01-01T08:00:01.040+08:00| 50|14.0|2024-11-19|text40| +|2025-10-10T22:20:51.000+08:00| b| c|1041|5.1|v41|1970-01-01T08:00:01.041+08:00| 51|14.1|2024-11-20|text41| +|2025-10-10T22:20:52.000+08:00| b| c|1042|5.2|v42|1970-01-01T08:00:01.042+08:00| 52|14.2|2024-11-21|text42| +|2025-10-10T22:20:53.000+08:00| b| c|1043|5.3|v43|1970-01-01T08:00:01.043+08:00| 53|14.3|2024-11-22|text43| +|2025-10-10T22:20:54.000+08:00| b| c|1044|5.4|v44|1970-01-01T08:00:01.044+08:00| 54|14.4|2024-11-23|text44| +|2025-10-10T22:20:55.000+08:00| b| c|1045|5.5|v45|1970-01-01T08:00:01.045+08:00| 55|14.5|2024-11-24|text45| +|2025-10-10T22:20:56.000+08:00| b| c|null|5.6|v46|1970-01-01T08:00:01.046+08:00|null|14.6|2024-11-25| null| +|2025-10-10T22:20:57.000+08:00| b| c|1047|5.7|v47|1970-01-01T08:00:01.047+08:00| 57|14.7|2024-11-26|text47| +|2025-10-10T22:20:58.000+08:00| b| c|1048|5.8|v48|1970-01-01T08:00:01.048+08:00| 58|14.8|2024-11-27|text48| +|2025-10-10T22:20:59.000+08:00| b| c|1049|5.9|v49|1970-01-01T08:00:01.049+08:00| 59|14.9|2024-11-28|text49| +|2025-10-10T22:21:00.000+08:00| b| c|1050|6.0|v50|1970-01-01T08:00:01.050+08:00| 60|15.0|2024-11-29|text50| +|2025-10-10T22:21:01.000+08:00| b| c|1051|6.1|v51|1970-01-01T08:00:01.051+08:00| 61|15.1|2024-11-30|text51| +|2025-10-10T22:21:02.000+08:00| b| c|1052|6.2|v52|1970-01-01T08:00:01.052+08:00| 62|15.2|2024-12-01|text52| +|2025-10-10T22:21:03.000+08:00| b| c|null|6.3|v53|1970-01-01T08:00:01.053+08:00|null|15.3|2024-12-02| null| +|2025-10-10T22:21:04.000+08:00| b| c|1054|6.4|v54|1970-01-01T08:00:01.054+08:00| 64|15.4|2024-12-03|text54| +|2025-10-10T22:21:05.000+08:00| b| c|1055|6.5|v55|1970-01-01T08:00:01.055+08:00| 65|15.5|2024-12-04|text55| +|2025-10-10T22:21:06.000+08:00| b| c|1056|6.6|v56|1970-01-01T08:00:01.056+08:00| 66|15.6|2024-12-05|text56| +|2025-10-10T22:21:07.000+08:00| b| c|1057|6.7|v57|1970-01-01T08:00:01.057+08:00| 67|15.7|2024-12-06|text57| +|2025-10-10T22:21:08.000+08:00| b| c|1058|6.8|v58|1970-01-01T08:00:01.058+08:00| 68|15.8|2024-12-07|text58| +|2025-10-10T22:21:09.000+08:00| b| c|null|6.9|v59|1970-01-01T08:00:01.059+08:00|null|15.9|2024-12-08| null| +|2025-10-10T22:21:10.000+08:00| b| c|1060|7.0|v60|1970-01-01T08:00:01.060+08:00| 70|16.0|2024-12-09|text60| +|2025-10-10T22:21:11.000+08:00| b| c|1061|7.1|v61|1970-01-01T08:00:01.061+08:00| 71|16.1|2024-12-10|text61| +|2025-10-10T22:21:12.000+08:00| b| c|1062|7.2|v62|1970-01-01T08:00:01.062+08:00| 72|16.2|2024-12-11|text62| +|2025-10-10T22:21:13.000+08:00| b| c|1063|7.3|v63|1970-01-01T08:00:01.063+08:00| 73|16.3|2024-12-12|text63| +|2025-10-10T22:21:14.000+08:00| b| c|1064|7.4|v64|1970-01-01T08:00:01.064+08:00| 74|16.4|2024-12-13|text64| +|2025-10-10T22:21:15.000+08:00| b| c|1065|7.5|v65|1970-01-01T08:00:01.065+08:00| 75|16.5|2024-12-14|text65| +|2025-10-10T22:21:16.000+08:00| b| c|null|7.6|v66|1970-01-01T08:00:01.066+08:00|null|16.6|2024-12-15| null| +|2025-10-10T22:21:17.000+08:00| b| c|1067|7.7|v67|1970-01-01T08:00:01.067+08:00| 77|16.7|2024-12-16|text67| +|2025-10-10T22:21:18.000+08:00| b| c|1068|7.8|v68|1970-01-01T08:00:01.068+08:00| 78|16.8|2024-12-17|text68| +|2025-10-10T22:21:19.000+08:00| b| c|1069|7.9|v69|1970-01-01T08:00:01.069+08:00| 79|16.9|2024-12-18|text69| ++-----------------------------+--+--+----+---+---+-----------------------------+----+----+----------+------+ +Total line number = 40 +``` \ No newline at end of file diff --git a/python/tests/resources/simple_table_t1.tsfile b/python/tests/resources/simple_table_t1.tsfile new file mode 100644 index 00000000..6ca6647a Binary files /dev/null and b/python/tests/resources/simple_table_t1.tsfile differ diff --git a/python/tests/resources/simple_table_t2.tsfile b/python/tests/resources/simple_table_t2.tsfile new file mode 100644 index 00000000..2cfa12d5 Binary files /dev/null and b/python/tests/resources/simple_table_t2.tsfile differ diff --git a/python/tests/resources/simple_tree.tsfile b/python/tests/resources/simple_tree.tsfile new file mode 100644 index 00000000..f281a907 Binary files /dev/null and b/python/tests/resources/simple_tree.tsfile differ diff --git a/python/tests/test_load_tsfile_from_iotdb.py b/python/tests/test_load_tsfile_from_iotdb.py new file mode 100644 index 00000000..d865dd35 --- /dev/null +++ b/python/tests/test_load_tsfile_from_iotdb.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import os + +import numpy as np + +import tsfile as ts + + +def test_load_tsfile_from_iotdb(): + test_path = os.path.dirname(os.path.abspath(__file__)) + dir_path = os.path.join(test_path, 'resources') + simple_tree_path = os.path.join(dir_path, 'simple_tree.tsfile') + df = ts.to_dataframe(simple_tree_path) + + ## -------- + assert len(df) == 105, "row count mismatch" + assert df["time"].isna().sum() == 0 + assert int(df["time"].sum()) == 15960 + assert df["temperature"].isna().sum() == 5 + assert df["status"].isna().sum() == 5 + assert (df["status"] == True).sum() == 50 + assert (df["status"] == False).sum() == 50 + ## --------- + + # + simple_tabl1_path = os.path.join(dir_path, 'simple_table_t1.tsfile') + df = ts.to_dataframe(simple_tabl1_path) + ## --------- + assert len(df) == 60 + assert df["time"].isna().sum() == 0 + assert df["time"].sum() == ( + (1760106020000 + 1760106049000) * 30 // 2 + + (1760106080000 + 1760106109000) * 30 // 2 + ) + assert df["s0"].isna().sum() == 0 + assert df["s1"].isna().sum() == 0 + assert df["s2"].isna().sum() == 8 + assert df["s3"].isna().sum() == 0 + assert df["s4"].isna().sum() == 0 + assert df["s4"].nunique() == 60 + assert df["s5"].isna().sum() == 0 + + assert df["s5"].sum() == ( + (1010 + 1039) * 30 // 2 + + (1070 + 1099) * 30 // 2 + ) + assert df["s6"].isna().sum() == 8 + + assert df["s6"].sum(skipna=True) == ( + (20 + 49) * 30 // 2 - (26 + 33 + 39 + 46) + + + (80 + 109) * 30 // 2 - (86 + 93 + 99 + 106) + ) + assert df["s7"].isna().sum() == 0 + assert df["s8"].isna().sum() == 0 + assert df["s8"].nunique() == 60 + assert df["s9"].isna().sum() == 8 + ## --------- + + simple_tabl2_path = os.path.join(dir_path, 'simple_table_t2.tsfile') + df = ts.to_dataframe(simple_tabl2_path) + ## --------- + assert len(df) == 40 + assert df["time"].isna().sum() == 0 + assert int(df["time"].sum()) == 70404242080000 + + assert df["s0"].isna().sum() == 0 + assert df["s1"].isna().sum() == 0 + assert df["s0"].nunique() == 2 + assert df["s1"].nunique() == 2 + + assert df["s2"].isna().sum() == 5 + assert int(df["s2"].sum(skipna=True)) == 36450 + + assert df["s3"].isna().sum() == 0 + assert np.isclose(float(df["s3"].sum()), 208.0, rtol=1e-6, atol=1e-6) + + assert df["s4"].isna().sum() == 0 + assert df["s4"].nunique() == 40 + + assert df["s5"].isna().sum() == 0 + assert int(df["s5"].sum()) == 41680 + + assert df["s6"].isna().sum() == 5 + assert int(df["s6"].sum(skipna=True)) == 1800 + + assert df["s7"].isna().sum() == 0 + assert np.isclose(float(df["s7"].sum()), 568.0, rtol=1e-6, atol=1e-6) + + assert df["s8"].isna().sum() == 0 + assert df["s8"].nunique() == 40 + + assert df["s9"].isna().sum() == 5 + ## --------- diff --git a/python/tsfile/constants.py b/python/tsfile/constants.py index 2f1ad982..7d1f5ff5 100644 --- a/python/tsfile/constants.py +++ b/python/tsfile/constants.py @@ -56,7 +56,7 @@ class TSDataType(IntEnum): Convert datatype to pandas dtype """ if self == TSDataType.BOOLEAN: - return "bool" + return "boolean" elif self == TSDataType.INT32: return "Int32" elif self == TSDataType.INT64: diff --git a/python/tsfile/tsfile_reader.pyx b/python/tsfile/tsfile_reader.pyx index 041fe76d..359492d6 100644 --- a/python/tsfile/tsfile_reader.pyx +++ b/python/tsfile/tsfile_reader.pyx @@ -19,6 +19,7 @@ #cython: language_level=3 import weakref +from email.contentmanager import raw_data_manager from typing import List import pandas as pd
