This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch fix_demos_multi_device in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit b845b7929ad30145b36efac0b5c005a9ce967bb7 Author: Tian Jiang <[email protected]> AuthorDate: Fri Mar 7 13:08:27 2025 +0800 fix demos & fix insertion with multiple devices --- cpp/examples/c_examples/demo_read.c | 24 +++++----- cpp/examples/c_examples/demo_write.c | 1 + cpp/examples/cpp_examples/demo_read.cpp | 4 +- cpp/src/writer/tsfile_writer.cc | 53 ++++++++-------------- cpp/src/writer/tsfile_writer.h | 16 +++---- .../reader/table_view/tsfile_reader_table_test.cc | 11 ++++- 6 files changed, 51 insertions(+), 58 deletions(-) diff --git a/cpp/examples/c_examples/demo_read.c b/cpp/examples/c_examples/demo_read.c index 8e4e7acf..d3c17a12 100644 --- a/cpp/examples/c_examples/demo_read.c +++ b/cpp/examples/c_examples/demo_read.c @@ -43,11 +43,11 @@ ERRNO read_tsfile() { // Get query result metadata: column name and datatype ResultSetMetaData metadata = tsfile_result_set_get_metadata(ret); - int column_num = metadata.column_num; + int column_num = tsfile_result_set_metadata_get_column_num(metadata); - for (int i = 0; i < column_num; i++) { - printf("column:%s, datatype:%d\n", metadata.column_names[i], - metadata.data_types[i]); + for (int i = 1; i <= column_num; i++) { + printf("column:%s, datatype:%d\n", tsfile_result_set_metadata_get_column_name(metadata, i), + tsfile_result_set_metadata_get_data_type(metadata, i)); } // Get data by column name or index. @@ -55,37 +55,37 @@ ERRNO read_tsfile() { // Timestamp at column 1 and column index begin from 1. Timestamp timestamp = tsfile_result_set_get_value_by_index_int64_t(ret, 1); - printf("%ld ", timestamp); + printf("%ld\n", timestamp); for (int i = 1; i <= column_num; i++) { if (tsfile_result_set_is_null_by_index(ret, i)) { printf(" null "); } else { - switch (metadata.data_types[i]) { + switch (tsfile_result_set_metadata_get_data_type(metadata, i)) { case TS_DATATYPE_BOOLEAN: - printf("%d", tsfile_result_set_get_value_by_index_bool( + printf("%d\n", tsfile_result_set_get_value_by_index_bool( ret, i)); break; case TS_DATATYPE_INT32: - printf("%d", + printf("%d\n", tsfile_result_set_get_value_by_index_int32_t(ret, i)); break; case TS_DATATYPE_INT64: - printf("%ld", + printf("%ld\n", tsfile_result_set_get_value_by_index_int64_t(ret, i)); break; case TS_DATATYPE_FLOAT: - printf("%f", tsfile_result_set_get_value_by_index_float( + printf("%f\n", tsfile_result_set_get_value_by_index_float( ret, i)); break; case TS_DATATYPE_DOUBLE: - printf("%lf", + printf("%lf\n", tsfile_result_set_get_value_by_index_double(ret, i)); break; case TS_DATATYPE_STRING: - printf("%s", + printf("%s\n", tsfile_result_set_get_value_by_index_string(ret, i)); break; diff --git a/cpp/examples/c_examples/demo_write.c b/cpp/examples/c_examples/demo_write.c index fffe3f18..ebfc99df 100644 --- a/cpp/examples/c_examples/demo_write.c +++ b/cpp/examples/c_examples/demo_write.c @@ -54,6 +54,7 @@ ERRNO write_tsfile() { .encoding = TS_ENCODING_PLAIN, .column_category = FIELD}; + remove("test_c.tsfile"); // Create a file with specify path to write tsfile. WriteFile file = write_file_new("test_c.tsfile", &code); HANDLE_ERROR(code); diff --git a/cpp/examples/cpp_examples/demo_read.cpp b/cpp/examples/cpp_examples/demo_read.cpp index 4951e8ca..667fbfa1 100644 --- a/cpp/examples/cpp_examples/demo_read.cpp +++ b/cpp/examples/cpp_examples/demo_read.cpp @@ -54,7 +54,7 @@ int demo_read() { for (int i = 1; i <= column_num; i++) { std::cout << "column name: " << metadata->get_column_name(i) << std::endl; - std::cout << "column type: " << metadata->get_column_type(i) + std::cout << "column type: " << std::to_string(metadata->get_column_type(i)) << std::endl; } @@ -84,7 +84,7 @@ int demo_read() { std::cout << ret->get_value<double>(i) << std::endl; break; case common::STRING: - std::cout << ret->get_value<common::String*>(i) + std::cout << *(ret->get_value<common::String*>(i)) << std::endl; break; default:; diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 88e2927d..8d7f050c 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -671,7 +671,8 @@ int TsFileWriter::write_tablet_aligned(const Tablet &tablet) { if (IS_NULL(value_chunk_writer)) { continue; } - value_write_column(value_chunk_writer, tablet, c); + value_write_column(value_chunk_writer, tablet, c, 0, + tablet.get_cur_row_size()); } return ret; } @@ -716,7 +717,8 @@ int TsFileWriter::write_table(Tablet &tablet) { int start_idx = 0; for (auto &device_id_end_index_pair : device_id_end_index_pairs) { auto device_id = device_id_end_index_pair.first; - if (device_id_end_index_pair.second == 0) continue; + int end_idx = device_id_end_index_pair.second; + if (end_idx == 0) continue; if (table_aligned_) { SimpleVector<ValueChunkWriter *> value_chunk_writers; TimeChunkWriter *time_chunk_writer = nullptr; @@ -725,7 +727,7 @@ int TsFileWriter::write_table(Tablet &tablet) { value_chunk_writers))) { return ret; } - for (uint32_t i = 0; i < tablet.get_cur_row_size(); i++) { + for (uint32_t i = start_idx; i < end_idx; i++) { time_chunk_writer->write(tablet.timestamps_[i]); } uint32_t field_col_count = 0; @@ -737,10 +739,12 @@ int TsFileWriter::write_table(Tablet &tablet) { if (IS_NULL(value_chunk_writer)) { continue; } - value_write_column(value_chunk_writer, tablet, i); + value_write_column(value_chunk_writer, tablet, i, start_idx, + end_idx); field_col_count++; } } + start_idx = end_idx; } else { MeasurementNamesFromTablet mnames_getter(tablet); SimpleVector<ChunkWriter *> chunk_writers; @@ -851,23 +855,23 @@ int TsFileWriter::value_write_column(ValueChunkWriter *value_chunk_writer, if (data_type == common::BOOLEAN) { ret = write_typed_column(value_chunk_writer, timestamps, (bool *)col_values.bool_data, - col_notnull_bitmap, row_count); + col_notnull_bitmap, start_idx, end_idx); } else if (data_type == common::INT32) { ret = write_typed_column(value_chunk_writer, timestamps, (int32_t *)col_values.int32_data, - col_notnull_bitmap, row_count); + col_notnull_bitmap, start_idx, end_idx); } else if (data_type == common::INT64) { ret = write_typed_column(value_chunk_writer, timestamps, (int64_t *)col_values.int64_data, - col_notnull_bitmap, row_count); + col_notnull_bitmap, start_idx, end_idx); } else if (data_type == common::FLOAT) { ret = write_typed_column(value_chunk_writer, timestamps, (float *)col_values.float_data, - col_notnull_bitmap, row_count); + col_notnull_bitmap, start_idx, end_idx); } else if (data_type == common::DOUBLE) { ret = write_typed_column(value_chunk_writer, timestamps, (double *)col_values.double_data, - col_notnull_bitmap, row_count); + col_notnull_bitmap, start_idx, end_idx); } else { return E_NOT_SUPPORT; } @@ -888,7 +892,7 @@ int TsFileWriter::value_write_column(ValueChunkWriter *value_chunk_writer, #define DO_VALUE_WRITE_TYPED_COLUMN() \ do { \ int ret = E_OK; \ - for (uint32_t r = 0; r < row_count; r++) { \ + for (uint32_t r = start_idx; r < end_idx; r++) { \ if (LIKELY(col_notnull_bitmap.test(r))) { \ ret = value_chunk_writer->write(timestamps[r], col_values[r], \ true); \ @@ -946,35 +950,35 @@ int TsFileWriter::write_typed_column(ChunkWriter *chunk_writer, int TsFileWriter::write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, bool *col_values, BitMap &col_notnull_bitmap, - uint32_t row_count) { + uint32_t start_idx, uint32_t end_idx) { DO_VALUE_WRITE_TYPED_COLUMN(); } int TsFileWriter::write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, int32_t *col_values, BitMap &col_notnull_bitmap, - uint32_t row_count) { + uint32_t start_idx, uint32_t end_idx) { DO_VALUE_WRITE_TYPED_COLUMN(); } int TsFileWriter::write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, int64_t *col_values, BitMap &col_notnull_bitmap, - uint32_t row_count) { + uint32_t start_idx, uint32_t end_idx) { DO_VALUE_WRITE_TYPED_COLUMN(); } int TsFileWriter::write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, float *col_values, BitMap &col_notnull_bitmap, - uint32_t row_count) { + uint32_t start_idx, uint32_t end_idx) { DO_VALUE_WRITE_TYPED_COLUMN(); } int TsFileWriter::write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, double *col_values, BitMap &col_notnull_bitmap, - uint32_t row_count) { + uint32_t start_idx, uint32_t end_idx) { DO_VALUE_WRITE_TYPED_COLUMN(); } @@ -993,25 +997,6 @@ int TsFileWriter::flush() { DeviceSchemasMapIter device_iter; for (device_iter = schemas_.begin(); device_iter != schemas_.end(); device_iter++) { // cppcheck-suppress postfixOperator - if (device_iter->second->is_aligned_) { - SimpleVector<ValueChunkWriter *> value_chunk_writers; - TimeChunkWriter *time_chunk_writer; - MeasurementSchemaMapNamesGetter mnames_getter( - device_iter->second->measurement_schema_map_); - if (RET_FAIL(do_check_schema_aligned( - device_iter->first, mnames_getter, time_chunk_writer, - value_chunk_writers))) { - return ret; - } - } else { - SimpleVector<ChunkWriter *> chunk_writers; - MeasurementSchemaMapNamesGetter mnames_getter( - device_iter->second->measurement_schema_map_); - if (RET_FAIL(do_check_schema(device_iter->first, mnames_getter, - chunk_writers))) { - return ret; - } - } if (check_chunk_group_empty(device_iter->second, device_iter->second->is_aligned_)) { continue; diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h index bfaf70f0..1d2368fe 100644 --- a/cpp/src/writer/tsfile_writer.h +++ b/cpp/src/writer/tsfile_writer.h @@ -182,36 +182,36 @@ class TsFileWriter { int write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, bool *col_values, common::BitMap &col_notnull_bitmap, - uint32_t row_count); + uint32_t start_idx, uint32_t end_idx); int write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, double *col_values, common::BitMap &col_notnull_bitmap, - uint32_t row_count); + uint32_t start_idx, uint32_t end_idx); int write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, common::String *col_values, common::BitMap &col_notnull_bitmap, - int32_t row_count); + uint32_t start_idx, uint32_t end_idx); int write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, float *col_values, common::BitMap &col_notnull_bitmap, - uint32_t row_count); + uint32_t start_idx, uint32_t end_idx); int write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, int32_t *col_values, common::BitMap &col_notnull_bitmap, - uint32_t row_count); + uint32_t start_idx, uint32_t end_idx); int write_typed_column(ValueChunkWriter *value_chunk_writer, int64_t *timestamps, int64_t *col_values, common::BitMap &col_notnull_bitmap, - uint32_t row_count); + uint32_t start_idx, uint32_t end_idx); int value_write_column(ValueChunkWriter *value_chunk_writer, const Tablet &tablet, int col_idx, - uint32_t start_idx = 0, - uint32_t end_idx = UINT32_MAX); + uint32_t start_idx, + uint32_t end_idx); }; } // end namespace storage diff --git a/cpp/test/reader/table_view/tsfile_reader_table_test.cc b/cpp/test/reader/table_view/tsfile_reader_table_test.cc index b1d7c257..a98a00f3 100644 --- a/cpp/test/reader/table_view/tsfile_reader_table_test.cc +++ b/cpp/test/reader/table_view/tsfile_reader_table_test.cc @@ -135,12 +135,12 @@ class TsFileTableReaderTest : public ::testing::Test { return tablet; } - void test_table_model_query(uint32_t points_per_device = 10) { + void test_table_model_query(uint32_t points_per_device = 10, uint32_t device_num = 1) { auto table_schema = gen_table_schema(0); auto tsfile_table_writer_ = std::make_shared<TsFileTableWriter>(&write_file_, table_schema); - auto tablet = gen_tablet(table_schema, 0, 1, points_per_device); + auto tablet = gen_tablet(table_schema, 0, device_num, points_per_device); ASSERT_EQ(tsfile_table_writer_->write_table(tablet), common::E_OK); ASSERT_EQ(tsfile_table_writer_->flush(), common::E_OK); ASSERT_EQ(tsfile_table_writer_->close(), common::E_OK); @@ -221,6 +221,13 @@ TEST_F(TsFileTableReaderTest, TableModelQueryMultiLargePage) { g_config_value_.page_writer_max_point_num_ = prev_config; } +TEST_F(TsFileTableReaderTest, TableModelQueryMultiDevices) { + int prev_config = g_config_value_.page_writer_max_point_num_; + g_config_value_.page_writer_max_point_num_ = 10000; + test_table_model_query(g_config_value_.page_writer_max_point_num_, 10); + g_config_value_.page_writer_max_point_num_ = prev_config; +} + TEST_F(TsFileTableReaderTest, TableModelResultMetadata) { auto table_schema = gen_table_schema(0); auto tsfile_table_writer_ =
