This is an automated email from the ASF dual-hosted git repository.
colinlee pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new ef11ec92 fix writer add memory threshold. (#446)
ef11ec92 is described below
commit ef11ec9229410b9b1adc90f4dd7af22ea8a63a5b
Author: Colin Lee <[email protected]>
AuthorDate: Thu Mar 20 20:49:46 2025 +0800
fix writer add memory threshold. (#446)
---
cpp/src/cwrapper/tsfile_cwrapper.h | 4 ++--
cpp/src/writer/tsfile_table_writer.h | 8 +++++---
cpp/test/writer/table_view/tsfile_writer_table_test.cc | 9 +++++++++
3 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h
b/cpp/src/cwrapper/tsfile_cwrapper.h
index b152782b..d43e5dce 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.h
+++ b/cpp/src/cwrapper/tsfile_cwrapper.h
@@ -155,8 +155,8 @@ TsFileWriter tsfile_writer_new(WriteFile file, TableSchema*
schema,
* @param file Target file where the table data will be written.
* @param schema Table schema definition.
* - Ownership: Should be free it by Caller.
- * @param memory_threshold used to limit the memory size
- * of objects. If set to 0, no memory limit is enforced.
+ * @param memory_threshold When the size of written data exceeds
+ * this value, the data will be automatically flushed to the disk.
* @param err_code [out] E_OK(0), or check error code in errno_define_c.h.
*
* @return TsFileWriter Valid handle on success, NULL on failure.
diff --git a/cpp/src/writer/tsfile_table_writer.h
b/cpp/src/writer/tsfile_table_writer.h
index 2b37395f..72d2833e 100644
--- a/cpp/src/writer/tsfile_table_writer.h
+++ b/cpp/src/writer/tsfile_table_writer.h
@@ -45,12 +45,13 @@ class TsFileTableWriter {
* not be null.
* @param table_schema Used to construct table structures. Defines the
* schema of the table being written.
- * @param memory_threshold Optional parameter used to limit the memory size
- * of objects. If set to 0, no memory limit is enforced.
+ * @param memory_threshold Optional parameter. When the size of written
+ * data exceeds this value, the data will be automatically flushed to the
+ * disk. Default value is 128MB.
*/
template <typename T>
explicit TsFileTableWriter(storage::WriteFile* writer_file, T*
table_schema,
- uint64_t memory_threshold = 0) {
+ uint64_t memory_threshold = 128 * 1024 * 1024) {
static_assert(!std::is_same<T, std::nullptr_t>::value,
"table_schema cannot be nullptr");
tsfile_writer_ = std::make_shared<TsFileWriter>();
@@ -62,6 +63,7 @@ class TsFileTableWriter {
auto table_schema_ptr = std::make_shared<TableSchema>(*table_schema);
tsfile_writer_->register_table(table_schema_ptr);
exclusive_table_name_ = table_schema->get_table_name();
+ common::g_config_value_.chunk_group_size_threshold_ = memory_threshold;
}
~TsFileTableWriter();
diff --git a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
index 78d9c31b..3fe1d461 100644
--- a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
+++ b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
@@ -180,6 +180,15 @@ TEST_F(TsFileWriterTableTest, WriteNonExistTableTest) {
delete table_schema;
}
+TEST_F(TsFileWriterTableTest, WriterWithMemoryThreshold) {
+ auto table_schema = gen_table_schema(0);
+ auto tsfile_table_writer_ =
+ std::make_shared<TsFileTableWriter>(&write_file_, table_schema, 256 *
1024 * 1024);
+ ASSERT_EQ(common::g_config_value_.chunk_group_size_threshold_, 256 * 1024
* 1024);
+ tsfile_table_writer_->close();
+ delete table_schema;
+}
+
TEST_F(TsFileWriterTableTest, WriteAndReadSimple) {
std::vector<MeasurementSchema*> measurement_schemas;
std::vector<ColumnCategory> column_categories;
