This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 9ec0abf57 Add C++ and Python TsFile properties support (#897)
9ec0abf57 is described below
commit 9ec0abf57835ec38873e9e7e4e84e70fd745eeb3
Author: Colin Lee <[email protected]>
AuthorDate: Thu Aug 6 12:31:50 2026 +0800
Add C++ and Python TsFile properties support (#897)
* Add C++ and Python TsFile properties support
* Fix TsFile properties metadata compatibility
* Fix Windows TsFile binary file mode
* Preserve legacy empty BloomFilter metadata
---
cpp/README-zh.md | 19 +++
cpp/README.md | 20 +++
cpp/src/common/tsfile_common.cc | 163 ++++++++++++++++---
cpp/src/common/tsfile_common.h | 32 +++-
cpp/src/cwrapper/tsfile_cwrapper.cc | 148 +++++++++++++++++
cpp/src/cwrapper/tsfile_cwrapper.h | 42 +++++
cpp/src/file/tsfile_io_writer.cc | 57 +++++--
cpp/src/file/tsfile_io_writer.h | 5 +
cpp/src/file/write_file.cc | 6 +
cpp/src/reader/bloom_filter.cc | 39 ++++-
cpp/src/reader/tsfile_reader.cc | 9 ++
cpp/src/reader/tsfile_reader.h | 3 +
cpp/src/writer/tsfile_table_writer.cc | 17 ++
cpp/src/writer/tsfile_table_writer.h | 6 +
cpp/src/writer/tsfile_writer.cc | 17 ++
cpp/src/writer/tsfile_writer.h | 6 +
cpp/test/common/tsfile_common_test.cc | 78 ++++++++-
cpp/test/cwrapper/cwrapper_properties_test.cc | 177 +++++++++++++++++++++
cpp/test/reader/bloom_filter_test.cc | 49 ++++++
.../writer/table_view/tsfile_writer_table_test.cc | 24 +++
cpp/test/writer/tsfile_properties_test.cc | 112 +++++++++++++
python/README-zh.md | 18 ++-
python/README.md | 16 ++
python/tests/test_tsfile_properties.py | 98 ++++++++++++
python/tsfile/tsfile_cpp.pxd | 16 ++
python/tsfile/tsfile_reader.pyx | 42 ++++-
python/tsfile/tsfile_table_writer.py | 6 +
python/tsfile/tsfile_writer.pyx | 37 ++++-
28 files changed, 1208 insertions(+), 54 deletions(-)
diff --git a/cpp/README-zh.md b/cpp/README-zh.md
index a8af952b1..b0d76f506 100644
--- a/cpp/README-zh.md
+++ b/cpp/README-zh.md
@@ -188,3 +188,22 @@ bash build.sh
```
即可在 `./examples/build` 目录下生成可执行文件。
+
+### 文件级 Properties
+
+`TsFileWriter` 和 `TsFileTableWriter` 可以在 writer 打开期间新增或覆盖二进制
+property。传入的数据会立即复制，调用 `flush()` 后仍可继续修改；文件关闭后不能修改。
+
+```cpp
+std::vector<uint8_t> value = {0x01, 0x00, 0xFF};
+writer.add_tsfile_property("binary-property", value);
+
+// nullptr 且长度为 0 表示 null；空 vector 表示非 null 的零长度值。
+writer.add_tsfile_property("null-property", nullptr, 0);
+writer.add_tsfile_property("empty-property", std::vector<uint8_t>());
+
+storage::TsFileProperties properties = reader.get_tsfile_properties();
+```
+
+Property value 本身不保存数据类型。整数、浮点数或结构体应由应用使用明确、可跨语言的
+字节编码进行转换。
diff --git a/cpp/README.md b/cpp/README.md
index 918eff68f..5f62afe24 100644
--- a/cpp/README.md
+++ b/cpp/README.md
@@ -203,3 +203,23 @@ By default, parallel write is enabled when the machine has
more than one CPU cor
## Use TsFile
You can find examples on how to read and write data in `demo_read.cpp` and
`demo_write.cpp` located under `./examples/cpp_examples`. There are also
examples under `./examples/c_examples` on how to use a C-style API to read and
write data in a C environment. The examples will be built automatically when
you run the main build command.
+
+### File-level properties
+
+`TsFileWriter` and `TsFileTableWriter` can add or replace binary properties
+while the writer is open. Values are copied immediately and may still be
+changed after `flush()`; a closed file cannot be modified.
+
+```cpp
+std::vector<uint8_t> value = {0x01, 0x00, 0xFF};
+writer.add_tsfile_property("binary-property", value);
+
+// nullptr with length 0 is null; an empty vector is a non-null empty value.
+writer.add_tsfile_property("null-property", nullptr, 0);
+writer.add_tsfile_property("empty-property", std::vector<uint8_t>());
+
+storage::TsFileProperties properties = reader.get_tsfile_properties();
+```
+
+Property values do not store a data type. Applications should define their own
+portable byte encoding for integers, floating-point values, or structures.
diff --git a/cpp/src/common/tsfile_common.cc b/cpp/src/common/tsfile_common.cc
index a3fcc0a70..3cac258ff 100644
--- a/cpp/src/common/tsfile_common.cc
+++ b/cpp/src/common/tsfile_common.cc
@@ -20,6 +20,7 @@
#include "common/tsfile_common.h"
#include <algorithm>
+#include <limits>
#include <map>
#include "common/logger/elog.h"
@@ -180,37 +181,101 @@ int TSMIterator::get_next(std::shared_ptr<IDeviceID>&
ret_device_id,
return ret;
}
-int TsFileMeta::serialize_to(common::ByteStream& out) {
+int TsFileMeta::serialize_to(common::ByteStream& out,
+ int32_t& serialized_size) {
+ serialized_size = 0;
+ const size_t max_property_size =
+ static_cast<size_t>(std::numeric_limits<int32_t>::max());
+ if (tsfile_properties_.size() > max_property_size) {
+ return common::E_OUT_OF_RANGE;
+ }
+ for (const auto& tsfile_property : tsfile_properties_) {
+ if (tsfile_property.first.size() > max_property_size ||
+ (!tsfile_property.second.is_null &&
+ tsfile_property.second.value.size() > max_property_size)) {
+ return common::E_OUT_OF_RANGE;
+ }
+ }
+
+ int ret = common::E_OK;
auto start_idx = out.total_size();
- common::SerializationUtil::write_var_uint(
- table_metadata_index_node_map_.size(), out);
+ if (RET_FAIL(common::SerializationUtil::write_var_uint(
+ table_metadata_index_node_map_.size(), out))) {
+ return ret;
+ }
for (auto& idx_nodes_iter : table_metadata_index_node_map_) {
- common::SerializationUtil::write_var_str(idx_nodes_iter.first, out);
- idx_nodes_iter.second->serialize_to(out);
+ if (RET_FAIL(common::SerializationUtil::write_var_str(
+ idx_nodes_iter.first, out))) {
+ return ret;
+ } else if (RET_FAIL(idx_nodes_iter.second->serialize_to(out))) {
+ return ret;
+ }
}
- common::SerializationUtil::write_var_uint(table_schemas_.size(), out);
+ if (RET_FAIL(common::SerializationUtil::write_var_uint(
+ table_schemas_.size(), out))) {
+ return ret;
+ }
for (auto& table_schema_iter : table_schemas_) {
- common::SerializationUtil::write_var_str(table_schema_iter.first, out);
- table_schema_iter.second->serialize_to(out);
+ if (RET_FAIL(common::SerializationUtil::write_var_str(
+ table_schema_iter.first, out))) {
+ return ret;
+ } else if (RET_FAIL(table_schema_iter.second->serialize_to(out))) {
+ return ret;
+ }
}
- common::SerializationUtil::write_i64(meta_offset_, out);
+ if (RET_FAIL(common::SerializationUtil::write_i64(meta_offset_, out))) {
+ return ret;
+ }
if (bloom_filter_ != nullptr) {
- bloom_filter_->serialize_to(out);
+ if (RET_FAIL(bloom_filter_->serialize_to(out))) {
+ return ret;
+ }
} else {
- common::SerializationUtil::write_ui8(0, out);
+ if (RET_FAIL(common::SerializationUtil::write_ui8(0, out))) {
+ return ret;
+ }
}
- common::SerializationUtil::write_var_int(tsfile_properties_.size(), out);
+ if (RET_FAIL(common::SerializationUtil::write_var_int(
+ static_cast<int32_t>(tsfile_properties_.size()), out))) {
+ return ret;
+ }
for (const auto& tsfile_property : tsfile_properties_) {
- common::SerializationUtil::write_var_str(tsfile_property.first, out);
- common::SerializationUtil::write_var_char_ptr(tsfile_property.second,
- out);
+ if (RET_FAIL(common::SerializationUtil::write_var_str(
+ tsfile_property.first, out))) {
+ return ret;
+ }
+ const TsFilePropertyValue& value = tsfile_property.second;
+ if (value.is_null) {
+ if (RET_FAIL(common::SerializationUtil::write_var_int(
+ NO_STR_TO_READ, out))) {
+ return ret;
+ }
+ } else {
+ if (RET_FAIL(common::SerializationUtil::write_var_int(
+ static_cast<int32_t>(value.value.size()), out))) {
+ return ret;
+ }
+ if (!value.value.empty()) {
+ if (RET_FAIL(out.write_buf(
+ value.value.data(),
+ static_cast<uint32_t>(value.value.size())))) {
+ return ret;
+ }
+ }
+ }
}
- return out.total_size() - start_idx;
+ const uint64_t total_size = out.total_size() - start_idx;
+ if (total_size >
+ static_cast<uint64_t>(std::numeric_limits<int32_t>::max())) {
+ return common::E_OUT_OF_RANGE;
+ }
+ serialized_size = static_cast<int32_t>(total_size);
+ return common::E_OK;
}
int TsFileMeta::deserialize_from(common::ByteStream& in) {
@@ -251,15 +316,67 @@ int TsFileMeta::deserialize_from(common::ByteStream& in) {
common::SerializationUtil::read_i64(meta_offset_, in);
- bloom_filter_->deserialize_from(in);
+ if (RET_FAIL(bloom_filter_->deserialize_from(in))) {
+ return ret;
+ }
int32_t tsfile_properties_size = 0;
- common::SerializationUtil::read_var_int(tsfile_properties_size, in);
+ if
(RET_FAIL(common::SerializationUtil::read_var_int(tsfile_properties_size,
+ in))) {
+ return ret;
+ }
+ if (tsfile_properties_size < 0) {
+ return common::E_TSFILE_CORRUPTED;
+ }
for (int i = 0; i < tsfile_properties_size; i++) {
- std::string key, *value;
- common::SerializationUtil::read_var_str(key, in);
- common::SerializationUtil::read_var_char_ptr(value, in);
- tsfile_properties_.emplace(key, value);
+ std::string key;
+ int32_t key_len = 0;
+ int32_t value_len = 0;
+ if (RET_FAIL(common::SerializationUtil::read_var_int(key_len, in))) {
+ return ret;
+ } else if (key_len < 0) {
+ return common::E_TSFILE_CORRUPTED;
+ }
+ if (static_cast<uint64_t>(key_len) > in.remaining_size()) {
+ return common::E_TSFILE_CORRUPTED;
+ }
+ key.resize(static_cast<size_t>(key_len));
+ if (key_len > 0) {
+ uint32_t read_len = 0;
+ if (RET_FAIL(in.read_buf(reinterpret_cast<uint8_t*>(&key[0]),
+ static_cast<uint32_t>(key_len),
+ read_len))) {
+ return ret;
+ } else if (read_len != static_cast<uint32_t>(key_len)) {
+ return common::E_BUF_NOT_ENOUGH;
+ }
+ }
+ if (RET_FAIL(common::SerializationUtil::read_var_int(value_len, in))) {
+ return ret;
+ }
+
+ TsFilePropertyValue value;
+ if (value_len == NO_STR_TO_READ) {
+ value.is_null = true;
+ } else if (value_len < 0) {
+ return common::E_TSFILE_CORRUPTED;
+ } else {
+ if (static_cast<uint64_t>(value_len) > in.remaining_size()) {
+ return common::E_TSFILE_CORRUPTED;
+ }
+ value.is_null = false;
+ value.value.resize(static_cast<size_t>(value_len));
+ if (value_len > 0) {
+ uint32_t read_len = 0;
+ if (RET_FAIL(
+ in.read_buf(value.value.data(), value_len, read_len)))
{
+ return ret;
+ } else if (read_len != static_cast<uint32_t>(value_len)) {
+ return common::E_BUF_NOT_ENOUGH;
+ }
+ }
+ }
+ tsfile_properties_.emplace(key, std::move(value));
}
return ret;
}
@@ -375,4 +492,4 @@ int MetaIndexNode::binary_search_children(const String
&name,
}
#endif
-} // end namespace storage
\ No newline at end of file
+} // end namespace storage
diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h
index fd3690200..d763acbbd 100644
--- a/cpp/src/common/tsfile_common.h
+++ b/cpp/src/common/tsfile_common.h
@@ -1126,13 +1126,35 @@ struct MetaIndexNode {
class TableSchema;
+struct TsFilePropertyValue {
+ /** A default-constructed property represents a null value. */
+ TsFilePropertyValue() : is_null(true), value() {}
+
+ /** A vector, including an empty vector, represents a non-null value. */
+ explicit TsFilePropertyValue(const std::vector<uint8_t>& value)
+ : is_null(false), value(value) {}
+
+ /** nullptr represents null; a non-null pointer with length 0 is empty. */
+ TsFilePropertyValue(const uint8_t* data, uint32_t value_len)
+ : is_null(data == nullptr), value() {
+ if (data != nullptr && value_len > 0) {
+ value.assign(data, data + value_len);
+ }
+ }
+
+ bool is_null;
+ std::vector<uint8_t> value;
+};
+
+using TsFileProperties = std::unordered_map<std::string, TsFilePropertyValue>;
+
struct TsFileMeta {
typedef std::map<std::shared_ptr<IDeviceID>,
std::shared_ptr<MetaIndexNode>,
IDeviceIDComparator>
DeviceNodeMap;
std::map<std::string, std::shared_ptr<MetaIndexNode>>
table_metadata_index_node_map_;
- std::unordered_map<std::string, std::string*> tsfile_properties_;
+ TsFileProperties tsfile_properties_;
typedef std::unordered_map<std::string, std::shared_ptr<TableSchema>>
TableSchemasMap;
TableSchemasMap table_schemas_;
@@ -1170,18 +1192,12 @@ struct TsFileMeta {
if (bloom_filter_ != nullptr) {
bloom_filter_->destroy();
}
- for (auto properties : tsfile_properties_) {
- if (properties.second != nullptr) {
- delete properties.second;
- properties.second = nullptr;
- }
- }
tsfile_properties_.clear();
table_metadata_index_node_map_.clear();
table_schemas_.clear();
}
- int serialize_to(common::ByteStream& out);
+ int serialize_to(common::ByteStream& out, int32_t& serialized_size);
int deserialize_from(common::ByteStream& in);
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc
b/cpp/src/cwrapper/tsfile_cwrapper.cc
index ffe2f59b6..3e27f60d0 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.cc
+++ b/cpp/src/cwrapper/tsfile_cwrapper.cc
@@ -31,6 +31,8 @@
#endif
#include <cstring>
+#include <limits>
+#include <new>
#include <set>
#include <vector>
@@ -258,6 +260,27 @@ ERRNO tsfile_writer_close(TsFileWriter writer) {
return ret;
}
+ERRNO tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key,
+ uint32_t key_len, const uint8_t* value,
+ uint32_t value_len) {
+ if (writer == nullptr || key == nullptr ||
+ (value == nullptr && value_len > 0)) {
+ return common::E_INVALID_ARG;
+ }
+ if (key_len > static_cast<uint32_t>(std::numeric_limits<int32_t>::max()) ||
+ value_len >
+ static_cast<uint32_t>(std::numeric_limits<int32_t>::max())) {
+ return common::E_OUT_OF_RANGE;
+ }
+ try {
+ auto* w = static_cast<storage::TsFileTableWriter*>(writer);
+ return w->add_tsfile_property(std::string(key, key_len), value,
+ value_len);
+ } catch (const std::bad_alloc&) {
+ return common::E_OOM;
+ }
+}
+
ERRNO tsfile_reader_close(TsFileReader reader) {
auto* ts_reader = static_cast<storage::TsFileReader*>(reader);
delete ts_reader;
@@ -1432,6 +1455,110 @@ void tsfile_free_device_timeseries_metadata_map(
map->device_count = 0;
}
+void tsfile_free_tsfile_properties(TsFileProperty* properties,
+ uint32_t length) {
+ if (properties == nullptr) {
+ return;
+ }
+ for (uint32_t i = 0; i < length; i++) {
+ free(properties[i].key);
+ properties[i].key = nullptr;
+ free(properties[i].value);
+ properties[i].value = nullptr;
+ properties[i].key_len = 0;
+ properties[i].value_len = 0;
+ properties[i].is_null = false;
+ }
+ free(properties);
+}
+
+ERRNO tsfile_reader_get_tsfile_properties(TsFileReader reader,
+ TsFileProperty** out_properties,
+ uint32_t* out_length) {
+ if (out_properties == nullptr || out_length == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+ *out_properties = nullptr;
+ *out_length = 0;
+ if (reader == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+
+ try {
+ auto* r = static_cast<storage::TsFileReader*>(reader);
+ storage::TsFileProperties cpp_properties = r->get_tsfile_properties();
+ if (cpp_properties.size() >
+ static_cast<size_t>(std::numeric_limits<uint32_t>::max()) ||
+ cpp_properties.size() >
+ std::numeric_limits<size_t>::max() / sizeof(TsFileProperty)) {
+ return common::E_OUT_OF_RANGE;
+ }
+ if (cpp_properties.empty()) {
+ return common::E_OK;
+ }
+
+ auto* properties = static_cast<TsFileProperty*>(
+ malloc(sizeof(TsFileProperty) * cpp_properties.size()));
+ if (properties == nullptr) {
+ return common::E_OOM;
+ }
+ memset(properties, 0, sizeof(TsFileProperty) * cpp_properties.size());
+
+ uint32_t property_index = 0;
+ for (const auto& cpp_property : cpp_properties) {
+ TsFileProperty& property = properties[property_index];
+ if (cpp_property.first.size() >
+ static_cast<size_t>(std::numeric_limits<uint32_t>::max())) {
+ tsfile_free_tsfile_properties(properties, property_index);
+ return common::E_OUT_OF_RANGE;
+ }
+ property.key_len =
static_cast<uint32_t>(cpp_property.first.size());
+ property.key = static_cast<char*>(
+ malloc(static_cast<size_t>(property.key_len) + 1U));
+ if (property.key == nullptr) {
+ tsfile_free_tsfile_properties(properties, property_index + 1);
+ return common::E_OOM;
+ }
+ if (property.key_len > 0) {
+ memcpy(property.key, cpp_property.first.data(),
+ property.key_len);
+ }
+ property.key[property.key_len] = '\0';
+
+ const storage::TsFilePropertyValue& cpp_value =
cpp_property.second;
+ property.is_null = cpp_value.is_null;
+ if (!cpp_value.is_null) {
+ if (cpp_value.value.size() >
+ static_cast<size_t>(std::numeric_limits<uint32_t>::max()))
{
+ tsfile_free_tsfile_properties(properties,
+ property_index + 1);
+ return common::E_OUT_OF_RANGE;
+ }
+ property.value_len =
+ static_cast<uint32_t>(cpp_value.value.size());
+ if (property.value_len > 0) {
+ property.value =
+ static_cast<uint8_t*>(malloc(property.value_len));
+ if (property.value == nullptr) {
+ tsfile_free_tsfile_properties(properties,
+ property_index + 1);
+ return common::E_OOM;
+ }
+ memcpy(property.value, cpp_value.value.data(),
+ property.value_len);
+ }
+ }
+ property_index++;
+ }
+
+ *out_properties = properties;
+ *out_length = static_cast<uint32_t>(cpp_properties.size());
+ return common::E_OK;
+ } catch (const std::bad_alloc&) {
+ return common::E_OOM;
+ }
+}
+
// delete pointer
void _free_tsfile_ts_record(TsRecord* record) {
if (*record != nullptr) {
@@ -1640,6 +1767,27 @@ ERRNO _tsfile_writer_flush(TsFileWriter writer) {
return w->flush();
}
+ERRNO _tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key,
+ uint32_t key_len, const uint8_t*
value,
+ uint32_t value_len) {
+ if (writer == nullptr || key == nullptr ||
+ (value == nullptr && value_len > 0)) {
+ return common::E_INVALID_ARG;
+ }
+ if (key_len > static_cast<uint32_t>(std::numeric_limits<int32_t>::max()) ||
+ value_len >
+ static_cast<uint32_t>(std::numeric_limits<int32_t>::max())) {
+ return common::E_OUT_OF_RANGE;
+ }
+ try {
+ auto* w = static_cast<storage::TsFileWriter*>(writer);
+ return w->add_tsfile_property(std::string(key, key_len), value,
+ value_len);
+ } catch (const std::bad_alloc&) {
+ return common::E_OOM;
+ }
+}
+
ResultSet _tsfile_reader_query_device(TsFileReader reader,
const char* device_name,
char** sensor_name, uint32_t sensor_num,
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h
b/cpp/src/cwrapper/tsfile_cwrapper.h
index 768aec962..16d392bae 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.h
+++ b/cpp/src/cwrapper/tsfile_cwrapper.h
@@ -230,6 +230,21 @@ typedef struct DeviceTimeseriesMetadataMap {
uint32_t device_count;
} DeviceTimeseriesMetadataMap;
+/**
+ * @brief One file-level property with length-aware binary storage.
+ *
+ * @p key is allocated with one trailing NUL for convenience, while @p key_len
+ * is authoritative and preserves embedded NUL bytes. @p is_null distinguishes
+ * a null value from a non-null zero-length value.
+ */
+typedef struct TsFileProperty {
+ char* key;
+ uint32_t key_len;
+ uint8_t* value;
+ uint32_t value_len;
+ bool is_null;
+} TsFileProperty;
+
/** Frees path, table_name, and segments inside @p d; zeros @p d. */
void tsfile_device_id_free_contents(DeviceID* d);
@@ -435,6 +450,17 @@ TsFileReader tsfile_reader_new(const char* pathname,
ERRNO* err_code);
*/
ERRNO tsfile_writer_close(TsFileWriter writer);
+/**
+ * @brief Adds or replaces a file-level property while the table writer is
open.
+ *
+ * The key and value are copied immediately. A NULL value with value_len == 0
+ * represents a null property; a non-NULL value with value_len == 0 represents
+ * an empty byte array.
+ */
+ERRNO tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key,
+ uint32_t key_len, const uint8_t* value,
+ uint32_t value_len);
+
/**
* @brief Releases resources associated with a TsFileReader.
*
@@ -477,6 +503,17 @@ ERRNO tsfile_reader_get_timeseries_metadata_for_devices(
void tsfile_free_device_timeseries_metadata_map(
DeviceTimeseriesMetadataMap* map);
+/**
+ * @brief Returns a heap-allocated array containing all file-level properties.
+ *
+ * Caller must release the result with tsfile_free_tsfile_properties().
+ */
+ERRNO tsfile_reader_get_tsfile_properties(TsFileReader reader,
+ TsFileProperty** out_properties,
+ uint32_t* out_length);
+
+void tsfile_free_tsfile_properties(TsFileProperty* properties, uint32_t
length);
+
/*--------------------------Tablet API------------------------ */
/**
@@ -1055,6 +1092,11 @@ ERRNO _tsfile_writer_close(TsFileWriter writer);
// Flush Chunk into tsfile from current tsFileWriter
ERRNO _tsfile_writer_flush(TsFileWriter writer);
+// Add or replace a file-level property on the generic writer used by Python.
+ERRNO _tsfile_writer_add_tsfile_property(TsFileWriter writer, const char* key,
+ uint32_t key_len, const uint8_t*
value,
+ uint32_t value_len);
+
// Queries time-series data for a specific device within a given time range.
ResultSet _tsfile_reader_query_device(TsFileReader reader,
const char* device_name,
diff --git a/cpp/src/file/tsfile_io_writer.cc b/cpp/src/file/tsfile_io_writer.cc
index 8c207ca82..29ddf0d90 100644
--- a/cpp/src/file/tsfile_io_writer.cc
+++ b/cpp/src/file/tsfile_io_writer.cc
@@ -23,6 +23,7 @@
#include <chrono>
#include <iomanip>
+#include <limits>
#include <memory>
#include "common/device_id.h"
@@ -93,6 +94,7 @@ void TsFileIOWriter::destroy() {
use_prev_alloc_cgm_ = false;
is_aligned_ = false;
file_base_offset_ = 0;
+ tsfile_properties_.clear();
destroyed_ = true;
meta_allocator_.destroy();
@@ -103,6 +105,38 @@ void TsFileIOWriter::destroy() {
}
}
+int TsFileIOWriter::add_tsfile_property(const std::string& key,
+ const uint8_t* value,
+ uint32_t value_len) {
+ if (file_ == nullptr || file_->get_fd() < 0) {
+ return common::E_FILE_WRITE_ERR;
+ }
+ if (value_len > 0 && value == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+ if (key.size() > static_cast<size_t>(std::numeric_limits<int32_t>::max())
||
+ value_len >
+ static_cast<uint32_t>(std::numeric_limits<int32_t>::max())) {
+ return common::E_OUT_OF_RANGE;
+ }
+ tsfile_properties_[key] = TsFilePropertyValue(value, value_len);
+ return common::E_OK;
+}
+
+int TsFileIOWriter::add_tsfile_property(const std::string& key,
+ const std::vector<uint8_t>& value) {
+ if (file_ == nullptr || file_->get_fd() < 0) {
+ return common::E_FILE_WRITE_ERR;
+ }
+ if (key.size() > static_cast<size_t>(std::numeric_limits<int32_t>::max())
||
+ value.size() >
+ static_cast<size_t>(std::numeric_limits<int32_t>::max())) {
+ return common::E_OUT_OF_RANGE;
+ }
+ tsfile_properties_[key] = TsFilePropertyValue(value);
+ return common::E_OK;
+}
+
int TsFileIOWriter::start_file() {
int ret = E_OK;
if (RET_FAIL(write_buf(MAGIC_STRING_TSFILE, MAGIC_STRING_TSFILE_LEN))) {
@@ -472,18 +506,23 @@ int TsFileIOWriter::write_file_index() {
}
tsfile_meta.table_metadata_index_node_map_ = table_nodes_map;
tsfile_meta.table_schemas_ = schema_->table_schema_map_;
- tsfile_meta.tsfile_properties_.insert(
- std::make_pair("encryptLevel", new std::string(encrypt_level_)));
- tsfile_meta.tsfile_properties_.insert(
- std::make_pair("encryptType", new std::string(encrypt_type_)));
- tsfile_meta.tsfile_properties_.insert(
- std::make_pair("encryptKey", nullptr));
+ tsfile_meta.tsfile_properties_ = tsfile_properties_;
+ tsfile_meta.tsfile_properties_["encryptLevel"] = TsFilePropertyValue(
+ reinterpret_cast<const uint8_t*>(encrypt_level_.data()),
+ static_cast<uint32_t>(encrypt_level_.size()));
+ tsfile_meta.tsfile_properties_["encryptType"] = TsFilePropertyValue(
+ reinterpret_cast<const uint8_t*>(encrypt_type_.data()),
+ static_cast<uint32_t>(encrypt_type_.size()));
+ tsfile_meta.tsfile_properties_["encryptKey"] = TsFilePropertyValue();
#if DEBUG_SE
auto tsfile_meta_offset = write_stream_.total_size();
#endif
- auto total_write_size = tsfile_meta.serialize_to(write_stream_);
- if (RET_FAIL(common::SerializationUtil::write_i32(total_write_size,
- write_stream_))) {
+ int32_t total_write_size = 0;
+ if (RET_FAIL(
+ tsfile_meta.serialize_to(write_stream_, total_write_size))) {
+ return ret;
+ } else if (RET_FAIL(common::SerializationUtil::write_i32(
+ total_write_size, write_stream_))) {
return ret;
}
tsfile_meta.bloom_filter_ = nullptr;
diff --git a/cpp/src/file/tsfile_io_writer.h b/cpp/src/file/tsfile_io_writer.h
index f041a1c57..bbb1e4988 100644
--- a/cpp/src/file/tsfile_io_writer.h
+++ b/cpp/src/file/tsfile_io_writer.h
@@ -89,6 +89,10 @@ class TsFileIOWriter {
void destroy();
void set_generate_table_schema(bool generate_table_schema);
+ int add_tsfile_property(const std::string& key, const uint8_t* value,
+ uint32_t value_len);
+ int add_tsfile_property(const std::string& key,
+ const std::vector<uint8_t>& value);
int start_file();
int start_flush_chunk_group(std::shared_ptr<IDeviceID> device_id,
bool is_aligned = false);
@@ -242,6 +246,7 @@ class TsFileIOWriter {
std::string encrypt_level_;
std::string encrypt_type_;
std::string encrypt_key_;
+ TsFileProperties tsfile_properties_;
bool is_aligned_;
/** Recovery only: absolute file offset at which write_stream_ logically
* begins. Normal (non-recovery) path keeps this at 0. */
diff --git a/cpp/src/file/write_file.cc b/cpp/src/file/write_file.cc
index 227520b71..68ac127ac 100644
--- a/cpp/src/file/write_file.cc
+++ b/cpp/src/file/write_file.cc
@@ -52,6 +52,12 @@ int WriteFile::create(const std::string& file_path, int
flags, mode_t mode) {
int WriteFile::do_create(int flags, mode_t mode) {
int ret = E_OK;
+#ifdef _WIN32
+ // TsFile is a binary format. Callers of the C++ API may pass ordinary
+ // POSIX-style flags without O_BINARY; leaving the descriptor in text mode
+ // would translate byte 0x0A to 0x0D 0x0A and corrupt serialized metadata.
+ flags |= O_BINARY;
+#endif
// TODO make sure no same file exists
fd_ = ::open(path_.c_str(), flags, mode);
if (fd_ < 0) {
diff --git a/cpp/src/reader/bloom_filter.cc b/cpp/src/reader/bloom_filter.cc
index 4aff4ecd3..09e2c9a15 100644
--- a/cpp/src/reader/bloom_filter.cc
+++ b/cpp/src/reader/bloom_filter.cc
@@ -235,11 +235,12 @@ int BloomFilter::serialize_to(ByteStream& out) {
bitset_.to_bytes(filter_data_bytes, filter_data_bytes_len);
if (RET_FAIL(
SerializationUtil::write_var_uint(filter_data_bytes_len, out))) {
- } else if (RET_FAIL(
- out.write_buf(filter_data_bytes, filter_data_bytes_len))) {
- } else if (RET_FAIL(SerializationUtil::write_var_uint(size_, out))) {
- } else if (RET_FAIL(
- SerializationUtil::write_var_uint(hash_func_count_, out))) {
+ } else if (filter_data_bytes_len > 0) {
+ if (RET_FAIL(out.write_buf(filter_data_bytes, filter_data_bytes_len)))
{
+ } else if (RET_FAIL(SerializationUtil::write_var_uint(size_, out))) {
+ } else if (RET_FAIL(SerializationUtil::write_var_uint(hash_func_count_,
+ out))) {
+ }
}
if (filter_data_bytes_len > 0) {
bitset_.revert_bytes(filter_data_bytes);
@@ -253,6 +254,31 @@ int BloomFilter::deserialize_from(ByteStream& in) {
uint32_t ret_read_len = 0;
uint8_t* filter_data = nullptr;
if (RET_FAIL(SerializationUtil::read_var_uint(filter_data_bytes_len, in)))
{
+ } else if (filter_data_bytes_len == 0) {
+ // Older C++ writers serialized an empty filter as three zero varints:
+ // byte length, bit count, and hash-function count. The Java-compatible
+ // encoding contains only the byte length. Probe the two legacy fields
+ // and restore the cursor when the following data is instead the
+ // TsFile property count from the current encoding.
+ const uint64_t legacy_fields_pos = in.read_pos();
+ if (in.remaining_size() >= 2) {
+ uint32_t legacy_size = 0;
+ uint32_t legacy_hash_func_count = 0;
+ int probe_ret = SerializationUtil::read_var_uint(legacy_size, in);
+ if (probe_ret == E_OK && legacy_size == 0) {
+ probe_ret = SerializationUtil::read_var_uint(
+ legacy_hash_func_count, in);
+ }
+ if (probe_ret != E_OK || legacy_size != 0 ||
+ legacy_hash_func_count != 0) {
+ in.set_read_pos(legacy_fields_pos);
+ }
+ }
+ size_ = 0;
+ hash_func_count_ = 0;
+ return E_OK;
+ } else if (filter_data_bytes_len > in.remaining_size()) {
+ ret = E_TSFILE_CORRUPTED;
} else if (UNLIKELY(nullptr ==
(filter_data = (uint8_t*)mem_alloc(
filter_data_bytes_len, MOD_BLOOM_FILTER)))) {
@@ -264,6 +290,9 @@ int BloomFilter::deserialize_from(ByteStream& in) {
} else if (RET_FAIL(SerializationUtil::read_var_uint(size_, in))) {
} else if (RET_FAIL(
SerializationUtil::read_var_uint(hash_func_count_, in))) {
+ } else if (size_ == 0 || hash_func_count_ == 0 ||
+ hash_func_count_ > MAX_HASH_FUNC_COUNT) {
+ ret = E_TSFILE_CORRUPTED;
} else {
for (uint32_t i = 0; i < hash_func_count_; i++) {
hash_func_arr_[i].init(size_, SEEDS[i]);
diff --git a/cpp/src/reader/tsfile_reader.cc b/cpp/src/reader/tsfile_reader.cc
index 33d0d8967..fb5f8fd92 100644
--- a/cpp/src/reader/tsfile_reader.cc
+++ b/cpp/src/reader/tsfile_reader.cc
@@ -527,6 +527,15 @@ DeviceTimeseriesMetadataMap
TsFileReader::get_timeseries_metadata() {
return result;
}
+TsFileProperties TsFileReader::get_tsfile_properties() {
+ if (tsfile_executor_ == nullptr) {
+ return TsFileProperties();
+ }
+ TsFileMeta* file_metadata = tsfile_executor_->get_tsfile_meta();
+ return file_metadata == nullptr ? TsFileProperties()
+ : file_metadata->tsfile_properties_;
+}
+
ResultSet* TsFileReader::read_timeseries(
const std::shared_ptr<IDeviceID>& device_id,
const std::vector<std::string>& measurement_name) {
diff --git a/cpp/src/reader/tsfile_reader.h b/cpp/src/reader/tsfile_reader.h
index e2f9f3496..3ba490045 100644
--- a/cpp/src/reader/tsfile_reader.h
+++ b/cpp/src/reader/tsfile_reader.h
@@ -216,6 +216,9 @@ class TsFileReader {
*/
DeviceTimeseriesMetadataMap get_timeseries_metadata();
+ /** Return a copy of all file-level properties, preserving null values. */
+ TsFileProperties get_tsfile_properties();
+
/**
* @brief get the table schema by the table name
*
diff --git a/cpp/src/writer/tsfile_table_writer.cc
b/cpp/src/writer/tsfile_table_writer.cc
index b1b7911bd..5432aff5c 100644
--- a/cpp/src/writer/tsfile_table_writer.cc
+++ b/cpp/src/writer/tsfile_table_writer.cc
@@ -92,6 +92,23 @@ int storage::TsFileTableWriter::flush() {
return tsfile_writer_->flush();
}
+int storage::TsFileTableWriter::add_tsfile_property(const std::string& key,
+ const uint8_t* value,
+ uint32_t value_len) {
+ if (closed_ || !tsfile_writer_) {
+ return common::E_FILE_WRITE_ERR;
+ }
+ return tsfile_writer_->add_tsfile_property(key, value, value_len);
+}
+
+int storage::TsFileTableWriter::add_tsfile_property(
+ const std::string& key, const std::vector<uint8_t>& value) {
+ if (closed_ || !tsfile_writer_) {
+ return common::E_FILE_WRITE_ERR;
+ }
+ return tsfile_writer_->add_tsfile_property(key, value);
+}
+
int storage::TsFileTableWriter::close() {
if (closed_) {
return common::E_OK;
diff --git a/cpp/src/writer/tsfile_table_writer.h
b/cpp/src/writer/tsfile_table_writer.h
index a2d2a5fd9..d7c79254f 100644
--- a/cpp/src/writer/tsfile_table_writer.h
+++ b/cpp/src/writer/tsfile_table_writer.h
@@ -106,6 +106,12 @@ class TsFileTableWriter {
* @return Returns 0 on success, or a non-zero error code on failure.
*/
int flush();
+
+ /** Add or replace a binary file-level property while the writer is open.
*/
+ int add_tsfile_property(const std::string& key, const uint8_t* value,
+ uint32_t value_len);
+ int add_tsfile_property(const std::string& key,
+ const std::vector<uint8_t>& value);
/**
* Closes the writer and releases any resources held by it.
* After calling this method, no further operations should be performed on
diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc
index 0b4c8668c..aa0e555f8 100644
--- a/cpp/src/writer/tsfile_writer.cc
+++ b/cpp/src/writer/tsfile_writer.cc
@@ -1960,4 +1960,21 @@ int TsFileWriter::close() {
return io_writer_->end_file();
}
+int TsFileWriter::add_tsfile_property(const std::string& key,
+ const uint8_t* value,
+ uint32_t value_len) {
+ if (io_writer_ == nullptr) {
+ return E_FILE_WRITE_ERR;
+ }
+ return io_writer_->add_tsfile_property(key, value, value_len);
+}
+
+int TsFileWriter::add_tsfile_property(const std::string& key,
+ const std::vector<uint8_t>& value) {
+ if (io_writer_ == nullptr) {
+ return E_FILE_WRITE_ERR;
+ }
+ return io_writer_->add_tsfile_property(key, value);
+}
+
} // end namespace storage
diff --git a/cpp/src/writer/tsfile_writer.h b/cpp/src/writer/tsfile_writer.h
index e0b102c97..55e9e7f3a 100644
--- a/cpp/src/writer/tsfile_writer.h
+++ b/cpp/src/writer/tsfile_writer.h
@@ -84,6 +84,12 @@ class TsFileWriter {
int write_tree(const TsRecord& record);
int write_table(Tablet& tablet);
+ /** Add or replace a binary file-level property while the writer is open.
*/
+ int add_tsfile_property(const std::string& key, const uint8_t* value,
+ uint32_t value_len);
+ int add_tsfile_property(const std::string& key,
+ const std::vector<uint8_t>& value);
+
typedef std::map<std::shared_ptr<IDeviceID>, MeasurementSchemaGroup*,
IDeviceIDComparator>
DeviceSchemasMap;
diff --git a/cpp/test/common/tsfile_common_test.cc
b/cpp/test/common/tsfile_common_test.cc
index 2108b2d02..5704afd19 100644
--- a/cpp/test/common/tsfile_common_test.cc
+++ b/cpp/test/common/tsfile_common_test.cc
@@ -449,19 +449,23 @@ TEST_F(TsFileMetaTest, SerializeDeserialize) {
table_name, column_schemas, column_categories);
meta_.table_schemas_.insert(std::make_pair(table_name, table_schema));
+ meta_.tsfile_properties_.insert(std::make_pair(
+ "key",
+ TsFilePropertyValue(std::vector<uint8_t>{'v', 'a', 'l', 'u', 'e'})));
meta_.tsfile_properties_.insert(
- std::make_pair("key", new std::string("value")));
- meta_.tsfile_properties_.insert(std::make_pair("null_key", nullptr));
+ std::make_pair("null_key", TsFilePropertyValue()));
meta_.meta_offset_ = 456;
void* buf = pa_.alloc(sizeof(BloomFilter));
meta_.bloom_filter_ = new (buf) BloomFilter();
meta_.bloom_filter_->init(0.1, 100);
- meta_.serialize_to(*out_);
+ int32_t serialized_size = 0;
+ ASSERT_EQ(common::E_OK, meta_.serialize_to(*out_, serialized_size));
+ ASSERT_EQ(serialized_size, out_->total_size());
TsFileMeta new_meta(&pa_);
- new_meta.deserialize_from(*out_);
+ ASSERT_EQ(common::E_OK, new_meta.deserialize_from(*out_));
ASSERT_EQ(new_meta.meta_offset_, 456);
ASSERT_EQ(new_meta.table_metadata_index_node_map_.size(), 1);
@@ -471,8 +475,70 @@ TEST_F(TsFileMetaTest, SerializeDeserialize) {
ASSERT_EQ(new_meta.table_schemas_.size(), 1);
ASSERT_EQ(
new_meta.table_schemas_[table_name]->get_column_categories().size(),
1);
- ASSERT_EQ(*new_meta.tsfile_properties_["key"], std::string("value"));
- ASSERT_EQ(new_meta.tsfile_properties_["null_key"], nullptr);
+ ASSERT_FALSE(new_meta.tsfile_properties_["key"].is_null);
+ ASSERT_EQ(new_meta.tsfile_properties_["key"].value,
+ (std::vector<uint8_t>{'v', 'a', 'l', 'u', 'e'}));
+ ASSERT_TRUE(new_meta.tsfile_properties_["null_key"].is_null);
+ ASSERT_TRUE(new_meta.tsfile_properties_["null_key"].value.empty());
+}
+
+TEST_F(TsFileMetaTest, DeserializesLegacyEmptyBloomFilterEncoding) {
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_i64(0, *out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1,
*out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_str("legacy", *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(3,
*out_));
+ ASSERT_EQ(common::E_OK, out_->write_buf("old", 3));
+
+ TsFileMeta meta(&pa_);
+ ASSERT_EQ(common::E_OK, meta.deserialize_from(*out_));
+ ASSERT_EQ(1U, meta.tsfile_properties_.size());
+ ASSERT_FALSE(meta.tsfile_properties_["legacy"].is_null);
+ EXPECT_EQ((std::vector<uint8_t>{'o', 'l', 'd'}),
+ meta.tsfile_properties_["legacy"].value);
+ EXPECT_EQ(0U, out_->remaining_size());
+}
+
+TEST_F(TsFileMetaTest, RejectsPropertyKeyLengthBeyondRemainingInput) {
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_i64(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_ui8(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1,
*out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_int(1024, *out_));
+
+ TsFileMeta meta(&pa_);
+ EXPECT_EQ(common::E_TSFILE_CORRUPTED, meta.deserialize_from(*out_));
+}
+
+TEST_F(TsFileMetaTest, RejectsPropertyValueLengthBeyondRemainingInput) {
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_i64(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_ui8(0, *out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1,
*out_));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_int(1,
*out_));
+ ASSERT_EQ(common::E_OK, out_->write_buf("k", 1));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_int(1024, *out_));
+
+ TsFileMeta meta(&pa_);
+ EXPECT_EQ(common::E_TSFILE_CORRUPTED, meta.deserialize_from(*out_));
}
// Regression: the default-compression configuration must name a compressor
diff --git a/cpp/test/cwrapper/cwrapper_properties_test.cc
b/cpp/test/cwrapper/cwrapper_properties_test.cc
new file mode 100644
index 000000000..7b2e43116
--- /dev/null
+++ b/cpp/test/cwrapper/cwrapper_properties_test.cc
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <cstdio>
+#include <cstring>
+#include <limits>
+#include <string>
+
+#include "cwrapper/tsfile_cwrapper.h"
+#include "utils/errno_define.h"
+
+namespace {
+
+const TsFileProperty* FindProperty(const TsFileProperty* properties,
+ uint32_t property_count,
+ const std::string& key) {
+ for (uint32_t i = 0; i < property_count; i++) {
+ if (properties[i].key_len == key.size() &&
+ std::memcmp(properties[i].key, key.data(), key.size()) == 0) {
+ return &properties[i];
+ }
+ }
+ return nullptr;
+}
+
+TEST(CWrapperPropertiesTest, GenericWriterRoundTripsLengthAwareValues) {
+ const char* file_name = "cwrapper_properties_test.tsfile";
+ std::remove(file_name);
+
+ ERRNO error_code = common::E_OK;
+ TsFileWriter writer =
+ _tsfile_writer_new(file_name, 128 * 1024 * 1024, &error_code);
+ ASSERT_NE(nullptr, writer);
+ ASSERT_EQ(common::E_OK, error_code);
+
+ const uint8_t first[] = {'f', 'i', 'r', 's', 't'};
+ const uint8_t binary[] = {0x00, 0xFF, 0x80, 0x01, 0x00};
+ const char embedded_null_key[] = {'k', '\0', 'y'};
+ const uint8_t empty_marker = 0;
+ EXPECT_EQ(common::E_INVALID_ARG,
+ _tsfile_writer_add_tsfile_property(nullptr, "key", 3, binary,
+ sizeof(binary)));
+ EXPECT_EQ(common::E_INVALID_ARG,
+ _tsfile_writer_add_tsfile_property(writer, nullptr, 0, binary,
+ sizeof(binary)));
+ EXPECT_EQ(common::E_INVALID_ARG,
+ _tsfile_writer_add_tsfile_property(writer, "key", 3, nullptr,
1));
+ const uint32_t oversized_len =
+ static_cast<uint32_t>(std::numeric_limits<int32_t>::max()) + 1U;
+ EXPECT_EQ(common::E_OUT_OF_RANGE,
+ _tsfile_writer_add_tsfile_property(writer, "key", 3, binary,
+ oversized_len));
+ ASSERT_EQ(common::E_OK,
+ _tsfile_writer_add_tsfile_property(writer, "overwritten", 11,
+ first, sizeof(first)));
+ ASSERT_EQ(common::E_OK, _tsfile_writer_flush(writer));
+ ASSERT_EQ(common::E_OK,
+ _tsfile_writer_add_tsfile_property(writer, "overwritten", 11,
+ binary, sizeof(binary)));
+ ASSERT_EQ(common::E_OK,
+ _tsfile_writer_add_tsfile_property(writer, embedded_null_key,
+ sizeof(embedded_null_key),
+ binary, sizeof(binary)));
+ ASSERT_EQ(common::E_OK, _tsfile_writer_add_tsfile_property(
+ writer, "empty", 5, &empty_marker, 0));
+ ASSERT_EQ(common::E_OK, _tsfile_writer_add_tsfile_property(writer, "null",
+ 4, nullptr, 0));
+ ASSERT_EQ(common::E_OK, _tsfile_writer_close(writer));
+
+ TsFileReader reader = tsfile_reader_new(file_name, &error_code);
+ ASSERT_NE(nullptr, reader);
+ ASSERT_EQ(common::E_OK, error_code);
+ TsFileProperty* properties = nullptr;
+ uint32_t property_count = 0;
+ ASSERT_EQ(common::E_OK, tsfile_reader_get_tsfile_properties(
+ reader, &properties, &property_count));
+
+ const TsFileProperty* overwritten =
+ FindProperty(properties, property_count, "overwritten");
+ ASSERT_NE(nullptr, overwritten);
+ EXPECT_FALSE(overwritten->is_null);
+ ASSERT_EQ(sizeof(binary), overwritten->value_len);
+ EXPECT_EQ(0, std::memcmp(binary, overwritten->value, sizeof(binary)));
+
+ const TsFileProperty* embedded_key_property =
+ FindProperty(properties, property_count,
+ std::string(embedded_null_key,
sizeof(embedded_null_key)));
+ ASSERT_NE(nullptr, embedded_key_property);
+ ASSERT_EQ(sizeof(binary), embedded_key_property->value_len);
+ EXPECT_EQ(
+ 0, std::memcmp(binary, embedded_key_property->value, sizeof(binary)));
+
+ const TsFileProperty* empty =
+ FindProperty(properties, property_count, "empty");
+ ASSERT_NE(nullptr, empty);
+ EXPECT_FALSE(empty->is_null);
+ EXPECT_EQ(0U, empty->value_len);
+
+ const TsFileProperty* null_value =
+ FindProperty(properties, property_count, "null");
+ ASSERT_NE(nullptr, null_value);
+ EXPECT_TRUE(null_value->is_null);
+ EXPECT_EQ(0U, null_value->value_len);
+
+ tsfile_free_tsfile_properties(properties, property_count);
+ TsFileProperty sentinel{};
+ properties = &sentinel;
+ property_count = 1;
+ EXPECT_EQ(common::E_INVALID_ARG,
+ tsfile_reader_get_tsfile_properties(nullptr, &properties,
+ &property_count));
+ EXPECT_EQ(nullptr, properties);
+ EXPECT_EQ(0U, property_count);
+ EXPECT_EQ(common::E_OK, tsfile_reader_close(reader));
+ EXPECT_EQ(0, std::remove(file_name));
+}
+
+TEST(CWrapperPropertiesTest, TableWriterSetterUsesExplicitLengths) {
+ const char* file_name = "cwrapper_table_properties_test.tsfile";
+ std::remove(file_name);
+
+ ERRNO error_code = common::E_OK;
+ WriteFile file = write_file_new(file_name, &error_code);
+ ASSERT_NE(nullptr, file);
+ ASSERT_EQ(common::E_OK, error_code);
+
+ ColumnSchema column = {const_cast<char*>("value"), TS_DATATYPE_INT64,
+ FIELD};
+ TableSchema schema = {const_cast<char*>("table"), &column, 1};
+ TsFileWriter writer = tsfile_writer_new(file, &schema, &error_code);
+ ASSERT_NE(nullptr, writer);
+ const uint8_t binary[] = {0xAA, 0x00, 0xBB};
+ const uint32_t oversized_len =
+ static_cast<uint32_t>(std::numeric_limits<int32_t>::max()) + 1U;
+ EXPECT_EQ(common::E_OUT_OF_RANGE,
+ tsfile_writer_add_tsfile_property(writer, "binary", 6, binary,
+ oversized_len));
+ ASSERT_EQ(common::E_OK, tsfile_writer_add_tsfile_property(
+ writer, "binary", 6, binary, sizeof(binary)));
+ ASSERT_EQ(common::E_OK, tsfile_writer_close(writer));
+ free_write_file(&file);
+
+ TsFileReader reader = tsfile_reader_new(file_name, &error_code);
+ ASSERT_NE(nullptr, reader);
+ TsFileProperty* properties = nullptr;
+ uint32_t property_count = 0;
+ ASSERT_EQ(common::E_OK, tsfile_reader_get_tsfile_properties(
+ reader, &properties, &property_count));
+ const TsFileProperty* property =
+ FindProperty(properties, property_count, "binary");
+ ASSERT_NE(nullptr, property);
+ ASSERT_EQ(sizeof(binary), property->value_len);
+ EXPECT_EQ(0, std::memcmp(binary, property->value, sizeof(binary)));
+ tsfile_free_tsfile_properties(properties, property_count);
+ EXPECT_EQ(common::E_OK, tsfile_reader_close(reader));
+ EXPECT_EQ(0, std::remove(file_name));
+}
+
+} // namespace
diff --git a/cpp/test/reader/bloom_filter_test.cc
b/cpp/test/reader/bloom_filter_test.cc
index 29b24db97..46a43e466 100644
--- a/cpp/test/reader/bloom_filter_test.cc
+++ b/cpp/test/reader/bloom_filter_test.cc
@@ -64,3 +64,52 @@ TEST(BloomfilterTest, BloomFilter) {
common::mem_free(filter_data_bytes);
common::mem_free(filter_data_bytes2);
}
+
+TEST(BloomfilterTest, EmptyFilterUsesJavaCompatibleEncoding) {
+ BloomFilter filter;
+ ASSERT_EQ(common::E_OK, filter.init(0.1, 0));
+
+ common::ByteStream out(1024, common::MOD_DEFAULT);
+ ASSERT_EQ(common::E_OK, filter.serialize_to(out));
+ ASSERT_EQ(1U, out.total_size());
+
+ BloomFilter deserialized;
+ ASSERT_EQ(common::E_OK, deserialized.deserialize_from(out));
+ EXPECT_TRUE(deserialized.is_empty());
+ EXPECT_EQ(0U, out.remaining_size());
+}
+
+TEST(BloomfilterTest, DeserializesLegacyEmptyFilterEncoding) {
+ common::ByteStream out(1024, common::MOD_DEFAULT);
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, out));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, out));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(0, out));
+
+ BloomFilter deserialized;
+ ASSERT_EQ(common::E_OK, deserialized.deserialize_from(out));
+ EXPECT_TRUE(deserialized.is_empty());
+ EXPECT_EQ(0U, out.remaining_size());
+}
+
+TEST(BloomfilterTest, RejectsInvalidHashFunctionCount) {
+ common::ByteStream out(1024, common::MOD_DEFAULT);
+ const uint8_t filter_byte = 1;
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(1, out));
+ ASSERT_EQ(common::E_OK, out.write_buf(&filter_byte, 1));
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(256, out));
+ ASSERT_EQ(common::E_OK, common::SerializationUtil::write_var_uint(
+ BloomFilter::MAX_HASH_FUNC_COUNT + 1, out));
+
+ BloomFilter filter;
+ EXPECT_EQ(common::E_TSFILE_CORRUPTED, filter.deserialize_from(out));
+}
+
+TEST(BloomfilterTest, RejectsFilterLengthBeyondRemainingInput) {
+ common::ByteStream out(1024, common::MOD_DEFAULT);
+ ASSERT_EQ(common::E_OK,
+ common::SerializationUtil::write_var_uint(1024, out));
+
+ BloomFilter filter;
+ EXPECT_EQ(common::E_TSFILE_CORRUPTED, filter.deserialize_from(out));
+}
diff --git a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
index 0dfaccc06..2dd9b5643 100644
--- a/cpp/test/writer/table_view/tsfile_writer_table_test.cc
+++ b/cpp/test/writer/table_view/tsfile_writer_table_test.cc
@@ -144,6 +144,30 @@ TEST_F(TsFileWriterTableTest, WriteTableTest) {
delete table_schema;
}
+TEST_F(TsFileWriterTableTest, AddTsFilePropertyDelegatesToWriter) {
+ auto table_schema = gen_table_schema(0);
+ TsFileTableWriter writer(&write_file_, table_schema);
+ const std::vector<uint8_t> before_flush = {'b', 'e', 'f', 'o', 'r', 'e'};
+ const std::vector<uint8_t> after_flush = {0x00, 0xFF, 0x01};
+
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("table-property", before_flush));
+ ASSERT_EQ(common::E_OK, writer.flush());
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("table-property", after_flush));
+ ASSERT_EQ(common::E_OK, writer.close());
+ ASSERT_EQ(common::E_FILE_WRITE_ERR,
+ writer.add_tsfile_property("closed", after_flush));
+
+ TsFileReader reader;
+ ASSERT_EQ(common::E_OK, reader.open(file_name_));
+ TsFileProperties properties = reader.get_tsfile_properties();
+ ASSERT_FALSE(properties.at("table-property").is_null);
+ EXPECT_EQ(after_flush, properties.at("table-property").value);
+ EXPECT_EQ(common::E_OK, reader.close());
+ delete table_schema;
+}
+
TEST_F(TsFileWriterTableTest, WithoutTagAndMultiPage) {
std::vector<MeasurementSchema*> measurement_schemas;
std::vector<ColumnCategory> column_categories;
diff --git a/cpp/test/writer/tsfile_properties_test.cc
b/cpp/test/writer/tsfile_properties_test.cc
new file mode 100644
index 000000000..4590c1bcd
--- /dev/null
+++ b/cpp/test/writer/tsfile_properties_test.cc
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <cstdio>
+#include <vector>
+
+#include "common/global.h"
+#include "reader/tsfile_reader.h"
+#include "writer/tsfile_writer.h"
+
+namespace storage {
+
+namespace {
+
+std::vector<uint8_t> Bytes(const std::string& value) {
+ return std::vector<uint8_t>(value.begin(), value.end());
+}
+
+class TsFilePropertiesTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ libtsfile_init();
+ file_name_ = "tsfile_properties_test.tsfile";
+ std::remove(file_name_.c_str());
+ }
+
+ void TearDown() override {
+ std::remove(file_name_.c_str());
+ libtsfile_destroy();
+ }
+
+ std::string file_name_;
+};
+
+TEST_F(TsFilePropertiesTest, WriterPreservesBinaryNullAndEmptyValues) {
+ TsFileWriter writer;
+ ASSERT_EQ(common::E_OK, writer.open(file_name_));
+
+ const std::vector<uint8_t> first_value = {'f', 'i', 'r', 's', 't'};
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("overwritten", first_value));
+ ASSERT_EQ(common::E_OK, writer.flush());
+
+ const std::vector<uint8_t> binary_value = {0x00, 0x7F, 0x80, 0xFF, 0x00};
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("overwritten", binary_value));
+ std::vector<uint8_t> copied_value = {0x10, 0x20, 0x30};
+ ASSERT_EQ(common::E_OK, writer.add_tsfile_property("copied",
copied_value));
+ copied_value[0] = 0xFF;
+ const std::string embedded_null_key("embedded\0key", 12);
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property(embedded_null_key, binary_value));
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("empty", std::vector<uint8_t>()));
+ ASSERT_EQ(common::E_OK, writer.add_tsfile_property("null", nullptr, 0));
+ ASSERT_EQ(common::E_INVALID_ARG,
+ writer.add_tsfile_property("invalid", nullptr, 1));
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("encryptLevel", Bytes("custom")));
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("encryptType", Bytes("custom")));
+ ASSERT_EQ(common::E_OK,
+ writer.add_tsfile_property("encryptKey", Bytes("custom")));
+ ASSERT_EQ(common::E_OK, writer.close());
+ ASSERT_EQ(common::E_FILE_WRITE_ERR,
+ writer.add_tsfile_property("closed", binary_value));
+
+ TsFileReader reader;
+ ASSERT_EQ(common::E_OK, reader.open(file_name_));
+ TsFileProperties properties = reader.get_tsfile_properties();
+
+ ASSERT_FALSE(properties.at("overwritten").is_null);
+ EXPECT_EQ(binary_value, properties.at("overwritten").value);
+ ASSERT_FALSE(properties.at("empty").is_null);
+ EXPECT_TRUE(properties.at("empty").value.empty());
+ EXPECT_EQ((std::vector<uint8_t>{0x10, 0x20, 0x30}),
+ properties.at("copied").value);
+ EXPECT_EQ(binary_value, properties.at(embedded_null_key).value);
+ EXPECT_TRUE(properties.at("null").is_null);
+ EXPECT_TRUE(properties.at("null").value.empty());
+
+ ASSERT_FALSE(properties.at("encryptLevel").is_null);
+ EXPECT_EQ(Bytes("0"), properties.at("encryptLevel").value);
+ ASSERT_FALSE(properties.at("encryptType").is_null);
+ EXPECT_EQ(Bytes("org.apache.tsfile.encrypt.UNENCRYPTED"),
+ properties.at("encryptType").value);
+ EXPECT_TRUE(properties.at("encryptKey").is_null);
+ EXPECT_TRUE(properties.at("encryptKey").value.empty());
+ EXPECT_EQ(common::E_OK, reader.close());
+}
+
+} // namespace
+
+} // namespace storage
diff --git a/python/README-zh.md b/python/README-zh.md
index 660c001e8..dd3a59ea4 100644
--- a/python/README-zh.md
+++ b/python/README-zh.md
@@ -66,4 +66,20 @@ mvn -P with-cpp,with-python clean verify
```sh
python setup.py build_ext --inplace
-```
\ No newline at end of file
+```
+
+## 文件级 Properties
+
+`TsFileWriter` 和 `TsFileTableWriter` 可以在打开期间写入二进制 property。
+setter 仅接受 `bytes`。reader 返回 `dict[str, bytes | None]`，并区分 null 与
+零长度 bytes。
+
+```python
+with TsFileWriter("example.tsfile") as writer:
+ writer.add_tsfile_property("binary-property", b"\x01\x00\xff")
+
+with TsFileReader("example.tsfile") as reader:
+ properties = reader.get_tsfile_properties()
+```
+
+Property value 不携带数据类型；保存数字或结构体时应使用明确、可跨语言的字节编码。
diff --git a/python/README.md b/python/README.md
index 51cb498ec..8e2716a2c 100644
--- a/python/README.md
+++ b/python/README.md
@@ -61,3 +61,19 @@ Build by python command:
python setup.py build_ext --inplace
```
+## File-level properties
+
+`TsFileWriter` and `TsFileTableWriter` accept binary properties while they are
+open. The setter accepts `bytes` only. Readers return `dict[str, bytes |
None]`,
+preserving null and empty values separately.
+
+```python
+with TsFileWriter("example.tsfile") as writer:
+ writer.add_tsfile_property("binary-property", b"\x01\x00\xff")
+
+with TsFileReader("example.tsfile") as reader:
+ properties = reader.get_tsfile_properties()
+```
+
+Values do not carry a data type; use an explicit portable encoding when storing
+numbers or structures.
diff --git a/python/tests/test_tsfile_properties.py
b/python/tests/test_tsfile_properties.py
new file mode 100644
index 000000000..de2d8c88f
--- /dev/null
+++ b/python/tests/test_tsfile_properties.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import pytest
+
+from tsfile import (
+ ColumnCategory,
+ ColumnSchema,
+ FileWriteError,
+ TableSchema,
+ TSDataType,
+ TsFileCorruptedError,
+ TsFileReader,
+ TsFileTableWriter,
+ TsFileWriter,
+)
+
+
+def test_tsfile_writer_properties_round_trip(tmp_path):
+ path = os.fspath(tmp_path / "writer-properties.tsfile")
+ writer = TsFileWriter(path)
+ writer.add_tsfile_property("overwritten", b"first")
+ writer.flush()
+ writer.add_tsfile_property("overwritten", b"\x00\xff\x80\x00")
+ writer.add_tsfile_property("empty", b"")
+ writer.add_tsfile_property("embedded\x00key", b"binary-key")
+
+ with pytest.raises(TypeError):
+ writer.add_tsfile_property("text", "not-bytes")
+ with pytest.raises(TypeError):
+ writer.add_tsfile_property("bytearray", bytearray(b"not-bytes"))
+ with pytest.raises(TypeError):
+ writer.add_tsfile_property("bytes-subclass", type("B", (bytes,),
{})(b"value"))
+
+ writer.close()
+ with pytest.raises(FileWriteError):
+ writer.add_tsfile_property("closed", b"value")
+
+ with TsFileReader(path) as reader:
+ properties = reader.get_tsfile_properties()
+ assert properties["overwritten"] == b"\x00\xff\x80\x00"
+ assert properties["empty"] == b""
+ assert properties["embedded\x00key"] == b"binary-key"
+ assert properties["encryptLevel"] == b"0"
+ assert properties["encryptType"] ==
b"org.apache.tsfile.encrypt.UNENCRYPTED"
+ assert properties["encryptKey"] is None
+
+
+def test_tsfile_table_writer_property_delegation(tmp_path):
+ path = os.fspath(tmp_path / "table-writer-properties.tsfile")
+ schema = TableSchema(
+ "table",
+ [ColumnSchema("value", TSDataType.INT64, ColumnCategory.FIELD)],
+ )
+ writer = TsFileTableWriter(path, schema)
+ writer.add_tsfile_property("table-property", b"before")
+ writer.flush()
+ writer.add_tsfile_property("table-property", b"after\x00\xff")
+ writer.close()
+ with pytest.raises(FileWriteError):
+ writer.add_tsfile_property("closed", b"value")
+
+ with TsFileReader(path) as reader:
+ assert reader.get_tsfile_properties()["table-property"] ==
b"after\x00\xff"
+
+
+def test_reader_reports_invalid_utf8_property_key(tmp_path):
+ path = tmp_path / "invalid-property-key.tsfile"
+ writer = TsFileWriter(os.fspath(path))
+ writer.add_tsfile_property("invalid-key", b"value")
+ writer.close()
+
+ file_bytes = path.read_bytes()
+ assert file_bytes.count(b"invalid-key") == 1
+ path.write_bytes(file_bytes.replace(b"invalid-key", b"invalid-\xffey", 1))
+
+ with TsFileReader(os.fspath(path)) as reader:
+ with pytest.raises(
+ TsFileCorruptedError,
+ match="TsFile property key is not valid UTF-8",
+ ):
+ reader.get_tsfile_properties()
diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd
index cc14f4034..4daf0b153 100644
--- a/python/tsfile/tsfile_cpp.pxd
+++ b/python/tsfile/tsfile_cpp.pxd
@@ -25,6 +25,7 @@ cdef extern from "cwrapper/errno_define_c.h":
enum:
RET_OK
RET_NO_MORE_DATA
+ RET_FILE_WRITE_ERR
# import symbols from tsfile_cwrapper.h
cdef extern from "cwrapper/tsfile_cwrapper.h":
@@ -181,6 +182,13 @@ cdef extern from "cwrapper/tsfile_cwrapper.h":
DeviceTimeseriesMetadataEntry * entries
uint32_t device_count
+ ctypedef struct TsFileProperty:
+ char * key
+ uint32_t key_len
+ uint8_t * value
+ uint32_t value_len
+ bint is_null
+
ctypedef struct ResultSetMetaData:
char** column_names
TSDataType * data_types
@@ -201,6 +209,9 @@ cdef extern from "cwrapper/tsfile_cwrapper.h":
# writer : flush
ErrorCode _tsfile_writer_flush(TsFileWriter writer);
+ ErrorCode _tsfile_writer_add_tsfile_property(
+ TsFileWriter writer, const char * key, uint32_t key_len,
+ const uint8_t * value, uint32_t value_len);
# writer : register table, device and timeseries
ErrorCode _tsfile_writer_register_table(TsFileWriter writer, TableSchema *
schema);
@@ -316,6 +327,11 @@ cdef extern from "cwrapper/tsfile_cwrapper.h":
DeviceTimeseriesMetadataMap * out_map);
void tsfile_free_device_timeseries_metadata_map(
DeviceTimeseriesMetadataMap * map);
+ ErrorCode tsfile_reader_get_tsfile_properties(
+ TsFileReader reader, TsFileProperty ** out_properties,
+ uint32_t * out_length);
+ void tsfile_free_tsfile_properties(TsFileProperty * properties,
+ uint32_t length);
# Tag filter types and functions
diff --git a/python/tsfile/tsfile_reader.pyx b/python/tsfile/tsfile_reader.pyx
index 36374adde..a2e8fe263 100644
--- a/python/tsfile/tsfile_reader.pyx
+++ b/python/tsfile/tsfile_reader.pyx
@@ -26,10 +26,11 @@ from libc.string cimport strlen
from cpython.bytes cimport PyBytes_FromStringAndSize
from libc.string cimport memset
import pyarrow as pa
-from libc.stdint cimport INT64_MIN, INT64_MAX, uintptr_t
+from libc.stdint cimport INT64_MIN, INT64_MAX, uint32_t, uintptr_t
from tsfile.schema import TSDataType as TSDataTypePy
from tsfile.schema import DeviceID, DeviceTimeseriesMetadataGroup
+from tsfile.exceptions import TsFileCorruptedError
from tsfile.tag_filter import ComparisonTagFilter, BetweenTagFilter,
AndTagFilter, OrTagFilter, NotTagFilter
from .date_utils import parse_int_to_date
from .tsfile_cpp cimport *
@@ -518,6 +519,45 @@ cdef class TsFileReaderPy:
"""
return reader_get_timeseries_metadata_c(self.reader, device_ids)
+ def get_tsfile_properties(self) -> Dict[str, Optional[bytes]]:
+ """
+ Return file-level properties as ``dict[str, bytes | None]``.
+
+ Null property values are returned as ``None`` and remain distinct from
+ non-null zero-length byte strings.
+ """
+ cdef TsFileProperty * properties = NULL
+ cdef uint32_t property_count = 0
+ cdef uint32_t i
+ cdef ErrorCode err_code
+ cdef object key
+ cdef dict result = {}
+
+ err_code = tsfile_reader_get_tsfile_properties(
+ self.reader, &properties, &property_count
+ )
+ check_error(err_code)
+ try:
+ for i in range(property_count):
+ try:
+ key = PyBytes_FromStringAndSize(
+ properties[i].key, properties[i].key_len
+ ).decode('utf-8')
+ except UnicodeDecodeError:
+ raise TsFileCorruptedError(
+ context="TsFile property key is not valid UTF-8"
+ ) from None
+ if properties[i].is_null:
+ result[key] = None
+ else:
+ result[key] = PyBytes_FromStringAndSize(
+ <const char *> properties[i].value,
+ properties[i].value_len,
+ )
+ finally:
+ tsfile_free_tsfile_properties(properties, property_count)
+ return result
+
def close(self):
"""
Close TsFile Reader, if reader has result sets, invalid them.
diff --git a/python/tsfile/tsfile_table_writer.py
b/python/tsfile/tsfile_table_writer.py
index 9f3a257e6..e4fceef97 100644
--- a/python/tsfile/tsfile_table_writer.py
+++ b/python/tsfile/tsfile_table_writer.py
@@ -232,6 +232,12 @@ class TsFileTableWriter:
"""
self.writer.close()
+ def add_tsfile_property(self, key: str, value: bytes):
+ """
+ Add or replace a binary file-level property while the writer is open.
+ """
+ self.writer.add_tsfile_property(key, value)
+
def flush(self):
"""
Flush current data to tsfile.
diff --git a/python/tsfile/tsfile_writer.pyx b/python/tsfile/tsfile_writer.pyx
index 9e84d83c0..4df03b449 100644
--- a/python/tsfile/tsfile_writer.pyx
+++ b/python/tsfile/tsfile_writer.pyx
@@ -23,7 +23,8 @@ from tsfile.schema import TableSchema as TableSchemaPy
from tsfile.schema import TimeseriesSchema as TimeseriesSchemaPy, DeviceSchema
as DeviceSchemaPy
from tsfile.tablet import Tablet as TabletPy
from libc.string cimport memset
-from libc.stdint cimport uintptr_t
+from libc.stdint cimport uint32_t, uint8_t, uintptr_t
+from cpython.bytes cimport PyBytes_AsStringAndSize
from .tsfile_cpp cimport *
from .tsfile_py_cpp cimport *
@@ -163,6 +164,40 @@ cdef class TsFileWriterPy:
if arrow_schema.release != NULL:
arrow_schema.release(&arrow_schema)
+ def add_tsfile_property(self, key: str, value: bytes):
+ """
+ Add or replace a binary file-level property while the writer is open.
+
+ ``value`` must be ``bytes``. The data is copied immediately, and a
+ later call with the same key replaces the previous value.
+ """
+ if not isinstance(key, str):
+ raise TypeError("TsFile property key must be str")
+ if type(value) is not bytes:
+ raise TypeError("TsFile property value must be bytes")
+ if self.writer == NULL:
+ check_error(RET_FILE_WRITE_ERR, b"TsFile writer is closed")
+
+ cdef bytes encoded_key = key.encode('utf-8')
+ cdef char * value_ptr = NULL
+ cdef Py_ssize_t value_len = 0
+ cdef ErrorCode errno
+ if len(encoded_key) > 0x7FFFFFFF:
+ raise OverflowError("TsFile property key is too large")
+ if PyBytes_AsStringAndSize(value, &value_ptr, &value_len) < 0:
+ raise TypeError("TsFile property value must be bytes")
+ if value_len > 0x7FFFFFFF:
+ raise OverflowError("TsFile property value is too large")
+
+ errno = _tsfile_writer_add_tsfile_property(
+ self.writer,
+ <const char *> encoded_key,
+ <uint32_t> len(encoded_key),
+ <const uint8_t *> value_ptr,
+ <uint32_t> value_len,
+ )
+ check_error(errno)
+
cpdef close(self):
"""
Flush data and Close tsfile writer.
