This is an automated email from the ASF dual-hosted git repository.
hongzhigao pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new af35aeec Feat/c python timeseries metadata (#767)
af35aeec is described below
commit af35aeeca40a0b818562b1caa4dfcb2ca752ab9d
Author: Hongzhi Gao <[email protected]>
AuthorDate: Tue Apr 7 14:49:33 2026 +0800
Feat/c python timeseries metadata (#767)
* c and python get_metadata interface wrapper
* python c metadata interface
* spotless apply
* fix DeviceTimeseriesMetadataEntry details
* replace DeviceID to TsDeviceDetails
* fix c/python statistic
* mvn spotless:apply
---
cpp/src/cwrapper/tsfile_cwrapper.cc | 567 ++++++++++++++++++++++++++++
cpp/src/cwrapper/tsfile_cwrapper.h | 159 ++++++++
cpp/test/cwrapper/cwrapper_metadata_test.cc | 290 ++++++++++++++
python/tests/test_reader_metadata.py | 212 +++++++++++
python/tsfile/schema.py | 100 ++++-
python/tsfile/tsfile_cpp.pxd | 88 +++++
python/tsfile/tsfile_py_cpp.pxd | 3 +
python/tsfile/tsfile_py_cpp.pyx | 202 ++++++++++
python/tsfile/tsfile_reader.pyx | 22 +-
9 files changed, 1641 insertions(+), 2 deletions(-)
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.cc
b/cpp/src/cwrapper/tsfile_cwrapper.cc
index 8cf7b622..e6ecef2a 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.cc
+++ b/cpp/src/cwrapper/tsfile_cwrapper.cc
@@ -26,8 +26,12 @@
#include <cstring>
#include <set>
+#include <vector>
+#include "common/device_id.h"
+#include "common/statistic.h"
#include "common/tablet.h"
+#include "common/tsfile_common.h"
#include "reader/result_set.h"
#include "reader/table_result_set.h"
#include "reader/tsfile_reader.h"
@@ -695,6 +699,569 @@ DeviceSchema*
tsfile_reader_get_all_timeseries_schemas(TsFileReader reader,
return device_schema;
}
+void tsfile_device_id_free_contents(DeviceID* d) {
+ if (d == nullptr) {
+ return;
+ }
+ free(d->path);
+ d->path = nullptr;
+ free(d->table_name);
+ d->table_name = nullptr;
+ if (d->segments != nullptr) {
+ for (uint32_t k = 0; k < d->segment_count; k++) {
+ free(d->segments[k]);
+ }
+ free(d->segments);
+ d->segments = nullptr;
+ }
+ d->segment_count = 0;
+}
+
+namespace {
+
+char* dup_common_string_to_cstr(const common::String& s) {
+ if (s.buf_ == nullptr || s.len_ == 0) {
+ return strdup("");
+ }
+ char* p = static_cast<char*>(malloc(static_cast<size_t>(s.len_) + 1U));
+ if (p == nullptr) {
+ return nullptr;
+ }
+ memcpy(p, s.buf_, static_cast<size_t>(s.len_));
+ p[s.len_] = '\0';
+ return p;
+}
+
+static TSDataType cpp_stat_type_to_c(common::TSDataType t) {
+ return static_cast<TSDataType>(static_cast<uint8_t>(t));
+}
+
+void free_timeseries_statistic_heap(TimeseriesStatistic* s) {
+ if (s == nullptr) {
+ return;
+ }
+ TsFileStatisticBase* b = tsfile_statistic_base(s);
+ if (!b->has_statistic) {
+ return;
+ }
+ switch (b->type) {
+ case TS_DATATYPE_STRING:
+ free(s->u.string_s.str_min);
+ s->u.string_s.str_min = nullptr;
+ free(s->u.string_s.str_max);
+ s->u.string_s.str_max = nullptr;
+ free(s->u.string_s.str_first);
+ s->u.string_s.str_first = nullptr;
+ free(s->u.string_s.str_last);
+ s->u.string_s.str_last = nullptr;
+ break;
+ case TS_DATATYPE_TEXT:
+ free(s->u.text_s.str_first);
+ s->u.text_s.str_first = nullptr;
+ free(s->u.text_s.str_last);
+ s->u.text_s.str_last = nullptr;
+ break;
+ default:
+ break;
+ }
+}
+
+void clear_timeseries_statistic(TimeseriesStatistic* s) {
+ memset(s, 0, sizeof(*s));
+ tsfile_statistic_base(s)->type = TS_DATATYPE_INVALID;
+}
+
+/**
+ * Fills @p out from C++ Statistic. On allocation failure returns E_OOM and
+ * clears/frees any partial string fields in @p out.
+ */
+int fill_timeseries_statistic(storage::Statistic* st,
+ TimeseriesStatistic* out) {
+ clear_timeseries_statistic(out);
+ if (st == nullptr) {
+ return common::E_OK;
+ }
+ const common::TSDataType t = st->get_type();
+ switch (t) {
+ case common::BOOLEAN: {
+ auto* bs = static_cast<storage::BooleanStatistic*>(st);
+ TsFileBoolStatistic* p = &out->u.bool_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::BOOLEAN);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->sum = static_cast<double>(bs->sum_value_);
+ p->first_bool = bs->first_value_;
+ p->last_bool = bs->last_value_;
+ break;
+ }
+ case common::INT32: {
+ auto* is = static_cast<storage::Int32Statistic*>(st);
+ TsFileIntStatistic* p = &out->u.int_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::INT32);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->sum = static_cast<double>(is->sum_value_);
+ if (p->base.row_count > 0) {
+ p->min_int64 = static_cast<int64_t>(is->min_value_);
+ p->max_int64 = static_cast<int64_t>(is->max_value_);
+ p->first_int64 = static_cast<int64_t>(is->first_value_);
+ p->last_int64 = static_cast<int64_t>(is->last_value_);
+ }
+ break;
+ }
+ case common::DATE: {
+ auto* is = static_cast<storage::Int32Statistic*>(st);
+ TsFileIntStatistic* p = &out->u.int_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::DATE);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->sum = static_cast<double>(is->sum_value_);
+ if (p->base.row_count > 0) {
+ p->min_int64 = static_cast<int64_t>(is->min_value_);
+ p->max_int64 = static_cast<int64_t>(is->max_value_);
+ p->first_int64 = static_cast<int64_t>(is->first_value_);
+ p->last_int64 = static_cast<int64_t>(is->last_value_);
+ }
+ break;
+ }
+ case common::INT64: {
+ auto* ls = static_cast<storage::Int64Statistic*>(st);
+ TsFileIntStatistic* p = &out->u.int_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::INT64);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->sum = ls->sum_value_;
+ if (p->base.row_count > 0) {
+ p->min_int64 = ls->min_value_;
+ p->max_int64 = ls->max_value_;
+ p->first_int64 = ls->first_value_;
+ p->last_int64 = ls->last_value_;
+ }
+ break;
+ }
+ case common::TIMESTAMP: {
+ auto* ls = static_cast<storage::Int64Statistic*>(st);
+ TsFileIntStatistic* p = &out->u.int_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::TIMESTAMP);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->sum = ls->sum_value_;
+ if (p->base.row_count > 0) {
+ p->min_int64 = ls->min_value_;
+ p->max_int64 = ls->max_value_;
+ p->first_int64 = ls->first_value_;
+ p->last_int64 = ls->last_value_;
+ }
+ break;
+ }
+ case common::FLOAT: {
+ auto* fs = static_cast<storage::FloatStatistic*>(st);
+ TsFileFloatStatistic* p = &out->u.float_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::FLOAT);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->sum = static_cast<double>(fs->sum_value_);
+ if (p->base.row_count > 0) {
+ p->min_float64 = static_cast<double>(fs->min_value_);
+ p->max_float64 = static_cast<double>(fs->max_value_);
+ p->first_float64 = static_cast<double>(fs->first_value_);
+ p->last_float64 = static_cast<double>(fs->last_value_);
+ }
+ break;
+ }
+ case common::DOUBLE: {
+ auto* ds = static_cast<storage::DoubleStatistic*>(st);
+ TsFileFloatStatistic* p = &out->u.float_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::DOUBLE);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->sum = ds->sum_value_;
+ if (p->base.row_count > 0) {
+ p->min_float64 = ds->min_value_;
+ p->max_float64 = ds->max_value_;
+ p->first_float64 = ds->first_value_;
+ p->last_float64 = ds->last_value_;
+ }
+ break;
+ }
+ case common::STRING: {
+ auto* ss = static_cast<storage::StringStatistic*>(st);
+ TsFileStringStatistic* p = &out->u.string_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::STRING);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->str_min = dup_common_string_to_cstr(ss->min_value_);
+ if (p->str_min == nullptr) {
+ free_timeseries_statistic_heap(out);
+ clear_timeseries_statistic(out);
+ return common::E_OOM;
+ }
+ p->str_max = dup_common_string_to_cstr(ss->max_value_);
+ if (p->str_max == nullptr) {
+ free_timeseries_statistic_heap(out);
+ clear_timeseries_statistic(out);
+ return common::E_OOM;
+ }
+ p->str_first = dup_common_string_to_cstr(ss->first_value_);
+ if (p->str_first == nullptr) {
+ free_timeseries_statistic_heap(out);
+ clear_timeseries_statistic(out);
+ return common::E_OOM;
+ }
+ p->str_last = dup_common_string_to_cstr(ss->last_value_);
+ if (p->str_last == nullptr) {
+ free_timeseries_statistic_heap(out);
+ clear_timeseries_statistic(out);
+ return common::E_OOM;
+ }
+ break;
+ }
+ case common::TEXT: {
+ auto* ts = static_cast<storage::TextStatistic*>(st);
+ TsFileTextStatistic* p = &out->u.text_s;
+ p->base.has_statistic = true;
+ p->base.type = cpp_stat_type_to_c(common::TEXT);
+ p->base.row_count = st->get_count();
+ p->base.start_time = st->start_time_;
+ p->base.end_time = st->get_end_time();
+ p->str_first = dup_common_string_to_cstr(ts->first_value_);
+ if (p->str_first == nullptr) {
+ free_timeseries_statistic_heap(out);
+ clear_timeseries_statistic(out);
+ return common::E_OOM;
+ }
+ p->str_last = dup_common_string_to_cstr(ts->last_value_);
+ if (p->str_last == nullptr) {
+ free_timeseries_statistic_heap(out);
+ clear_timeseries_statistic(out);
+ return common::E_OOM;
+ }
+ break;
+ }
+ default: {
+ TsFileStatisticBase* b = tsfile_statistic_base(out);
+ b->has_statistic = true;
+ b->type = TS_DATATYPE_INVALID;
+ b->row_count = st->get_count();
+ b->start_time = st->start_time_;
+ b->end_time = st->get_end_time();
+ break;
+ }
+ }
+ return common::E_OK;
+}
+
+void free_device_timeseries_metadata_entries_partial(
+ DeviceTimeseriesMetadataEntry* entries, size_t filled_count) {
+ if (entries == nullptr) {
+ return;
+ }
+ for (size_t i = 0; i < filled_count; i++) {
+ tsfile_device_id_free_contents(&entries[i].device);
+ if (entries[i].timeseries != nullptr) {
+ for (uint32_t j = 0; j < entries[i].timeseries_count; j++) {
+ free_timeseries_statistic_heap(
+ &entries[i].timeseries[j].statistic);
+ free(entries[i].timeseries[j].measurement_name);
+ }
+ free(entries[i].timeseries);
+ entries[i].timeseries = nullptr;
+ }
+ }
+ free(entries);
+}
+
+/**
+ * Copies path, table name, and segment strings from IDeviceID into heap
+ * buffers. On failure, frees any partial allocations and returns E_OOM.
+ */
+int duplicate_ideviceid_to_device_fields(storage::IDeviceID* id,
+ char** out_path, char**
out_table_name,
+ uint32_t* out_segment_count,
+ char*** out_segments) {
+ *out_path = nullptr;
+ *out_table_name = nullptr;
+ *out_segment_count = 0;
+ *out_segments = nullptr;
+ if (id == nullptr) {
+ *out_path = strdup("");
+ *out_table_name = strdup("");
+ if (*out_path == nullptr || *out_table_name == nullptr) {
+ free(*out_path);
+ free(*out_table_name);
+ *out_path = nullptr;
+ *out_table_name = nullptr;
+ return common::E_OOM;
+ }
+ return common::E_OK;
+ }
+ const std::string dname = id->get_device_name();
+ *out_path = strdup(dname.c_str());
+ if (*out_path == nullptr) {
+ return common::E_OOM;
+ }
+ const std::string tname = id->get_table_name();
+ *out_table_name = strdup(tname.c_str());
+ if (*out_table_name == nullptr) {
+ free(*out_path);
+ *out_path = nullptr;
+ return common::E_OOM;
+ }
+ const int n = id->segment_num();
+ if (n <= 0) {
+ return common::E_OK;
+ }
+ auto* seg_arr =
+ static_cast<char**>(malloc(sizeof(char*) * static_cast<size_t>(n)));
+ if (seg_arr == nullptr) {
+ free(*out_table_name);
+ *out_table_name = nullptr;
+ free(*out_path);
+ *out_path = nullptr;
+ return common::E_OOM;
+ }
+ memset(seg_arr, 0, sizeof(char*) * static_cast<size_t>(n));
+ const auto& segs = id->get_segments();
+ for (int i = 0; i < n; i++) {
+ const std::string* ps =
+ (static_cast<size_t>(i) < segs.size()) ? segs[i] : nullptr;
+ const char* lit = (ps != nullptr) ? ps->c_str() : "null";
+ seg_arr[i] = strdup(lit);
+ if (seg_arr[i] == nullptr) {
+ for (int j = 0; j < i; j++) {
+ free(seg_arr[j]);
+ }
+ free(seg_arr);
+ free(*out_table_name);
+ *out_table_name = nullptr;
+ free(*out_path);
+ *out_path = nullptr;
+ return common::E_OOM;
+ }
+ }
+ *out_segment_count = static_cast<uint32_t>(n);
+ *out_segments = seg_arr;
+ return common::E_OK;
+}
+
+int fill_device_id_from_ideviceid(storage::IDeviceID* id, DeviceID* out) {
+ memset(out, 0, sizeof(*out));
+ return duplicate_ideviceid_to_device_fields(
+ id, &out->path, &out->table_name, &out->segment_count, &out->segments);
+}
+
+void clear_metadata_entry_device_only(DeviceTimeseriesMetadataEntry* e) {
+ if (e == nullptr) {
+ return;
+ }
+ tsfile_device_id_free_contents(&e->device);
+}
+
+ERRNO populate_c_metadata_map_from_cpp(
+ storage::DeviceTimeseriesMetadataMap& cpp_map,
+ DeviceTimeseriesMetadataMap* out_map) {
+ if (cpp_map.empty()) {
+ return common::E_OK;
+ }
+ const uint32_t dev_n = static_cast<uint32_t>(cpp_map.size());
+ auto* entries = static_cast<DeviceTimeseriesMetadataEntry*>(
+ malloc(sizeof(DeviceTimeseriesMetadataEntry) * dev_n));
+ if (entries == nullptr) {
+ return common::E_OOM;
+ }
+ memset(entries, 0, sizeof(DeviceTimeseriesMetadataEntry) * dev_n);
+ size_t di = 0;
+ for (const auto& kv : cpp_map) {
+ DeviceTimeseriesMetadataEntry& e = entries[di];
+ const int dup_rc = fill_device_id_from_ideviceid(
+ kv.first ? kv.first.get() : nullptr, &e.device);
+ if (dup_rc != common::E_OK) {
+ free_device_timeseries_metadata_entries_partial(entries, di);
+ return dup_rc;
+ }
+ const auto& vec = kv.second;
+ uint32_t n_ts = 0;
+ for (const auto& idx_nz : vec) {
+ if (idx_nz != nullptr) {
+ n_ts++;
+ }
+ }
+ e.timeseries_count = n_ts;
+ if (e.timeseries_count == 0) {
+ e.timeseries = nullptr;
+ di++;
+ continue;
+ }
+ e.timeseries = static_cast<TimeseriesMetadata*>(
+ malloc(sizeof(TimeseriesMetadata) * e.timeseries_count));
+ if (e.timeseries == nullptr) {
+ clear_metadata_entry_device_only(&e);
+ free_device_timeseries_metadata_entries_partial(entries, di);
+ return common::E_OOM;
+ }
+ memset(e.timeseries, 0,
+ sizeof(TimeseriesMetadata) * e.timeseries_count);
+ uint32_t slot = 0;
+ for (const auto& idx : vec) {
+ if (idx == nullptr) {
+ continue;
+ }
+ TimeseriesMetadata& m = e.timeseries[slot];
+ common::String mn = idx->get_measurement_name();
+ m.measurement_name = strdup(mn.to_std_string().c_str());
+ if (m.measurement_name == nullptr) {
+ for (uint32_t u = 0; u < slot; u++) {
+ free_timeseries_statistic_heap(&e.timeseries[u].statistic);
+ free(e.timeseries[u].measurement_name);
+ }
+ free(e.timeseries);
+ e.timeseries = nullptr;
+ clear_metadata_entry_device_only(&e);
+ free_device_timeseries_metadata_entries_partial(entries, di);
+ return common::E_OOM;
+ }
+ m.data_type = static_cast<TSDataType>(idx->get_data_type());
+ storage::Statistic* st = idx->get_statistic();
+ int32_t chunk_cnt = 0;
+ auto* cl = idx->get_chunk_meta_list();
+ if (cl != nullptr) {
+ chunk_cnt = static_cast<int32_t>(cl->size());
+ }
+ m.chunk_meta_count = chunk_cnt;
+ const int st_rc = fill_timeseries_statistic(st, &m.statistic);
+ if (st_rc != common::E_OK) {
+ for (uint32_t u = 0; u < slot; u++) {
+ free_timeseries_statistic_heap(&e.timeseries[u].statistic);
+ free(e.timeseries[u].measurement_name);
+ }
+ free_timeseries_statistic_heap(&m.statistic);
+ free(m.measurement_name);
+ free(e.timeseries);
+ e.timeseries = nullptr;
+ clear_metadata_entry_device_only(&e);
+ free_device_timeseries_metadata_entries_partial(entries, di);
+ return st_rc;
+ }
+ slot++;
+ }
+ di++;
+ }
+ out_map->entries = entries;
+ out_map->device_count = dev_n;
+ return common::E_OK;
+}
+
+} // namespace
+
+void tsfile_free_device_id_array(DeviceID* devices, uint32_t length) {
+ if (devices == nullptr) {
+ return;
+ }
+ for (uint32_t i = 0; i < length; i++) {
+ tsfile_device_id_free_contents(&devices[i]);
+ }
+ free(devices);
+}
+
+ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID**
out_devices,
+ uint32_t* out_length) {
+ if (reader == nullptr || out_devices == nullptr || out_length == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+ *out_devices = nullptr;
+ *out_length = 0;
+ auto* r = static_cast<storage::TsFileReader*>(reader);
+ const auto ids = r->get_all_devices();
+ if (ids.empty()) {
+ return common::E_OK;
+ }
+ auto* arr = static_cast<DeviceID*>(malloc(sizeof(DeviceID) * ids.size()));
+ if (arr == nullptr) {
+ return common::E_OOM;
+ }
+ memset(arr, 0, sizeof(DeviceID) * ids.size());
+ for (size_t i = 0; i < ids.size(); i++) {
+ const int rc = fill_device_id_from_ideviceid(ids[i].get(), &arr[i]);
+ if (rc != common::E_OK) {
+ tsfile_free_device_id_array(arr, static_cast<uint32_t>(i));
+ return rc;
+ }
+ }
+ *out_devices = arr;
+ *out_length = static_cast<uint32_t>(ids.size());
+ return common::E_OK;
+}
+
+ERRNO tsfile_reader_get_timeseries_metadata_all(
+ TsFileReader reader, DeviceTimeseriesMetadataMap* out_map) {
+ if (reader == nullptr || out_map == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+ out_map->entries = nullptr;
+ out_map->device_count = 0;
+ auto* r = static_cast<storage::TsFileReader*>(reader);
+ storage::DeviceTimeseriesMetadataMap cpp_map =
r->get_timeseries_metadata();
+ return populate_c_metadata_map_from_cpp(cpp_map, out_map);
+}
+
+ERRNO tsfile_reader_get_timeseries_metadata_for_devices(
+ TsFileReader reader, const DeviceID* devices, uint32_t length,
+ DeviceTimeseriesMetadataMap* out_map) {
+ if (reader == nullptr || out_map == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+ out_map->entries = nullptr;
+ out_map->device_count = 0;
+ if (length == 0) {
+ return common::E_OK;
+ }
+ if (devices == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+ for (uint32_t i = 0; i < length; i++) {
+ if (devices[i].path == nullptr) {
+ return common::E_INVALID_ARG;
+ }
+ }
+ auto* r = static_cast<storage::TsFileReader*>(reader);
+ std::vector<std::shared_ptr<storage::IDeviceID>> query_ids;
+ query_ids.reserve(length);
+ for (uint32_t i = 0; i < length; i++) {
+ query_ids.push_back(std::make_shared<storage::StringArrayDeviceID>(
+ std::string(devices[i].path)));
+ }
+ storage::DeviceTimeseriesMetadataMap cpp_map =
+ r->get_timeseries_metadata(query_ids);
+ return populate_c_metadata_map_from_cpp(cpp_map, out_map);
+}
+
+void tsfile_free_device_timeseries_metadata_map(
+ DeviceTimeseriesMetadataMap* map) {
+ if (map == nullptr) {
+ return;
+ }
+ free_device_timeseries_metadata_entries_partial(map->entries,
+ map->device_count);
+ map->entries = nullptr;
+ map->device_count = 0;
+}
+
// delete pointer
void _free_tsfile_ts_record(TsRecord* record) {
if (*record != nullptr) {
diff --git a/cpp/src/cwrapper/tsfile_cwrapper.h
b/cpp/src/cwrapper/tsfile_cwrapper.h
index 4f4ce8d6..6c0e6d2c 100644
--- a/cpp/src/cwrapper/tsfile_cwrapper.h
+++ b/cpp/src/cwrapper/tsfile_cwrapper.h
@@ -104,6 +104,134 @@ typedef struct device_schema {
int timeseries_num;
} DeviceSchema;
+/**
+ * @brief Common header for all statistic variants (first member of each
+ * TsFile*Statistic struct; also aliases the start of TimeseriesStatistic::u).
+ *
+ * When @p has_statistic is false, @p type is undefined. Otherwise @p type
+ * selects which @ref TimeseriesStatisticUnion member is active (INT32/DATE/
+ * INT64/TIMESTAMP share @c int_s). @c sum exists only on @c bool_s, @c int_s,
+ * and @c float_s. Heap strings in string_s/text_s are
+ * freed by tsfile_free_device_timeseries_metadata_map only.
+ */
+typedef struct TsFileStatisticBase {
+ bool has_statistic;
+ TSDataType type;
+ int32_t row_count;
+ int64_t start_time;
+ int64_t end_time;
+} TsFileStatisticBase;
+
+typedef struct TsFileBoolStatistic {
+ TsFileStatisticBase base;
+ double sum;
+ bool first_bool;
+ bool last_bool;
+} TsFileBoolStatistic;
+
+typedef struct TsFileIntStatistic {
+ TsFileStatisticBase base;
+ double sum;
+ int64_t min_int64;
+ int64_t max_int64;
+ int64_t first_int64;
+ int64_t last_int64;
+} TsFileIntStatistic;
+
+typedef struct TsFileFloatStatistic {
+ TsFileStatisticBase base;
+ double sum;
+ double min_float64;
+ double max_float64;
+ double first_float64;
+ double last_float64;
+} TsFileFloatStatistic;
+
+typedef struct TsFileStringStatistic {
+ TsFileStatisticBase base;
+ char* str_min;
+ char* str_max;
+ char* str_first;
+ char* str_last;
+} TsFileStringStatistic;
+
+typedef struct TsFileTextStatistic {
+ TsFileStatisticBase base;
+ char* str_first;
+ char* str_last;
+} TsFileTextStatistic;
+
+/**
+ * @brief One of the typed layouts; active member follows @c base.type.
+ */
+typedef union TimeseriesStatisticUnion {
+ TsFileBoolStatistic bool_s;
+ TsFileIntStatistic int_s;
+ TsFileFloatStatistic float_s;
+ TsFileStringStatistic string_s;
+ TsFileTextStatistic text_s;
+} TimeseriesStatisticUnion;
+
+/**
+ * @brief Aggregated statistic for one timeseries (subset of C++ Statistic).
+ *
+ * Read common fields via @c tsfile_statistic_base(s). Type-specific fields
+ * via @c s->u.int_s, @c s->u.float_s, etc., per @c base.type.
+ */
+typedef struct TimeseriesStatistic {
+ TimeseriesStatisticUnion u;
+} TimeseriesStatistic;
+
+/** Pointer to the common header at the start of @p s->u (any active arm). */
+#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u)
+
+/**
+ * @brief One measurement's metadata as exposed to C.
+ */
+typedef struct TimeseriesMetadata {
+ char* measurement_name;
+ TSDataType data_type;
+ int32_t chunk_meta_count;
+ TimeseriesStatistic statistic;
+} TimeseriesMetadata;
+
+/**
+ * @brief Device identity from IDeviceID (path, table name, segments).
+ *
+ * Heap fields are freed by tsfile_device_id_free_contents or
+ * tsfile_free_device_id_array, or as part of
+ * tsfile_free_device_timeseries_metadata_map for entries.
+ */
+typedef struct DeviceID {
+ char* path;
+ char* table_name;
+ uint32_t segment_count;
+ char** segments;
+} DeviceID;
+
+/**
+ * @brief One device's timeseries metadata list plus DeviceID.
+ *
+ * @p device heap fields freed by tsfile_free_device_timeseries_metadata_map.
+ */
+typedef struct DeviceTimeseriesMetadataEntry {
+ DeviceID device;
+ TimeseriesMetadata* timeseries;
+ uint32_t timeseries_count;
+} DeviceTimeseriesMetadataEntry;
+
+/**
+ * @brief Map device -> list of TimeseriesMetadata (C layout with explicit
+ * counts).
+ */
+typedef struct DeviceTimeseriesMetadataMap {
+ DeviceTimeseriesMetadataEntry* entries;
+ uint32_t device_count;
+} DeviceTimeseriesMetadataMap;
+
+/** Frees path, table_name, and segments inside @p d; zeros @p d. */
+void tsfile_device_id_free_contents(DeviceID* d);
+
typedef struct result_set_meta_data {
char** column_names;
TSDataType* data_types;
@@ -316,6 +444,37 @@ ERRNO tsfile_writer_close(TsFileWriter writer);
*/
ERRNO tsfile_reader_close(TsFileReader reader);
+/**
+ * @brief Lists all devices (path, table name, segments from IDeviceID).
+ *
+ * @param out_devices [out] Allocated array; caller frees with
+ * tsfile_free_device_id_array.
+ */
+ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID**
out_devices,
+ uint32_t* out_length);
+
+void tsfile_free_device_id_array(DeviceID* devices, uint32_t length);
+
+/**
+ * @brief Timeseries metadata for all devices in the file.
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_all(
+ TsFileReader reader, DeviceTimeseriesMetadataMap* out_map);
+
+/**
+ * @brief Timeseries metadata for a subset of devices.
+ *
+ * @param devices NULL and length>0 is E_INVALID_ARG. length==0: empty result
+ * (E_OK); @p devices is not read.
+ * For each entry, @p path must be non-NULL (canonical device path).
+ */
+ERRNO tsfile_reader_get_timeseries_metadata_for_devices(
+ TsFileReader reader, const DeviceID* devices, uint32_t length,
+ DeviceTimeseriesMetadataMap* out_map);
+
+void tsfile_free_device_timeseries_metadata_map(
+ DeviceTimeseriesMetadataMap* map);
+
/*--------------------------Tablet API------------------------ */
/**
diff --git a/cpp/test/cwrapper/cwrapper_metadata_test.cc
b/cpp/test/cwrapper/cwrapper_metadata_test.cc
new file mode 100644
index 00000000..57fca4de
--- /dev/null
+++ b/cpp/test/cwrapper/cwrapper_metadata_test.cc
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include <cstring>
+#include <string>
+
+extern "C" {
+#include "cwrapper/errno_define_c.h"
+#include "cwrapper/tsfile_cwrapper.h"
+}
+
+namespace cwrapper_metadata {
+
+class CWrapperMetadataTest : public testing::Test {};
+
+TEST_F(CWrapperMetadataTest, GetAllDevicesAndMetadataWithStatistic) {
+ ERRNO code = RET_OK;
+ const char* filename = "cwrapper_metadata_stat.tsfile";
+ remove(filename);
+
+ const char* device = "root.sg.d1";
+ char* m_int = strdup("s_int");
+ timeseries_schema sch{};
+ sch.timeseries_name = m_int;
+ sch.data_type = TS_DATATYPE_INT32;
+ sch.encoding = TS_ENCODING_PLAIN;
+ sch.compression = TS_COMPRESSION_UNCOMPRESSED;
+
+ auto* writer = static_cast<void*>(
+ _tsfile_writer_new(filename, 128 * 1024 * 1024, &code));
+ ASSERT_EQ(RET_OK, code);
+ ASSERT_EQ(RET_OK, _tsfile_writer_register_timeseries(writer, device,
&sch));
+
+ for (int row = 0; row < 3; row++) {
+ auto* record = static_cast<TsRecord>(
+ _ts_record_new(device, static_cast<int64_t>(row + 1), 1));
+ const int32_t v = static_cast<int32_t>((row + 1) * 10);
+ ASSERT_EQ(RET_OK, _insert_data_into_ts_record_by_name_int32_t(
+ record, m_int, v));
+ ASSERT_EQ(RET_OK, _tsfile_writer_write_ts_record(writer, record));
+ _free_tsfile_ts_record(reinterpret_cast<TsRecord*>(&record));
+ }
+ ASSERT_EQ(RET_OK, _tsfile_writer_close(writer));
+
+ TsFileReader reader = tsfile_reader_new(filename, &code);
+ ASSERT_EQ(RET_OK, code);
+ ASSERT_NE(nullptr, reader);
+
+ DeviceID* details = nullptr;
+ uint32_t n_det = 0;
+ ASSERT_EQ(RET_OK, tsfile_reader_get_all_devices(reader, &details, &n_det));
+ ASSERT_EQ(1u, n_det);
+ ASSERT_NE(nullptr, details);
+ ASSERT_STREQ(device, details[0].path);
+ ASSERT_NE(nullptr, details[0].table_name);
+ EXPECT_STREQ("root.sg", details[0].table_name);
+ EXPECT_EQ(2u, details[0].segment_count);
+ ASSERT_NE(nullptr, details[0].segments);
+ EXPECT_STREQ("root.sg", details[0].segments[0]);
+ EXPECT_STREQ("d1", details[0].segments[1]);
+ tsfile_free_device_id_array(details, n_det);
+
+ DeviceTimeseriesMetadataMap map{};
+ ASSERT_EQ(RET_OK, tsfile_reader_get_timeseries_metadata_all(reader, &map));
+ ASSERT_EQ(1u, map.device_count);
+ ASSERT_NE(nullptr, map.entries);
+ ASSERT_STREQ(device, map.entries[0].device.path);
+ ASSERT_NE(nullptr, map.entries[0].device.table_name);
+ EXPECT_STREQ("root.sg", map.entries[0].device.table_name);
+ EXPECT_EQ(2u, map.entries[0].device.segment_count);
+ ASSERT_NE(nullptr, map.entries[0].device.segments);
+ EXPECT_STREQ("root.sg", map.entries[0].device.segments[0]);
+ EXPECT_STREQ("d1", map.entries[0].device.segments[1]);
+ ASSERT_EQ(1u, map.entries[0].timeseries_count);
+ ASSERT_NE(nullptr, map.entries[0].timeseries);
+ TimeseriesMetadata& tm = map.entries[0].timeseries[0];
+ ASSERT_STREQ(m_int, tm.measurement_name);
+ ASSERT_EQ(TS_DATATYPE_INT32, tm.data_type);
+ TsFileStatisticBase* sb = tsfile_statistic_base(&tm.statistic);
+ ASSERT_TRUE(sb->has_statistic);
+ EXPECT_EQ(3, sb->row_count);
+ EXPECT_EQ(1, sb->start_time);
+ EXPECT_EQ(3, sb->end_time);
+ EXPECT_DOUBLE_EQ(60.0, tm.statistic.u.int_s.sum);
+ ASSERT_EQ(TS_DATATYPE_INT32, sb->type);
+ EXPECT_EQ(10, tm.statistic.u.int_s.min_int64);
+ EXPECT_EQ(30, tm.statistic.u.int_s.max_int64);
+ EXPECT_EQ(10, tm.statistic.u.int_s.first_int64);
+ EXPECT_EQ(30, tm.statistic.u.int_s.last_int64);
+
+ tsfile_free_device_timeseries_metadata_map(&map);
+
+ DeviceTimeseriesMetadataMap empty{};
+ ASSERT_EQ(RET_OK, tsfile_reader_get_timeseries_metadata_for_devices(
+ reader, nullptr, 0, &empty));
+ EXPECT_EQ(0u, empty.device_count);
+ EXPECT_EQ(nullptr, empty.entries);
+
+ DeviceID q{};
+ q.path = const_cast<char*>(device);
+ q.table_name = nullptr;
+ q.segment_count = 0;
+ q.segments = nullptr;
+ DeviceTimeseriesMetadataMap one{};
+ ASSERT_EQ(RET_OK, tsfile_reader_get_timeseries_metadata_for_devices(
+ reader, &q, 1, &one));
+ ASSERT_EQ(1u, one.device_count);
+ tsfile_free_device_timeseries_metadata_map(&one);
+
+ ASSERT_EQ(RET_OK, tsfile_reader_close(reader));
+ free(m_int);
+ remove(filename);
+}
+
+TEST_F(CWrapperMetadataTest, GetTimeseriesMetadataBooleanStatistic) {
+ ERRNO code = RET_OK;
+ const char* filename = "cwrapper_metadata_bool.tsfile";
+ remove(filename);
+
+ const char* device = "root.sg.bool";
+ char* m_b = strdup("s_bool");
+ timeseries_schema sch{};
+ sch.timeseries_name = m_b;
+ sch.data_type = TS_DATATYPE_BOOLEAN;
+ sch.encoding = TS_ENCODING_PLAIN;
+ sch.compression = TS_COMPRESSION_UNCOMPRESSED;
+
+ auto* writer = static_cast<void*>(
+ _tsfile_writer_new(filename, 128 * 1024 * 1024, &code));
+ ASSERT_EQ(RET_OK, code);
+ ASSERT_EQ(RET_OK, _tsfile_writer_register_timeseries(writer, device,
&sch));
+
+ const bool vals[] = {true, false, true};
+ for (int row = 0; row < 3; row++) {
+ auto* record = static_cast<TsRecord>(
+ _ts_record_new(device, static_cast<int64_t>(row + 1), 1));
+ ASSERT_EQ(RET_OK, _insert_data_into_ts_record_by_name_bool(record, m_b,
+ vals[row]));
+ ASSERT_EQ(RET_OK, _tsfile_writer_write_ts_record(writer, record));
+ _free_tsfile_ts_record(reinterpret_cast<TsRecord*>(&record));
+ }
+ ASSERT_EQ(RET_OK, _tsfile_writer_close(writer));
+
+ TsFileReader reader = tsfile_reader_new(filename, &code);
+ ASSERT_EQ(RET_OK, code);
+
+ DeviceTimeseriesMetadataMap map{};
+ ASSERT_EQ(RET_OK, tsfile_reader_get_timeseries_metadata_all(reader, &map));
+ TimeseriesMetadata& tm = map.entries[0].timeseries[0];
+ ASSERT_STREQ(m_b, tm.measurement_name);
+ ASSERT_EQ(TS_DATATYPE_BOOLEAN, tm.data_type);
+ TsFileStatisticBase* sb = tsfile_statistic_base(&tm.statistic);
+ ASSERT_TRUE(sb->has_statistic);
+ EXPECT_DOUBLE_EQ(2.0, tm.statistic.u.bool_s.sum);
+ ASSERT_EQ(TS_DATATYPE_BOOLEAN, sb->type);
+ EXPECT_TRUE(tm.statistic.u.bool_s.first_bool);
+ EXPECT_TRUE(tm.statistic.u.bool_s.last_bool);
+
+ tsfile_free_device_timeseries_metadata_map(&map);
+ ASSERT_EQ(RET_OK, tsfile_reader_close(reader));
+ free(m_b);
+ remove(filename);
+}
+
+TEST_F(CWrapperMetadataTest, GetTimeseriesMetadataStringStatistic) {
+ ERRNO code = RET_OK;
+ const char* filename = "cwrapper_metadata_str.tsfile";
+ remove(filename);
+
+ const char* device = "root.sg.str";
+ char* m_str = strdup("s_str");
+ timeseries_schema sch{};
+ sch.timeseries_name = m_str;
+ sch.data_type = TS_DATATYPE_STRING;
+ sch.encoding = TS_ENCODING_PLAIN;
+ sch.compression = TS_COMPRESSION_UNCOMPRESSED;
+
+ auto* writer = static_cast<void*>(
+ _tsfile_writer_new(filename, 128 * 1024 * 1024, &code));
+ ASSERT_EQ(RET_OK, code);
+ ASSERT_EQ(RET_OK, _tsfile_writer_register_timeseries(writer, device,
&sch));
+
+ const char* vals[] = {"aa", "cc", "bb"};
+ for (int row = 0; row < 3; row++) {
+ auto* record = static_cast<TsRecord>(
+ _ts_record_new(device, static_cast<int64_t>(row + 1), 1));
+ ASSERT_EQ(RET_OK, _insert_data_into_ts_record_by_name_string_with_len(
+ record, m_str, vals[row],
+ static_cast<int>(std::strlen(vals[row]))));
+ ASSERT_EQ(RET_OK, _tsfile_writer_write_ts_record(writer, record));
+ _free_tsfile_ts_record(reinterpret_cast<TsRecord*>(&record));
+ }
+ ASSERT_EQ(RET_OK, _tsfile_writer_close(writer));
+
+ TsFileReader reader = tsfile_reader_new(filename, &code);
+ ASSERT_EQ(RET_OK, code);
+
+ DeviceTimeseriesMetadataMap map{};
+ ASSERT_EQ(RET_OK, tsfile_reader_get_timeseries_metadata_all(reader, &map));
+ ASSERT_EQ(1u, map.device_count);
+ TimeseriesMetadata& tm = map.entries[0].timeseries[0];
+ ASSERT_STREQ(m_str, tm.measurement_name);
+ ASSERT_EQ(TS_DATATYPE_STRING, tm.data_type);
+ TsFileStatisticBase* sb = tsfile_statistic_base(&tm.statistic);
+ ASSERT_TRUE(sb->has_statistic);
+ ASSERT_EQ(TS_DATATYPE_STRING, sb->type);
+ ASSERT_NE(nullptr, tm.statistic.u.string_s.str_min);
+ ASSERT_NE(nullptr, tm.statistic.u.string_s.str_max);
+ ASSERT_NE(nullptr, tm.statistic.u.string_s.str_first);
+ ASSERT_NE(nullptr, tm.statistic.u.string_s.str_last);
+ EXPECT_STREQ("aa", tm.statistic.u.string_s.str_min);
+ EXPECT_STREQ("cc", tm.statistic.u.string_s.str_max);
+ EXPECT_STREQ("aa", tm.statistic.u.string_s.str_first);
+ EXPECT_STREQ("bb", tm.statistic.u.string_s.str_last);
+
+ tsfile_free_device_timeseries_metadata_map(&map);
+ ASSERT_EQ(RET_OK, tsfile_reader_close(reader));
+ free(m_str);
+ remove(filename);
+}
+
+TEST_F(CWrapperMetadataTest, GetTimeseriesMetadataNullDevicePath) {
+ ERRNO code = RET_OK;
+ const char* filename = "cwrapper_metadata_null_path.tsfile";
+ remove(filename);
+
+ auto* writer = static_cast<void*>(
+ _tsfile_writer_new(filename, 128 * 1024 * 1024, &code));
+ ASSERT_EQ(RET_OK, code);
+ ASSERT_EQ(RET_OK, _tsfile_writer_close(writer));
+
+ TsFileReader reader = tsfile_reader_new(filename, &code);
+ ASSERT_EQ(RET_OK, code);
+
+ DeviceID bad{};
+ bad.path = nullptr;
+ bad.table_name = nullptr;
+ bad.segment_count = 0;
+ bad.segments = nullptr;
+ DeviceTimeseriesMetadataMap map{};
+ EXPECT_EQ(RET_INVALID_ARG,
+ tsfile_reader_get_timeseries_metadata_for_devices(reader, &bad,
1,
+ &map));
+
+ ASSERT_EQ(RET_OK, tsfile_reader_close(reader));
+ remove(filename);
+}
+
+TEST_F(CWrapperMetadataTest, GetTimeseriesMetadataInvalidArgs) {
+ ERRNO code = RET_OK;
+ const char* filename = "cwrapper_metadata_empty.tsfile";
+ remove(filename);
+
+ auto* writer = static_cast<void*>(
+ _tsfile_writer_new(filename, 128 * 1024 * 1024, &code));
+ ASSERT_EQ(RET_OK, code);
+ ASSERT_EQ(RET_OK, _tsfile_writer_close(writer));
+
+ TsFileReader reader = tsfile_reader_new(filename, &code);
+ ASSERT_EQ(RET_OK, code);
+
+ DeviceTimeseriesMetadataMap map{};
+ EXPECT_NE(RET_OK, tsfile_reader_get_timeseries_metadata_all(nullptr,
&map));
+ EXPECT_NE(RET_OK,
+ tsfile_reader_get_timeseries_metadata_all(reader, nullptr));
+
+ ASSERT_EQ(RET_OK, tsfile_reader_close(reader));
+ remove(filename);
+}
+
+} // namespace cwrapper_metadata
diff --git a/python/tests/test_reader_metadata.py
b/python/tests/test_reader_metadata.py
new file mode 100644
index 00000000..558fcbb1
--- /dev/null
+++ b/python/tests/test_reader_metadata.py
@@ -0,0 +1,212 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os
+import tempfile
+
+import pytest
+
+from tsfile import Field, RowRecord, TimeseriesSchema, TsFileReader,
TsFileWriter
+from tsfile import TSDataType
+from tsfile.schema import (
+ BoolTimeseriesStatistic,
+ DeviceID,
+ IntTimeseriesStatistic,
+ StringTimeseriesStatistic,
+)
+
+
+def test_get_all_devices_segments():
+ path = os.path.join(tempfile.gettempdir(),
"py_reader_metadata_details.tsfile")
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+ device = "root.sg.py_details"
+ writer = TsFileWriter(path)
+ writer.register_timeseries(
+ device, TimeseriesSchema("m", TSDataType.INT32))
+ writer.write_row_record(
+ RowRecord(device, 1, [Field("m", 1, TSDataType.INT32)]))
+ writer.close()
+
+ reader = TsFileReader(path)
+ try:
+ details = reader.get_all_devices()
+ assert len(details) == 1
+ d0 = details[0]
+ assert d0.path == device
+ assert d0.table_name == "root.sg"
+ assert d0.segments == ("root.sg", "py_details")
+
+ grp = reader.get_timeseries_metadata(None)[device]
+ assert grp.table_name == "root.sg"
+ assert grp.segments == ("root.sg", "py_details")
+ assert len(grp.timeseries) == 1
+ finally:
+ reader.close()
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+
+def test_get_all_devices_and_timeseries_metadata_statistic():
+ path = os.path.join(tempfile.gettempdir(),
"py_reader_metadata_stat.tsfile")
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+ device = "root.sg.py_meta"
+ writer = TsFileWriter(path)
+ writer.register_timeseries(
+ device, TimeseriesSchema("m_int", TSDataType.INT32))
+ for row in range(3):
+ v = (row + 1) * 10
+ writer.write_row_record(
+ RowRecord(
+ device,
+ row + 1,
+ [Field("m_int", v, TSDataType.INT32)],
+ )
+ )
+ writer.close()
+
+ reader = TsFileReader(path)
+ try:
+ devices = reader.get_all_devices()
+ assert len(devices) == 1
+ assert devices[0].path == device
+
+ meta_all = reader.get_timeseries_metadata(None)
+ assert list(meta_all.keys()) == [device]
+ grp = meta_all[device]
+ assert grp.table_name == "root.sg"
+ assert grp.segments == ("root.sg", "py_meta")
+ series = grp.timeseries
+ assert len(series) == 1
+ m = series[0]
+ assert m.measurement_name == "m_int"
+ assert m.data_type == TSDataType.INT32
+ st = m.statistic
+ assert isinstance(st, IntTimeseriesStatistic)
+ assert st.has_statistic
+ assert st.row_count == 3
+ assert st.start_time == 1
+ assert st.end_time == 3
+ assert st.sum == pytest.approx(60.0)
+ assert st.min_int64 == 10
+ assert st.max_int64 == 30
+ assert st.first_int64 == 10
+ assert st.last_int64 == 30
+
+ assert reader.get_timeseries_metadata([]) == {}
+
+ sub = reader.get_timeseries_metadata([DeviceID(device, None, ())])
+ assert device in sub
+ assert len(sub[device].timeseries) == 1
+
+ sub_str = reader.get_timeseries_metadata([device])
+ assert device in sub_str
+ finally:
+ reader.close()
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+
+def test_get_timeseries_metadata_boolean_statistic():
+ path = os.path.join(tempfile.gettempdir(),
"py_reader_metadata_bool.tsfile")
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+ device = "root.sg.py_bool"
+ writer = TsFileWriter(path)
+ writer.register_timeseries(
+ device, TimeseriesSchema("m_b", TSDataType.BOOLEAN))
+ for row, b in enumerate([True, False, True]):
+ writer.write_row_record(
+ RowRecord(
+ device,
+ row + 1,
+ [Field("m_b", b, TSDataType.BOOLEAN)],
+ )
+ )
+ writer.close()
+
+ reader = TsFileReader(path)
+ try:
+ meta_all = reader.get_timeseries_metadata(None)
+ st = meta_all[device].timeseries[0].statistic
+ assert isinstance(st, BoolTimeseriesStatistic)
+ assert st.has_statistic
+ assert st.sum == pytest.approx(2.0)
+ assert st.first_bool is True
+ assert st.last_bool is True
+ finally:
+ reader.close()
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+
+def test_get_timeseries_metadata_string_statistic():
+ path = os.path.join(tempfile.gettempdir(), "py_reader_metadata_str.tsfile")
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
+
+ device = "root.sg.py_str"
+ writer = TsFileWriter(path)
+ writer.register_timeseries(
+ device, TimeseriesSchema("m_str", TSDataType.STRING))
+ for row, s in enumerate(["aa", "cc", "bb"]):
+ writer.write_row_record(
+ RowRecord(
+ device,
+ row + 1,
+ [Field("m_str", s, TSDataType.STRING)],
+ )
+ )
+ writer.close()
+
+ reader = TsFileReader(path)
+ try:
+ meta_all = reader.get_timeseries_metadata(None)
+ m = meta_all[device].timeseries[0]
+ assert m.measurement_name == "m_str"
+ assert m.data_type == TSDataType.STRING
+ st = m.statistic
+ assert isinstance(st, StringTimeseriesStatistic)
+ assert st.has_statistic
+ assert st.str_min == "aa"
+ assert st.str_max == "cc"
+ assert st.str_first == "aa"
+ assert st.str_last == "bb"
+ finally:
+ reader.close()
+ try:
+ os.unlink(path)
+ except OSError:
+ pass
diff --git a/python/tsfile/schema.py b/python/tsfile/schema.py
index c89649bf..ce85f683 100644
--- a/python/tsfile/schema.py
+++ b/python/tsfile/schema.py
@@ -15,12 +15,110 @@
# specific language governing permissions and limitations
# under the License.
#
-from typing import List
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
from .exceptions import TypeMismatchError
from .constants import TSDataType, ColumnCategory, TSEncoding, Compressor
+@dataclass(frozen=True)
+class TimeseriesStatistic:
+ """Common statistic fields from the C API (no type-specific payload)."""
+
+ has_statistic: bool
+ row_count: int
+ start_time: int
+ end_time: int
+
+
+@dataclass(frozen=True)
+class IntTimeseriesStatistic(TimeseriesStatistic):
+ """INT32, DATE, INT64, TIMESTAMP chunk statistics."""
+
+ sum: float
+ min_int64: int
+ max_int64: int
+ first_int64: int
+ last_int64: int
+
+
+@dataclass(frozen=True)
+class FloatTimeseriesStatistic(TimeseriesStatistic):
+ """FLOAT, DOUBLE chunk statistics."""
+
+ sum: float
+ min_float64: float
+ max_float64: float
+ first_float64: float
+ last_float64: float
+
+
+@dataclass(frozen=True)
+class BoolTimeseriesStatistic(TimeseriesStatistic):
+ """BOOLEAN chunk statistics."""
+
+ sum: float
+ first_bool: bool
+ last_bool: bool
+
+
+@dataclass(frozen=True)
+class StringTimeseriesStatistic(TimeseriesStatistic):
+ """STRING: lexicographic min/max and time-ordered first/last."""
+
+ str_min: Optional[str]
+ str_max: Optional[str]
+ str_first: Optional[str]
+ str_last: Optional[str]
+
+
+@dataclass(frozen=True)
+class TextTimeseriesStatistic(TimeseriesStatistic):
+ """TEXT: first/last only (no min/max)."""
+
+ str_first: Optional[str]
+ str_last: Optional[str]
+
+
+TimeseriesStatisticType = Union[
+ TimeseriesStatistic,
+ IntTimeseriesStatistic,
+ FloatTimeseriesStatistic,
+ BoolTimeseriesStatistic,
+ StringTimeseriesStatistic,
+ TextTimeseriesStatistic,
+]
+
+
+@dataclass(frozen=True)
+class TimeseriesMetadata:
+ """Per-measurement metadata from get_timeseries_metadata (includes
statistic when present)."""
+
+ measurement_name: str
+ data_type: TSDataType
+ chunk_meta_count: int
+ statistic: TimeseriesStatisticType
+
+
+@dataclass(frozen=True)
+class DeviceID:
+ """Device identity from the native reader (path, table name, segments).
NULL C fields become None."""
+
+ path: Optional[str]
+ table_name: Optional[str]
+ segments: Tuple[Optional[str], ...]
+
+
+@dataclass(frozen=True)
+class DeviceTimeseriesMetadataGroup:
+ """One device's timeseries list plus table name and path segments (dict
key is device path)."""
+
+ table_name: Optional[str]
+ segments: Tuple[Optional[str], ...]
+ timeseries: List[TimeseriesMetadata]
+
+
class TimeseriesSchema:
"""
Metadata schema for a time series (name, data type, encoding, compression).
diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd
index 29008148..74f3a7d9 100644
--- a/python/tsfile/tsfile_cpp.pxd
+++ b/python/tsfile/tsfile_cpp.pxd
@@ -103,6 +103,78 @@ cdef extern from "cwrapper/tsfile_cwrapper.h":
TimeseriesSchema * timeseries_schema
int timeseries_num
+ ctypedef struct TsFileStatisticBase:
+ bint has_statistic
+ TSDataType type
+ int32_t row_count
+ int64_t start_time
+ int64_t end_time
+
+ ctypedef struct TsFileBoolStatistic:
+ TsFileStatisticBase base
+ double sum
+ bint first_bool
+ bint last_bool
+
+ ctypedef struct TsFileIntStatistic:
+ TsFileStatisticBase base
+ double sum
+ int64_t min_int64
+ int64_t max_int64
+ int64_t first_int64
+ int64_t last_int64
+
+ ctypedef struct TsFileFloatStatistic:
+ TsFileStatisticBase base
+ double sum
+ double min_float64
+ double max_float64
+ double first_float64
+ double last_float64
+
+ ctypedef struct TsFileStringStatistic:
+ TsFileStatisticBase base
+ char* str_min
+ char* str_max
+ char* str_first
+ char* str_last
+
+ ctypedef struct TsFileTextStatistic:
+ TsFileStatisticBase base
+ char* str_first
+ char* str_last
+
+ ctypedef union TimeseriesStatisticUnion:
+ TsFileBoolStatistic bool_s
+ TsFileIntStatistic int_s
+ TsFileFloatStatistic float_s
+ TsFileStringStatistic string_s
+ TsFileTextStatistic text_s
+
+ ctypedef struct TimeseriesStatistic:
+ TimeseriesStatisticUnion u
+
+ ctypedef struct TimeseriesMetadata:
+ char * measurement_name
+ TSDataType data_type
+ int32_t chunk_meta_count
+ TimeseriesStatistic statistic
+
+ ctypedef struct DeviceID:
+ char * path
+ char * table_name
+ uint32_t segment_count
+ char ** segments
+
+ ctypedef struct DeviceTimeseriesMetadataEntry:
+ DeviceID device
+ TimeseriesMetadata * timeseries
+ uint32_t timeseries_count
+
+ ctypedef struct DeviceTimeseriesMetadataMap:
+ DeviceTimeseriesMetadataEntry * entries
+ uint32_t device_count
+
ctypedef struct ResultSetMetaData:
char** column_names
TSDataType * data_types
@@ -218,6 +290,22 @@ cdef extern from "cwrapper/tsfile_cwrapper.h":
DeviceSchema * tsfile_reader_get_all_timeseries_schemas(TsFileReader
reader,
uint32_t * size);
+ void tsfile_device_id_free_contents(DeviceID * d)
+
+ ErrorCode tsfile_reader_get_all_devices(TsFileReader reader,
+ DeviceID ** out_devices,
+ uint32_t * out_length);
+ void tsfile_free_device_id_array(DeviceID * devices,
+ uint32_t length);
+
+ ErrorCode tsfile_reader_get_timeseries_metadata_all(
+ TsFileReader reader, DeviceTimeseriesMetadataMap * out_map);
+ ErrorCode tsfile_reader_get_timeseries_metadata_for_devices(
+ TsFileReader reader, const DeviceID * devices, uint32_t length,
+ DeviceTimeseriesMetadataMap * out_map);
+ void tsfile_free_device_timeseries_metadata_map(
+ DeviceTimeseriesMetadataMap * map);
+
# resultSet : get data from resultSet
bint tsfile_result_set_next(ResultSet result_set, ErrorCode * err_code);
bint tsfile_result_set_is_null_by_index(ResultSet result_set, uint32_t
column_index);
diff --git a/python/tsfile/tsfile_py_cpp.pxd b/python/tsfile/tsfile_py_cpp.pxd
index 197a4ec8..b6baee80 100644
--- a/python/tsfile/tsfile_py_cpp.pxd
+++ b/python/tsfile/tsfile_py_cpp.pxd
@@ -67,5 +67,8 @@ cdef public api ResultSet
tsfile_reader_query_table_by_row_c(TsFileReader reader
cdef public api object get_table_schema(TsFileReader reader, object table_name)
cdef public api object get_all_table_schema(TsFileReader reader)
cdef public api object get_all_timeseries_schema(TsFileReader reader)
+cdef public api object reader_get_all_devices_c(TsFileReader reader)
+cdef public api object reader_get_timeseries_metadata_c(TsFileReader reader,
+ object device_ids)
cpdef public api object get_tsfile_config()
cpdef public api void set_tsfile_config(dict new_config)
\ No newline at end of file
diff --git a/python/tsfile/tsfile_py_cpp.pyx b/python/tsfile/tsfile_py_cpp.pyx
index 4febeb73..c564304f 100644
--- a/python/tsfile/tsfile_py_cpp.pyx
+++ b/python/tsfile/tsfile_py_cpp.pyx
@@ -26,6 +26,7 @@ import numpy as np
from libc.stdlib cimport free
from libc.stdlib cimport malloc
from libc.string cimport strdup
+from libc.string cimport memset
from cpython.exc cimport PyErr_SetObject
from cpython.unicode cimport PyUnicode_AsUTF8String, PyUnicode_AsUTF8,
PyUnicode_AsUTF8AndSize
from cpython.bytes cimport PyBytes_AsString, PyBytes_AsStringAndSize
@@ -36,6 +37,15 @@ from tsfile.schema import TSDataType as TSDataTypePy,
TSEncoding as TSEncodingPy
from tsfile.schema import Compressor as CompressorPy, ColumnCategory as
CategoryPy
from tsfile.schema import TableSchema as TableSchemaPy, ColumnSchema as
ColumnSchemaPy
from tsfile.schema import DeviceSchema as DeviceSchemaPy, TimeseriesSchema as
TimeseriesSchemaPy
+from tsfile.schema import BoolTimeseriesStatistic as BoolTimeseriesStatisticPy
+from tsfile.schema import DeviceID as DeviceIDPy
+from tsfile.schema import DeviceTimeseriesMetadataGroup as
DeviceTimeseriesMetadataGroupPy
+from tsfile.schema import FloatTimeseriesStatistic as
FloatTimeseriesStatisticPy
+from tsfile.schema import IntTimeseriesStatistic as IntTimeseriesStatisticPy
+from tsfile.schema import StringTimeseriesStatistic as
StringTimeseriesStatisticPy
+from tsfile.schema import TextTimeseriesStatistic as TextTimeseriesStatisticPy
+from tsfile.schema import TimeseriesStatistic as TimeseriesStatisticPy
+from tsfile.schema import TimeseriesMetadata as TimeseriesMetadataPy
# check exception and set py exception object
cdef inline void check_error(int errcode, const char * context=NULL) except*:
@@ -922,3 +932,195 @@ cdef object get_all_timeseries_schema(TsFileReader
reader):
device_schemas.update([(schema_py.get_device_name(), schema_py)])
free(schemas)
return device_schemas
+
+cdef object _c_str_to_py_utf8_or_none(char* p):
+ if p == NULL:
+ return None
+ return p.decode('utf-8')
+
+cdef object timeseries_statistic_c_to_py(TimeseriesStatistic* s):
+ cdef TsFileStatisticBase* b
+ cdef TSDataType dt
+ if s == NULL:
+ return TimeseriesStatisticPy(False, 0, 0, 0)
+ b = <TsFileStatisticBase*>&s.u
+ if not b.has_statistic:
+ return TimeseriesStatisticPy(
+ False, int(b.row_count), int(b.start_time), int(b.end_time))
+ dt = b.type
+ if dt == TS_DATATYPE_INVALID:
+ return TimeseriesStatisticPy(
+ True, int(b.row_count), int(b.start_time), int(b.end_time))
+ if (dt == TS_DATATYPE_INT32 or dt == TS_DATATYPE_DATE or
+ dt == TS_DATATYPE_INT64 or dt == TS_DATATYPE_TIMESTAMP):
+ return IntTimeseriesStatisticPy(
+ True, int(b.row_count), int(b.start_time), int(b.end_time),
+ float(s.u.int_s.sum),
+ int(s.u.int_s.min_int64),
+ int(s.u.int_s.max_int64),
+ int(s.u.int_s.first_int64),
+ int(s.u.int_s.last_int64),
+ )
+ if dt == TS_DATATYPE_FLOAT or dt == TS_DATATYPE_DOUBLE:
+ return FloatTimeseriesStatisticPy(
+ True, int(b.row_count), int(b.start_time), int(b.end_time),
+ float(s.u.float_s.sum),
+ float(s.u.float_s.min_float64),
+ float(s.u.float_s.max_float64),
+ float(s.u.float_s.first_float64),
+ float(s.u.float_s.last_float64),
+ )
+ if dt == TS_DATATYPE_BOOLEAN:
+ return BoolTimeseriesStatisticPy(
+ True, int(b.row_count), int(b.start_time), int(b.end_time),
+ float(s.u.bool_s.sum),
+ bool(s.u.bool_s.first_bool),
+ bool(s.u.bool_s.last_bool),
+ )
+ if dt == TS_DATATYPE_STRING:
+ return StringTimeseriesStatisticPy(
+ True, int(b.row_count), int(b.start_time), int(b.end_time),
+ _c_str_to_py_utf8_or_none(s.u.string_s.str_min),
+ _c_str_to_py_utf8_or_none(s.u.string_s.str_max),
+ _c_str_to_py_utf8_or_none(s.u.string_s.str_first),
+ _c_str_to_py_utf8_or_none(s.u.string_s.str_last),
+ )
+ if dt == TS_DATATYPE_TEXT:
+ return TextTimeseriesStatisticPy(
+ True, int(b.row_count), int(b.start_time), int(b.end_time),
+ _c_str_to_py_utf8_or_none(s.u.text_s.str_first),
+ _c_str_to_py_utf8_or_none(s.u.text_s.str_last),
+ )
+ return TimeseriesStatisticPy(
+ True, int(b.row_count), int(b.start_time), int(b.end_time))
+
+cdef object timeseries_metadata_c_to_py(TimeseriesMetadata* m):
+ cdef str name_py
+ if m == NULL or m.measurement_name == NULL:
+ name_py = ""
+ else:
+ name_py = m.measurement_name.decode('utf-8')
+ cdef object stat = timeseries_statistic_c_to_py(&m.statistic)
+ return TimeseriesMetadataPy(
+ name_py,
+ TSDataTypePy(m.data_type),
+ int(m.chunk_meta_count),
+ stat,
+ )
+
+cdef tuple c_device_segments_to_tuple(char** segs, uint32_t n):
+ cdef uint32_t i
+ cdef list out = []
+ for i in range(n):
+ if segs == NULL or segs[i] == NULL:
+ out.append(None)
+ else:
+ out.append(segs[i].decode('utf-8'))
+ return tuple(out)
+
+cdef dict device_timeseries_metadata_map_to_py(DeviceTimeseriesMetadataMap*
mmap):
+ cdef dict out = {}
+ cdef uint32_t di, ti
+ cdef char* p
+ cdef char* tnp
+ cdef object key
+ cdef object table_py
+ cdef tuple segs_py
+ cdef list series
+ for di in range(mmap.device_count):
+ p = mmap.entries[di].device.path
+ if p == NULL:
+ key = None
+ else:
+ key = p.decode('utf-8')
+ tnp = mmap.entries[di].device.table_name
+ if tnp == NULL:
+ table_py = None
+ else:
+ table_py = tnp.decode('utf-8')
+ segs_py = c_device_segments_to_tuple(
+ mmap.entries[di].device.segments,
+ mmap.entries[di].device.segment_count)
+ series = []
+ for ti in range(mmap.entries[di].timeseries_count):
+ series.append(
+ timeseries_metadata_c_to_py(
+ &mmap.entries[di].timeseries[ti]))
+ out[key] = DeviceTimeseriesMetadataGroupPy(
+ table_py, segs_py, series)
+ return out
+
+cdef public api object reader_get_all_devices_c(TsFileReader reader):
+ cdef DeviceID* arr = NULL
+ cdef uint32_t n = 0
+ cdef int err
+ cdef list out = []
+ cdef uint32_t i
+ cdef object path_py
+ cdef object tname_py
+ cdef tuple segs_py
+ err = tsfile_reader_get_all_devices(reader, &arr, &n)
+ check_error(err)
+ try:
+ for i in range(n):
+ if arr[i].path == NULL:
+ path_py = None
+ else:
+ path_py = arr[i].path.decode('utf-8')
+ if arr[i].table_name == NULL:
+ tname_py = None
+ else:
+ tname_py = arr[i].table_name.decode('utf-8')
+ segs_py = c_device_segments_to_tuple(arr[i].segments,
+ arr[i].segment_count)
+ out.append(DeviceIDPy(path_py, tname_py, segs_py))
+ finally:
+ tsfile_free_device_id_array(arr, n)
+ return out
+
+cdef public api object reader_get_timeseries_metadata_c(TsFileReader reader,
+ object device_ids):
+ cdef DeviceTimeseriesMetadataMap mmap
+ cdef DeviceID* q = NULL
+ cdef uint32_t qlen = 0
+ cdef uint32_t i
+ cdef int err
+ cdef bytes bpath
+ cdef const char* raw
+ memset(&mmap, 0, sizeof(DeviceTimeseriesMetadataMap))
+ if device_ids is None:
+ err = tsfile_reader_get_timeseries_metadata_all(reader, &mmap)
+ check_error(err)
+ elif len(device_ids) == 0:
+ err = tsfile_reader_get_timeseries_metadata_for_devices(
+ reader, NULL, 0, &mmap)
+ check_error(err)
+ else:
+ qlen = <uint32_t> len(device_ids)
+ q = <DeviceID*> malloc(sizeof(DeviceID) * qlen)
+ if q == NULL:
+ raise MemoryError()
+ memset(q, 0, sizeof(DeviceID) * qlen)
+ try:
+ for i in range(qlen):
+ dev = device_ids[i]
+ try:
+ path_s = dev.path
+ except AttributeError:
+ path_s = str(dev)
+ bpath = path_s.encode('utf-8')
+ raw = PyBytes_AsString(bpath)
+ q[i].path = strdup(raw)
+ if q[i].path == NULL:
+ raise MemoryError()
+ err = tsfile_reader_get_timeseries_metadata_for_devices(
+ reader, q, qlen, &mmap)
+ check_error(err)
+ finally:
+ for i in range(qlen):
+ free(q[i].path)
+ free(q)
+ try:
+ return device_timeseries_metadata_map_to_py(&mmap)
+ finally:
+ tsfile_free_device_timeseries_metadata_map(&mmap)
diff --git a/python/tsfile/tsfile_reader.pyx b/python/tsfile/tsfile_reader.pyx
index 3a1a15d4..2259f770 100644
--- a/python/tsfile/tsfile_reader.pyx
+++ b/python/tsfile/tsfile_reader.pyx
@@ -19,7 +19,7 @@
#cython: language_level=3
import weakref
-from typing import List
+from typing import List, Optional, Dict
import pandas as pd
from libc.stdint cimport INT64_MIN, INT64_MAX
@@ -30,6 +30,7 @@ import pyarrow as pa
from libc.stdint cimport INT64_MIN, INT64_MAX, uintptr_t
from tsfile.schema import TSDataType as TSDataTypePy
+from tsfile.schema import DeviceID, DeviceTimeseriesMetadataGroup
from .date_utils import parse_int_to_date
from .tsfile_cpp cimport *
from .tsfile_py_cpp cimport *
@@ -427,6 +428,25 @@ cdef class TsFileReaderPy:
"""
return get_all_timeseries_schema(self.reader)
+ def get_all_devices(self) -> List[DeviceID]:
+ """
+ Return all devices (path, table name, segments) as
+ :class:`tsfile.schema.DeviceID`. NULL C fields become None.
+ """
+ return reader_get_all_devices_c(self.reader)
+
+ def get_timeseries_metadata(
+ self, device_ids: Optional[List] = None
+ ) -> Dict[str, DeviceTimeseriesMetadataGroup]:
+ """
+ Return map device path ->
:class:`tsfile.schema.DeviceTimeseriesMetadataGroup`
+ (table name, segments, and list of
:class:`tsfile.schema.TimeseriesMetadata`).
+
+ ``device_ids is None``: all devices. ``device_ids == []``: empty map.
+ Non-empty list restricts to those devices (only existing devices
appear).
+ """
+ return reader_get_timeseries_metadata_c(self.reader, device_ids)
+
def close(self):
"""
Close TsFile Reader, if reader has result sets, invalid them.
