This is an automated email from the ASF dual-hosted git repository. ColinLeeo pushed a commit to branch doc/update-interfaces-from-develop in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit c44b3897e8965ff233ff65daea14dc831275d7bf Author: ColinLee <[email protected]> AuthorDate: Thu Jul 9 18:06:58 2026 +0800 Update interface docs from develop --- src/UserGuide/develop/DataFrame/TsFileDataFrame.md | 21 +- .../InterfaceDefinition/InterfaceDefinition-C.md | 211 +++++++++++++++++++- .../InterfaceDefinition/InterfaceDefinition-CPP.md | 147 +++++++++++++- .../InterfaceDefinition-Python.md | 171 ++++++++++++++++- src/UserGuide/latest/DataFrame/TsFileDataFrame.md | 21 +- .../InterfaceDefinition/InterfaceDefinition-C.md | 211 +++++++++++++++++++- .../InterfaceDefinition/InterfaceDefinition-CPP.md | 147 +++++++++++++- .../InterfaceDefinition-Python.md | 171 ++++++++++++++++- .../UserGuide/develop/DataFrame/TsFileDataFrame.md | 21 +- .../InterfaceDefinition/InterfaceDefinition-C.md | 213 ++++++++++++++++++++- .../InterfaceDefinition/InterfaceDefinition-CPP.md | 145 +++++++++++++- .../InterfaceDefinition-Python.md | 165 +++++++++++++++- .../UserGuide/latest/DataFrame/TsFileDataFrame.md | 21 +- .../InterfaceDefinition/InterfaceDefinition-C.md | 213 ++++++++++++++++++++- .../InterfaceDefinition/InterfaceDefinition-CPP.md | 145 +++++++++++++- .../InterfaceDefinition-Python.md | 165 +++++++++++++++- 16 files changed, 2102 insertions(+), 86 deletions(-) diff --git a/src/UserGuide/develop/DataFrame/TsFileDataFrame.md b/src/UserGuide/develop/DataFrame/TsFileDataFrame.md index 91a32ee8e..dbd88160e 100644 --- a/src/UserGuide/develop/DataFrame/TsFileDataFrame.md +++ b/src/UserGuide/develop/DataFrame/TsFileDataFrame.md @@ -64,9 +64,11 @@ In the table below, `df` is a `TsFileDataFrame` instance, created with | Example | Operation | Returns | |---|---|---| -| `TsFileDataFrame(paths)` | Load a file / list of files / directory | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | Load a file / list of files / directory. Set `show_progress=False` to silence loading progress on stderr | `TsFileDataFrame` | | `len(df)` | Number of time series | `int` | +| `df.model` | Loaded file model: `"table"` or `"tree"` | `str` | | `df.list_timeseries("weather")` | Series names, optionally filtered by prefix | `List[str]` | +| `df.list_timeseries_metadata("weather")` | Series metadata, optionally filtered by prefix | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`, `df[0]`, `df[-1]` | One series | `Timeseries` | | `df["city"]` | A metadata column (a tag / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` | | `df[0:3]`, `df[[0, 2, 5]]` | Subset view by integer position: a contiguous range (`0:3`), or the listed positions (`[0, 2, 5]`); positions are the printed `index` column | `TsFileDataFrame` | @@ -150,6 +152,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # recursively find every .tsfile under the directory +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -203,7 +206,21 @@ optionally filtered by a prefix. Calling it with no argument returns all series. ``` To inspect metadata such as start/end time and count, print the DataFrame (or a -subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe). +subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe) — or call +`list_timeseries_metadata()`. + +## Inspecting metadata + +`list_timeseries_metadata(path_prefix="")` returns a pandas DataFrame indexed by +series name. It includes `field`, `start_time`, `end_time`, `count`, and the +device tag columns. For table-model files it also includes the `table` column. + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## Selecting series diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index 0fb0c1e36..262cb5148 100644 --- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -44,10 +46,18 @@ typedef enum { TS_ENCODING_PLAIN = 0, TS_ENCODING_DICTIONARY = 1, TS_ENCODING_RLE = 2, + TS_ENCODING_DIFF = 3, TS_ENCODING_TS_2DIFF = 4, + TS_ENCODING_BITMAP = 5, + TS_ENCODING_GORILLA_V1 = 6, + TS_ENCODING_REGULAR = 7, TS_ENCODING_GORILLA = 8, TS_ENCODING_ZIGZAG = 9, + TS_ENCODING_FREQ = 10, + TS_ENCODING_CHIMP = 11, TS_ENCODING_SPRINTZ = 12, + TS_ENCODING_RLBE = 13, + TS_ENCODING_CAMEL = 14, TS_ENCODING_INVALID = 255 } TSEncoding; @@ -57,7 +67,12 @@ typedef enum { TS_COMPRESSION_SNAPPY = 1, TS_COMPRESSION_GZIP = 2, TS_COMPRESSION_LZO = 3, + TS_COMPRESSION_SDT = 4, + TS_COMPRESSION_PAA = 5, + TS_COMPRESSION_PLA = 6, TS_COMPRESSION_LZ4 = 7, + TS_COMPRESSION_ZSTD = 8, + TS_COMPRESSION_LZMA2 = 9, TS_COMPRESSION_INVALID = 255 } CompressionType; @@ -86,6 +101,19 @@ typedef struct table_schema { int column_num; } TableSchema; +typedef struct timeseries_schema { + char* timeseries_name; + TSDataType data_type; + TSEncoding encoding; + CompressionType compression; +} TimeseriesSchema; + +typedef struct device_schema { + char* device_name; + TimeseriesSchema* timeseries_schema; + int timeseries_num; +} DeviceSchema; + // ResultSetMetaData: Contains metadata for a result set, // such as column names and their data types. typedef struct result_set_meta_data { @@ -93,11 +121,73 @@ typedef struct result_set_meta_data { TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct DeviceID { + char* path; + char* table_name; + uint32_t segment_count; + char** segments; +} DeviceID; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) + +typedef struct TimeseriesMetadata { + char* measurement_name; + TSDataType data_type; + int32_t chunk_meta_count; + TimeseriesStatistic statistic; + TimeseriesStatistic timeline_statistic; +} TimeseriesMetadata; + +typedef struct DeviceTimeseriesMetadataEntry { + DeviceID device; + TimeseriesMetadata* timeseries; + uint32_t timeseries_count; +} DeviceTimeseriesMetadataEntry; + +typedef struct DeviceTimeseriesMetadataMap { + DeviceTimeseriesMetadataEntry* entries; + uint32_t device_count; +} DeviceTimeseriesMetadataMap; ``` > `ColumnSchema` does not carry encoding/compression: on write, columns follow > the > global defaults (see [Configuration](#configuration-encoding--compression)); > on > read, each column is decoded with the file's actual settings. +> +> `TimeseriesStatistic` is a tagged union in `tsfile_cwrapper.h`. Read common +> fields through `tsfile_statistic_base(&metadata.statistic)` and then use the +> active typed member (`int_s`, `float_s`, `bool_s`, `string_s`, or `text_s`) +> according to the statistic data type. ## Write Interface @@ -220,9 +310,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * @param value [in] Null-terminated string. Ownership remains with caller. * @return ERRNO. */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // Supports multiple data types ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -250,9 +342,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [in] Null-terminated string. Copied internally. */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); // Supports multiple data types @@ -329,11 +423,13 @@ Allowed encodings per data type, and the default used when you do not change it: | Data type | Allowed encodings | Default | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` | -| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`). ```C // e.g. write every column with LZ4 compression @@ -412,6 +508,31 @@ void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree queries + +```C +/** + * @brief Query tree-model data by measurement names within a time range. + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief Query tree-model data by row with offset/limit. + * + * @param device_ids Array of device identifiers. + * @param measurement_names Array of measurement names. + * @param offset Leading rows to skip (>= 0). + * @param limit Max rows to return; < 0 means unlimited. + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### Filtering by tag @@ -436,6 +557,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // column >= value TAG_FILTER_REGEXP = 6, // column matches the regex value TAG_FILTER_NOT_REGEXP = 7, // column does not match the regex value + TAG_FILTER_IS_NULL = 8, // column is null + TAG_FILTER_IS_NOT_NULL = 9, // column is not null } TagFilterOp; /** @@ -444,7 +567,7 @@ typedef enum { * @param reader [in] Valid TsFileReader handle. * @param table_name [in] Table whose schema defines the TAG columns. * @param column_name [in] Name of the TAG column to filter on. - * @param value [in] Comparison value (TAG columns are STRING). + * @param value [in] Comparison value (ignored for IS NULL / IS NOT NULL). * @param op [in] Comparison operator (TagFilterOp). * @param err_code [out] RET_OK(0) on success, or error code in errno_define_c.h. * @return TagFilterHandle on success; NULL on failure. @@ -465,6 +588,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// Convenience builders for common single-column predicates. +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // Combine predicates. AND/OR/NOT take ownership of their children; free the root only. TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -522,6 +659,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### Read batch results as Arrow + +Batch query result sets (`batch_size > 0`) can be fetched as Arrow C Data +Interface arrays and schemas. The caller owns the returned Arrow objects and +must call their `release` callbacks when finished. + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + Example — read `temperature` only for devices whose `region` TAG equals `shanghai`: @@ -664,17 +813,59 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader, TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, uint32_t* size); +/** + * @brief Gets all timeseries schema in the tsfile. + * @param size[out] number of DeviceSchema elements in the returned array. + * @return DeviceSchema*, an array of device schemas. + * @note The caller must call free_device_schema() on each element + * and free() the array pointer. + */ +DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader, + uint32_t* size); + /** * @brief Free the tableschema's space. * @param schema [in] the table schema to be freed. */ void free_table_schema(TableSchema schema); + +void free_device_schema(DeviceSchema schema); ``` +### Get Devices and Timeseries Metadata +```C +/** + * @brief Lists all devices in the file. + * + * @param out_devices[out] allocated array; free with tsfile_free_device_id_array(). + * @param out_length[out] number of devices in the returned array. + */ +ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices, + uint32_t* out_length); +void tsfile_free_device_id_array(DeviceID* devices, uint32_t length); +void tsfile_device_id_free_contents(DeviceID* d); +/** + * @brief Timeseries metadata for all devices in the file. + */ +ERRNO tsfile_reader_get_timeseries_metadata_all( + TsFileReader reader, DeviceTimeseriesMetadataMap* out_map); +/** + * @brief Timeseries metadata for the specified devices. + * + * length == 0 returns an empty map. For non-empty input, each DeviceID.path + * should contain the canonical device path. + */ +ERRNO tsfile_reader_get_timeseries_metadata_for_devices( + TsFileReader reader, const DeviceID* devices, uint32_t length, + DeviceTimeseriesMetadataMap* out_map); + +void tsfile_free_device_timeseries_metadata_map( + DeviceTimeseriesMetadataMap* map); +``` diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 496e48aca..d37e00777 100644 --- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // Value encoding. See the table below for which encodings apply to which types. @@ -42,19 +46,35 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, + CHIMP = 11, SPRINTZ = 12, + RLBE = 13, + CAMEL = 14, + INVALID_ENCODING = 255, }; -// Compression type. SNAPPY/GZIP/LZO/LZ4 depend on build options; LZ4 is the default. +// Compression type. SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 depend on build options; +// LZ4 is the default. enum CompressionType : uint8_t { UNCOMPRESSED = 0, SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + ZSTD = 8, + LZMA2 = 9, + INVALID_COMPRESSION = 255, }; // Column role within a table schema. @@ -72,6 +92,8 @@ Encodings applicable to each data type: | `GORILLA` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` | | `ZIGZAG` | `INT32`, `INT64` | | `SPRINTZ` | `INT32`, `INT64`, `FLOAT`, `DOUBLE` | +| `CHIMP`, `RLBE` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` | +| `CAMEL` | `DOUBLE` | Default value encoding per type: `BOOLEAN → PLAIN`, `INT32 / INT64 → TS_2DIFF`, `FLOAT / DOUBLE → GORILLA`, `TEXT / STRING / BLOB → PLAIN`. The default @@ -134,6 +156,65 @@ class TsFileTableWriter { }; ``` +### TsFileWriter + +`TsFileWriter` is the lower-level writer interface from `tsfile_writer.h`. It +supports both tree-model and table-model writes. + +```cpp +extern int libtsfile_init(); +extern void libtsfile_destroy(); + +// Writer configuration. Invalid values return common::E_INVALID_ARG and leave +// the previous value unchanged. +extern int set_page_max_point_count(uint32_t page_max_point_count); +extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node); + +class TsFileWriter { + public: + TsFileWriter(); + ~TsFileWriter(); + void destroy(); + + int open(const std::string& file_path, int flags, mode_t mode); + int open(const std::string& file_path); + int init(storage::WriteFile* write_file); + int init(storage::RestorableTsFileIOWriter* rw); + + void set_generate_table_schema(bool generate_table_schema); + + int register_timeseries(const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_timeseries( + const std::string& device_path, + const std::vector<MeasurementSchema*>& measurement_schema_vec); + int register_aligned_timeseries( + const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_aligned_timeseries( + const std::string& device_id, + const std::vector<MeasurementSchema*>& measurement_schemas); + int register_table(const std::shared_ptr<TableSchema>& table_schema); + + int write_record(const TsRecord& record); + int write_tablet(const Tablet& tablet); + int write_record_aligned(const TsRecord& record); + int write_tablet_aligned(const Tablet& tablet); + int write_tree(const Tablet& tablet); + int write_tree(const TsRecord& record); + int write_table(Tablet& tablet); + + DeviceSchemasMap* get_schema_group_map(); + std::shared_ptr<TableSchema> get_table_schema( + const std::string& table_name) const; + int64_t calculate_mem_size_for_all_group(); + int64_t calculate_meta_mem_size() const; + int check_memory_size_and_may_flush_chunks(); + int flush(); + int close(); +}; +``` + ### TableSchema Describe the data structure of the table schema @@ -300,8 +381,14 @@ uint8_t common::get_global_compression(); // Time-column encoding/compression (the data type is fixed to INT64). int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +Global compression accepts `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `LZ4`, +`ZSTD`, and `LZMA2`. The codec enum also contains legacy values such as `SDT`, +`PAA`, and `PLA`, but the global compression setter rejects them. + ## Read Interface ### Tsfile Reader ```cpp @@ -339,6 +426,17 @@ class TsFileReader { * @return Returns 0 on success, or a non-zero error code on failure. */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief query the tsfile by the path list, start time and end time. + * This method is used to query the tree model. + * + * @param [in] path_list the full path list + * @param [in] start_time the start time + * @param [in] end_time the end time + * @param [out] result_set the result set + */ + int query(std::vector<std::string>& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief query the tsfile by the table name, columns names, start time * and end time. @@ -351,7 +449,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector<std::string> &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief query the tsfile by the table name, columns names, start time @@ -366,7 +464,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector<std::string>& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief query tree-model time series by row with offset and limit. + */ + int queryByRow(std::vector<std::string>& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief query a table by row, with offset/limit pushdown and an optional @@ -386,6 +491,13 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief query tree-model data by measurement names within a time range. + */ + int query_table_on_tree(const std::vector<std::string>& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief destroy the result set, this method should be called after the * query is finished and result_set @@ -393,6 +505,26 @@ class TsFileReader { * @param qds the result set */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr<IDeviceID>& device_id, + const std::vector<std::string>& measurement_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices( + std::string table_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids(); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id, + std::vector<MeasurementSchema>& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector<std::shared_ptr<IDeviceID>>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief get the table schema by the table name * @@ -409,6 +541,11 @@ class TsFileReader { std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` is the metadata map returned by +`get_timeseries_metadata()`: `std::map<std::shared_ptr<IDeviceID>, +std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`. + ### ResultSet ```cpp /** @@ -540,6 +677,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +689,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index bbcde49cf..806fa9115 100644 --- a/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -47,10 +47,17 @@ class TSEncoding(IntEnum): PLAIN = 0 # all types DICTIONARY = 1 # STRING, TEXT RLE = 2 # INT32, INT64, TIMESTAMP, DATE + DIFF = 3 TS_2DIFF = 4 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE ZIGZAG = 9 # INT32, INT64 + CHIMP = 11 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE SPRINTZ = 12 # INT32, INT64, FLOAT, DOUBLE + RLBE = 13 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE + CAMEL = 14 # DOUBLE class Compressor(IntEnum): """ @@ -60,7 +67,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -103,13 +115,39 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` ## Write interface -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -162,6 +200,48 @@ class TsFileTableWriter: def __exit__(self, exc_type, exc_val, exc_tb) ``` +### TsFileWriter + +`TsFileWriter` is the lower-level writer exposed from `writer.pyx`. It can write +tree-model data (`register_timeseries`, `register_device`, `write_tablet`, +`write_row_record`) and table-model data (`register_table`, `write_table`, +`write_dataframe`, `write_arrow_batch`). + +```python +class TsFileWriter: + """ + :param pathname: Destination TsFile path. + :param memory_threshold: bytes buffered before an automatic flush (default 128MB). + """ + def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024) + + def register_timeseries(self, device_name: str, + timeseries_schema: TimeseriesSchema) + + def register_device(self, device_schema: DeviceSchema) + + def register_table(self, table_schema: TableSchema) + + def write_tablet(self, tablet: Tablet) + + def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame, + tableschema: TableSchema) + + def write_row_record(self, record: RowRecord) + + def write_table(self, tablet: Tablet) + + def write_arrow_batch(self, table_name: str, data, + time_col_index: int = -1) + + def flush(self) + + def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) +``` + ### Tablet definition @@ -231,11 +311,15 @@ encodings per data type, and the default used when you do not change it: | Data type | Allowed encodings | Default | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` | -| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`). The enum also exposes legacy values +such as `SDT`, `PAA`, and `PLA`, but the global configuration setter rejects +them. ## Read Interface @@ -262,11 +346,39 @@ class TsFileReader: :param column_names: A list of column names to retrieve. :param start_time: The start time of the query range (default: minimum int64 value). :param end_time: The end time of the query range (default: maximum int64 value). + :param tag_filter: Optional tag predicate for table-model TAG columns. + :param batch_size: <= 0 returns rows one by one; > 0 returns blocks of that size. :return: A query result set handler. """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + Execute a time range query on tree-model measurement columns. + + :param column_names: Measurement names to retrieve. + :param start_time: The start time of the query range. + :param end_time: The end time of the query range. + :return: A query result set handler. + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + Execute tree-model query by row with offset/limit. + + :param device_ids: Device identifiers to query. + :param measurement_names: Measurement names to retrieve. + :param offset: Number of leading rows to skip. + :param limit: Maximum number of rows to return; < 0 means unlimited. + :return: A query result set handler. + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ Execute a table query by row, with offset/limit pushdown and an optional @@ -287,6 +399,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + Execute a tree-model time range query for one device. + + :param device_name: Device identifier. + :param sensor_list: Measurement names to retrieve. + :param start_time: Query start time. + :param end_time: Query end time. + :return: A query result set handler. + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ Retrieves the schema of the specified table. @@ -303,6 +427,31 @@ class TsFileReader: """ def get_all_table_schemas(self) ->dict[str, TableSchema] + """ + Retrieves all tree-model timeseries schemas grouped by device. + + :return: A list of DeviceSchema objects. + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + Retrieves all device identifiers in the file. + + :return: A list of DeviceID(path, table_name, segments). + """ + def get_all_devices(self) -> List[DeviceID] + + """ + Retrieves per-timeseries metadata for all devices, or only the specified + devices. + + :param device_ids: None for all devices, [] for an empty result, or a list + of DeviceID / path-compatible device identifiers. + :return: dict mapping device segment tuples to DeviceTimeseriesMetadataGroup. + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] """ Closes the TsFile reader. If the reader has active result sets, they will be invalidated. @@ -346,6 +495,13 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + Fetches the next batch result as a pyarrow.Table. Returns None when no more + TsBlock batches are available. This is only valid for result sets created + with batch_size > 0. + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ Retrieves the value at the specified index from the query result set. @@ -397,6 +553,9 @@ class ResultSet: Closes the result set and releases any associated resources. """ def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) ``` ### to_dataframe @@ -467,4 +626,4 @@ def to_dataframe(file_path: str, ColumnNotExistError If any specified column does not exist in the table schema. """ -``` \ No newline at end of file +``` diff --git a/src/UserGuide/latest/DataFrame/TsFileDataFrame.md b/src/UserGuide/latest/DataFrame/TsFileDataFrame.md index 91a32ee8e..dbd88160e 100644 --- a/src/UserGuide/latest/DataFrame/TsFileDataFrame.md +++ b/src/UserGuide/latest/DataFrame/TsFileDataFrame.md @@ -64,9 +64,11 @@ In the table below, `df` is a `TsFileDataFrame` instance, created with | Example | Operation | Returns | |---|---|---| -| `TsFileDataFrame(paths)` | Load a file / list of files / directory | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | Load a file / list of files / directory. Set `show_progress=False` to silence loading progress on stderr | `TsFileDataFrame` | | `len(df)` | Number of time series | `int` | +| `df.model` | Loaded file model: `"table"` or `"tree"` | `str` | | `df.list_timeseries("weather")` | Series names, optionally filtered by prefix | `List[str]` | +| `df.list_timeseries_metadata("weather")` | Series metadata, optionally filtered by prefix | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`, `df[0]`, `df[-1]` | One series | `Timeseries` | | `df["city"]` | A metadata column (a tag / `field` / `start_time` / `end_time` / `count`) | `pandas.Series` | | `df[0:3]`, `df[[0, 2, 5]]` | Subset view by integer position: a contiguous range (`0:3`), or the listed positions (`[0, 2, 5]`); positions are the printed `index` column | `TsFileDataFrame` | @@ -150,6 +152,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # recursively find every .tsfile under the directory +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -203,7 +206,21 @@ optionally filtered by a prefix. Calling it with no argument returns all series. ``` To inspect metadata such as start/end time and count, print the DataFrame (or a -subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe). +subset of it) — see [Displaying a DataFrame](#displaying-a-dataframe) — or call +`list_timeseries_metadata()`. + +## Inspecting metadata + +`list_timeseries_metadata(path_prefix="")` returns a pandas DataFrame indexed by +series name. It includes `field`, `start_time`, `end_time`, `count`, and the +device tag columns. For table-model files it also includes the `table` column. + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## Selecting series diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index 0fb0c1e36..262cb5148 100644 --- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -44,10 +46,18 @@ typedef enum { TS_ENCODING_PLAIN = 0, TS_ENCODING_DICTIONARY = 1, TS_ENCODING_RLE = 2, + TS_ENCODING_DIFF = 3, TS_ENCODING_TS_2DIFF = 4, + TS_ENCODING_BITMAP = 5, + TS_ENCODING_GORILLA_V1 = 6, + TS_ENCODING_REGULAR = 7, TS_ENCODING_GORILLA = 8, TS_ENCODING_ZIGZAG = 9, + TS_ENCODING_FREQ = 10, + TS_ENCODING_CHIMP = 11, TS_ENCODING_SPRINTZ = 12, + TS_ENCODING_RLBE = 13, + TS_ENCODING_CAMEL = 14, TS_ENCODING_INVALID = 255 } TSEncoding; @@ -57,7 +67,12 @@ typedef enum { TS_COMPRESSION_SNAPPY = 1, TS_COMPRESSION_GZIP = 2, TS_COMPRESSION_LZO = 3, + TS_COMPRESSION_SDT = 4, + TS_COMPRESSION_PAA = 5, + TS_COMPRESSION_PLA = 6, TS_COMPRESSION_LZ4 = 7, + TS_COMPRESSION_ZSTD = 8, + TS_COMPRESSION_LZMA2 = 9, TS_COMPRESSION_INVALID = 255 } CompressionType; @@ -86,6 +101,19 @@ typedef struct table_schema { int column_num; } TableSchema; +typedef struct timeseries_schema { + char* timeseries_name; + TSDataType data_type; + TSEncoding encoding; + CompressionType compression; +} TimeseriesSchema; + +typedef struct device_schema { + char* device_name; + TimeseriesSchema* timeseries_schema; + int timeseries_num; +} DeviceSchema; + // ResultSetMetaData: Contains metadata for a result set, // such as column names and their data types. typedef struct result_set_meta_data { @@ -93,11 +121,73 @@ typedef struct result_set_meta_data { TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct DeviceID { + char* path; + char* table_name; + uint32_t segment_count; + char** segments; +} DeviceID; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) + +typedef struct TimeseriesMetadata { + char* measurement_name; + TSDataType data_type; + int32_t chunk_meta_count; + TimeseriesStatistic statistic; + TimeseriesStatistic timeline_statistic; +} TimeseriesMetadata; + +typedef struct DeviceTimeseriesMetadataEntry { + DeviceID device; + TimeseriesMetadata* timeseries; + uint32_t timeseries_count; +} DeviceTimeseriesMetadataEntry; + +typedef struct DeviceTimeseriesMetadataMap { + DeviceTimeseriesMetadataEntry* entries; + uint32_t device_count; +} DeviceTimeseriesMetadataMap; ``` > `ColumnSchema` does not carry encoding/compression: on write, columns follow > the > global defaults (see [Configuration](#configuration-encoding--compression)); > on > read, each column is decoded with the file's actual settings. +> +> `TimeseriesStatistic` is a tagged union in `tsfile_cwrapper.h`. Read common +> fields through `tsfile_statistic_base(&metadata.statistic)` and then use the +> active typed member (`int_s`, `float_s`, `bool_s`, `string_s`, or `text_s`) +> according to the statistic data type. ## Write Interface @@ -220,9 +310,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * @param value [in] Null-terminated string. Ownership remains with caller. * @return ERRNO. */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // Supports multiple data types ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -250,9 +342,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [in] Null-terminated string. Copied internally. */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); // Supports multiple data types @@ -329,11 +423,13 @@ Allowed encodings per data type, and the default used when you do not change it: | Data type | Allowed encodings | Default | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` | -| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`). ```C // e.g. write every column with LZ4 compression @@ -412,6 +508,31 @@ void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree queries + +```C +/** + * @brief Query tree-model data by measurement names within a time range. + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief Query tree-model data by row with offset/limit. + * + * @param device_ids Array of device identifiers. + * @param measurement_names Array of measurement names. + * @param offset Leading rows to skip (>= 0). + * @param limit Max rows to return; < 0 means unlimited. + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### Filtering by tag @@ -436,6 +557,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // column >= value TAG_FILTER_REGEXP = 6, // column matches the regex value TAG_FILTER_NOT_REGEXP = 7, // column does not match the regex value + TAG_FILTER_IS_NULL = 8, // column is null + TAG_FILTER_IS_NOT_NULL = 9, // column is not null } TagFilterOp; /** @@ -444,7 +567,7 @@ typedef enum { * @param reader [in] Valid TsFileReader handle. * @param table_name [in] Table whose schema defines the TAG columns. * @param column_name [in] Name of the TAG column to filter on. - * @param value [in] Comparison value (TAG columns are STRING). + * @param value [in] Comparison value (ignored for IS NULL / IS NOT NULL). * @param op [in] Comparison operator (TagFilterOp). * @param err_code [out] RET_OK(0) on success, or error code in errno_define_c.h. * @return TagFilterHandle on success; NULL on failure. @@ -465,6 +588,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// Convenience builders for common single-column predicates. +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // Combine predicates. AND/OR/NOT take ownership of their children; free the root only. TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -522,6 +659,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### Read batch results as Arrow + +Batch query result sets (`batch_size > 0`) can be fetched as Arrow C Data +Interface arrays and schemas. The caller owns the returned Arrow objects and +must call their `release` callbacks when finished. + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + Example — read `temperature` only for devices whose `region` TAG equals `shanghai`: @@ -664,17 +813,59 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader, TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, uint32_t* size); +/** + * @brief Gets all timeseries schema in the tsfile. + * @param size[out] number of DeviceSchema elements in the returned array. + * @return DeviceSchema*, an array of device schemas. + * @note The caller must call free_device_schema() on each element + * and free() the array pointer. + */ +DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader, + uint32_t* size); + /** * @brief Free the tableschema's space. * @param schema [in] the table schema to be freed. */ void free_table_schema(TableSchema schema); + +void free_device_schema(DeviceSchema schema); ``` +### Get Devices and Timeseries Metadata +```C +/** + * @brief Lists all devices in the file. + * + * @param out_devices[out] allocated array; free with tsfile_free_device_id_array(). + * @param out_length[out] number of devices in the returned array. + */ +ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices, + uint32_t* out_length); +void tsfile_free_device_id_array(DeviceID* devices, uint32_t length); +void tsfile_device_id_free_contents(DeviceID* d); +/** + * @brief Timeseries metadata for all devices in the file. + */ +ERRNO tsfile_reader_get_timeseries_metadata_all( + TsFileReader reader, DeviceTimeseriesMetadataMap* out_map); +/** + * @brief Timeseries metadata for the specified devices. + * + * length == 0 returns an empty map. For non-empty input, each DeviceID.path + * should contain the canonical device path. + */ +ERRNO tsfile_reader_get_timeseries_metadata_for_devices( + TsFileReader reader, const DeviceID* devices, uint32_t length, + DeviceTimeseriesMetadataMap* out_map); + +void tsfile_free_device_timeseries_metadata_map( + DeviceTimeseriesMetadataMap* map); +``` diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 496e48aca..d37e00777 100644 --- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // Value encoding. See the table below for which encodings apply to which types. @@ -42,19 +46,35 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, + CHIMP = 11, SPRINTZ = 12, + RLBE = 13, + CAMEL = 14, + INVALID_ENCODING = 255, }; -// Compression type. SNAPPY/GZIP/LZO/LZ4 depend on build options; LZ4 is the default. +// Compression type. SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 depend on build options; +// LZ4 is the default. enum CompressionType : uint8_t { UNCOMPRESSED = 0, SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + ZSTD = 8, + LZMA2 = 9, + INVALID_COMPRESSION = 255, }; // Column role within a table schema. @@ -72,6 +92,8 @@ Encodings applicable to each data type: | `GORILLA` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` | | `ZIGZAG` | `INT32`, `INT64` | | `SPRINTZ` | `INT32`, `INT64`, `FLOAT`, `DOUBLE` | +| `CHIMP`, `RLBE` | `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE` | +| `CAMEL` | `DOUBLE` | Default value encoding per type: `BOOLEAN → PLAIN`, `INT32 / INT64 → TS_2DIFF`, `FLOAT / DOUBLE → GORILLA`, `TEXT / STRING / BLOB → PLAIN`. The default @@ -134,6 +156,65 @@ class TsFileTableWriter { }; ``` +### TsFileWriter + +`TsFileWriter` is the lower-level writer interface from `tsfile_writer.h`. It +supports both tree-model and table-model writes. + +```cpp +extern int libtsfile_init(); +extern void libtsfile_destroy(); + +// Writer configuration. Invalid values return common::E_INVALID_ARG and leave +// the previous value unchanged. +extern int set_page_max_point_count(uint32_t page_max_point_count); +extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node); + +class TsFileWriter { + public: + TsFileWriter(); + ~TsFileWriter(); + void destroy(); + + int open(const std::string& file_path, int flags, mode_t mode); + int open(const std::string& file_path); + int init(storage::WriteFile* write_file); + int init(storage::RestorableTsFileIOWriter* rw); + + void set_generate_table_schema(bool generate_table_schema); + + int register_timeseries(const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_timeseries( + const std::string& device_path, + const std::vector<MeasurementSchema*>& measurement_schema_vec); + int register_aligned_timeseries( + const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_aligned_timeseries( + const std::string& device_id, + const std::vector<MeasurementSchema*>& measurement_schemas); + int register_table(const std::shared_ptr<TableSchema>& table_schema); + + int write_record(const TsRecord& record); + int write_tablet(const Tablet& tablet); + int write_record_aligned(const TsRecord& record); + int write_tablet_aligned(const Tablet& tablet); + int write_tree(const Tablet& tablet); + int write_tree(const TsRecord& record); + int write_table(Tablet& tablet); + + DeviceSchemasMap* get_schema_group_map(); + std::shared_ptr<TableSchema> get_table_schema( + const std::string& table_name) const; + int64_t calculate_mem_size_for_all_group(); + int64_t calculate_meta_mem_size() const; + int check_memory_size_and_may_flush_chunks(); + int flush(); + int close(); +}; +``` + ### TableSchema Describe the data structure of the table schema @@ -300,8 +381,14 @@ uint8_t common::get_global_compression(); // Time-column encoding/compression (the data type is fixed to INT64). int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +Global compression accepts `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `LZ4`, +`ZSTD`, and `LZMA2`. The codec enum also contains legacy values such as `SDT`, +`PAA`, and `PLA`, but the global compression setter rejects them. + ## Read Interface ### Tsfile Reader ```cpp @@ -339,6 +426,17 @@ class TsFileReader { * @return Returns 0 on success, or a non-zero error code on failure. */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief query the tsfile by the path list, start time and end time. + * This method is used to query the tree model. + * + * @param [in] path_list the full path list + * @param [in] start_time the start time + * @param [in] end_time the end time + * @param [out] result_set the result set + */ + int query(std::vector<std::string>& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief query the tsfile by the table name, columns names, start time * and end time. @@ -351,7 +449,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector<std::string> &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief query the tsfile by the table name, columns names, start time @@ -366,7 +464,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector<std::string>& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief query tree-model time series by row with offset and limit. + */ + int queryByRow(std::vector<std::string>& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief query a table by row, with offset/limit pushdown and an optional @@ -386,6 +491,13 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief query tree-model data by measurement names within a time range. + */ + int query_table_on_tree(const std::vector<std::string>& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief destroy the result set, this method should be called after the * query is finished and result_set @@ -393,6 +505,26 @@ class TsFileReader { * @param qds the result set */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr<IDeviceID>& device_id, + const std::vector<std::string>& measurement_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices( + std::string table_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids(); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id, + std::vector<MeasurementSchema>& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector<std::shared_ptr<IDeviceID>>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief get the table schema by the table name * @@ -409,6 +541,11 @@ class TsFileReader { std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` is the metadata map returned by +`get_timeseries_metadata()`: `std::map<std::shared_ptr<IDeviceID>, +std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`. + ### ResultSet ```cpp /** @@ -540,6 +677,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +689,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index bbcde49cf..806fa9115 100644 --- a/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -47,10 +47,17 @@ class TSEncoding(IntEnum): PLAIN = 0 # all types DICTIONARY = 1 # STRING, TEXT RLE = 2 # INT32, INT64, TIMESTAMP, DATE + DIFF = 3 TS_2DIFF = 4 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE ZIGZAG = 9 # INT32, INT64 + CHIMP = 11 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE SPRINTZ = 12 # INT32, INT64, FLOAT, DOUBLE + RLBE = 13 # INT32, INT64, TIMESTAMP, DATE, FLOAT, DOUBLE + CAMEL = 14 # DOUBLE class Compressor(IntEnum): """ @@ -60,7 +67,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -103,13 +115,39 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` ## Write interface -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -162,6 +200,48 @@ class TsFileTableWriter: def __exit__(self, exc_type, exc_val, exc_tb) ``` +### TsFileWriter + +`TsFileWriter` is the lower-level writer exposed from `writer.pyx`. It can write +tree-model data (`register_timeseries`, `register_device`, `write_tablet`, +`write_row_record`) and table-model data (`register_table`, `write_table`, +`write_dataframe`, `write_arrow_batch`). + +```python +class TsFileWriter: + """ + :param pathname: Destination TsFile path. + :param memory_threshold: bytes buffered before an automatic flush (default 128MB). + """ + def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024) + + def register_timeseries(self, device_name: str, + timeseries_schema: TimeseriesSchema) + + def register_device(self, device_schema: DeviceSchema) + + def register_table(self, table_schema: TableSchema) + + def write_tablet(self, tablet: Tablet) + + def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame, + tableschema: TableSchema) + + def write_row_record(self, record: RowRecord) + + def write_table(self, tablet: Tablet) + + def write_arrow_batch(self, table_name: str, data, + time_col_index: int = -1) + + def flush(self) + + def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) +``` + ### Tablet definition @@ -231,11 +311,15 @@ encodings per data type, and the default used when you do not change it: | Data type | Allowed encodings | Default | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`, `INT64`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ` | `TS_2DIFF` | -| `FLOAT`, `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ` | `GORILLA` | +| `INT32`, `INT64`, `TIMESTAMP`, `DATE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `ZIGZAG`, `RLE`, `SPRINTZ`, `CHIMP`, `RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`, `TS_2DIFF`, `GORILLA`, `SPRINTZ`, `CHIMP`, `RLBE`, `CAMEL` | `GORILLA` | | `STRING`, `TEXT` | `PLAIN`, `DICTIONARY` | `PLAIN` | -Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, or `LZ4` (default `LZ4`). +Compression applies to any data type: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, +`LZ4`, `ZSTD`, or `LZMA2` (default `LZ4`). The enum also exposes legacy values +such as `SDT`, `PAA`, and `PLA`, but the global configuration setter rejects +them. ## Read Interface @@ -262,11 +346,39 @@ class TsFileReader: :param column_names: A list of column names to retrieve. :param start_time: The start time of the query range (default: minimum int64 value). :param end_time: The end time of the query range (default: maximum int64 value). + :param tag_filter: Optional tag predicate for table-model TAG columns. + :param batch_size: <= 0 returns rows one by one; > 0 returns blocks of that size. :return: A query result set handler. """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + Execute a time range query on tree-model measurement columns. + + :param column_names: Measurement names to retrieve. + :param start_time: The start time of the query range. + :param end_time: The end time of the query range. + :return: A query result set handler. + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + Execute tree-model query by row with offset/limit. + + :param device_ids: Device identifiers to query. + :param measurement_names: Measurement names to retrieve. + :param offset: Number of leading rows to skip. + :param limit: Maximum number of rows to return; < 0 means unlimited. + :return: A query result set handler. + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ Execute a table query by row, with offset/limit pushdown and an optional @@ -287,6 +399,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + Execute a tree-model time range query for one device. + + :param device_name: Device identifier. + :param sensor_list: Measurement names to retrieve. + :param start_time: Query start time. + :param end_time: Query end time. + :return: A query result set handler. + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ Retrieves the schema of the specified table. @@ -303,6 +427,31 @@ class TsFileReader: """ def get_all_table_schemas(self) ->dict[str, TableSchema] + """ + Retrieves all tree-model timeseries schemas grouped by device. + + :return: A list of DeviceSchema objects. + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + Retrieves all device identifiers in the file. + + :return: A list of DeviceID(path, table_name, segments). + """ + def get_all_devices(self) -> List[DeviceID] + + """ + Retrieves per-timeseries metadata for all devices, or only the specified + devices. + + :param device_ids: None for all devices, [] for an empty result, or a list + of DeviceID / path-compatible device identifiers. + :return: dict mapping device segment tuples to DeviceTimeseriesMetadataGroup. + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] """ Closes the TsFile reader. If the reader has active result sets, they will be invalidated. @@ -346,6 +495,13 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + Fetches the next batch result as a pyarrow.Table. Returns None when no more + TsBlock batches are available. This is only valid for result sets created + with batch_size > 0. + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ Retrieves the value at the specified index from the query result set. @@ -397,6 +553,9 @@ class ResultSet: Closes the result set and releases any associated resources. """ def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) ``` ### to_dataframe @@ -467,4 +626,4 @@ def to_dataframe(file_path: str, ColumnNotExistError If any specified column does not exist in the table schema. """ -``` \ No newline at end of file +``` diff --git a/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md b/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md index 816a8cca6..2a7bf9616 100644 --- a/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md +++ b/src/zh/UserGuide/develop/DataFrame/TsFileDataFrame.md @@ -59,9 +59,11 @@ data.values # -> np.ndarray, shape (N, 2)：N | 示例 | 操作 | 返回类型 | |---|---|---| -| `TsFileDataFrame(paths)` | 加载文件 / 文件列表 / 目录 | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | 加载文件 / 文件列表 / 目录。设置 `show_progress=False` 可关闭 stderr 上的加载进度 | `TsFileDataFrame` | | `len(df)` | 时间序列总数 | `int` | +| `df.model` | 已加载文件模型：`"table"` 或 `"tree"` | `str` | | `df.list_timeseries("weather")` | 获取序列名，可按前缀筛选 | `List[str]` | +| `df.list_timeseries_metadata("weather")` | 获取序列元数据，可按前缀筛选 | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`、`df[0]`、`df[-1]` | 获取单条序列 | `Timeseries` | | `df["city"]` | 获取某元数据列（标签 / `field` / `start_time` / `end_time` / `count`） | `pandas.Series` | | `df[0:3]`、`df[[0, 2, 5]]` | 按整数位置取子集视图：连续区间（`0:3`）或所列位置（`[0, 2, 5]`）；位置即打印的 `index` 列 | `TsFileDataFrame` | @@ -132,6 +134,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # 递归查找目录下所有 .tsfile +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -172,7 +175,21 @@ TsFileDataFrame(table model, 972 time series, 5 files) ['weather.Beijing.humidity', 'weather.Beijing.temperature'] ``` -若需查看起止时间、点数等元信息，可打印 DataFrame（或其子集）——见[DataFrame 的展示](#dataframe-的展示)。 +若需查看起止时间、点数等元信息，可打印 DataFrame（或其子集）——见[DataFrame 的展示](#dataframe-的展示)——也可以调用 +`list_timeseries_metadata()`。 + +## 查看元数据 + +`list_timeseries_metadata(path_prefix="")` 返回以序列名为索引的 pandas DataFrame。 +其中包含 `field`、`start_time`、`end_time`、`count` 以及设备标签列。对于 +table 模型文件，还会包含 `table` 列。 + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## 选取序列 diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index de6d39796..1edc34e54 100644 --- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -44,10 +46,18 @@ typedef enum { TS_ENCODING_PLAIN = 0, TS_ENCODING_DICTIONARY = 1, TS_ENCODING_RLE = 2, + TS_ENCODING_DIFF = 3, TS_ENCODING_TS_2DIFF = 4, + TS_ENCODING_BITMAP = 5, + TS_ENCODING_GORILLA_V1 = 6, + TS_ENCODING_REGULAR = 7, TS_ENCODING_GORILLA = 8, TS_ENCODING_ZIGZAG = 9, + TS_ENCODING_FREQ = 10, + TS_ENCODING_CHIMP = 11, TS_ENCODING_SPRINTZ = 12, + TS_ENCODING_RLBE = 13, + TS_ENCODING_CAMEL = 14, TS_ENCODING_INVALID = 255 } TSEncoding; @@ -57,7 +67,12 @@ typedef enum { TS_COMPRESSION_SNAPPY = 1, TS_COMPRESSION_GZIP = 2, TS_COMPRESSION_LZO = 3, + TS_COMPRESSION_SDT = 4, + TS_COMPRESSION_PAA = 5, + TS_COMPRESSION_PLA = 6, TS_COMPRESSION_LZ4 = 7, + TS_COMPRESSION_ZSTD = 8, + TS_COMPRESSION_LZMA2 = 9, TS_COMPRESSION_INVALID = 255 } CompressionType; @@ -83,15 +98,89 @@ typedef struct table_schema { int column_num; } TableSchema; +typedef struct timeseries_schema { + char* timeseries_name; + TSDataType data_type; + TSEncoding encoding; + CompressionType compression; +} TimeseriesSchema; + +typedef struct device_schema { + char* device_name; + TimeseriesSchema* timeseries_schema; + int timeseries_num; +} DeviceSchema; + // ResultSetMetaData：结果集的元数据，包括列名和数据类型。 typedef struct result_set_meta_data { char** column_names; TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct DeviceID { + char* path; + char* table_name; + uint32_t segment_count; + char** segments; +} DeviceID; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) + +typedef struct TimeseriesMetadata { + char* measurement_name; + TSDataType data_type; + int32_t chunk_meta_count; + TimeseriesStatistic statistic; + TimeseriesStatistic timeline_statistic; +} TimeseriesMetadata; + +typedef struct DeviceTimeseriesMetadataEntry { + DeviceID device; + TimeseriesMetadata* timeseries; + uint32_t timeseries_count; +} DeviceTimeseriesMetadataEntry; + +typedef struct DeviceTimeseriesMetadataMap { + DeviceTimeseriesMetadataEntry* entries; + uint32_t device_count; +} DeviceTimeseriesMetadataMap; ``` > `ColumnSchema` 不携带编码/压缩：写入时列遵循全局默认值（见[配置](#配置编码与压缩)），读取时按文件中的实际配置解码。 +> +> `TimeseriesStatistic` 是 `tsfile_cwrapper.h` 中的带标签 union。可通过 +> `tsfile_statistic_base(&metadata.statistic)` 读取通用字段，再根据数据类型读取 +> `int_s`、`float_s`、`bool_s`、`string_s` 或 `text_s`。 ## 写入接口 @@ -207,9 +296,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * * @param value [输入] 以 '\0' 结尾的字符串，调用者保留所有权。 */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // 支持多种数据类型插入 ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -236,9 +327,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [输入] 字符串会被内部复制。 */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); ERRNO tablet_add_value_by_index_int32_t(Tablet tablet, uint32_t row_index, @@ -310,11 +403,12 @@ uint8_t get_global_time_compression(); | 数据类型 | 允许的编码 | 默认值 | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` | -| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | +| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | -压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`（默认 `LZ4`）。 +压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`（默认 `LZ4`）。 ```C // 例如：所有列均以 LZ4 压缩写入 @@ -387,6 +481,31 @@ bool tsfile_result_set_next(ResultSet result_set, ERRNO* error_code); void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree 查询 + +```C +/** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief 按行查询 tree 模型数据，支持 offset/limit。 + * + * @param device_ids 设备标识数组。 + * @param measurement_names 测点名数组。 + * @param offset 需要跳过的起始行数（>= 0）。 + * @param limit 最多返回行数；< 0 表示不限制。 + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### 按标签过滤 @@ -409,6 +528,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // 列 >= 值 TAG_FILTER_REGEXP = 6, // 列匹配正则 值 TAG_FILTER_NOT_REGEXP = 7, // 列不匹配正则 值 + TAG_FILTER_IS_NULL = 8, // 列为空 + TAG_FILTER_IS_NOT_NULL = 9, // 列不为空 } TagFilterOp; /** @@ -417,7 +538,7 @@ typedef enum { * @param reader [输入] 有效的 TsFileReader 句柄。 * @param table_name [输入] 其 schema 定义了这些标签列的表名。 * @param column_name [输入] 要过滤的标签列名。 - * @param value [输入] 比较值（标签列为 STRING 类型）。 + * @param value [输入] 比较值（IS NULL / IS NOT NULL 会忽略该参数）。 * @param op [输入] 比较运算符（TagFilterOp）。 * @param err_code [输出] 成功返回 RET_OK(0)，否则返回 errno_define_c.h 中的错误码。 * @return 成功返回 TagFilterHandle，失败返回 NULL。 @@ -437,6 +558,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// 常用单列谓词的便捷构造函数。 +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // 组合谓词。AND/OR/NOT 会接管其子节点的所有权，只需释放根节点。 TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -493,6 +628,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### 将批结果读取为 Arrow + +对 `batch_size > 0` 创建的批查询结果集，可以使用 Arrow C Data Interface 读取 +`ArrowArray` 和 `ArrowSchema`。调用者拥有返回的 Arrow 对象，使用完后必须调用 +它们的 `release` 回调。 + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + 示例——只读取 `region` 标签等于 `shanghai` 的设备的 `temperature`： ```C @@ -640,6 +787,17 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader, TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, uint32_t* size); +/** + * @brief 获取 TsFile 中所有 timeseries schema。 + * + * @param size [输出] 返回的 DeviceSchema 数组中的元素数量。 + * @return DeviceSchema* 设备 schema 数组指针。 + * @note 调用者必须对数组中的每个元素调用 free_device_schema()， + * 并对整个数组指针调用 free() 进行释放。 + */ +DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader, + uint32_t* size); + /** * @brief 释放 TableSchema 占用的内存空间。 * @@ -647,5 +805,40 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, */ void free_table_schema(TableSchema schema); + +void free_device_schema(DeviceSchema schema); ``` +### 获取设备与 Timeseries Metadata + +```C +/** + * @brief 列出文件中的所有设备。 + * + * @param out_devices [输出] 分配出的设备数组；用 tsfile_free_device_id_array() 释放。 + * @param out_length [输出] 返回数组中的设备数量。 + */ +ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices, + uint32_t* out_length); + +void tsfile_free_device_id_array(DeviceID* devices, uint32_t length); +void tsfile_device_id_free_contents(DeviceID* d); + +/** + * @brief 获取文件中所有设备的 timeseries metadata。 + */ +ERRNO tsfile_reader_get_timeseries_metadata_all( + TsFileReader reader, DeviceTimeseriesMetadataMap* out_map); + +/** + * @brief 获取指定设备的 timeseries metadata。 + * + * length == 0 返回空 map。非空输入中每个 DeviceID.path 应包含规范设备路径。 + */ +ERRNO tsfile_reader_get_timeseries_metadata_for_devices( + TsFileReader reader, const DeviceID* devices, uint32_t length, + DeviceTimeseriesMetadataMap* out_map); + +void tsfile_free_device_timeseries_metadata_map( + DeviceTimeseriesMetadataMap* map); +``` diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 50de78b47..f337c46ec 100644 --- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // 值编码。各编码适用于哪些类型见下表。 @@ -42,19 +46,35 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, + CHIMP = 11, SPRINTZ = 12, + RLBE = 13, + CAMEL = 14, + INVALID_ENCODING = 255, }; -// 压缩类型。SNAPPY/GZIP/LZO/LZ4 取决于构建选项；默认压缩为 LZ4。 +// 压缩类型。SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 取决于构建选项； +// 默认压缩为 LZ4。 enum CompressionType : uint8_t { UNCOMPRESSED = 0, SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + ZSTD = 8, + LZMA2 = 9, + INVALID_COMPRESSION = 255, }; // 列在表 schema 内的角色。 @@ -72,6 +92,8 @@ enum class ColumnCategory { TAG = 0, FIELD = 1, ATTRIBUTE = 2, TIME = 3 }; | `GORILLA` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` | | `ZIGZAG` | `INT32`、`INT64` | | `SPRINTZ` | `INT32`、`INT64`、`FLOAT`、`DOUBLE` | +| `CHIMP`、`RLBE` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` | +| `CAMEL` | `DOUBLE` | 各类型的默认值编码：`BOOLEAN → PLAIN`、`INT32 / INT64 → TS_2DIFF`、 `FLOAT / DOUBLE → GORILLA`、`TEXT / STRING / BLOB → PLAIN`。默认压缩为 `LZ4`。 @@ -135,6 +157,64 @@ class TsFileTableWriter { }; ``` +### TsFileWriter + +`TsFileWriter` 是 `tsfile_writer.h` 中的低层写入接口，支持 tree 模型和 table +模型写入。 + +```cpp +extern int libtsfile_init(); +extern void libtsfile_destroy(); + +// 写入器配置。非法值返回 common::E_INVALID_ARG，并保持原配置不变。 +extern int set_page_max_point_count(uint32_t page_max_point_count); +extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node); + +class TsFileWriter { + public: + TsFileWriter(); + ~TsFileWriter(); + void destroy(); + + int open(const std::string& file_path, int flags, mode_t mode); + int open(const std::string& file_path); + int init(storage::WriteFile* write_file); + int init(storage::RestorableTsFileIOWriter* rw); + + void set_generate_table_schema(bool generate_table_schema); + + int register_timeseries(const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_timeseries( + const std::string& device_path, + const std::vector<MeasurementSchema*>& measurement_schema_vec); + int register_aligned_timeseries( + const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_aligned_timeseries( + const std::string& device_id, + const std::vector<MeasurementSchema*>& measurement_schemas); + int register_table(const std::shared_ptr<TableSchema>& table_schema); + + int write_record(const TsRecord& record); + int write_tablet(const Tablet& tablet); + int write_record_aligned(const TsRecord& record); + int write_tablet_aligned(const Tablet& tablet); + int write_tree(const Tablet& tablet); + int write_tree(const TsRecord& record); + int write_table(Tablet& tablet); + + DeviceSchemasMap* get_schema_group_map(); + std::shared_ptr<TableSchema> get_table_schema( + const std::string& table_name) const; + int64_t calculate_mem_size_for_all_group(); + int64_t calculate_meta_mem_size() const; + int check_memory_size_and_may_flush_chunks(); + int flush(); + int close(); +}; +``` + ### TableSchema 描述表模式（schema）的数据结构。 @@ -297,8 +377,14 @@ uint8_t common::get_global_compression(); // 时间列的编码/压缩（数据类型固定为 INT64）。 int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +全局压缩支持 `UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD` +和 `LZMA2`。压缩枚举中也包含 `SDT`、`PAA`、`PLA` 等历史值，但全局压缩 +setter 会拒绝这些值。 + ## 读取接口 ### Tsfile Reader ```cpp @@ -335,6 +421,16 @@ class TsFileReader { * @return 成功时返回 0，失败时返回 errno_define.h 中的非零错误码。 */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief 通过路径列表、起始时间和结束时间查询 tsfile。用于 tree 模型。 + * + * @param [in] path_list 完整路径列表。 + * @param [in] start_time 起始时间。 + * @param [in] end_time 结束时间。 + * @param [out] result_set 查询结果集。 + */ + int query(std::vector<std::string>& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief 通过表名、列名、起始时间和结束时间查询 tsfile。 * @@ -346,7 +442,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector<std::string> &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief 通过表名、列名、开始时间、结束时间和标签过滤器查询 tsfile。 @@ -360,7 +456,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector<std::string>& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief 按行查询 tree 模型序列，支持 offset/limit。 + */ + int queryByRow(std::vector<std::string>& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief 按行查询表，支持偏移量/行数限制下推与可选的标签过滤。 @@ -379,12 +482,39 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ + int query_table_on_tree(const std::vector<std::string>& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief 销毁结果集，该方法应在查询完成并使用完 result_set 后调用。 * * @param qds 查询结果集。 */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr<IDeviceID>& device_id, + const std::vector<std::string>& measurement_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices( + std::string table_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids(); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id, + std::vector<MeasurementSchema>& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector<std::shared_ptr<IDeviceID>>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief 根据表名获取表的模式信息。 * @@ -401,6 +531,11 @@ class TsFileReader { std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` 是 `get_timeseries_metadata()` 返回的元数据 +map：`std::map<std::shared_ptr<IDeviceID>, +std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`。 + ### ResultSet ```cpp /** @@ -540,6 +675,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +687,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index c94b690c7..1d3e875b1 100644 --- a/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/zh/UserGuide/develop/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -45,10 +45,17 @@ class TSEncoding(IntEnum): PLAIN = 0 # 所有类型 DICTIONARY = 1 # STRING、TEXT RLE = 2 # INT32、INT64、TIMESTAMP、DATE + DIFF = 3 TS_2DIFF = 4 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE ZIGZAG = 9 # INT32、INT64 + CHIMP = 11 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE SPRINTZ = 12 # INT32、INT64、FLOAT、DOUBLE + RLBE = 13 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE + CAMEL = 14 # DOUBLE class Compressor(IntEnum): """ @@ -58,7 +65,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -99,6 +111,32 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` @@ -106,7 +144,7 @@ class ResultSetMetaData: ## 写入接口 -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -157,6 +195,48 @@ class TsFileTableWriter: ``` +### TsFileWriter + +`TsFileWriter` 是 `writer.pyx` 暴露的低层写入接口。它可以写入 tree 模型数据 +（`register_timeseries`、`register_device`、`write_tablet`、`write_row_record`）， +也可以写入 table 模型数据（`register_table`、`write_table`、`write_dataframe`、 +`write_arrow_batch`）。 + +```python +class TsFileWriter: + """ + :param pathname: 目标 TsFile 路径。 + :param memory_threshold: 触发自动刷盘前缓冲的字节数（默认 128MB）。 + """ + def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024) + + def register_timeseries(self, device_name: str, + timeseries_schema: TimeseriesSchema) + + def register_device(self, device_schema: DeviceSchema) + + def register_table(self, table_schema: TableSchema) + + def write_tablet(self, tablet: Tablet) + + def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame, + tableschema: TableSchema) + + def write_row_record(self, record: RowRecord) + + def write_table(self, tablet: Tablet) + + def write_arrow_batch(self, table_name: str, data, + time_col_index: int = -1) + + def flush(self) + + def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) +``` + ### Tablet definition @@ -223,11 +303,12 @@ set_tsfile_config({ | 数据类型 | 允许的编码 | 默认值 | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` | -| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | +| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | -压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`（默认 `LZ4`）。 +压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`（默认 `LZ4`）。枚举还暴露 `SDT`、`PAA`、`PLA` 等历史值，但全局配置 setter 会拒绝这些值。 ## 读取接口 @@ -253,11 +334,39 @@ class TsFileReader: :param column_names: 要检索的列名列表。 :param start_time: 查询范围的起始时间（默认：int64 最小值）。 :param end_time: 查询范围的结束时间（默认：int64 最大值）。 + :param tag_filter: 可选的 table 模型 TAG 列谓词。 + :param batch_size: <= 0 逐行返回；> 0 按该大小返回数据块。 :return: 查询结果集处理器。 """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + 对 tree 模型测点列执行时间范围查询。 + + :param column_names: 要检索的测点名列表。 + :param start_time: 查询范围的起始时间。 + :param end_time: 查询范围的结束时间。 + :return: 查询结果集处理器。 + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + 按行查询 tree 模型数据，支持 offset/limit。 + + :param device_ids: 要查询的设备标识列表。 + :param measurement_names: 要检索的测点名列表。 + :param offset: 需要跳过的起始行数。 + :param limit: 最多返回行数；< 0 表示不限制。 + :return: 查询结果集处理器。 + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ 按行查询表，支持偏移量/行数限制下推与可选的标签过滤。标签谓词把查询限定到 @@ -277,6 +386,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + 对单个设备执行 tree 模型时间范围查询。 + + :param device_name: 设备标识。 + :param sensor_list: 要检索的测点名列表。 + :param start_time: 查询起始时间。 + :param end_time: 查询结束时间。 + :return: 查询结果集处理器。 + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ 获取指定表的模式信息。 @@ -292,6 +413,31 @@ class TsFileReader: """ def get_all_table_schemas(self) -> dict[str, TableSchema] + """ + 获取所有 tree 模型 timeseries schema，按设备分组。 + + :return: DeviceSchema 对象列表。 + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + 获取文件中的所有设备标识。 + + :return: DeviceID(path, table_name, segments) 列表。 + """ + def get_all_devices(self) -> List[DeviceID] + + """ + 获取所有设备或指定设备的 per-timeseries metadata。 + + :param device_ids: None 表示全部设备，[] 表示空结果，或传入 DeviceID / + 路径兼容的设备标识列表。 + :return: dict，key 为设备 segment tuple，value 为 DeviceTimeseriesMetadataGroup。 + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] + """ 关闭 TsFile 读取器。如果读取器中有活动的结果集，它们将失效。 """ @@ -332,6 +478,12 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + 将下一个批结果读取为 pyarrow.Table。没有更多 TsBlock 批时返回 None。 + 仅适用于通过 batch_size > 0 创建的结果集。 + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ 从查询结果集中按索引获取值。 @@ -376,6 +528,9 @@ class ResultSet: """ def close(self) + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) + ``` diff --git a/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md b/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md index 816a8cca6..2a7bf9616 100644 --- a/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md +++ b/src/zh/UserGuide/latest/DataFrame/TsFileDataFrame.md @@ -59,9 +59,11 @@ data.values # -> np.ndarray, shape (N, 2)：N | 示例 | 操作 | 返回类型 | |---|---|---| -| `TsFileDataFrame(paths)` | 加载文件 / 文件列表 / 目录 | `TsFileDataFrame` | +| `TsFileDataFrame(paths, show_progress=True)` | 加载文件 / 文件列表 / 目录。设置 `show_progress=False` 可关闭 stderr 上的加载进度 | `TsFileDataFrame` | | `len(df)` | 时间序列总数 | `int` | +| `df.model` | 已加载文件模型：`"table"` 或 `"tree"` | `str` | | `df.list_timeseries("weather")` | 获取序列名，可按前缀筛选 | `List[str]` | +| `df.list_timeseries_metadata("weather")` | 获取序列元数据，可按前缀筛选 | `pandas.DataFrame` | | `df["weather.Beijing.humidity"]`、`df[0]`、`df[-1]` | 获取单条序列 | `Timeseries` | | `df["city"]` | 获取某元数据列（标签 / `field` / `start_time` / `end_time` / `count`） | `pandas.Series` | | `df[0:3]`、`df[[0, 2, 5]]` | 按整数位置取子集视图：连续区间（`0:3`）或所列位置（`[0, 2, 5]`）；位置即打印的 `index` 列 | `TsFileDataFrame` | @@ -132,6 +134,7 @@ from tsfile import TsFileDataFrame df = TsFileDataFrame(["data/weather.tsfile", "data/sensor.tsfile"]) df = TsFileDataFrame("data/") # 递归查找目录下所有 .tsfile +df = TsFileDataFrame("data/", show_progress=False) print(df) ``` @@ -172,7 +175,21 @@ TsFileDataFrame(table model, 972 time series, 5 files) ['weather.Beijing.humidity', 'weather.Beijing.temperature'] ``` -若需查看起止时间、点数等元信息，可打印 DataFrame（或其子集）——见[DataFrame 的展示](#dataframe-的展示)。 +若需查看起止时间、点数等元信息，可打印 DataFrame（或其子集）——见[DataFrame 的展示](#dataframe-的展示)——也可以调用 +`list_timeseries_metadata()`。 + +## 查看元数据 + +`list_timeseries_metadata(path_prefix="")` 返回以序列名为索引的 pandas DataFrame。 +其中包含 `field`、`start_time`、`end_time`、`count` 以及设备标签列。对于 +table 模型文件，还会包含 `table` 列。 + +```python +meta = df.list_timeseries_metadata() +weather_meta = df.list_timeseries_metadata("weather") + +meta[["field", "start_time", "end_time", "count"]].head() +``` ## 选取序列 diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md index de6d39796..1edc34e54 100644 --- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md +++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-C.md @@ -32,10 +32,12 @@ typedef enum { TS_DATATYPE_FLOAT = 3, TS_DATATYPE_DOUBLE = 4, TS_DATATYPE_TEXT = 5, + TS_DATATYPE_VECTOR = 6, TS_DATATYPE_TIMESTAMP = 8, TS_DATATYPE_DATE = 9, TS_DATATYPE_BLOB = 10, TS_DATATYPE_STRING = 11, + TS_DATATYPE_NULL_TYPE = 254, TS_DATATYPE_INVALID = 255 } TSDataType; @@ -44,10 +46,18 @@ typedef enum { TS_ENCODING_PLAIN = 0, TS_ENCODING_DICTIONARY = 1, TS_ENCODING_RLE = 2, + TS_ENCODING_DIFF = 3, TS_ENCODING_TS_2DIFF = 4, + TS_ENCODING_BITMAP = 5, + TS_ENCODING_GORILLA_V1 = 6, + TS_ENCODING_REGULAR = 7, TS_ENCODING_GORILLA = 8, TS_ENCODING_ZIGZAG = 9, + TS_ENCODING_FREQ = 10, + TS_ENCODING_CHIMP = 11, TS_ENCODING_SPRINTZ = 12, + TS_ENCODING_RLBE = 13, + TS_ENCODING_CAMEL = 14, TS_ENCODING_INVALID = 255 } TSEncoding; @@ -57,7 +67,12 @@ typedef enum { TS_COMPRESSION_SNAPPY = 1, TS_COMPRESSION_GZIP = 2, TS_COMPRESSION_LZO = 3, + TS_COMPRESSION_SDT = 4, + TS_COMPRESSION_PAA = 5, + TS_COMPRESSION_PLA = 6, TS_COMPRESSION_LZ4 = 7, + TS_COMPRESSION_ZSTD = 8, + TS_COMPRESSION_LZMA2 = 9, TS_COMPRESSION_INVALID = 255 } CompressionType; @@ -83,15 +98,89 @@ typedef struct table_schema { int column_num; } TableSchema; +typedef struct timeseries_schema { + char* timeseries_name; + TSDataType data_type; + TSEncoding encoding; + CompressionType compression; +} TimeseriesSchema; + +typedef struct device_schema { + char* device_name; + TimeseriesSchema* timeseries_schema; + int timeseries_num; +} DeviceSchema; + // ResultSetMetaData：结果集的元数据，包括列名和数据类型。 typedef struct result_set_meta_data { char** column_names; TSDataType* data_types; int column_num; } ResultSetMetaData; + +typedef struct arrow_schema ArrowSchema; +typedef struct arrow_array ArrowArray; + +typedef struct DeviceID { + char* path; + char* table_name; + uint32_t segment_count; + char** segments; +} DeviceID; + +typedef struct TsFileStatisticBase { + bool has_statistic; + TSDataType type; + int32_t row_count; + int64_t start_time; + int64_t end_time; +} TsFileStatisticBase; + +typedef struct TsFileBoolStatistic { TsFileStatisticBase base; double sum; bool first_bool; bool last_bool; } TsFileBoolStatistic; +typedef struct TsFileIntStatistic { TsFileStatisticBase base; double sum; int64_t min_int64; int64_t max_int64; int64_t first_int64; int64_t last_int64; } TsFileIntStatistic; +typedef struct TsFileFloatStatistic { TsFileStatisticBase base; double sum; double min_float64; double max_float64; double first_float64; double last_float64; } TsFileFloatStatistic; +typedef struct TsFileStringStatistic { TsFileStatisticBase base; char* str_min; char* str_max; char* str_first; char* str_last; } TsFileStringStatistic; +typedef struct TsFileTextStatistic { TsFileStatisticBase base; char* str_first; char* str_last; } TsFileTextStatistic; + +typedef union TimeseriesStatisticUnion { + TsFileBoolStatistic bool_s; + TsFileIntStatistic int_s; + TsFileFloatStatistic float_s; + TsFileStringStatistic string_s; + TsFileTextStatistic text_s; +} TimeseriesStatisticUnion; + +typedef struct TimeseriesStatistic { + TimeseriesStatisticUnion u; +} TimeseriesStatistic; + +#define tsfile_statistic_base(s) ((TsFileStatisticBase*)&(s)->u) + +typedef struct TimeseriesMetadata { + char* measurement_name; + TSDataType data_type; + int32_t chunk_meta_count; + TimeseriesStatistic statistic; + TimeseriesStatistic timeline_statistic; +} TimeseriesMetadata; + +typedef struct DeviceTimeseriesMetadataEntry { + DeviceID device; + TimeseriesMetadata* timeseries; + uint32_t timeseries_count; +} DeviceTimeseriesMetadataEntry; + +typedef struct DeviceTimeseriesMetadataMap { + DeviceTimeseriesMetadataEntry* entries; + uint32_t device_count; +} DeviceTimeseriesMetadataMap; ``` > `ColumnSchema` 不携带编码/压缩：写入时列遵循全局默认值（见[配置](#配置编码与压缩)），读取时按文件中的实际配置解码。 +> +> `TimeseriesStatistic` 是 `tsfile_cwrapper.h` 中的带标签 union。可通过 +> `tsfile_statistic_base(&metadata.statistic)` 读取通用字段，再根据数据类型读取 +> `int_s`、`float_s`、`bool_s`、`string_s` 或 `text_s`。 ## 写入接口 @@ -207,9 +296,11 @@ ERRNO tablet_add_timestamp(Tablet tablet, uint32_t row_index, * * @param value [输入] 以 '\0' 结尾的字符串，调用者保留所有权。 */ -ERRNO tablet_add_value_by_name_string(Tablet tablet, uint32_t row_index, - const char* column_name, - const char* value); +ERRNO tablet_add_value_by_name_string_with_len(Tablet tablet, + uint32_t row_index, + const char* column_name, + const char* value, + int value_len); // 支持多种数据类型插入 ERRNO tablet_add_value_by_name_int32_t(Tablet tablet, uint32_t row_index, @@ -236,9 +327,11 @@ ERRNO tablet_add_value_by_name_bool(Tablet tablet, uint32_t row_index, * * @param value [输入] 字符串会被内部复制。 */ -ERRNO tablet_add_value_by_index_string(Tablet tablet, uint32_t row_index, - uint32_t column_index, - const char* value); +ERRNO tablet_add_value_by_index_string_with_len(Tablet tablet, + uint32_t row_index, + uint32_t column_index, + const char* value, + int value_len); ERRNO tablet_add_value_by_index_int32_t(Tablet tablet, uint32_t row_index, @@ -310,11 +403,12 @@ uint8_t get_global_time_compression(); | 数据类型 | 允许的编码 | 默认值 | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` | -| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | +| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | -压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`（默认 `LZ4`）。 +压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`（默认 `LZ4`）。 ```C // 例如：所有列均以 LZ4 压缩写入 @@ -387,6 +481,31 @@ bool tsfile_result_set_next(ResultSet result_set, ERRNO* error_code); void free_tsfile_result_set(ResultSet* result_set); ``` +### Tree 查询 + +```C +/** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ +ResultSet tsfile_query_table_on_tree(TsFileReader reader, char** columns, + uint32_t column_num, Timestamp start_time, + Timestamp end_time, ERRNO* err_code); + +/** + * @brief 按行查询 tree 模型数据，支持 offset/limit。 + * + * @param device_ids 设备标识数组。 + * @param measurement_names 测点名数组。 + * @param offset 需要跳过的起始行数（>= 0）。 + * @param limit 最多返回行数；< 0 表示不限制。 + */ +ResultSet tsfile_reader_query_tree_by_row(TsFileReader reader, + char** device_ids, int device_ids_len, + char** measurement_names, + int measurement_names_len, int offset, + int limit, ERRNO* err_code); +``` + ### 按标签过滤 @@ -409,6 +528,8 @@ typedef enum { TAG_FILTER_GTEQ = 5, // 列 >= 值 TAG_FILTER_REGEXP = 6, // 列匹配正则 值 TAG_FILTER_NOT_REGEXP = 7, // 列不匹配正则 值 + TAG_FILTER_IS_NULL = 8, // 列为空 + TAG_FILTER_IS_NOT_NULL = 9, // 列不为空 } TagFilterOp; /** @@ -417,7 +538,7 @@ typedef enum { * @param reader [输入] 有效的 TsFileReader 句柄。 * @param table_name [输入] 其 schema 定义了这些标签列的表名。 * @param column_name [输入] 要过滤的标签列名。 - * @param value [输入] 比较值（标签列为 STRING 类型）。 + * @param value [输入] 比较值（IS NULL / IS NOT NULL 会忽略该参数）。 * @param op [输入] 比较运算符（TagFilterOp）。 * @param err_code [输出] 成功返回 RET_OK(0)，否则返回 errno_define_c.h 中的错误码。 * @return 成功返回 TagFilterHandle，失败返回 NULL。 @@ -437,6 +558,20 @@ TagFilterHandle tsfile_tag_filter_between(TsFileReader reader, const char* lower, const char* upper, bool is_not, ERRNO* err_code); +// 常用单列谓词的便捷构造函数。 +TagFilterHandle tsfile_tag_filter_eq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_neq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_lteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gt(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); +TagFilterHandle tsfile_tag_filter_gteq(TsFileReader reader, const char* table_name, + const char* column_name, const char* value); + // 组合谓词。AND/OR/NOT 会接管其子节点的所有权，只需释放根节点。 TagFilterHandle tsfile_tag_filter_and(TagFilterHandle left, TagFilterHandle right); TagFilterHandle tsfile_tag_filter_or(TagFilterHandle left, TagFilterHandle right); @@ -493,6 +628,18 @@ ResultSet tsfile_query_table_with_tag_filter( TagFilterHandle tag_filter, int batch_size, ERRNO* err_code); ``` +### 将批结果读取为 Arrow + +对 `batch_size > 0` 创建的批查询结果集，可以使用 Arrow C Data Interface 读取 +`ArrowArray` 和 `ArrowSchema`。调用者拥有返回的 Arrow 对象，使用完后必须调用 +它们的 `release` 回调。 + +```C +ERRNO tsfile_result_set_get_next_tsblock_as_arrow(ResultSet result_set, + ArrowArray* out_array, + ArrowSchema* out_schema); +``` + 示例——只读取 `region` 标签等于 `shanghai` 的设备的 `temperature`： ```C @@ -640,6 +787,17 @@ TableSchema tsfile_reader_get_table_schema(TsFileReader reader, TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, uint32_t* size); +/** + * @brief 获取 TsFile 中所有 timeseries schema。 + * + * @param size [输出] 返回的 DeviceSchema 数组中的元素数量。 + * @return DeviceSchema* 设备 schema 数组指针。 + * @note 调用者必须对数组中的每个元素调用 free_device_schema()， + * 并对整个数组指针调用 free() 进行释放。 + */ +DeviceSchema* tsfile_reader_get_all_timeseries_schemas(TsFileReader reader, + uint32_t* size); + /** * @brief 释放 TableSchema 占用的内存空间。 * @@ -647,5 +805,40 @@ TableSchema* tsfile_reader_get_all_table_schemas(TsFileReader reader, */ void free_table_schema(TableSchema schema); + +void free_device_schema(DeviceSchema schema); ``` +### 获取设备与 Timeseries Metadata + +```C +/** + * @brief 列出文件中的所有设备。 + * + * @param out_devices [输出] 分配出的设备数组；用 tsfile_free_device_id_array() 释放。 + * @param out_length [输出] 返回数组中的设备数量。 + */ +ERRNO tsfile_reader_get_all_devices(TsFileReader reader, DeviceID** out_devices, + uint32_t* out_length); + +void tsfile_free_device_id_array(DeviceID* devices, uint32_t length); +void tsfile_device_id_free_contents(DeviceID* d); + +/** + * @brief 获取文件中所有设备的 timeseries metadata。 + */ +ERRNO tsfile_reader_get_timeseries_metadata_all( + TsFileReader reader, DeviceTimeseriesMetadataMap* out_map); + +/** + * @brief 获取指定设备的 timeseries metadata。 + * + * length == 0 返回空 map。非空输入中每个 DeviceID.path 应包含规范设备路径。 + */ +ERRNO tsfile_reader_get_timeseries_metadata_for_devices( + TsFileReader reader, const DeviceID* devices, uint32_t length, + DeviceTimeseriesMetadataMap* out_map); + +void tsfile_free_device_timeseries_metadata_map( + DeviceTimeseriesMetadataMap* map); +``` diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md index 50de78b47..f337c46ec 100644 --- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md +++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-CPP.md @@ -31,10 +31,14 @@ enum TSDataType : uint8_t { FLOAT = 3, DOUBLE = 4, TEXT = 5, + VECTOR = 6, + UNKNOWN = 7, TIMESTAMP = 8, DATE = 9, BLOB = 10, STRING = 11, + NULL_TYPE = 254, + INVALID_DATATYPE = 255, }; // 值编码。各编码适用于哪些类型见下表。 @@ -42,19 +46,35 @@ enum TSEncoding : uint8_t { PLAIN = 0, DICTIONARY = 1, RLE = 2, + DIFF = 3, TS_2DIFF = 4, + BITMAP = 5, + GORILLA_V1 = 6, + REGULAR = 7, GORILLA = 8, ZIGZAG = 9, + FREQ = 10, + CHIMP = 11, SPRINTZ = 12, + RLBE = 13, + CAMEL = 14, + INVALID_ENCODING = 255, }; -// 压缩类型。SNAPPY/GZIP/LZO/LZ4 取决于构建选项；默认压缩为 LZ4。 +// 压缩类型。SNAPPY/GZIP/LZO/LZ4/ZSTD/LZMA2 取决于构建选项； +// 默认压缩为 LZ4。 enum CompressionType : uint8_t { UNCOMPRESSED = 0, SNAPPY = 1, GZIP = 2, LZO = 3, + SDT = 4, + PAA = 5, + PLA = 6, LZ4 = 7, + ZSTD = 8, + LZMA2 = 9, + INVALID_COMPRESSION = 255, }; // 列在表 schema 内的角色。 @@ -72,6 +92,8 @@ enum class ColumnCategory { TAG = 0, FIELD = 1, ATTRIBUTE = 2, TIME = 3 }; | `GORILLA` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` | | `ZIGZAG` | `INT32`、`INT64` | | `SPRINTZ` | `INT32`、`INT64`、`FLOAT`、`DOUBLE` | +| `CHIMP`、`RLBE` | `INT32`、`INT64`、`TIMESTAMP`、`DATE`、`FLOAT`、`DOUBLE` | +| `CAMEL` | `DOUBLE` | 各类型的默认值编码：`BOOLEAN → PLAIN`、`INT32 / INT64 → TS_2DIFF`、 `FLOAT / DOUBLE → GORILLA`、`TEXT / STRING / BLOB → PLAIN`。默认压缩为 `LZ4`。 @@ -135,6 +157,64 @@ class TsFileTableWriter { }; ``` +### TsFileWriter + +`TsFileWriter` 是 `tsfile_writer.h` 中的低层写入接口，支持 tree 模型和 table +模型写入。 + +```cpp +extern int libtsfile_init(); +extern void libtsfile_destroy(); + +// 写入器配置。非法值返回 common::E_INVALID_ARG，并保持原配置不变。 +extern int set_page_max_point_count(uint32_t page_max_point_count); +extern int set_max_degree_of_index_node(uint32_t max_degree_of_index_node); + +class TsFileWriter { + public: + TsFileWriter(); + ~TsFileWriter(); + void destroy(); + + int open(const std::string& file_path, int flags, mode_t mode); + int open(const std::string& file_path); + int init(storage::WriteFile* write_file); + int init(storage::RestorableTsFileIOWriter* rw); + + void set_generate_table_schema(bool generate_table_schema); + + int register_timeseries(const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_timeseries( + const std::string& device_path, + const std::vector<MeasurementSchema*>& measurement_schema_vec); + int register_aligned_timeseries( + const std::string& device_id, + const MeasurementSchema& measurement_schema); + int register_aligned_timeseries( + const std::string& device_id, + const std::vector<MeasurementSchema*>& measurement_schemas); + int register_table(const std::shared_ptr<TableSchema>& table_schema); + + int write_record(const TsRecord& record); + int write_tablet(const Tablet& tablet); + int write_record_aligned(const TsRecord& record); + int write_tablet_aligned(const Tablet& tablet); + int write_tree(const Tablet& tablet); + int write_tree(const TsRecord& record); + int write_table(Tablet& tablet); + + DeviceSchemasMap* get_schema_group_map(); + std::shared_ptr<TableSchema> get_table_schema( + const std::string& table_name) const; + int64_t calculate_mem_size_for_all_group(); + int64_t calculate_meta_mem_size() const; + int check_memory_size_and_may_flush_chunks(); + int flush(); + int close(); +}; +``` + ### TableSchema 描述表模式（schema）的数据结构。 @@ -297,8 +377,14 @@ uint8_t common::get_global_compression(); // 时间列的编码/压缩（数据类型固定为 INT64）。 int common::set_global_time_encoding(uint8_t encoding); int common::set_global_time_compression(uint8_t compression); +uint8_t common::get_global_time_encoding(); +uint8_t common::get_global_time_compression(); ``` +全局压缩支持 `UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD` +和 `LZMA2`。压缩枚举中也包含 `SDT`、`PAA`、`PLA` 等历史值，但全局压缩 +setter 会拒绝这些值。 + ## 读取接口 ### Tsfile Reader ```cpp @@ -335,6 +421,16 @@ class TsFileReader { * @return 成功时返回 0，失败时返回 errno_define.h 中的非零错误码。 */ int query(storage::QueryExpression *qe, ResultSet *&ret_qds); + /** + * @brief 通过路径列表、起始时间和结束时间查询 tsfile。用于 tree 模型。 + * + * @param [in] path_list 完整路径列表。 + * @param [in] start_time 起始时间。 + * @param [in] end_time 结束时间。 + * @param [out] result_set 查询结果集。 + */ + int query(std::vector<std::string>& path_list, int64_t start_time, + int64_t end_time, ResultSet*& result_set); /** * @brief 通过表名、列名、起始时间和结束时间查询 tsfile。 * @@ -346,7 +442,7 @@ class TsFileReader { */ int query(const std::string &table_name, const std::vector<std::string> &columns_names, int64_t start_time, - int64_t end_time, ResultSet *&result_set); + int64_t end_time, ResultSet *&result_set, int batch_size = -1); /** * @brief 通过表名、列名、开始时间、结束时间和标签过滤器查询 tsfile。 @@ -360,7 +456,14 @@ class TsFileReader { */ int query(const std::string& table_name, const std::vector<std::string>& columns_names, int64_t start_time, - int64_t end_time, ResultSet*& result_set, Filter* tag_filter); + int64_t end_time, ResultSet*& result_set, Filter* tag_filter, + int batch_size = 0); + + /** + * @brief 按行查询 tree 模型序列，支持 offset/limit。 + */ + int queryByRow(std::vector<std::string>& path_list, int offset, int limit, + ResultSet*& result_set); /** * @brief 按行查询表，支持偏移量/行数限制下推与可选的标签过滤。 @@ -379,12 +482,39 @@ class TsFileReader { int limit, ResultSet*& result_set, Filter* tag_filter = nullptr, int batch_size = 0); + /** + * @brief 按测点名在时间范围内查询 tree 模型数据。 + */ + int query_table_on_tree(const std::vector<std::string>& measurement_names, + int64_t start_time, int64_t end_time, + ResultSet*& result_set); + /** * @brief 销毁结果集，该方法应在查询完成并使用完 result_set 后调用。 * * @param qds 查询结果集。 */ void destroy_query_data_set(ResultSet *qds); + + ResultSet* read_timeseries( + const std::shared_ptr<IDeviceID>& device_id, + const std::vector<std::string>& measurement_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices( + std::string table_name); + + std::vector<std::shared_ptr<IDeviceID>> get_all_device_ids(); + + std::vector<std::shared_ptr<IDeviceID>> get_all_devices(); + + int get_timeseries_schema(std::shared_ptr<IDeviceID> device_id, + std::vector<MeasurementSchema>& result); + + DeviceTimeseriesMetadataMap get_timeseries_metadata( + const std::vector<std::shared_ptr<IDeviceID>>& device_ids); + + DeviceTimeseriesMetadataMap get_timeseries_metadata(); + /** * @brief 根据表名获取表的模式信息。 * @@ -401,6 +531,11 @@ class TsFileReader { std::vector<std::shared_ptr<TableSchema>> get_all_table_schemas(); }; ``` + +`DeviceTimeseriesMetadataMap` 是 `get_timeseries_metadata()` 返回的元数据 +map：`std::map<std::shared_ptr<IDeviceID>, +std::vector<std::shared_ptr<ITimeseriesIndex>>, IDeviceIDComparator>`。 + ### ResultSet ```cpp /** @@ -540,6 +675,8 @@ class TagFilterBuilder { Filter* reg_exp(const std::string& columnName, const std::string& value); Filter* not_reg_exp(const std::string& columnName, const std::string& value); + Filter* is_null(const std::string& columnName); + Filter* is_not_null(const std::string& columnName); Filter* between_and(const std::string& columnName, const std::string& lower, const std::string& upper); Filter* not_between_and(const std::string& columnName, @@ -550,4 +687,4 @@ class TagFilterBuilder { static Filter* or_filter(Filter* left, Filter* right); static Filter* not_filter(Filter* filter); }; -``` \ No newline at end of file +``` diff --git a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md index c94b690c7..1d3e875b1 100644 --- a/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md +++ b/src/zh/UserGuide/latest/QuickStart/InterfaceDefinition/InterfaceDefinition-Python.md @@ -45,10 +45,17 @@ class TSEncoding(IntEnum): PLAIN = 0 # 所有类型 DICTIONARY = 1 # STRING、TEXT RLE = 2 # INT32、INT64、TIMESTAMP、DATE + DIFF = 3 TS_2DIFF = 4 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE + BITMAP = 5 + GORILLA_V1 = 6 + REGULAR = 7 GORILLA = 8 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE ZIGZAG = 9 # INT32、INT64 + CHIMP = 11 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE SPRINTZ = 12 # INT32、INT64、FLOAT、DOUBLE + RLBE = 13 # INT32、INT64、TIMESTAMP、DATE、FLOAT、DOUBLE + CAMEL = 14 # DOUBLE class Compressor(IntEnum): """ @@ -58,7 +65,12 @@ class Compressor(IntEnum): SNAPPY = 1 GZIP = 2 LZO = 3 + SDT = 4 + PAA = 5 + PLA = 6 LZ4 = 7 + ZSTD = 8 + LZMA2 = 9 class ColumnCategory(IntEnum): """ @@ -99,6 +111,32 @@ class ResultSetMetaData: def __init__(self, column_list: List[str], data_types: List[TSDataType]) +@dataclass(frozen=True) +class DeviceID: + path: Optional[str] + table_name: Optional[str] + segments: tuple[Optional[str], ...] + +@dataclass(frozen=True) +class TimeseriesStatistic: + has_statistic: bool + row_count: int + start_time: int + end_time: int + +@dataclass(frozen=True) +class TimeseriesMetadata: + measurement_name: str + data_type: TSDataType + chunk_meta_count: int + statistic: TimeseriesStatistic + timeline_statistic: TimeseriesStatistic + +@dataclass(frozen=True) +class DeviceTimeseriesMetadataGroup: + table_name: Optional[str] + segments: tuple[Optional[str], ...] + timeseries: list[TimeseriesMetadata] ``` @@ -106,7 +144,7 @@ class ResultSetMetaData: ## 写入接口 -### TsFileWriter +### TsFileTableWriter ```python class TsFileTableWriter: @@ -157,6 +195,48 @@ class TsFileTableWriter: ``` +### TsFileWriter + +`TsFileWriter` 是 `writer.pyx` 暴露的低层写入接口。它可以写入 tree 模型数据 +（`register_timeseries`、`register_device`、`write_tablet`、`write_row_record`）， +也可以写入 table 模型数据（`register_table`、`write_table`、`write_dataframe`、 +`write_arrow_batch`）。 + +```python +class TsFileWriter: + """ + :param pathname: 目标 TsFile 路径。 + :param memory_threshold: 触发自动刷盘前缓冲的字节数（默认 128MB）。 + """ + def __init__(self, pathname: str, memory_threshold: int = 128 * 1024 * 1024) + + def register_timeseries(self, device_name: str, + timeseries_schema: TimeseriesSchema) + + def register_device(self, device_schema: DeviceSchema) + + def register_table(self, table_schema: TableSchema) + + def write_tablet(self, tablet: Tablet) + + def write_dataframe(self, target_table: str, dataframe: pandas.DataFrame, + tableschema: TableSchema) + + def write_row_record(self, record: RowRecord) + + def write_table(self, tablet: Tablet) + + def write_arrow_batch(self, table_name: str, data, + time_col_index: int = -1) + + def flush(self) + + def close(self) + + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) +``` + ### Tablet definition @@ -223,11 +303,12 @@ set_tsfile_config({ | 数据类型 | 允许的编码 | 默认值 | |---|---|---| | `BOOLEAN` | `PLAIN` | `PLAIN` | -| `INT32`、`INT64`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ` | `TS_2DIFF` | -| `FLOAT`、`DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ` | `GORILLA` | +| `INT32`、`INT64`、`TIMESTAMP`、`DATE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`ZIGZAG`、`RLE`、`SPRINTZ`、`CHIMP`、`RLBE` | `TS_2DIFF` | +| `FLOAT` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE` | `GORILLA` | +| `DOUBLE` | `PLAIN`、`TS_2DIFF`、`GORILLA`、`SPRINTZ`、`CHIMP`、`RLBE`、`CAMEL` | `GORILLA` | | `STRING`、`TEXT` | `PLAIN`、`DICTIONARY` | `PLAIN` | -压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`（默认 `LZ4`）。 +压缩适用于所有数据类型：`UNCOMPRESSED`、`SNAPPY`、`GZIP`、`LZO`、`LZ4`、`ZSTD`、`LZMA2`（默认 `LZ4`）。枚举还暴露 `SDT`、`PAA`、`PLA` 等历史值，但全局配置 setter 会拒绝这些值。 ## 读取接口 @@ -253,11 +334,39 @@ class TsFileReader: :param column_names: 要检索的列名列表。 :param start_time: 查询范围的起始时间（默认：int64 最小值）。 :param end_time: 查询范围的结束时间（默认：int64 最大值）。 + :param tag_filter: 可选的 table 模型 TAG 列谓词。 + :param batch_size: <= 0 逐行返回；> 0 按该大小返回数据块。 :return: 查询结果集处理器。 """ def query_table(self, table_name : str, column_names : List[str], start_time : int = np.iinfo(np.int64).min, - end_time: int = np.iinfo(np.int64).max) -> ResultSet + end_time: int = np.iinfo(np.int64).max, + tag_filter = None, batch_size : int = 0) -> ResultSet + + """ + 对 tree 模型测点列执行时间范围查询。 + + :param column_names: 要检索的测点名列表。 + :param start_time: 查询范围的起始时间。 + :param end_time: 查询范围的结束时间。 + :return: 查询结果集处理器。 + """ + def query_table_on_tree(self, column_names : List[str], + start_time : int = np.iinfo(np.int64).min, + end_time : int = np.iinfo(np.int64).max) -> ResultSet + + """ + 按行查询 tree 模型数据，支持 offset/limit。 + + :param device_ids: 要查询的设备标识列表。 + :param measurement_names: 要检索的测点名列表。 + :param offset: 需要跳过的起始行数。 + :param limit: 最多返回行数；< 0 表示不限制。 + :return: 查询结果集处理器。 + """ + def query_tree_by_row(self, device_ids : List[str], + measurement_names : List[str], + offset : int = 0, limit : int = -1) -> ResultSet """ 按行查询表，支持偏移量/行数限制下推与可选的标签过滤。标签谓词把查询限定到 @@ -277,6 +386,18 @@ class TsFileReader: offset : int = 0, limit : int = -1, tag_filter = None, batch_size : int = 0) -> ResultSet + """ + 对单个设备执行 tree 模型时间范围查询。 + + :param device_name: 设备标识。 + :param sensor_list: 要检索的测点名列表。 + :param start_time: 查询起始时间。 + :param end_time: 查询结束时间。 + :return: 查询结果集处理器。 + """ + def query_timeseries(self, device_name : str, sensor_list : List[str], + start_time : int = 0, end_time : int = 0) -> ResultSet + """ 获取指定表的模式信息。 @@ -292,6 +413,31 @@ class TsFileReader: """ def get_all_table_schemas(self) -> dict[str, TableSchema] + """ + 获取所有 tree 模型 timeseries schema，按设备分组。 + + :return: DeviceSchema 对象列表。 + """ + def get_all_timeseries_schemas(self) -> list[DeviceSchema] + + """ + 获取文件中的所有设备标识。 + + :return: DeviceID(path, table_name, segments) 列表。 + """ + def get_all_devices(self) -> List[DeviceID] + + """ + 获取所有设备或指定设备的 per-timeseries metadata。 + + :param device_ids: None 表示全部设备，[] 表示空结果，或传入 DeviceID / + 路径兼容的设备标识列表。 + :return: dict，key 为设备 segment tuple，value 为 DeviceTimeseriesMetadataGroup。 + """ + def get_timeseries_metadata( + self, device_ids: Optional[List] = None + ) -> Dict[tuple, DeviceTimeseriesMetadataGroup] + """ 关闭 TsFile 读取器。如果读取器中有活动的结果集，它们将失效。 """ @@ -332,6 +478,12 @@ class ResultSet: """ def read_data_frame(self, max_row_num : int = 1024) -> DataFrame + """ + 将下一个批结果读取为 pyarrow.Table。没有更多 TsBlock 批时返回 None。 + 仅适用于通过 batch_size > 0 创建的结果集。 + """ + def read_arrow_batch(self) -> Optional[pyarrow.Table] + """ 从查询结果集中按索引获取值。 @@ -376,6 +528,9 @@ class ResultSet: """ def close(self) + def __enter__(self) + def __exit__(self, exc_type, exc_val, exc_tb) + ```
