This is an automated email from the ASF dual-hosted git repository.
ColinLeeo pushed a commit to branch dataframe_desc
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/dataframe_desc by this push:
new e968332f8 feat(python): align dataframe description schema
e968332f8 is described below
commit e968332f8a0e6fd5656f72c0fb89ac4d57b288d8
Author: ColinLee <[email protected]>
AuthorDate: Sun Aug 9 13:31:26 2026 +0800
feat(python): align dataframe description schema
---
python/README-zh.md | 19 +++++---
python/README.md | 20 ++++++---
python/tests/test_tsfile_dataset.py | 90 +++++++++++++++++++++++++++++--------
python/tsfile/dataset/dataframe.py | 44 +++++++++++++-----
4 files changed, 133 insertions(+), 40 deletions(-)
diff --git a/python/README-zh.md b/python/README-zh.md
index 672ec1342..62b1f67fc 100644
--- a/python/README-zh.md
+++ b/python/README-zh.md
@@ -44,20 +44,26 @@
```json
{
- "weather": {
- "temperature": "C",
- "humidity": "T"
+ "multivariate": {
+ "weather": {
+ "columns": {
+ "temperature": "C",
+ "humidity": "T"
+ }
+ }
}
}
```
+未作为多变量训练数据使用的表可以不写入 `multivariate`。
+
创建 dataframe 时传入描述文件路径。`get` 和 `set` 提供对 JSON 顶层键的字典式
访问，其中 `set` 会将更新后的描述写回文件；另外还可以按表获取协变量列、目标
变量列及其数量：
```python
df = TsFileDataFrame("weather.tsfile", description_path="description.json")
-df.get("weather")
+df.get("multivariate")
df.get_covariate_columns("weather")
df.get_target_columns("weather")
df.get_covariate_column_count("weather")
@@ -69,7 +75,10 @@ df.get_device_statistics("weather")
df.get_device_point_count("weather", "device_a")
df.get_device_stats("weather", {"device": "device_a"})
df.list_device_metadata("weather")
-df.set("weather", {"temperature": "T", "humidity": "T"})
+df.set(
+ "multivariate",
+ {"weather": {"columns": {"temperature": "T", "humidity": "T"}}},
+)
df.close()
```
diff --git a/python/README.md b/python/README.md
index 424006e57..e068c09ad 100644
--- a/python/README.md
+++ b/python/README.md
@@ -41,20 +41,27 @@ covariate and `T` for a target column:
```json
{
- "weather": {
- "temperature": "C",
- "humidity": "T"
+ "multivariate": {
+ "weather": {
+ "columns": {
+ "temperature": "C",
+ "humidity": "T"
+ }
+ }
}
}
```
+Tables that are not used as multivariate training data may be omitted from
+`multivariate`.
+
Pass the description path when opening the dataframe. `get` and `set` provide
dictionary-style access to top-level JSON values; `set` writes the updated
description back to disk.
```python
df = TsFileDataFrame("weather.tsfile", description_path="description.json")
-df.get("weather")
+df.get("multivariate")
df.get_covariate_columns("weather")
df.get_target_columns("weather")
df.get_covariate_column_count("weather")
@@ -66,7 +73,10 @@ df.get_device_statistics("weather")
df.get_device_point_count("weather", "device_a")
df.get_device_stats("weather", {"device": "device_a"})
df.list_device_metadata("weather")
-df.set("weather", {"temperature": "T", "humidity": "T"})
+df.set(
+ "multivariate",
+ {"weather": {"columns": {"temperature": "T", "humidity": "T"}}},
+)
df.close()
```
diff --git a/python/tests/test_tsfile_dataset.py
b/python/tests/test_tsfile_dataset.py
index 3e42f9061..384751a68 100644
--- a/python/tests/test_tsfile_dataset.py
+++ b/python/tests/test_tsfile_dataset.py
@@ -266,9 +266,13 @@ def
test_dataset_description_get_set_and_column_roles(tmp_path):
description_path.write_text(
json.dumps(
{
- "weather": {
- "temperature": "C",
- "humidity": "T",
+ "multivariate": {
+ "weather": {
+ "columns": {
+ "temperature": "C",
+ "humidity": "T",
+ }
+ }
}
}
),
@@ -280,9 +284,13 @@ def
test_dataset_description_get_set_and_column_roles(tmp_path):
show_progress=False,
description_path=description_path,
) as tsdf:
- assert tsdf.get("weather") == {
- "temperature": "C",
- "humidity": "T",
+ assert tsdf.get("multivariate") == {
+ "weather": {
+ "columns": {
+ "temperature": "C",
+ "humidity": "T",
+ }
+ }
}
assert tsdf.get("missing") is None
assert tsdf.get("missing", {"default": True}) == {"default": True}
@@ -293,8 +301,8 @@ def
test_dataset_description_get_set_and_column_roles(tmp_path):
assert tsdf.get_covariate_columns("missing") == []
assert tsdf.get_target_column_count("missing") == 0
- description = tsdf.get("weather")
- description["temperature"] = "T"
+ description = tsdf.get("multivariate")
+ description["weather"]["columns"]["temperature"] = "T"
assert tsdf.get_covariate_columns("weather") == ["temperature"]
with pytest.raises(TypeError, match="must be JSON serializable"):
@@ -303,19 +311,27 @@ def
test_dataset_description_get_set_and_column_roles(tmp_path):
subset = tsdf[:1]
subset.set(
- "weather",
+ "multivariate",
{
- "temperature": "T",
- "humidity": "T",
+ "weather": {
+ "columns": {
+ "temperature": "T",
+ "humidity": "T",
+ }
+ }
},
)
assert tsdf.get_covariate_column_count("weather") == 0
assert tsdf.get_target_columns("weather") == ["temperature",
"humidity"]
assert json.loads(description_path.read_text(encoding="utf-8")) == {
- "weather": {
- "temperature": "T",
- "humidity": "T",
+ "multivariate": {
+ "weather": {
+ "columns": {
+ "temperature": "T",
+ "humidity": "T",
+ }
+ }
}
}
@@ -330,14 +346,46 @@ def
test_dataset_description_set_creates_missing_file(tmp_path):
show_progress=False,
description_path=description_path,
) as tsdf:
- assert tsdf.get("weather") is None
- tsdf.set("weather", {"temperature": "C", "humidity": "T"})
+ assert tsdf.get("multivariate") is None
+ tsdf.set(
+ "multivariate",
+ {
+ "weather": {
+ "columns": {
+ "temperature": "C",
+ "humidity": "T",
+ }
+ }
+ },
+ )
assert json.loads(description_path.read_text(encoding="utf-8")) == {
- "weather": {"temperature": "C", "humidity": "T"}
+ "multivariate": {
+ "weather": {
+ "columns": {
+ "temperature": "C",
+ "humidity": "T",
+ }
+ }
+ }
}
+def
test_dataset_description_allows_empty_or_unlisted_multivariate_tables(tmp_path):
+ path = tmp_path / "weather.tsfile"
+ description_path = tmp_path / "description.json"
+ _write_weather_file(path, 0)
+ description_path.write_text(json.dumps({"multivariate": {}}),
encoding="utf-8")
+
+ with TsFileDataFrame(
+ str(path),
+ show_progress=False,
+ description_path=description_path,
+ ) as tsdf:
+ assert tsdf.get_covariate_columns("weather") == []
+ assert tsdf.get_target_columns("weather") == []
+
+
def test_dataset_description_rejects_invalid_json_and_roles(tmp_path):
path = tmp_path / "weather.tsfile"
description_path = tmp_path / "description.json"
@@ -352,7 +400,8 @@ def
test_dataset_description_rejects_invalid_json_and_roles(tmp_path):
)
description_path.write_text(
- json.dumps({"weather": {"temperature": "X"}}), encoding="utf-8"
+ json.dumps({"multivariate": {"weather": {"columns": {"temperature":
"X"}}}}),
+ encoding="utf-8",
)
with TsFileDataFrame(
str(path),
@@ -362,7 +411,10 @@ def
test_dataset_description_rejects_invalid_json_and_roles(tmp_path):
with pytest.raises(ValueError, match="expected 'C' or 'T'"):
tsdf.get_covariate_columns("weather")
- tsdf.set("weather", {"temperature": ["C"]})
+ tsdf.set(
+ "multivariate",
+ {"weather": {"columns": {"temperature": ["C"]}}},
+ )
with pytest.raises(ValueError, match="expected 'C' or 'T'"):
tsdf.get_target_columns("weather")
diff --git a/python/tsfile/dataset/dataframe.py
b/python/tsfile/dataset/dataframe.py
index 0e61f1e15..52ea88b36 100644
--- a/python/tsfile/dataset/dataframe.py
+++ b/python/tsfile/dataset/dataframe.py
@@ -61,6 +61,8 @@ _OVERLAP_ROW_CHUNK_SIZE = 256
_COVARIATE_ROLE = "C"
_TARGET_ROLE = "T"
_DESCRIPTION_ROLES = {_COVARIATE_ROLE, _TARGET_ROLE}
+_MULTIVARIATE_KEY = "multivariate"
+_COLUMNS_KEY = "columns"
def _load_dataframe_description(description_path: Optional[str]) -> dict:
@@ -674,18 +676,24 @@ class TsFileDataFrame:
"""Lazy-loaded unified numeric dataset view over multiple TsFile shards.
``description_path`` optionally points to a JSON object describing the
- columns in each table. A description uses the following shape::
+ columns in multivariate tables. A description uses the following shape::
{
- "weather": {
- "temperature": "C",
- "humidity": "T"
+ "multivariate": {
+ "weather": {
+ "columns": {
+ "temperature": "C",
+ "humidity": "T"
+ }
+ }
}
}
- ``C`` marks a covariate column and ``T`` marks a target column. The
- description is independent of the TsFile data and is shared by dataframe
- subsets created through slicing or boolean selection.
+ Tables that are not used as multivariate training data do not need to be
+ listed under ``multivariate``. ``C`` marks a covariate column and ``T``
+ marks a target column. The description is independent of the TsFile data
+ and is shared by dataframe subsets created through slicing or boolean
+ selection.
"""
def __init__(
@@ -783,17 +791,31 @@ class TsFileDataFrame:
f"Description table name must be a string, got
{type(table_name)}"
)
- table_description = self._owner()._description.get(table_name)
+ multivariate_description =
self._owner()._description.get(_MULTIVARIATE_KEY)
+ if multivariate_description is None:
+ return []
+ if not isinstance(multivariate_description, dict):
+ raise ValueError("Description field 'multivariate' must be a JSON
object")
+
+ table_description = multivariate_description.get(table_name)
if table_description is None:
return []
if not isinstance(table_description, dict):
raise ValueError(
- f"Description for table '{table_name}' must be a JSON object"
+ f"Description for multivariate table '{table_name}' must be a
JSON object"
+ )
+
+ columns_description = table_description.get(_COLUMNS_KEY)
+ if columns_description is None:
+ return []
+ if not isinstance(columns_description, dict):
+ raise ValueError(
+ f"Description field 'multivariate.{table_name}.columns' must
be a JSON object"
)
invalid_roles = {
column: column_role
- for column, column_role in table_description.items()
+ for column, column_role in columns_description.items()
if not isinstance(column_role, str) or column_role not in
_DESCRIPTION_ROLES
}
if invalid_roles:
@@ -804,7 +826,7 @@ class TsFileDataFrame:
)
return [
column
- for column, column_role in table_description.items()
+ for column, column_role in columns_description.items()
if column_role == role
]
