JingsongLi commented on code in PR #11056:
URL: https://github.com/apache/paimon/pull/11056#discussion_r4521846327


##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,332 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _append_arrow_tables, _commit_metadata, _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64) for key in _STATS}
+              for item in items]
+    first = arrays[0]
+    for item in arrays:
+        if any(item[key].shape != first[key].shape for key in _STATS):
+            raise ValueError("LeRobot statistics shapes must match.")
+        count = item["count"]
+        if count.shape != (1,) or not np.all(np.isfinite(count)) \
+                or np.any(count <= 0) or np.any(count != np.floor(count)):
+            raise ValueError("LeRobot statistics count must be one positive 
integer.")
+        if any(item[key].shape != item["mean"].shape for key in ("min", "max", 
"std")):
+            raise ValueError("LeRobot statistic shapes must match their mean.")
+    count = sum(item["count"] for item in arrays)
+    mean = sum(item["mean"] * item["count"] for item in arrays) / count
+    variance = sum(item["count"] * (
+        item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / 
count
+    return {
+        "min": np.minimum.reduce([item["min"] for item in arrays]).tolist(),
+        "max": np.maximum.reduce([item["max"] for item in arrays]).tolist(),
+        "mean": mean.tolist(), "std": np.sqrt(variance).tolist(),
+        "count": count.astype(np.int64).tolist(),
+    }
+
+
+def _batch_stats(values):
+    values = np.asarray(values, dtype=np.float64)
+    return {"min": [values.min()], "max": [values.max()],
+            "mean": [values.mean()], "std": [values.std()],
+            "count": [len(values)]}
+
+
+def _vocabulary(plans, kind):
+    key = kind + "_table"
+    tables = [plan["metadata"][key] for plan in plans]
+    if all(table is None for table in tables):
+        return None
+    if any(table is None for table in tables):
+        raise ValueError("LeRobot sources disagree on %s metadata." % kind)
+    singular = kind[:-1]
+    index_name = singular + "_index"
+    schema = tables[0].schema
+    label = _pandas_index_column(schema, singular)
+    if set(schema.names) != {index_name, label}:
+        raise ValueError("Multi-source %s metadata requires only index and 
text." % kind)
+    rows, by_text = [], {}
+    for ordinal, (plan, table) in enumerate(zip(plans, tables)):
+        source_label = _pandas_index_column(table.schema, singular)
+        if (set(table.column_names) != {index_name, source_label}
+                or not table.schema.field(index_name).equals(
+                    schema.field(index_name), check_metadata=False)
+                or not 
table.schema.field(source_label).with_name(label).equals(
+                    schema.field(label), check_metadata=False)):
+            raise ValueError("LeRobot source %d has incompatible %s schema." % 
(ordinal, kind))
+        mapping = {}
+        for row in sorted(table.to_pylist(), key=lambda row: row[index_name]):
+            text = row[source_label]
+            if text not in by_text:
+                by_text[text] = len(rows)
+                rows.append({index_name: len(rows), label: text})
+            mapping[row[index_name]] = by_text[text]
+        plan[index_name] = mapping
+    return pa.Table.from_pylist(rows, schema=schema)
+
+
+def _feature_contract(features):
+    result = copy.deepcopy(features)
+    for feature in result.values():
+        if feature.get("dtype") == "video":
+            for key in ("info", "video_info"):
+                feature[key] = {k: v for k, v in feature.get(key, {}).items()
+                                if k not in _ENCODER_KEYS}
+    return result
+
+
+def _plan_merge(plans):
+    first = plans[0]
+    info = copy.deepcopy(first["info"])
+    metadata = dict(first["metadata"])
+    frame_offset = episode_offset = 0
+    for ordinal, plan in enumerate(plans):
+        other = plan["info"]
+        for key in (set(info) | set(other)) - _REBUILT_INFO:
+            if info.get(key) != other.get(key):
+                raise ValueError("LeRobot source %d has incompatible info.%s." 
% (ordinal, key))
+        expected_features = _feature_contract(info["features"])
+        actual_features = _feature_contract(other["features"])
+        for name in set(expected_features) | set(actual_features):
+            if expected_features.get(name) != actual_features.get(name):
+                raise ValueError("LeRobot source %d has incompatible 
features.%s." % (ordinal, name))
+        if not plan["schema"].equals(first["schema"], check_metadata=False):
+            raise ValueError("LeRobot source %d has incompatible frame 
schema." % ordinal)
+        if not plan["metadata"]["episodes_schema"].equals(
+                metadata["episodes_schema"], check_metadata=False):
+            raise ValueError("LeRobot source %d has incompatible Episode 
schema." % ordinal)
+        # Validate every Episode before creating any target table.
+        try:
+            for name in plan["videos"]:
+                _video_fps(other, other["features"][name], name)
+            for unused in _validated_episode_tables(plan["metadata"], 
first["videos"]):
+                pass
+        except ValueError as error:
+            raise ValueError("LeRobot source %d: %s" % (ordinal, error)) from 
error
+        plan["frame_offset"] = frame_offset
+        plan["episode_offset"] = episode_offset
+        plan["control_stats"] = {}
+        frame_offset += other["total_frames"]
+        episode_offset += other["total_episodes"]
+    metadata["tasks_table"] = _vocabulary(plans, "tasks")
+    metadata["subtasks_table"] = _vocabulary(plans, "subtasks")
+    info.update(total_frames=frame_offset, total_episodes=episode_offset,
+                total_tasks=metadata["tasks_table"].num_rows,
+                splits={"train": "0:%d" % episode_offset})
+    if metadata["subtasks_table"] is not None:
+        info["total_subtasks"] = metadata["subtasks_table"].num_rows
+    for key in ("total_chunks", "total_videos", "data_path", "video_path",
+                "chunks_size", "data_files_size_in_mb", 
"video_files_size_in_mb"):
+        info.pop(key, None)
+    # Encoder descriptions are not training features. Retain only shared 
values.
+    for name in first["videos"]:
+        for key in ("info", "video_info"):
+            details = [p["info"]["features"][name].get(key, {}) for p in plans]
+            info["features"][name][key] = {
+                k: v for k, v in details[0].items()
+                if all(d.get(k) == v for d in details[1:])}
+    stats = [p["metadata"]["stats_table"] for p in plans]
+    if any(s is None for s in stats) and not all(s is None for s in stats):
+        raise ValueError("LeRobot stats must be present in all sources or 
absent in all.")
+    merged_stats = None
+    if stats[0] is not None:
+        decoded = [_json_table(s) for s in stats]
+        if any(set(s) != set(decoded[0]) for s in decoded[1:]):
+            raise ValueError("LeRobot statistics features must match.")
+        merged_stats = {name: _merge_stats([s[name] for s in decoded])
+                        for name in decoded[0] if name not in _CONTROLS}
+    metadata["info_table"] = _metadata_table(info)
+    metadata["stats_table"] = (None if merged_stats is None
+                               else _metadata_table(merged_stats))
+    # Quantiles are omitted rather than presenting source summaries as merged 
values.
+    metadata["episodes_schema"] = pa.schema([
+        field for field in metadata["episodes_schema"]
+        if not any(field.name.startswith("stats/%s/q" % name) for name in 
_CONTROLS)
+    ], metadata=metadata["episodes_schema"].metadata)
+    return metadata, merged_stats
+
+
+def _transform_frames(plan, batch, episode):
+    for name in _CONTROLS:
+        if name not in batch.column_names:
+            continue
+        column = batch[name]
+        if name in ("index", "episode_index"):
+            offset = plan["frame_offset" if name == "index" else 
"episode_offset"]
+            values = pc.add_checked(column, pa.scalar(offset, 
type=column.type))
+        else:
+            mapping = plan[name]
+            values = pa.array([
+                None if name == "subtask_index" and value is None

Review Comment:
   [P1] The two nullable-statistics cases above now pass. I reproduced another 
partial commit on `d53f1b426f` with two five-frame FileIO sources. I added a 
`capture_time` column of type `timestamp[ms, tz=America/New_York]` to both 
Episode Parquet tables and called `load_from_lerobot` with a tag. The preflight 
validates against the source-derived schema, but the Episode writer compares it 
to the actual Paimon companion schema, where the field is `timestamp[ms, 
tz=UTC]`. The import raised `LeRobot component schema ... does not match target 
...`. After the exception, the target Frames table already contained all 10 
rows and the root tag was absent.
   
   Could the preflight validate or canonicalize against the created Episode 
table's Arrow schema before committing Frames? The original timezone can still 
be preserved in `source_episode_ipc`.



##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,475 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.arrow_utils import strict_arrow_table
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _EPISODE_FPS_COLUMN, _append_arrow_tables, _commit_metadata, _json_value,
+    _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+_DATASET_INDEX_COLUMN = "dataset_index"
+_SOURCE_FRAME_INDEX_COLUMN = "source_index"
+_DATASETS_SCHEMA = pa.schema([
+    pa.field(_DATASET_INDEX_COLUMN, pa.int64(), nullable=False),
+    pa.field("dataset_name", pa.string()),
+    pa.field("fps", pa.int64(), nullable=False),
+    pa.field("episode_from_index", pa.int64(), nullable=False),
+    pa.field("episode_to_index", pa.int64(), nullable=False),
+    pa.field("frame_from_index", pa.int64(), nullable=False),
+    pa.field("frame_to_index", pa.int64(), nullable=False),
+    pa.field("info_json", pa.string(), nullable=False),
+    pa.field("tasks_ipc", pa.binary(), nullable=False),
+    pa.field("subtasks_ipc", pa.binary()),
+    pa.field("stats_ipc", pa.binary()),
+    pa.field("episodes_schema_ipc", pa.binary(), nullable=False),
+])
+
+
+def _arrow_ipc(table):
+    if table is None:
+        return None
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64 if key in ("mean", 
"std") else None)
+               for key in _STATS}
+              for item in items]
+    first = arrays[0]
+    for item in arrays:
+        if any(item[key].shape != first[key].shape for key in _STATS):
+            raise ValueError("LeRobot statistics shapes must match.")
+        count = item["count"]
+        if count.shape != (1,) or not np.all(np.isfinite(count)) \
+                or np.any(count <= 0) or np.any(count != np.floor(count)):
+            raise ValueError("LeRobot statistics count must be one positive 
integer.")
+        if any(item[key].shape != item["mean"].shape for key in ("min", "max", 
"std")):
+            raise ValueError("LeRobot statistic shapes must match their mean.")
+    count = sum(int(item["count"][0]) for item in arrays)
+    mean = sum(item["mean"] * item["count"] for item in arrays) / count
+    variance = sum(item["count"] * (
+        item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / 
count
+
+    def extrema(key, reduction):
+        dtype = object if all(np.issubdtype(item[key].dtype, np.integer)
+                              for item in arrays) else np.float64
+        return reduction([item[key].astype(dtype) for item in arrays]).tolist()
+
+    return {
+        "min": extrema("min", np.minimum.reduce),
+        "max": extrema("max", np.maximum.reduce),
+        "mean": mean.tolist(), "std": np.sqrt(variance).tolist(),
+        "count": [count],
+    }
+
+
+def _batch_stats(values):
+    values = np.asarray(values)
+    return {"min": [values.min().item()], "max": [values.max().item()],
+            "mean": [values.mean()], "std": [values.std()],
+            "count": [len(values)]}
+
+
+def _vocabulary(plans, kind):
+    key = kind + "_table"
+    tables = [plan["metadata"][key] for plan in plans]
+    if all(table is None for table in tables):
+        return None
+    if any(table is None for table in tables):
+        raise ValueError("LeRobot sources disagree on %s metadata." % kind)
+    singular = kind[:-1]
+    index_name = singular + "_index"
+    schema = tables[0].schema
+    label = _pandas_index_column(schema, singular)
+    schema = tables[0].select([index_name, label]).schema
+    rows, by_text = [], {}
+    for ordinal, (plan, table) in enumerate(zip(plans, tables)):
+        source_label = _pandas_index_column(table.schema, singular)
+        if (not table.schema.field(index_name).equals(
+                schema.field(index_name), check_metadata=False)
+                or not 
(pa.types.is_string(table.schema.field(source_label).type)
+                        or 
pa.types.is_large_string(table.schema.field(source_label).type))):
+            raise ValueError("LeRobot source %d has incompatible %s schema." % 
(ordinal, kind))
+        mapping = {}
+        for row in sorted(table.select([index_name, source_label]).to_pylist(),
+                          key=lambda row: row[index_name]):
+            text = row[source_label]
+            if text not in by_text:
+                by_text[text] = len(rows)
+                rows.append({index_name: len(rows), label: text})
+            mapping[row[index_name]] = by_text[text]
+        plan[index_name] = mapping
+    return pa.Table.from_pylist(rows, schema=schema)
+
+
+def _feature_contract(features, fps_mode):
+    result = copy.deepcopy(features)
+    for feature in result.values():
+        if feature.get("dtype") == "video":
+            if fps_mode == "source":
+                feature.pop("fps", None)
+            for key in ("info", "video_info"):
+                feature[key] = {k: v for k, v in feature.get(key, {}).items()
+                                if k not in _ENCODER_KEYS
+                                and (fps_mode == "dataset" or k != 
"video.fps")}
+    return result
+
+
+def _same_fields(left, right):
+    return (set(left.names) == set(right.names)
+            and all(left.field(name).equals(right.field(name), 
check_metadata=False)
+                    for name in right.names))
+
+
+def _plan_merge(plans, fps_mode="dataset"):
+    first = plans[0]
+    reserved_frame_fields = {
+        _DATASET_INDEX_COLUMN, _SOURCE_FRAME_INDEX_COLUMN,
+    }
+    if reserved_frame_fields.intersection(first["schema"].names):
+        raise ValueError("LeRobot source frame features use reserved source 
index columns.")
+    info = copy.deepcopy(first["info"])
+    metadata = dict(first["metadata"])
+    # Quantiles are omitted rather than presenting source summaries as merged 
values.
+    metadata["episodes_schema"] = pa.schema([
+        field.with_nullable(True) if 
field.name.startswith("stats/subtask_index/") else field
+        for field in metadata["episodes_schema"]
+        if not any(field.name.startswith("stats/%s/q" % name) for name in 
_CONTROLS)
+        and not (fps_mode == "source" and field.name == _EPISODE_FPS_COLUMN)
+    ], metadata=metadata["episodes_schema"].metadata)
+    frame_offset = episode_offset = 0
+    for ordinal, plan in enumerate(plans):
+        other = plan["info"]
+        compared_keys = ("codebase_version", "robot_type")
+        if fps_mode == "dataset":
+            compared_keys += ("fps",)
+        for key in compared_keys:
+            if info.get(key) != other.get(key):
+                raise ValueError("LeRobot source %d has incompatible info.%s." 
% (ordinal, key))
+        expected_features = _feature_contract(info["features"], fps_mode)
+        actual_features = _feature_contract(other["features"], fps_mode)
+        for name in set(expected_features) | set(actual_features):
+            if expected_features.get(name) != actual_features.get(name):
+                raise ValueError("LeRobot source %d has incompatible 
features.%s." % (ordinal, name))
+        if not _same_fields(plan["schema"], first["schema"]):
+            raise ValueError("LeRobot source %d has incompatible frame 
schema." % ordinal)
+        episode_schema = pa.schema([
+            field.with_nullable(True) if 
field.name.startswith("stats/subtask_index/") else field
+            for field in plan["metadata"]["episodes_schema"]
+            if not any(field.name.startswith("stats/%s/q" % name) for name in 
_CONTROLS)
+            and not (fps_mode == "source" and field.name == 
_EPISODE_FPS_COLUMN)
+        ])
+        if not _same_fields(episode_schema, metadata["episodes_schema"]):
+            raise ValueError("LeRobot source %d has incompatible Episode 
schema." % ordinal)
+        # Validate every Episode before creating any target table.
+        plan["metadata"]["fps_mode"] = fps_mode
+        try:
+            for name in plan["videos"]:
+                _video_fps(other, other["features"][name], name)
+            for unused in _validated_episode_tables(plan["metadata"], 
first["videos"]):
+                pass
+        except ValueError as error:
+            raise ValueError("LeRobot source %d: %s" % (ordinal, error)) from 
error
+        plan["frame_offset"] = frame_offset
+        plan["episode_offset"] = episode_offset
+        plan["control_stats"] = {}
+        frame_offset += other["total_frames"]
+        episode_offset += other["total_episodes"]
+    for key in list(info):
+        if key not in _REBUILT_INFO and key not in ("codebase_version", 
"robot_type", "fps") \
+                and any(plan["info"].get(key) != info[key] for plan in plans):
+            del info[key]
+    metadata["tasks_table"] = _vocabulary(plans, "tasks")
+    metadata["subtasks_table"] = _vocabulary(plans, "subtasks")
+    reserved_episode_fields = {
+        _DATASET_INDEX_COLUMN,
+        "source_frame_from_index", "source_frame_to_index",
+        "source_episode_ipc",
+    }
+    if reserved_episode_fields.intersection(metadata["episodes_schema"].names):
+        raise ValueError("Source Episode metadata uses reserved source index 
columns.")
+    for field in (
+            pa.field(_DATASET_INDEX_COLUMN, pa.int64(), nullable=False),
+            pa.field("source_frame_from_index", pa.int64(), nullable=False),
+            pa.field("source_frame_to_index", pa.int64(), nullable=False),
+            pa.field("source_episode_ipc", pa.binary(), nullable=False)):
+        metadata["episodes_schema"] = metadata["episodes_schema"].append(field)
+    metadata["datasets_table"] = pa.Table.from_pylist([
+        {
+            _DATASET_INDEX_COLUMN: ordinal,
+            "dataset_name": plan["dataset_name"],
+            "fps": plan["metadata"]["fps"],
+            "episode_from_index": plan["episode_offset"],
+            "episode_to_index": (plan["episode_offset"]
+                                 + plan["info"]["total_episodes"]),
+            "frame_from_index": plan["frame_offset"],
+            "frame_to_index": plan["frame_offset"] + 
plan["info"]["total_frames"],
+            "info_json": json.dumps(_json_value(plan["info"]),
+                                    ensure_ascii=False, separators=(",", ":")),
+            "tasks_ipc": _arrow_ipc(plan["metadata"]["tasks_table"]),
+            "subtasks_ipc": _arrow_ipc(plan["metadata"]["subtasks_table"]),
+            "stats_ipc": _arrow_ipc(plan["metadata"]["stats_table"]),
+            "episodes_schema_ipc": 
plan["metadata"]["episodes_schema"].serialize().to_pybytes(),
+        }
+        for ordinal, plan in enumerate(plans)
+    ], schema=_DATASETS_SCHEMA)
+    info.update(total_frames=frame_offset, total_episodes=episode_offset,
+                total_tasks=metadata["tasks_table"].num_rows,
+                splits={"train": "0:%d" % episode_offset})
+    mixed_fps = len({plan["metadata"]["fps"] for plan in plans}) > 1
+    if fps_mode == "source":
+        info["fps_mode"] = "source"
+    if mixed_fps:
+        info["fps"] = None
+        for feature in info["features"].values():
+            if feature.get("dtype") != "video":
+                continue
+            feature.pop("fps", None)
+            for key in ("info", "video_info"):
+                if isinstance(feature.get(key), dict):
+                    feature[key].pop("video.fps", None)
+    if metadata["subtasks_table"] is not None:
+        info["total_subtasks"] = metadata["subtasks_table"].num_rows
+    for key in ("total_chunks", "total_videos", "data_path", "video_path",
+                "chunks_size", "data_files_size_in_mb", 
"video_files_size_in_mb"):
+        info.pop(key, None)
+    # Encoder descriptions are not training features. Retain only shared 
values.
+    for name in first["videos"]:
+        for key in ("info", "video_info"):
+            details = [p["info"]["features"][name].get(key, {}) for p in plans]
+            info["features"][name][key] = {
+                k: v for k, v in details[0].items()
+                if all(d.get(k) == v for d in details[1:])}
+    stats = [p["metadata"]["stats_table"] for p in plans]
+    if any(s is None for s in stats) and not all(s is None for s in stats):
+        raise ValueError("LeRobot stats must be present in all sources or 
absent in all.")
+    merged_stats = None
+    if stats[0] is not None:
+        decoded = [_json_table(s) for s in stats]
+        if any(set(s) != set(decoded[0]) for s in decoded[1:]):
+            raise ValueError("LeRobot statistics features must match.")
+        merged_stats = {name: _merge_stats([s[name] for s in decoded])
+                        for name in decoded[0] if name not in _CONTROLS}
+    metadata["info_table"] = _metadata_table(info)
+    metadata["stats_table"] = (None if merged_stats is None
+                               else _metadata_table(merged_stats))
+    return metadata, merged_stats
+
+
+def _transform_frames(plan, batch, episode):
+    batch = batch.append_column(
+        pa.field(_DATASET_INDEX_COLUMN, pa.int64(), nullable=False),
+        pa.array([plan[_DATASET_INDEX_COLUMN]] * batch.num_rows, 
type=pa.int64()))
+    batch = batch.append_column(pa.field(_SOURCE_FRAME_INDEX_COLUMN, 
pa.int64(),
+                                         nullable=False), batch["index"])
+    for name in _CONTROLS:
+        if name not in batch.column_names:
+            continue
+        column = batch[name]
+        if name == "index":
+            offset = plan["frame_offset"]
+            values = pc.add_checked(column, pa.scalar(offset, 
type=column.type))
+        elif name == "episode_index":
+            values = column
+        else:
+            mapping = plan[name]
+            values = pa.array([
+                None if name == "subtask_index" and value is None
+                else mapping[value]
+                for value in column.to_pylist()
+            ], type=column.type)

Review Comment:
   [P1] Preserve the original per-frame subtask ID when source labels repeat. A 
valid source vocabulary can contain IDs `0` and `1` with the same text, such as 
`reach`. In my repro, both IDs were remapped to `0`. Swapping the original IDs 
on two frames in the same Episode produced identical contents in all seven 
output tables, including the archived IPC. The saved vocabulary alone cannot 
recover which ID each source frame used.
   
   Could the import retain the original per-frame `subtask_index` alongside the 
merged ID, or reject ambiguous vocabularies before writing?



##########
paimon-python/pypaimon/multimodal/lerobot/dataset.py:
##########
@@ -1675,20 +1871,69 @@ def read_frames(decoder, windows):
     return {key: windows for key, windows in decoded if windows is not None}
 
 
+def _pack_nullable_labels(rows):
+    import torch
+
+    labels = [row.get("subtask_index") for row in rows]
+    if not any(_contains_none(value) for value in labels):
+        return
+    tensors = [value for label in labels
+               for value in (label if isinstance(label, list) else [label])
+               if torch.is_tensor(value)]
+    if not tensors:
+        return
+    # Disjoint views preserve isolation while sharing one storage across IPC.
+    packed = torch.cat([value.reshape(-1) for value in tensors])
+    views = iter(packed.split([value.numel() for value in tensors]))
+    for row, label in zip(rows, labels):
+        if isinstance(label, list):
+            row["subtask_index"] = [
+                next(views).reshape(value.shape) if torch.is_tensor(value) 
else value
+                for value in label]
+        elif torch.is_tensor(label):
+            row["subtask_index"] = next(views).reshape(label.shape)
+
+
 def _collate_lerobot_batch(rows, video_windows):
+    import torch
     from torch.utils.data import default_collate
 
-    if not video_windows:
+    nullable = {}
+    if "subtask_index" in rows[0]:
+        labels = [row["subtask_index"] for row in rows]
+        if any(_contains_none(value) for value in labels):
+            def values(label):
+                if isinstance(label, list):
+                    return [values(value) for value in label]
+                return -1 if label is None else label.tolist()
+
+            nullable["subtask_index"] = torch.tensor(
+                [values(label) for label in labels], dtype=torch.int64, 
device="cpu")
+            nullable["subtask_index_is_valid"] = nullable["subtask_index"] >= 0
+        else:
+            nullable["subtask_index"] = default_collate(labels)
+            nullable["subtask_index_is_valid"] = torch.ones_like(
+                nullable["subtask_index"], dtype=torch.bool)
+    if "subtask" in rows[0] and any(_contains_none(row["subtask"]) for row in 
rows):
+        nullable["subtask"] = [row["subtask"] for row in rows]
+    if not video_windows and not nullable:
         return default_collate(rows)
     remaining = [{
         key: value for key, value in row.items()
-        if key not in video_windows
+        if key not in video_windows and key not in nullable
     } for row in rows]
     batch = default_collate(remaining)
+    batch.update(nullable)

Review Comment:
   [P2] Preserve a source feature named `subtask_index_is_valid`. I imported a 
source with that boolean feature and two labelled frames. Individual reads 
returned `[False, True]` for the feature, while `to_dataloader(batch_size=2)` 
returned `[True, True]` for the same frames because the generated mask 
overwrote it. Could the generated mask use a separate namespace or otherwise 
preserve the original feature on a name collision?



##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,475 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.arrow_utils import strict_arrow_table
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _EPISODE_FPS_COLUMN, _append_arrow_tables, _commit_metadata, _json_value,
+    _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+_DATASET_INDEX_COLUMN = "dataset_index"
+_SOURCE_FRAME_INDEX_COLUMN = "source_index"
+_DATASETS_SCHEMA = pa.schema([
+    pa.field(_DATASET_INDEX_COLUMN, pa.int64(), nullable=False),
+    pa.field("dataset_name", pa.string()),
+    pa.field("fps", pa.int64(), nullable=False),
+    pa.field("episode_from_index", pa.int64(), nullable=False),
+    pa.field("episode_to_index", pa.int64(), nullable=False),
+    pa.field("frame_from_index", pa.int64(), nullable=False),
+    pa.field("frame_to_index", pa.int64(), nullable=False),
+    pa.field("info_json", pa.string(), nullable=False),
+    pa.field("tasks_ipc", pa.binary(), nullable=False),
+    pa.field("subtasks_ipc", pa.binary()),
+    pa.field("stats_ipc", pa.binary()),
+    pa.field("episodes_schema_ipc", pa.binary(), nullable=False),
+])
+
+
+def _arrow_ipc(table):
+    if table is None:
+        return None
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64 if key in ("mean", 
"std") else None)
+               for key in _STATS}
+              for item in items]
+    first = arrays[0]
+    for item in arrays:
+        if any(item[key].shape != first[key].shape for key in _STATS):
+            raise ValueError("LeRobot statistics shapes must match.")
+        count = item["count"]
+        if count.shape != (1,) or not np.all(np.isfinite(count)) \
+                or np.any(count <= 0) or np.any(count != np.floor(count)):
+            raise ValueError("LeRobot statistics count must be one positive 
integer.")
+        if any(item[key].shape != item["mean"].shape for key in ("min", "max", 
"std")):
+            raise ValueError("LeRobot statistic shapes must match their mean.")
+    count = sum(int(item["count"][0]) for item in arrays)
+    mean = sum(item["mean"] * item["count"] for item in arrays) / count
+    variance = sum(item["count"] * (
+        item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / 
count
+
+    def extrema(key, reduction):
+        dtype = object if all(np.issubdtype(item[key].dtype, np.integer)
+                              for item in arrays) else np.float64
+        return reduction([item[key].astype(dtype) for item in arrays]).tolist()
+
+    return {
+        "min": extrema("min", np.minimum.reduce),
+        "max": extrema("max", np.maximum.reduce),
+        "mean": mean.tolist(), "std": np.sqrt(variance).tolist(),
+        "count": [count],
+    }
+
+
+def _batch_stats(values):
+    values = np.asarray(values)
+    return {"min": [values.min().item()], "max": [values.max().item()],
+            "mean": [values.mean()], "std": [values.std()],
+            "count": [len(values)]}
+
+
+def _vocabulary(plans, kind):
+    key = kind + "_table"
+    tables = [plan["metadata"][key] for plan in plans]
+    if all(table is None for table in tables):
+        return None
+    if any(table is None for table in tables):
+        raise ValueError("LeRobot sources disagree on %s metadata." % kind)
+    singular = kind[:-1]
+    index_name = singular + "_index"
+    schema = tables[0].schema
+    label = _pandas_index_column(schema, singular)
+    schema = tables[0].select([index_name, label]).schema
+    rows, by_text = [], {}
+    for ordinal, (plan, table) in enumerate(zip(plans, tables)):
+        source_label = _pandas_index_column(table.schema, singular)
+        if (not table.schema.field(index_name).equals(
+                schema.field(index_name), check_metadata=False)
+                or not 
(pa.types.is_string(table.schema.field(source_label).type)
+                        or 
pa.types.is_large_string(table.schema.field(source_label).type))):
+            raise ValueError("LeRobot source %d has incompatible %s schema." % 
(ordinal, kind))
+        mapping = {}
+        for row in sorted(table.select([index_name, source_label]).to_pylist(),
+                          key=lambda row: row[index_name]):
+            text = row[source_label]
+            if text not in by_text:
+                by_text[text] = len(rows)
+                rows.append({index_name: len(rows), label: text})
+            mapping[row[index_name]] = by_text[text]
+        plan[index_name] = mapping
+    return pa.Table.from_pylist(rows, schema=schema)
+
+
+def _feature_contract(features, fps_mode):
+    result = copy.deepcopy(features)
+    for feature in result.values():
+        if feature.get("dtype") == "video":
+            if fps_mode == "source":
+                feature.pop("fps", None)
+            for key in ("info", "video_info"):
+                feature[key] = {k: v for k, v in feature.get(key, {}).items()
+                                if k not in _ENCODER_KEYS
+                                and (fps_mode == "dataset" or k != 
"video.fps")}
+    return result
+
+
+def _same_fields(left, right):
+    return (set(left.names) == set(right.names)
+            and all(left.field(name).equals(right.field(name), 
check_metadata=False)
+                    for name in right.names))
+
+
+def _plan_merge(plans, fps_mode="dataset"):
+    first = plans[0]
+    reserved_frame_fields = {
+        _DATASET_INDEX_COLUMN, _SOURCE_FRAME_INDEX_COLUMN,
+    }
+    if reserved_frame_fields.intersection(first["schema"].names):
+        raise ValueError("LeRobot source frame features use reserved source 
index columns.")
+    info = copy.deepcopy(first["info"])
+    metadata = dict(first["metadata"])
+    # Quantiles are omitted rather than presenting source summaries as merged 
values.
+    metadata["episodes_schema"] = pa.schema([
+        field.with_nullable(True) if 
field.name.startswith("stats/subtask_index/") else field
+        for field in metadata["episodes_schema"]
+        if not any(field.name.startswith("stats/%s/q" % name) for name in 
_CONTROLS)
+        and not (fps_mode == "source" and field.name == _EPISODE_FPS_COLUMN)
+    ], metadata=metadata["episodes_schema"].metadata)
+    frame_offset = episode_offset = 0
+    for ordinal, plan in enumerate(plans):
+        other = plan["info"]
+        compared_keys = ("codebase_version", "robot_type")
+        if fps_mode == "dataset":
+            compared_keys += ("fps",)
+        for key in compared_keys:
+            if info.get(key) != other.get(key):
+                raise ValueError("LeRobot source %d has incompatible info.%s." 
% (ordinal, key))
+        expected_features = _feature_contract(info["features"], fps_mode)
+        actual_features = _feature_contract(other["features"], fps_mode)
+        for name in set(expected_features) | set(actual_features):
+            if expected_features.get(name) != actual_features.get(name):
+                raise ValueError("LeRobot source %d has incompatible 
features.%s." % (ordinal, name))
+        if not _same_fields(plan["schema"], first["schema"]):
+            raise ValueError("LeRobot source %d has incompatible frame 
schema." % ordinal)
+        episode_schema = pa.schema([
+            field.with_nullable(True) if 
field.name.startswith("stats/subtask_index/") else field
+            for field in plan["metadata"]["episodes_schema"]
+            if not any(field.name.startswith("stats/%s/q" % name) for name in 
_CONTROLS)
+            and not (fps_mode == "source" and field.name == 
_EPISODE_FPS_COLUMN)
+        ])
+        if not _same_fields(episode_schema, metadata["episodes_schema"]):
+            raise ValueError("LeRobot source %d has incompatible Episode 
schema." % ordinal)
+        # Validate every Episode before creating any target table.
+        plan["metadata"]["fps_mode"] = fps_mode
+        try:
+            for name in plan["videos"]:
+                _video_fps(other, other["features"][name], name)
+            for unused in _validated_episode_tables(plan["metadata"], 
first["videos"]):
+                pass
+        except ValueError as error:
+            raise ValueError("LeRobot source %d: %s" % (ordinal, error)) from 
error
+        plan["frame_offset"] = frame_offset
+        plan["episode_offset"] = episode_offset
+        plan["control_stats"] = {}
+        frame_offset += other["total_frames"]
+        episode_offset += other["total_episodes"]
+    for key in list(info):
+        if key not in _REBUILT_INFO and key not in ("codebase_version", 
"robot_type", "fps") \
+                and any(plan["info"].get(key) != info[key] for plan in plans):
+            del info[key]
+    metadata["tasks_table"] = _vocabulary(plans, "tasks")
+    metadata["subtasks_table"] = _vocabulary(plans, "subtasks")
+    reserved_episode_fields = {
+        _DATASET_INDEX_COLUMN,
+        "source_frame_from_index", "source_frame_to_index",
+        "source_episode_ipc",
+    }
+    if reserved_episode_fields.intersection(metadata["episodes_schema"].names):
+        raise ValueError("Source Episode metadata uses reserved source index 
columns.")
+    for field in (
+            pa.field(_DATASET_INDEX_COLUMN, pa.int64(), nullable=False),
+            pa.field("source_frame_from_index", pa.int64(), nullable=False),
+            pa.field("source_frame_to_index", pa.int64(), nullable=False),
+            pa.field("source_episode_ipc", pa.binary(), nullable=False)):
+        metadata["episodes_schema"] = metadata["episodes_schema"].append(field)
+    metadata["datasets_table"] = pa.Table.from_pylist([
+        {
+            _DATASET_INDEX_COLUMN: ordinal,
+            "dataset_name": plan["dataset_name"],
+            "fps": plan["metadata"]["fps"],
+            "episode_from_index": plan["episode_offset"],
+            "episode_to_index": (plan["episode_offset"]
+                                 + plan["info"]["total_episodes"]),
+            "frame_from_index": plan["frame_offset"],
+            "frame_to_index": plan["frame_offset"] + 
plan["info"]["total_frames"],
+            "info_json": json.dumps(_json_value(plan["info"]),
+                                    ensure_ascii=False, separators=(",", ":")),
+            "tasks_ipc": _arrow_ipc(plan["metadata"]["tasks_table"]),
+            "subtasks_ipc": _arrow_ipc(plan["metadata"]["subtasks_table"]),

Review Comment:
   [P2] Archive the original Arrow vocabulary before normalization. With a 
two-column `tasks.parquet` written directly by Arrow, with no Pandas metadata, 
I found that `tasks_ipc` dropped the original schema and field metadata and 
changed a non-null label field to nullable. `_source_vocabulary_index` rebuilds 
the table through Pandas before it reaches this serialization step. Could we 
write the original Arrow table to IPC and normalize only the vocabulary used 
for training? A round-trip test with `check_metadata=True` would cover this 
case.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to