JingsongLi commented on code in PR #11056:
URL: https://github.com/apache/paimon/pull/11056#discussion_r4519856447


##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,332 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _append_arrow_tables, _commit_metadata, _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64) for key in _STATS}
+              for item in items]
+    first = arrays[0]
+    for item in arrays:
+        if any(item[key].shape != first[key].shape for key in _STATS):
+            raise ValueError("LeRobot statistics shapes must match.")
+        count = item["count"]
+        if count.shape != (1,) or not np.all(np.isfinite(count)) \
+                or np.any(count <= 0) or np.any(count != np.floor(count)):
+            raise ValueError("LeRobot statistics count must be one positive 
integer.")
+        if any(item[key].shape != item["mean"].shape for key in ("min", "max", 
"std")):
+            raise ValueError("LeRobot statistic shapes must match their mean.")
+    count = sum(item["count"] for item in arrays)
+    mean = sum(item["mean"] * item["count"] for item in arrays) / count
+    variance = sum(item["count"] * (
+        item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / 
count
+    return {
+        "min": np.minimum.reduce([item["min"] for item in arrays]).tolist(),
+        "max": np.maximum.reduce([item["max"] for item in arrays]).tolist(),
+        "mean": mean.tolist(), "std": np.sqrt(variance).tolist(),
+        "count": count.astype(np.int64).tolist(),
+    }
+
+
+def _batch_stats(values):
+    values = np.asarray(values, dtype=np.float64)
+    return {"min": [values.min()], "max": [values.max()],
+            "mean": [values.mean()], "std": [values.std()],
+            "count": [len(values)]}
+
+
+def _vocabulary(plans, kind):
+    key = kind + "_table"
+    tables = [plan["metadata"][key] for plan in plans]
+    if all(table is None for table in tables):
+        return None
+    if any(table is None for table in tables):
+        raise ValueError("LeRobot sources disagree on %s metadata." % kind)
+    singular = kind[:-1]
+    index_name = singular + "_index"
+    schema = tables[0].schema
+    label = _pandas_index_column(schema, singular)
+    if set(schema.names) != {index_name, label}:
+        raise ValueError("Multi-source %s metadata requires only index and 
text." % kind)
+    rows, by_text = [], {}
+    for ordinal, (plan, table) in enumerate(zip(plans, tables)):
+        source_label = _pandas_index_column(table.schema, singular)
+        if (set(table.column_names) != {index_name, source_label}
+                or not table.schema.field(index_name).equals(
+                    schema.field(index_name), check_metadata=False)
+                or not 
table.schema.field(source_label).with_name(label).equals(
+                    schema.field(label), check_metadata=False)):
+            raise ValueError("LeRobot source %d has incompatible %s schema." % 
(ordinal, kind))
+        mapping = {}
+        for row in sorted(table.to_pylist(), key=lambda row: row[index_name]):
+            text = row[source_label]
+            if text not in by_text:
+                by_text[text] = len(rows)
+                rows.append({index_name: len(rows), label: text})
+            mapping[row[index_name]] = by_text[text]
+        plan[index_name] = mapping
+    return pa.Table.from_pylist(rows, schema=schema)
+
+
+def _feature_contract(features):
+    result = copy.deepcopy(features)
+    for feature in result.values():
+        if feature.get("dtype") == "video":
+            for key in ("info", "video_info"):
+                feature[key] = {k: v for k, v in feature.get(key, {}).items()
+                                if k not in _ENCODER_KEYS}
+    return result
+
+
+def _plan_merge(plans):
+    first = plans[0]
+    info = copy.deepcopy(first["info"])
+    metadata = dict(first["metadata"])
+    frame_offset = episode_offset = 0
+    for ordinal, plan in enumerate(plans):
+        other = plan["info"]
+        for key in (set(info) | set(other)) - _REBUILT_INFO:
+            if info.get(key) != other.get(key):
+                raise ValueError("LeRobot source %d has incompatible info.%s." 
% (ordinal, key))
+        expected_features = _feature_contract(info["features"])
+        actual_features = _feature_contract(other["features"])
+        for name in set(expected_features) | set(actual_features):
+            if expected_features.get(name) != actual_features.get(name):
+                raise ValueError("LeRobot source %d has incompatible 
features.%s." % (ordinal, name))
+        if not plan["schema"].equals(first["schema"], check_metadata=False):
+            raise ValueError("LeRobot source %d has incompatible frame 
schema." % ordinal)
+        if not plan["metadata"]["episodes_schema"].equals(
+                metadata["episodes_schema"], check_metadata=False):
+            raise ValueError("LeRobot source %d has incompatible Episode 
schema." % ordinal)
+        # Validate every Episode before creating any target table.
+        try:
+            for name in plan["videos"]:
+                _video_fps(other, other["features"][name], name)
+            for unused in _validated_episode_tables(plan["metadata"], 
first["videos"]):
+                pass
+        except ValueError as error:
+            raise ValueError("LeRobot source %d: %s" % (ordinal, error)) from 
error
+        plan["frame_offset"] = frame_offset
+        plan["episode_offset"] = episode_offset
+        plan["control_stats"] = {}
+        frame_offset += other["total_frames"]
+        episode_offset += other["total_episodes"]
+    metadata["tasks_table"] = _vocabulary(plans, "tasks")
+    metadata["subtasks_table"] = _vocabulary(plans, "subtasks")
+    info.update(total_frames=frame_offset, total_episodes=episode_offset,
+                total_tasks=metadata["tasks_table"].num_rows,
+                splits={"train": "0:%d" % episode_offset})
+    if metadata["subtasks_table"] is not None:
+        info["total_subtasks"] = metadata["subtasks_table"].num_rows
+    for key in ("total_chunks", "total_videos", "data_path", "video_path",
+                "chunks_size", "data_files_size_in_mb", 
"video_files_size_in_mb"):
+        info.pop(key, None)
+    # Encoder descriptions are not training features. Retain only shared 
values.
+    for name in first["videos"]:
+        for key in ("info", "video_info"):
+            details = [p["info"]["features"][name].get(key, {}) for p in plans]
+            info["features"][name][key] = {
+                k: v for k, v in details[0].items()
+                if all(d.get(k) == v for d in details[1:])}
+    stats = [p["metadata"]["stats_table"] for p in plans]
+    if any(s is None for s in stats) and not all(s is None for s in stats):
+        raise ValueError("LeRobot stats must be present in all sources or 
absent in all.")
+    merged_stats = None
+    if stats[0] is not None:
+        decoded = [_json_table(s) for s in stats]
+        if any(set(s) != set(decoded[0]) for s in decoded[1:]):
+            raise ValueError("LeRobot statistics features must match.")
+        merged_stats = {name: _merge_stats([s[name] for s in decoded])
+                        for name in decoded[0] if name not in _CONTROLS}
+    metadata["info_table"] = _metadata_table(info)
+    metadata["stats_table"] = (None if merged_stats is None
+                               else _metadata_table(merged_stats))
+    # Quantiles are omitted rather than presenting source summaries as merged 
values.
+    metadata["episodes_schema"] = pa.schema([
+        field for field in metadata["episodes_schema"]
+        if not any(field.name.startswith("stats/%s/q" % name) for name in 
_CONTROLS)
+    ], metadata=metadata["episodes_schema"].metadata)
+    return metadata, merged_stats
+
+
+def _transform_frames(plan, batch, episode):
+    for name in _CONTROLS:
+        if name not in batch.column_names:
+            continue
+        column = batch[name]
+        if name in ("index", "episode_index"):
+            offset = plan["frame_offset" if name == "index" else 
"episode_offset"]
+            values = pc.add_checked(column, pa.scalar(offset, 
type=column.type))
+        else:
+            mapping = plan[name]
+            values = pa.array([
+                None if name == "subtask_index" and value is None

Review Comment:
   I reproduced a second failure after bypassing only the subtask control 
check. `_feature_field` declares `subtask_index` non-nullable, so strict Arrow 
conversion rejects the null before writing it. The training reader also 
requires an integer in both row validation and label lookup (`dataset.py`). 
Could we make this field nullable in the merged target, handle unlabelled 
frames in the reader, and extend the serial and Ray regression tests to read 
the imported frame through `PaimonLeRobotDataset`?



##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,332 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _append_arrow_tables, _commit_metadata, _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64) for key in _STATS}

Review Comment:
   [P2] Preserve exact integer extrema in merged statistics. I reproduced this 
with two FileIO sources whose `int64` observation values were `2**53 + 1` and 
`2**53 + 3`. The merged Frames table preserves both values, but `__stats` 
records a minimum of `2**53` and a maximum of `2**53 + 4` because this 
conversion rounds both extrema. Could we keep the native types for min, max, 
and count, and convert only the inputs to the moment calculation?



##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,332 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _append_arrow_tables, _commit_metadata, _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64) for key in _STATS}
+              for item in items]
+    first = arrays[0]
+    for item in arrays:
+        if any(item[key].shape != first[key].shape for key in _STATS):
+            raise ValueError("LeRobot statistics shapes must match.")
+        count = item["count"]
+        if count.shape != (1,) or not np.all(np.isfinite(count)) \
+                or np.any(count <= 0) or np.any(count != np.floor(count)):
+            raise ValueError("LeRobot statistics count must be one positive 
integer.")
+        if any(item[key].shape != item["mean"].shape for key in ("min", "max", 
"std")):
+            raise ValueError("LeRobot statistic shapes must match their mean.")
+    count = sum(item["count"] for item in arrays)
+    mean = sum(item["mean"] * item["count"] for item in arrays) / count
+    variance = sum(item["count"] * (
+        item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / 
count
+    return {
+        "min": np.minimum.reduce([item["min"] for item in arrays]).tolist(),
+        "max": np.maximum.reduce([item["max"] for item in arrays]).tolist(),
+        "mean": mean.tolist(), "std": np.sqrt(variance).tolist(),
+        "count": count.astype(np.int64).tolist(),
+    }
+
+
+def _batch_stats(values):
+    values = np.asarray(values, dtype=np.float64)
+    return {"min": [values.min()], "max": [values.max()],
+            "mean": [values.mean()], "std": [values.std()],
+            "count": [len(values)]}
+
+
+def _vocabulary(plans, kind):
+    key = kind + "_table"
+    tables = [plan["metadata"][key] for plan in plans]
+    if all(table is None for table in tables):
+        return None
+    if any(table is None for table in tables):
+        raise ValueError("LeRobot sources disagree on %s metadata." % kind)
+    singular = kind[:-1]
+    index_name = singular + "_index"
+    schema = tables[0].schema
+    label = _pandas_index_column(schema, singular)
+    if set(schema.names) != {index_name, label}:
+        raise ValueError("Multi-source %s metadata requires only index and 
text." % kind)
+    rows, by_text = [], {}
+    for ordinal, (plan, table) in enumerate(zip(plans, tables)):
+        source_label = _pandas_index_column(table.schema, singular)
+        if (set(table.column_names) != {index_name, source_label}
+                or not table.schema.field(index_name).equals(
+                    schema.field(index_name), check_metadata=False)
+                or not 
table.schema.field(source_label).with_name(label).equals(
+                    schema.field(label), check_metadata=False)):
+            raise ValueError("LeRobot source %d has incompatible %s schema." % 
(ordinal, kind))
+        mapping = {}
+        for row in sorted(table.to_pylist(), key=lambda row: row[index_name]):
+            text = row[source_label]
+            if text not in by_text:
+                by_text[text] = len(rows)
+                rows.append({index_name: len(rows), label: text})
+            mapping[row[index_name]] = by_text[text]
+        plan[index_name] = mapping
+    return pa.Table.from_pylist(rows, schema=schema)
+
+
+def _feature_contract(features):
+    result = copy.deepcopy(features)
+    for feature in result.values():
+        if feature.get("dtype") == "video":
+            for key in ("info", "video_info"):
+                feature[key] = {k: v for k, v in feature.get(key, {}).items()
+                                if k not in _ENCODER_KEYS}
+    return result
+
+
+def _plan_merge(plans):
+    first = plans[0]
+    info = copy.deepcopy(first["info"])
+    metadata = dict(first["metadata"])
+    frame_offset = episode_offset = 0
+    for ordinal, plan in enumerate(plans):
+        other = plan["info"]
+        for key in (set(info) | set(other)) - _REBUILT_INFO:
+            if info.get(key) != other.get(key):
+                raise ValueError("LeRobot source %d has incompatible info.%s." 
% (ordinal, key))
+        expected_features = _feature_contract(info["features"])
+        actual_features = _feature_contract(other["features"])
+        for name in set(expected_features) | set(actual_features):
+            if expected_features.get(name) != actual_features.get(name):
+                raise ValueError("LeRobot source %d has incompatible 
features.%s." % (ordinal, name))
+        if not plan["schema"].equals(first["schema"], check_metadata=False):
+            raise ValueError("LeRobot source %d has incompatible frame 
schema." % ordinal)
+        if not plan["metadata"]["episodes_schema"].equals(
+                metadata["episodes_schema"], check_metadata=False):
+            raise ValueError("LeRobot source %d has incompatible Episode 
schema." % ordinal)

Review Comment:
   [P2] Compare the fields that will be written by name. Reversing only the 
keys of `info.json`'s `features` object, or only the Episode Parquet column 
order, makes two individually importable sources fail to merge. The Episode 
check also rejects a source that differs only by remapped control quantile 
columns that `_plan_merge` later drops. Could we project out those columns 
before comparing Episode schemas, compare fields by name, and explicitly align 
frame batches to the target column order before strict validation?



##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,332 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _append_arrow_tables, _commit_metadata, _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64) for key in _STATS}
+              for item in items]
+    first = arrays[0]
+    for item in arrays:
+        if any(item[key].shape != first[key].shape for key in _STATS):
+            raise ValueError("LeRobot statistics shapes must match.")
+        count = item["count"]
+        if count.shape != (1,) or not np.all(np.isfinite(count)) \
+                or np.any(count <= 0) or np.any(count != np.floor(count)):
+            raise ValueError("LeRobot statistics count must be one positive 
integer.")
+        if any(item[key].shape != item["mean"].shape for key in ("min", "max", 
"std")):
+            raise ValueError("LeRobot statistic shapes must match their mean.")
+    count = sum(item["count"] for item in arrays)
+    mean = sum(item["mean"] * item["count"] for item in arrays) / count
+    variance = sum(item["count"] * (
+        item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / 
count
+    return {
+        "min": np.minimum.reduce([item["min"] for item in arrays]).tolist(),
+        "max": np.maximum.reduce([item["max"] for item in arrays]).tolist(),
+        "mean": mean.tolist(), "std": np.sqrt(variance).tolist(),
+        "count": count.astype(np.int64).tolist(),
+    }
+
+
+def _batch_stats(values):
+    values = np.asarray(values, dtype=np.float64)
+    return {"min": [values.min()], "max": [values.max()],
+            "mean": [values.mean()], "std": [values.std()],
+            "count": [len(values)]}
+
+
+def _vocabulary(plans, kind):
+    key = kind + "_table"
+    tables = [plan["metadata"][key] for plan in plans]
+    if all(table is None for table in tables):
+        return None
+    if any(table is None for table in tables):
+        raise ValueError("LeRobot sources disagree on %s metadata." % kind)
+    singular = kind[:-1]
+    index_name = singular + "_index"
+    schema = tables[0].schema
+    label = _pandas_index_column(schema, singular)
+    if set(schema.names) != {index_name, label}:
+        raise ValueError("Multi-source %s metadata requires only index and 
text." % kind)
+    rows, by_text = [], {}
+    for ordinal, (plan, table) in enumerate(zip(plans, tables)):
+        source_label = _pandas_index_column(table.schema, singular)
+        if (set(table.column_names) != {index_name, source_label}
+                or not table.schema.field(index_name).equals(
+                    schema.field(index_name), check_metadata=False)
+                or not 
table.schema.field(source_label).with_name(label).equals(
+                    schema.field(label), check_metadata=False)):

Review Comment:
   [P2] Accept equivalent Arrow string types for vocabulary labels. I 
reproduced a merge failure with two `tasks.parquet` files containing the same 
labels and IDs. Pandas exported one index as `string` and the other as 
`large_string`, and both passed single-source import. The merge already 
rebuilds the vocabulary from decoded labels, so could we accept both string 
representations after validating the labels and write them using the target 
vocabulary schema?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to