JingsongLi commented on code in PR #11056:
URL: https://github.com/apache/paimon/pull/11056#discussion_r4521847685


##########
paimon-python/pypaimon/multimodal/lerobot/dataset.py:
##########
@@ -1001,7 +1096,28 @@ def _episode_dataset(table):
         name for name in _target_schema(table).names
         if not name.startswith("stats/")
     ]
-    data = _read_arrow(table, projection).sort_by("episode_index")
+    data = _read_arrow(table, projection)

Review Comment:
   [P2] Exclude export IPC from the training projection. `source_episode_ipc` 
contains the original statistics and schema, so it brings back the statistics 
payload this projection is meant to skip. In the PR's four-Episode fixture, the 
Arrow training projection was 26,552 bytes with IPC and 344 bytes without it. 
The IPC also becomes part of the Episode metadata serialized for spawned 
DataLoader workers. Could we exclude `source_episode_ipc` here and project only 
the source metadata columns the training reader uses from `__datasets`?



##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -0,0 +1,475 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Merge source coordinates at the write boundary of a new table group."""
+
+import copy
+import json
+from contextlib import ExitStack
+
+import numpy as np
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from pypaimon.multimodal.arrow_utils import strict_arrow_table
+from pypaimon.multimodal.lerobot.loader import (
+    _commit_messages, _prepare_dataset, _video_fps,
+)
+from pypaimon.multimodal.lerobot.metadata import (
+    _EPISODE_FPS_COLUMN, _append_arrow_tables, _commit_metadata, _json_value,
+    _load_dataset_metadata,
+    _metadata_table, _prepare_metadata_tables,
+    _source_episode_tables, _validated_episode_tables,
+)
+from pypaimon.multimodal.lerobot.schema import (
+    _require_v3, _schema_from_info, _validate_v3_required_features,
+    _video_feature_names,
+)
+from pypaimon.multimodal.lerobot.source import (
+    _close_quietly, _import_lerobot_dataset, _load_hub_info,
+    _open_resolved_dataset, _pandas_index_column, _resolved_source,
+    _validate_info_paths,
+)
+from pypaimon.multimodal.source_utils import _validate_source_kerberos
+
+
+_CONTROLS = ("index", "episode_index", "task_index", "subtask_index")
+_STATS = ("min", "max", "mean", "std", "count")
+_ENCODER_KEYS = {
+    "video.codec", "video.pix_fmt", "video.gop_size", "video.crf",
+    "video.encoder", "video.extra_options", "video.video_backend",
+    "video.has_audio", "has_audio", "video.bit_rate",
+}
+# These properties describe source storage or are rebuilt for the merged 
dataset.
+_REBUILT_INFO = {
+    "total_frames", "total_episodes", "total_tasks", "total_subtasks", 
"total_videos",
+    "total_chunks", "splits", "data_path", "video_path", "chunks_size",
+    "data_files_size_in_mb", "video_files_size_in_mb", "features",
+}
+_DATASET_INDEX_COLUMN = "dataset_index"
+_SOURCE_FRAME_INDEX_COLUMN = "source_index"
+_DATASETS_SCHEMA = pa.schema([
+    pa.field(_DATASET_INDEX_COLUMN, pa.int64(), nullable=False),
+    pa.field("dataset_name", pa.string()),
+    pa.field("fps", pa.int64(), nullable=False),
+    pa.field("episode_from_index", pa.int64(), nullable=False),
+    pa.field("episode_to_index", pa.int64(), nullable=False),
+    pa.field("frame_from_index", pa.int64(), nullable=False),
+    pa.field("frame_to_index", pa.int64(), nullable=False),
+    pa.field("info_json", pa.string(), nullable=False),
+    pa.field("tasks_ipc", pa.binary(), nullable=False),
+    pa.field("subtasks_ipc", pa.binary()),
+    pa.field("stats_ipc", pa.binary()),
+    pa.field("episodes_schema_ipc", pa.binary(), nullable=False),
+])
+
+
+def _arrow_ipc(table):
+    if table is None:
+        return None
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()
+
+
+def _json_table(table):
+    return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()}
+
+
+def _merge_stats(items):
+    """Combine population moments; quantile summaries cannot be merged 
exactly."""
+    if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"}
+           for item in items):
+        raise ValueError("Unsupported LeRobot statistics fields.")
+    if any(not set(_STATS).issubset(item) for item in items):
+        raise ValueError("LeRobot statistics require min/max/mean/std/count.")
+    arrays = [{key: np.asarray(item[key], dtype=np.float64 if key in ("mean", 
"std") else None)
+               for key in _STATS}
+              for item in items]
+    first = arrays[0]
+    for item in arrays:
+        if any(item[key].shape != first[key].shape for key in _STATS):
+            raise ValueError("LeRobot statistics shapes must match.")
+        count = item["count"]
+        if count.shape != (1,) or not np.all(np.isfinite(count)) \
+                or np.any(count <= 0) or np.any(count != np.floor(count)):
+            raise ValueError("LeRobot statistics count must be one positive 
integer.")
+        if any(item[key].shape != item["mean"].shape for key in ("min", "max", 
"std")):
+            raise ValueError("LeRobot statistic shapes must match their mean.")
+    count = sum(int(item["count"][0]) for item in arrays)
+    mean = sum(item["mean"] * item["count"] for item in arrays) / count
+    variance = sum(item["count"] * (
+        item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / 
count
+
+    def extrema(key, reduction):
+        dtype = object if all(np.issubdtype(item[key].dtype, np.integer)
+                              for item in arrays) else np.float64
+        return reduction([item[key].astype(dtype) for item in arrays]).tolist()
+
+    return {
+        "min": extrema("min", np.minimum.reduce),
+        "max": extrema("max", np.maximum.reduce),
+        "mean": mean.tolist(), "std": np.sqrt(variance).tolist(),
+        "count": [count],
+    }
+
+
+def _batch_stats(values):
+    values = np.asarray(values)
+    return {"min": [values.min().item()], "max": [values.max().item()],
+            "mean": [values.mean()], "std": [values.std()],
+            "count": [len(values)]}
+
+
+def _vocabulary(plans, kind):
+    key = kind + "_table"
+    tables = [plan["metadata"][key] for plan in plans]
+    if all(table is None for table in tables):
+        return None
+    if any(table is None for table in tables):
+        raise ValueError("LeRobot sources disagree on %s metadata." % kind)
+    singular = kind[:-1]
+    index_name = singular + "_index"
+    schema = tables[0].schema
+    label = _pandas_index_column(schema, singular)
+    schema = tables[0].select([index_name, label]).schema
+    rows, by_text = [], {}
+    for ordinal, (plan, table) in enumerate(zip(plans, tables)):
+        source_label = _pandas_index_column(table.schema, singular)
+        if (not table.schema.field(index_name).equals(
+                schema.field(index_name), check_metadata=False)
+                or not 
(pa.types.is_string(table.schema.field(source_label).type)
+                        or 
pa.types.is_large_string(table.schema.field(source_label).type))):
+            raise ValueError("LeRobot source %d has incompatible %s schema." % 
(ordinal, kind))
+        mapping = {}
+        for row in sorted(table.select([index_name, source_label]).to_pylist(),
+                          key=lambda row: row[index_name]):
+            text = row[source_label]
+            if text not in by_text:
+                by_text[text] = len(rows)
+                rows.append({index_name: len(rows), label: text})
+            mapping[row[index_name]] = by_text[text]
+        plan[index_name] = mapping
+    return pa.Table.from_pylist(rows, schema=schema)
+
+
+def _feature_contract(features, fps_mode):
+    result = copy.deepcopy(features)
+    for feature in result.values():
+        if feature.get("dtype") == "video":
+            if fps_mode == "source":
+                feature.pop("fps", None)
+            for key in ("info", "video_info"):
+                feature[key] = {k: v for k, v in feature.get(key, {}).items()
+                                if k not in _ENCODER_KEYS
+                                and (fps_mode == "dataset" or k != 
"video.fps")}

Review Comment:
   [P2] Compare normalized video FPS in the default merge mode. I imported two 
real 10 FPS sources with matching video features. The only feature metadata 
difference was `video.fps` set to `10` in one source and `"10"` in the other. 
Each source imported successfully on its own, but the default merge rejected 
the second source with `incompatible features.camera`. Could the feature 
comparison normalize the FPS with the importer's parser, or omit the redundant 
feature FPS after `_video_fps` validates each source against its dataset FPS?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to