JingsongLi commented on code in PR #11113:
URL: https://github.com/apache/paimon/pull/11113#discussion_r4547721812


##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -82,7 +85,7 @@
     pa.field("subtask", pa.string(), nullable=False),
 ]))
 _DATASETS_SCHEMA = pa.schema([
-    pa.field("index", pa.int64(), nullable=False),
+    pa.field("index", pa.string(), nullable=False),

Review Comment:
   **[P1] Keep the capture writer compatible with this schema change**
   
   `PaimonLeRobotWriter` also imports `_DATASETS_SCHEMA`, but `flush()` still 
builds a Dataset row with integer `index=0` and passes it to 
`pa.Table.from_pylist(..., schema=_DATASETS_SCHEMA)` (`writer.py:693–701`). 
Changing this field to `pa.string()` therefore makes every non-empty writer 
flush/finalize fail with `ArrowTypeError: Expected bytes, got a 'int' object`. 
I reproduced this through the real writer and catalog, stubbing only the 
unavailable statistics dependency; the Episode snapshot was already committed, 
while Frames and Datasets had no committed snapshot. Restoring the integer 
schema made the same flow succeed.
   
   Please either preserve a separate schema for the capture writer or update 
its identifier generation, flush, and resume validation together; 
`_state_from_companion_tables()` also still requires `index == 0`.



##########
paimon-python/pypaimon/multimodal/lerobot/api.py:
##########
@@ -119,149 +91,17 @@ def load_from_lerobot(
     if extra_dataset_fields is not None:
         from pypaimon.multimodal.lerobot.aggregate import _DATASETS_SCHEMA
         extra_dataset_fields = _validated_text_fields(extra_dataset_fields, 
_DATASETS_SCHEMA.names)
-    if append or isinstance(source, (list, tuple)) or dataset_name is not None 
\
-            or extra_dataset_fields:
-        if not isinstance(source, (list, tuple)):
-            source = [source]
-        if not source:
-            raise ValueError("LeRobot sources must not be empty.")
-        if tag_name is not None:
-            _validate_tag_name(tag_name)
-        from pypaimon.multimodal.lerobot.aggregate import _import_sources
-        return _import_sources(
-            connection, table_name, source, batch_size, options,
-            _validated_source_options(source_options), tag_name, parallelism,
-            dataset_name, extra_dataset_fields, schema_evolution, append)
-    if parallelism > 1:
-        from pypaimon.multimodal.lerobot.ray_loader import _require_ray, 
_ray_source_uri
-        _require_ray()
-        source = _ray_source_uri(source)
-
+    if not isinstance(source, (list, tuple)):
+        source = [source]
+    if not source:
+        raise ValueError("LeRobot sources must not be empty.")
     if tag_name is not None:
         _validate_tag_name(tag_name)
-
-    validated_source_options = _validated_source_options(source_options)
-    _validate_source_kerberos(
-        [source], validated_source_options, "LeRobot")
-    with _resolved_source(source, validated_source_options) as (
-            resolved_source, local_info):
-        if local_info is None:
-            local_info = _load_hub_info(resolved_source)
-        _require_v3(local_info, resolved_source.path)
-        _validate_info_paths(local_info)
-        _schema_from_info(local_info)
-        _dataset_fps(local_info.get("fps"))
-        _validated_counts(local_info, resolved_source.path)
-        _validate_v3_required_features(local_info)
-        video_fields = _video_feature_names(local_info)
-        LeRobotDataset = (_import_lerobot_dataset()
-                          if parallelism == 1 and resolved_source.file_io is 
None
-                          else None)
-        dataset = _open_resolved_dataset(
-            LeRobotDataset,
-            resolved_source,
-            local_info,
-            download_videos=bool(video_fields),
-        )
-        try:
-            info = _info_dict(dataset.meta.info)
-            _require_v3(info, resolved_source.path)
-            _validated_counts(info, resolved_source.path)
-            _validate_v3_required_features(info)
-            video_fields = _video_feature_names(info)
-
-            lerobot_schema = _schema_from_info(info)
-            # Preserve source extensions omitted by LeRobot 
DatasetInfo.to_dict().
-            metadata = _load_dataset_metadata(
-                dataset, local_info, resolved_source)
-            from pypaimon.multimodal.lerobot.aggregate import (
-                _DATASETS_SCHEMA, _stats_json, _vocabulary_rows,
-            )
-            metadata["datasets_table"] = pa.Table.from_pylist([{
-                "index": 0,
-                "name": None,
-                "info": json.dumps(_json_value(local_info), 
ensure_ascii=False),
-                "tasks": _vocabulary_rows(metadata["tasks_table"], "task"),
-                "subtasks": _vocabulary_rows(metadata["subtasks_table"], 
"subtask"),
-                "stats": _stats_json(metadata["stats_table"]),
-            }], schema=_DATASETS_SCHEMA)
-            _import_dataset(
-                connection,
-                table_name,
-                dataset,
-                info,
-                resolved_source,
-                lerobot_schema,
-                batch_size,
-                options,
-                metadata,
-                tag_name,
-                video_fields,
-                parallelism,
-                validated_source_options,
-            )
-        finally:
-            close = getattr(dataset, "close", None)
-            if callable(close):
-                _close_quietly(dataset, "dataset")
-
-
-def _import_dataset(
-        connection,
-        table_name,
-        dataset,
-        info,
-        source,
-        source_schema,
-        batch_size,
-        options,
-        metadata,
-        tag_name,
-        video_fields,
-        parallelism=1,
-        source_options=None):
-    table = _create_target_table(
-        connection,
-        table_name,
-        source_schema,
-        options,
-        metadata,
-        video_fields,
-    )
-    tables = _prepare_metadata_tables(
-        connection, table.raw_table, metadata)
-    episodes_snapshot_id = _append_arrow_tables(
-        tables["episodes"],
-        _validated_episode_tables(metadata, video_fields),
-    )
-    frames_snapshot_id = None
-    if int(info["total_frames"]) > 0:
-        if parallelism > 1:
-            from pypaimon.multimodal.lerobot.ray_loader import 
_write_dataset_ray
-            write_dataset = partial(
-                _write_dataset_ray, parallelism=parallelism,
-                source_options=source_options)
-        else:
-            write_dataset = _write_dataset
-        frames_snapshot_id = write_dataset(
-            table,
-            dataset,
-            info,
-            source,
-            source_schema,
-            batch_size,
-            metadata,
-            video_fields,
-        )
-    _commit_metadata(
-        connection,
-        tables,
-        tag_name,
-        metadata,
-        table.identifier,
-        frames_snapshot_id,
-        episodes_snapshot_id,
-    )
+    from pypaimon.multimodal.lerobot.aggregate import _import_sources
+    return _import_sources(
+        connection, table_name, source, batch_size, options,
+        _validated_source_options(source_options), tag_name, parallelism,
+        dataset_name, extra_dataset_fields, schema_evolution, append)

Review Comment:
   **[P2] Preserve feature quantiles when importing a single source**
   
   Routing a single source string through `_import_sources` also switches its 
reader metadata to `_merged_source_stats()`. That function calls 
`_merge_stats()` even for one source, dropping `q01/q99` (and the other 
quantiles) from non-control features such as `action` and `observation.state`. 
I compared the same single-source import against the base commit: the base 
reader preserves these keys, whereas this version exposes only 
`min/max/mean/std/count`.
   
   This breaks existing single-source training with LeRobot's `QUANTILES` 
normalization, which requires `q01/q99` in `dataset.meta.stats` and raises when 
they are missing ([supported LeRobot 0.4.4 
implementation](https://github.com/huggingface/lerobot/blob/v0.4.4/src/lerobot/processor/normalize_processor.py)).
 The original quantiles remain in `_datasets.stats`, but the reader does not 
restore them. Please preserve the original non-control feature quantiles for a 
singleton source, where no statistical merge is needed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to