JingsongLi commented on code in PR #11113:
URL: https://github.com/apache/paimon/pull/11113#discussion_r4547721812
##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -82,7 +85,7 @@
pa.field("subtask", pa.string(), nullable=False),
]))
_DATASETS_SCHEMA = pa.schema([
- pa.field("index", pa.int64(), nullable=False),
+ pa.field("index", pa.string(), nullable=False),
Review Comment:
**[P1] Keep the capture writer compatible with this schema change**
`PaimonLeRobotWriter` also imports `_DATASETS_SCHEMA`, but `flush()` still
builds a Dataset row with integer `index=0` and passes it to
`pa.Table.from_pylist(..., schema=_DATASETS_SCHEMA)` (`writer.py:693–701`).
Changing this field to `pa.string()` therefore makes every non-empty writer
flush/finalize fail with `ArrowTypeError: Expected bytes, got a 'int' object`.
I reproduced this through the real writer and catalog, stubbing only the
unavailable statistics dependency; the Episode snapshot was already committed,
while Frames and Datasets had no committed snapshot. Restoring the integer
schema made the same flow succeed.
Please either preserve a separate schema for the capture writer or update
its identifier generation, flush, and resume validation together;
`_state_from_companion_tables()` also still requires `index == 0`.
##########
paimon-python/pypaimon/multimodal/lerobot/api.py:
##########
@@ -119,149 +91,17 @@ def load_from_lerobot(
if extra_dataset_fields is not None:
from pypaimon.multimodal.lerobot.aggregate import _DATASETS_SCHEMA
extra_dataset_fields = _validated_text_fields(extra_dataset_fields,
_DATASETS_SCHEMA.names)
- if append or isinstance(source, (list, tuple)) or dataset_name is not None
\
- or extra_dataset_fields:
- if not isinstance(source, (list, tuple)):
- source = [source]
- if not source:
- raise ValueError("LeRobot sources must not be empty.")
- if tag_name is not None:
- _validate_tag_name(tag_name)
- from pypaimon.multimodal.lerobot.aggregate import _import_sources
- return _import_sources(
- connection, table_name, source, batch_size, options,
- _validated_source_options(source_options), tag_name, parallelism,
- dataset_name, extra_dataset_fields, schema_evolution, append)
- if parallelism > 1:
- from pypaimon.multimodal.lerobot.ray_loader import _require_ray,
_ray_source_uri
- _require_ray()
- source = _ray_source_uri(source)
-
+ if not isinstance(source, (list, tuple)):
+ source = [source]
+ if not source:
+ raise ValueError("LeRobot sources must not be empty.")
if tag_name is not None:
_validate_tag_name(tag_name)
-
- validated_source_options = _validated_source_options(source_options)
- _validate_source_kerberos(
- [source], validated_source_options, "LeRobot")
- with _resolved_source(source, validated_source_options) as (
- resolved_source, local_info):
- if local_info is None:
- local_info = _load_hub_info(resolved_source)
- _require_v3(local_info, resolved_source.path)
- _validate_info_paths(local_info)
- _schema_from_info(local_info)
- _dataset_fps(local_info.get("fps"))
- _validated_counts(local_info, resolved_source.path)
- _validate_v3_required_features(local_info)
- video_fields = _video_feature_names(local_info)
- LeRobotDataset = (_import_lerobot_dataset()
- if parallelism == 1 and resolved_source.file_io is
None
- else None)
- dataset = _open_resolved_dataset(
- LeRobotDataset,
- resolved_source,
- local_info,
- download_videos=bool(video_fields),
- )
- try:
- info = _info_dict(dataset.meta.info)
- _require_v3(info, resolved_source.path)
- _validated_counts(info, resolved_source.path)
- _validate_v3_required_features(info)
- video_fields = _video_feature_names(info)
-
- lerobot_schema = _schema_from_info(info)
- # Preserve source extensions omitted by LeRobot
DatasetInfo.to_dict().
- metadata = _load_dataset_metadata(
- dataset, local_info, resolved_source)
- from pypaimon.multimodal.lerobot.aggregate import (
- _DATASETS_SCHEMA, _stats_json, _vocabulary_rows,
- )
- metadata["datasets_table"] = pa.Table.from_pylist([{
- "index": 0,
- "name": None,
- "info": json.dumps(_json_value(local_info),
ensure_ascii=False),
- "tasks": _vocabulary_rows(metadata["tasks_table"], "task"),
- "subtasks": _vocabulary_rows(metadata["subtasks_table"],
"subtask"),
- "stats": _stats_json(metadata["stats_table"]),
- }], schema=_DATASETS_SCHEMA)
- _import_dataset(
- connection,
- table_name,
- dataset,
- info,
- resolved_source,
- lerobot_schema,
- batch_size,
- options,
- metadata,
- tag_name,
- video_fields,
- parallelism,
- validated_source_options,
- )
- finally:
- close = getattr(dataset, "close", None)
- if callable(close):
- _close_quietly(dataset, "dataset")
-
-
-def _import_dataset(
- connection,
- table_name,
- dataset,
- info,
- source,
- source_schema,
- batch_size,
- options,
- metadata,
- tag_name,
- video_fields,
- parallelism=1,
- source_options=None):
- table = _create_target_table(
- connection,
- table_name,
- source_schema,
- options,
- metadata,
- video_fields,
- )
- tables = _prepare_metadata_tables(
- connection, table.raw_table, metadata)
- episodes_snapshot_id = _append_arrow_tables(
- tables["episodes"],
- _validated_episode_tables(metadata, video_fields),
- )
- frames_snapshot_id = None
- if int(info["total_frames"]) > 0:
- if parallelism > 1:
- from pypaimon.multimodal.lerobot.ray_loader import
_write_dataset_ray
- write_dataset = partial(
- _write_dataset_ray, parallelism=parallelism,
- source_options=source_options)
- else:
- write_dataset = _write_dataset
- frames_snapshot_id = write_dataset(
- table,
- dataset,
- info,
- source,
- source_schema,
- batch_size,
- metadata,
- video_fields,
- )
- _commit_metadata(
- connection,
- tables,
- tag_name,
- metadata,
- table.identifier,
- frames_snapshot_id,
- episodes_snapshot_id,
- )
+ from pypaimon.multimodal.lerobot.aggregate import _import_sources
+ return _import_sources(
+ connection, table_name, source, batch_size, options,
+ _validated_source_options(source_options), tag_name, parallelism,
+ dataset_name, extra_dataset_fields, schema_evolution, append)
Review Comment:
**[P2] Preserve feature quantiles when importing a single source**
Routing a single source string through `_import_sources` also switches its
reader metadata to `_merged_source_stats()`. That function calls
`_merge_stats()` even for one source, dropping `q01/q99` (and the other
quantiles) from non-control features such as `action` and `observation.state`.
I compared the same single-source import against the base commit: the base
reader preserves these keys, whereas this version exposes only
`min/max/mean/std/count`.
This breaks existing single-source training with LeRobot's `QUANTILES`
normalization, which requires `q01/q99` in `dataset.meta.stats` and raises when
they are missing ([supported LeRobot 0.4.4
implementation](https://github.com/huggingface/lerobot/blob/v0.4.4/src/lerobot/processor/normalize_processor.py)).
The original quantiles remain in `_datasets.stats`, but the reader does not
restore them. Please preserve the original non-control feature quantiles for a
singleton source, where no statistical merge is needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]