JingsongLi commented on code in PR #11056: URL: https://github.com/apache/paimon/pull/11056#discussion_r4520855302
########## paimon-python/pypaimon/multimodal/lerobot/aggregate.py: ########## @@ -0,0 +1,332 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Merge source coordinates at the write boundary of a new table group.""" + +import copy +import json +from contextlib import ExitStack + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc + +from pypaimon.multimodal.lerobot.loader import ( + _commit_messages, _prepare_dataset, _video_fps, +) +from pypaimon.multimodal.lerobot.metadata import ( + _append_arrow_tables, _commit_metadata, _load_dataset_metadata, + _metadata_table, _prepare_metadata_tables, + _source_episode_tables, _validated_episode_tables, +) +from pypaimon.multimodal.lerobot.schema import ( + _require_v3, _schema_from_info, _validate_v3_required_features, + _video_feature_names, +) +from pypaimon.multimodal.lerobot.source import ( + _close_quietly, _import_lerobot_dataset, _load_hub_info, + _open_resolved_dataset, _pandas_index_column, _resolved_source, + _validate_info_paths, +) +from pypaimon.multimodal.source_utils import _validate_source_kerberos + + +_CONTROLS = ("index", "episode_index", "task_index", "subtask_index") +_STATS = ("min", "max", "mean", "std", "count") +_ENCODER_KEYS = { + "video.codec", "video.pix_fmt", "video.gop_size", "video.crf", + "video.encoder", "video.extra_options", "video.video_backend", + "video.has_audio", "has_audio", "video.bit_rate", +} +# These properties describe source storage or are rebuilt for the merged dataset. +_REBUILT_INFO = { + "total_frames", "total_episodes", "total_tasks", "total_subtasks", "total_videos", + "total_chunks", "splits", "data_path", "video_path", "chunks_size", + "data_files_size_in_mb", "video_files_size_in_mb", "features", +} + + +def _json_table(table): + return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()} + + +def _merge_stats(items): + """Combine population moments; quantile summaries cannot be merged exactly.""" + if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"} + for item in items): + raise ValueError("Unsupported LeRobot statistics fields.") + if any(not set(_STATS).issubset(item) for item in items): + raise ValueError("LeRobot statistics require min/max/mean/std/count.") + arrays = [{key: np.asarray(item[key], dtype=np.float64) for key in _STATS} + for item in items] + first = arrays[0] + for item in arrays: + if any(item[key].shape != first[key].shape for key in _STATS): + raise ValueError("LeRobot statistics shapes must match.") + count = item["count"] + if count.shape != (1,) or not np.all(np.isfinite(count)) \ + or np.any(count <= 0) or np.any(count != np.floor(count)): + raise ValueError("LeRobot statistics count must be one positive integer.") + if any(item[key].shape != item["mean"].shape for key in ("min", "max", "std")): + raise ValueError("LeRobot statistic shapes must match their mean.") + count = sum(item["count"] for item in arrays) + mean = sum(item["mean"] * item["count"] for item in arrays) / count + variance = sum(item["count"] * ( + item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / count + return { + "min": np.minimum.reduce([item["min"] for item in arrays]).tolist(), + "max": np.maximum.reduce([item["max"] for item in arrays]).tolist(), + "mean": mean.tolist(), "std": np.sqrt(variance).tolist(), + "count": count.astype(np.int64).tolist(), + } + + +def _batch_stats(values): + values = np.asarray(values, dtype=np.float64) + return {"min": [values.min()], "max": [values.max()], + "mean": [values.mean()], "std": [values.std()], + "count": [len(values)]} + + +def _vocabulary(plans, kind): + key = kind + "_table" + tables = [plan["metadata"][key] for plan in plans] + if all(table is None for table in tables): + return None + if any(table is None for table in tables): + raise ValueError("LeRobot sources disagree on %s metadata." % kind) + singular = kind[:-1] + index_name = singular + "_index" + schema = tables[0].schema + label = _pandas_index_column(schema, singular) + if set(schema.names) != {index_name, label}: + raise ValueError("Multi-source %s metadata requires only index and text." % kind) + rows, by_text = [], {} + for ordinal, (plan, table) in enumerate(zip(plans, tables)): + source_label = _pandas_index_column(table.schema, singular) + if (set(table.column_names) != {index_name, source_label} + or not table.schema.field(index_name).equals( + schema.field(index_name), check_metadata=False) + or not table.schema.field(source_label).with_name(label).equals( + schema.field(label), check_metadata=False)): + raise ValueError("LeRobot source %d has incompatible %s schema." % (ordinal, kind)) + mapping = {} + for row in sorted(table.to_pylist(), key=lambda row: row[index_name]): + text = row[source_label] + if text not in by_text: + by_text[text] = len(rows) + rows.append({index_name: len(rows), label: text}) + mapping[row[index_name]] = by_text[text] + plan[index_name] = mapping + return pa.Table.from_pylist(rows, schema=schema) + + +def _feature_contract(features): + result = copy.deepcopy(features) + for feature in result.values(): + if feature.get("dtype") == "video": + for key in ("info", "video_info"): + feature[key] = {k: v for k, v in feature.get(key, {}).items() + if k not in _ENCODER_KEYS} + return result + + +def _plan_merge(plans): + first = plans[0] + info = copy.deepcopy(first["info"]) + metadata = dict(first["metadata"]) + frame_offset = episode_offset = 0 + for ordinal, plan in enumerate(plans): + other = plan["info"] + for key in (set(info) | set(other)) - _REBUILT_INFO: + if info.get(key) != other.get(key): + raise ValueError("LeRobot source %d has incompatible info.%s." % (ordinal, key)) + expected_features = _feature_contract(info["features"]) + actual_features = _feature_contract(other["features"]) + for name in set(expected_features) | set(actual_features): + if expected_features.get(name) != actual_features.get(name): + raise ValueError("LeRobot source %d has incompatible features.%s." % (ordinal, name)) + if not plan["schema"].equals(first["schema"], check_metadata=False): + raise ValueError("LeRobot source %d has incompatible frame schema." % ordinal) + if not plan["metadata"]["episodes_schema"].equals( + metadata["episodes_schema"], check_metadata=False): + raise ValueError("LeRobot source %d has incompatible Episode schema." % ordinal) + # Validate every Episode before creating any target table. + try: + for name in plan["videos"]: + _video_fps(other, other["features"][name], name) + for unused in _validated_episode_tables(plan["metadata"], first["videos"]): + pass + except ValueError as error: + raise ValueError("LeRobot source %d: %s" % (ordinal, error)) from error + plan["frame_offset"] = frame_offset + plan["episode_offset"] = episode_offset + plan["control_stats"] = {} + frame_offset += other["total_frames"] + episode_offset += other["total_episodes"] + metadata["tasks_table"] = _vocabulary(plans, "tasks") + metadata["subtasks_table"] = _vocabulary(plans, "subtasks") + info.update(total_frames=frame_offset, total_episodes=episode_offset, + total_tasks=metadata["tasks_table"].num_rows, + splits={"train": "0:%d" % episode_offset}) + if metadata["subtasks_table"] is not None: + info["total_subtasks"] = metadata["subtasks_table"].num_rows + for key in ("total_chunks", "total_videos", "data_path", "video_path", + "chunks_size", "data_files_size_in_mb", "video_files_size_in_mb"): + info.pop(key, None) + # Encoder descriptions are not training features. Retain only shared values. + for name in first["videos"]: + for key in ("info", "video_info"): + details = [p["info"]["features"][name].get(key, {}) for p in plans] + info["features"][name][key] = { + k: v for k, v in details[0].items() + if all(d.get(k) == v for d in details[1:])} + stats = [p["metadata"]["stats_table"] for p in plans] + if any(s is None for s in stats) and not all(s is None for s in stats): + raise ValueError("LeRobot stats must be present in all sources or absent in all.") + merged_stats = None + if stats[0] is not None: + decoded = [_json_table(s) for s in stats] + if any(set(s) != set(decoded[0]) for s in decoded[1:]): + raise ValueError("LeRobot statistics features must match.") + merged_stats = {name: _merge_stats([s[name] for s in decoded]) + for name in decoded[0] if name not in _CONTROLS} + metadata["info_table"] = _metadata_table(info) + metadata["stats_table"] = (None if merged_stats is None + else _metadata_table(merged_stats)) + # Quantiles are omitted rather than presenting source summaries as merged values. + metadata["episodes_schema"] = pa.schema([ + field for field in metadata["episodes_schema"] + if not any(field.name.startswith("stats/%s/q" % name) for name in _CONTROLS) + ], metadata=metadata["episodes_schema"].metadata) + return metadata, merged_stats + + +def _transform_frames(plan, batch, episode): + for name in _CONTROLS: + if name not in batch.column_names: + continue + column = batch[name] + if name in ("index", "episode_index"): + offset = plan["frame_offset" if name == "index" else "episode_offset"] + values = pc.add_checked(column, pa.scalar(offset, type=column.type)) + else: + mapping = plan[name] + values = pa.array([ + None if name == "subtask_index" and value is None Review Comment: [P1] I can now import an isolated null label, but I reproduced a partial commit on `d02ca187`. In two five-frame FileIO sources, I set both `subtask_index` values in the first Episode of source A to null and kept the `stats/subtask_index/*` list fields non-nullable in both Episode schemas. `load_from_lerobot` raised `Cannot write null to non-null column(stats/subtask_index/min)`, and a scan of the target Frames table returned all 10 rows after the exception. I reproduced a second failure after the Frames commit by leaving one label in that Episode and making its original `stats/subtask_index/mean` cell null. `_merged_episodes` used the null cell's shape to produce a scalar, which Arrow rejected for the list field. Could we derive the replacement shape and nullability from the output contract and validate the reconstructed Episodes before committing Frames? ########## paimon-python/pypaimon/multimodal/lerobot/dataset.py: ########## @@ -231,6 +232,21 @@ def _init_episodes(self, episodes): "dataset.") self._episode_ends = [end for _, end in self._episode_ranges] \ if self._episode_ranges is not None else None + self._episode_fps = _episode_fps( + self.meta, self._fps, self._total_episodes) + if self._fps is None and self._episode_fps \ + and len(set(self._episode_fps)) == 1: + self._fps = self._episode_fps[0] + for name in self._video_keys: + feature = self._features[name] + rates = ([feature["fps"]] if "fps" in feature else []) + for key in ("info", "video_info"): + details = feature.get(key, {}) + if "video.fps" in details: + rates.append(details["video.fps"]) + if any(any(_dataset_fps(rate) != fps for fps in self._episode_fps) + for rate in rates): + raise ValueError("LeRobot video feature %s fps conflicts with episode fps." % name) Review Comment: [P2] Keep the default FPS metadata compatible with the importer. I reproduced this by importing a real 10 FPS video whose feature metadata declared `video.fps` as `"10"` or `10.000005`. The default import succeeds because `_video_fps` accepts numeric strings and applies a tolerance, but the public reader then fails with `LeRobot metadata fps must be an integer`. There is a related default-mode regression in `metadata.py` at the `_EPISODE_FPS_COLUMN` check. Adding a Parquet Episode `fps` column containing `10.0` was accepted and preserved before this commit and now fails on default import. Could the reader share the importer's video FPS parsing and tolerance, and could the per-Episode checks apply only when Episode FPS is explicitly enabled for the table group? ########## paimon-python/pypaimon/multimodal/lerobot/aggregate.py: ########## @@ -0,0 +1,373 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Merge source coordinates at the write boundary of a new table group.""" + +import copy +import json +from contextlib import ExitStack + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc + +from pypaimon.multimodal.lerobot.loader import ( + _commit_messages, _prepare_dataset, _video_fps, +) +from pypaimon.multimodal.lerobot.metadata import ( + _EPISODE_FPS_COLUMN, _append_arrow_tables, _commit_metadata, _load_dataset_metadata, + _metadata_table, _prepare_metadata_tables, + _source_episode_tables, _use_episode_fps, _validated_episode_tables, +) +from pypaimon.multimodal.lerobot.schema import ( + _require_v3, _schema_from_info, _validate_v3_required_features, + _video_feature_names, +) +from pypaimon.multimodal.lerobot.source import ( + _close_quietly, _import_lerobot_dataset, _load_hub_info, + _open_resolved_dataset, _pandas_index_column, _resolved_source, + _validate_info_paths, +) +from pypaimon.multimodal.source_utils import _validate_source_kerberos + + +_CONTROLS = ("index", "episode_index", "task_index", "subtask_index") +_STATS = ("min", "max", "mean", "std", "count") +_ENCODER_KEYS = { + "video.codec", "video.pix_fmt", "video.gop_size", "video.crf", + "video.encoder", "video.extra_options", "video.video_backend", + "video.has_audio", "has_audio", "video.bit_rate", +} +# These properties describe source storage or are rebuilt for the merged dataset. +_REBUILT_INFO = { + "total_frames", "total_episodes", "total_tasks", "total_subtasks", "total_videos", + "total_chunks", "splits", "data_path", "video_path", "chunks_size", + "data_files_size_in_mb", "video_files_size_in_mb", "features", +} + + +def _json_table(table): + return {row["key"]: json.loads(row["value"]) for row in table.to_pylist()} + + +def _merge_stats(items): + """Combine population moments; quantile summaries cannot be merged exactly.""" + if any(set(item) - set(_STATS) - {"q01", "q10", "q50", "q90", "q99"} + for item in items): + raise ValueError("Unsupported LeRobot statistics fields.") + if any(not set(_STATS).issubset(item) for item in items): + raise ValueError("LeRobot statistics require min/max/mean/std/count.") + arrays = [{key: np.asarray(item[key], dtype=np.float64 if key in ("mean", "std") else None) + for key in _STATS} + for item in items] + first = arrays[0] + for item in arrays: + if any(item[key].shape != first[key].shape for key in _STATS): + raise ValueError("LeRobot statistics shapes must match.") + count = item["count"] + if count.shape != (1,) or not np.all(np.isfinite(count)) \ + or np.any(count <= 0) or np.any(count != np.floor(count)): + raise ValueError("LeRobot statistics count must be one positive integer.") + if any(item[key].shape != item["mean"].shape for key in ("min", "max", "std")): + raise ValueError("LeRobot statistic shapes must match their mean.") + count = sum(int(item["count"][0]) for item in arrays) + mean = sum(item["mean"] * item["count"] for item in arrays) / count + variance = sum(item["count"] * ( + item["std"] ** 2 + (item["mean"] - mean) ** 2) for item in arrays) / count + return { + "min": np.minimum.reduce([item["min"].astype(object) for item in arrays]).tolist(), + "max": np.maximum.reduce([item["max"].astype(object) for item in arrays]).tolist(), Review Comment: [P2] Preserve NaN propagation when merging floating-point extrema. I imported the same two FileIO sources in both orders. One source had NaN extrema for a feature coordinate and the other had finite extrema. When the NaN source came first, merged `min` and `max` were finite. When it came last, both were NaN. The previous numeric reduction propagated NaN in both orders. Could we use NumPy's floating-point reduction for float extrema while keeping the exact integer path? ########## paimon-python/pypaimon/multimodal/lerobot/dataset.py: ########## @@ -1672,17 +1728,30 @@ def read_frames(decoder, windows): def _collate_lerobot_batch(rows, video_windows): from torch.utils.data import default_collate - if not video_windows: + nullable = { + key: [row[key] for row in rows] + for key in ("subtask_index", "subtask") + if key in rows[0] and any( + _contains_none(row[key]) for row in rows) + } Review Comment: [P2] Keep the batch representation of nullable subtask indices stable. In one `to_dataloader(batch_size=2)` iteration, a batch containing an unlabelled frame returned `subtask_index` as `[tensor(0), None]`, and the next fully labelled batch returned `tensor([0, 1])`. Training code that moves this feature to a device or applies tensor operations then fails only on some batches. Could we use a consistent representation for the nullable feature across batches and delta windows, such as an integer tensor paired with a validity mask? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
