JingsongLi commented on code in PR #9708:
URL: https://github.com/apache/paimon/pull/9708#discussion_r3975742816
##########
paimon-python/pypaimon/multimodal/lerobot/writer.py:
##########
@@ -104,95 +328,222 @@ def __init__(
0,
"LeRobot",
)
+ self._metadata_tables = (
+ _prepare_metadata_tables(
+ connection, self._table.raw_table, metadata)
+ if created else self._open_metadata_tables(
+ connection, self._table.raw_table, metadata)
+ )
- self.num_frames, self.num_episodes, self._task_indices = \
- self._load_existing_state()
+ (self.num_frames, self.num_episodes, self._task_indices,
+ self._stats, stored_subtasks) = self._load_existing_state()
+ if stored_subtasks is not None:
+ if requested_subtasks is not None \
+ and requested_subtasks != stored_subtasks:
+ raise ValueError(
+ "subtasks do not match the existing LeRobot table.")
+ self.subtasks = stored_subtasks
+ else:
+ self.subtasks = requested_subtasks
+ if "subtask_index" in self.features and self.subtasks is None:
+ raise ValueError(
+ "subtasks are required when creating a table with "
+ "subtask_index.")
self._next_task_index = (
max(self._task_indices.values()) + 1
if self._task_indices else 0
)
self.pending_episodes = 0
self._episode_frames = []
+ self._pending_episode_rows = []
+ self._committed_task_count = len(self._task_indices)
+ self._subtasks_committed = self.num_frames > 0
self._table_write = None
self._table_commit = None
self._snapshot_recorder = None
self._finalized = False
self._failed = False
+ @staticmethod
+ def _writer_metadata(fps, features, subtasks):
+ info = {
+ "codebase_version": "v3.0",
+ "fps": fps,
+ "features": features,
+ "total_frames": 0,
+ "total_episodes": 0,
+ "total_tasks": 0,
+ "splits": {},
+ }
+ return {
+ "info_table": _metadata_table(info),
+ "episodes_schema": _episode_schema(features),
+ "tasks_table": pa.Table.from_pylist(
+ [], schema=_EMPTY_TASKS_SCHEMA),
Review Comment:
[P1] Preserve the text index metadata for tasks and subtasks
These companion tables store labels as ordinary columns without pandas index
metadata, but the `PaimonLeRobotDataset` reader merged in #9498 restores a
DataFrame and uses its index as the task/subtask text (`_component_dataframe`
and `_index_names`). Consequently, a recording with `task="pick"` is decoded as
task `"0"`, and its first frame fails `_validate_control_row` with `Paimon task
at LeRobot index 0 is not assigned to Episode 0`, because the episode correctly
lists `"pick"`. Subtask labels are likewise decoded as numeric strings. I
reproduced this using the PR writer's persisted output and the current master
reader; restoring the text indexes makes the same frame pass validation. Please
preserve the native text-index pandas metadata for both companion schemas using
the existing `_prepare_metadata_tables` mechanism, or make the reader
explicitly support these plain text columns, and add a writer-to-reader
regression test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
