YannByron commented on code in PR #9708:
URL: https://github.com/apache/paimon/pull/9708#discussion_r3975274177
##########
paimon-python/pypaimon/tests/multimodal_lerobot_writer_test.py:
##########
@@ -195,17 +364,150 @@ def
test_commits_multiple_completed_episodes_as_one_batch(self):
"timestamp",
"index",
"task_index",
- "task",
"observation.state",
"action",
]).to_arrow().sort_by("index").to_pylist()
self.assertEqual([0, 1], [row["episode_index"] for row in rows])
self.assertEqual([0, 0], [row["frame_index"] for row in rows])
self.assertEqual([0, 1], [row["index"] for row in rows])
self.assertEqual([0, 1], [row["task_index"] for row in rows])
- self.assertEqual(["pick", "place"], [row["task"] for row in rows])
self.assertEqual([0.0, 0.0], [row["timestamp"] for row in rows])
self.assertEqual([0.5, 0.0], [row["action"] for row in rows])
+ self.assertEqual([
+ {"task_index": 0, "task": "pick"},
+ {"task_index": 1, "task": "place"},
+ ], _catalog_rows(self.connection, "robot_data__tasks"))
+
+ def test_multiple_flushes_append_rows_and_replace_info(self):
+ writer = PaimonLeRobotWriter(
+ self.connection,
+ "multiple_flushes",
+ fps=10,
+ episodes_per_commit=1,
+ features={
+ "action": {
+ "dtype": "float32",
+ "shape": (1,),
+ "names": None,
+ },
+ },
+ )
+ for value, task in ((1.0, "pick"), (2.0, "place")):
+ writer.add_frame({
+ "action": np.array([value], dtype=np.float32),
+ "task": task,
+ })
+ writer.save_episode()
+ writer.finalize()
+
+ self.assertEqual(2, len(_catalog_rows(
+ self.connection, "multiple_flushes__episodes")))
+ self.assertEqual(2, len(_catalog_rows(
+ self.connection, "multiple_flushes__tasks")))
+ info_rows = _catalog_rows(self.connection, "multiple_flushes__info")
+ self.assertEqual(len({row["key"] for row in info_rows}),
+ len(info_rows))
+ info = {
+ row["key"]: json.loads(row["value"])
+ for row in info_rows
+ }
+ self.assertEqual(2, info["total_frames"])
+ self.assertEqual(2, info["total_episodes"])
+ self.assertEqual(2, info["total_tasks"])
+ self.assertEqual({"train": "0:2"}, info["splits"])
+
+ def test_persists_native_episode_and_global_stats(self):
+ from lerobot.datasets.compute_stats import (
Review Comment:
Fixed in 402e61fec. Writer tests now skip when the optional LeRobot
statistics module is unavailable, while the missing-dependency test remains
runnable. I also verified this with a simulated no-LeRobot environment: 22
tests skipped and the dependency test passed.
##########
paimon-python/pypaimon/multimodal/lerobot/writer.py:
##########
@@ -289,6 +556,12 @@ def save_episode(self):
raise ValueError("Cannot save an empty LeRobot episode.")
episode = self._episode_table()
+ episode_stats = _compute_stats(episode, self.features)
+ aggregate_stats = _lerobot_stats_functions()[0]
+ stats = (
+ aggregate_stats([self._stats, episode_stats])
Review Comment:
Fixed in 402e61fec. Statistics are now aggregated per feature using a
neutral internal key selected from the declared dtype, so numeric names
containing image are not validated as image tensors. Added a two-episode and
reopen regression test for observation.image_embedding.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
