JingsongLi commented on code in PR #11097:
URL: https://github.com/apache/paimon/pull/11097#discussion_r4548687709
##########
paimon-python/pypaimon/multimodal/lerobot/aggregate.py:
##########
@@ -84,13 +88,156 @@
_DATASETS_SCHEMA = pa.schema([
pa.field("index", pa.int64(), nullable=False),
pa.field("name", pa.string()),
+ pa.field("dataset_id", pa.string()),
+ pa.field("source_hash", pa.string()),
+ pa.field("control_counts", pa.string()),
pa.field("info", pa.string(), nullable=False),
pa.field("tasks", _SOURCE_TASKS_TYPE, nullable=False),
pa.field("subtasks", _SOURCE_SUBTASKS_TYPE),
pa.field("stats", pa.string()),
])
+def _dataset_physical_index(dataset_id):
+ """Stable positive BIGINT outside the sequential legacy Dataset range."""
+ digest = hashlib.sha256(dataset_id.encode("utf-8")).digest()
+ return _COORDINATED_INDEX_START | (
+ int.from_bytes(digest[:8], "big") & (_COORDINATED_INDEX_START - 1))
+
+
+def _completed_dataset_replay(connection, table_name, dataset_id, source_hash,
+ source_count, coordinated):
+ """Reject unsupported identity schemas and short-circuit published
replays."""
+ frames_name = _companion_identifier(
+ connection._identifier(table_name), "_frames")
+ frames = connection.get_table(frames_name).raw_table
+ identifiers = _companion_table_identifiers(frames)
+ tables = [frames] + [connection.catalog.get_table(identifiers[name])
+ for name in ("episodes", "datasets")]
+ datasets = tables[-1]
+ component_identity = {_COMPONENT_ID_COLUMN, _COMPONENT_HASH_COLUMN}
+ if (any(not component_identity.issubset(_target_schema(table).names)
+ for table in tables[:2]) or
+ not {"dataset_id",
"source_hash"}.issubset(_target_schema(datasets).names)):
+ raise ValueError(
+ "LeRobot dataset_id requires identity columns "
+ "in all three tables; create a new table group.")
+ builder = datasets.new_read_builder().with_projection(
+ ["index", "dataset_id", "source_hash"])
+ from pypaimon.common.predicate_builder import PredicateBuilder
+ predicate = PredicateBuilder(datasets.fields)
+ if coordinated:
+ builder.with_filter(predicate.equal("index",
_dataset_physical_index(dataset_id)))
+ else:
+ builder.with_filter(predicate.equal("dataset_id", dataset_id))
+ matches = [row for row in builder.new_read().to_arrow(
+ builder.new_scan().plan().splits()).to_pylist()
+ if row["dataset_id"] == dataset_id]
+ if not matches:
+ return False
+ if len(matches) != source_count or any(
+ row["source_hash"] != source_hash for row in matches):
+ raise ValueError(
+ "LeRobot dataset_id %r was used for different sources." %
dataset_id)
+ if not coordinated:
+ from pypaimon.multimodal.lerobot.dataset import (
+ _read_arrow, _source_dataset_ranges,
+ )
+ rows = _read_arrow(datasets, ["index",
"info"]).sort_by("index").to_pylist()
+ ranges = _source_dataset_ranges(rows)
+ expected = (ranges[-1][4], ranges[-1][2], len(rows))
+ for name, table, count in zip(("frames", "episodes", "datasets"),
+ tables, expected):
+ snapshot = table.snapshot_manager().get_latest_snapshot()
+ if snapshot is None or snapshot.next_row_id != count:
+ raise ValueError(
+ "LeRobot group has incomplete %s data; call "
+ "recover_lerobot_append before appending."
+ % name)
+ return True
+
+
+def _dataset_absent(table, physical_index, dataset_id, source_hash, count):
+ """Evaluate a Dataset's committed rows at each optimistic commit
attempt."""
+ def condition(snapshot):
+ if snapshot is None:
+ return True
+ pinned = table._copy_with_snapshot(snapshot)
+ key_column = ("dataset_index" if "dataset_index" in
+ _target_schema(pinned).names else "index")
Review Comment:
[P2] Select the Dataset key explicitly instead of inferring its table role
`extra_dataset_fields={"dataset_index": "notes.txt"}` is accepted and
creates a user string column on `_datasets`. Its presence makes this condition
treat the Dataset table as a Frames/Episodes table and filter the text column
instead of the physical key `index`. For ordinary notes text, every check
reports the Dataset absent. I reproduced two concurrent first attempts with the
same `dataset_id`, synchronizing them after Frame preparation: both succeeded,
leaving 10 Frames, 4 Episodes, and 3 Dataset rows, including two identical
physical Dataset keys. The training reader then fails with `index must be
unique and non-negative`. The early replay check cannot prevent this race
because neither attempt has published metadata yet. Please pass the table role
or key/identity columns explicitly; Dataset publication must always check
`index`, regardless of custom metadata columns.
##########
paimon-python/pypaimon/execution/python/write/file_store_commit.py:
##########
@@ -362,6 +366,13 @@ def _try_commit(self, commit_kind, commit_identifier,
commit_entries_plan,
commit_kind,
notify_callbacks=True):
break
+ if self.commit_condition is not None and not
self.commit_condition(latest_snapshot):
+ # No commit from this attempt can have succeeded after a
+ # definite failure. An uncertain response may have published
+ # these files, so leave them untouched in that case.
+ if not commit_result_may_be_uncertain and prepared_messages is
not None:
+ self.abort(prepared_messages)
Review Comment:
[P1] Preserve prepared files across uncertain commit retries
`commit_result_may_be_uncertain` is reset for each `_try_commit` call, so
this check cannot establish that the messages were never published by an
earlier call. I reproduced this through a conditional `StreamTableCommit`: with
`commit.max-retries=0`, publish the snapshot and then raise an `OSError` to
simulate a lost response. The first call raises while snapshot 1 and its data
file exist. Retrying the same checkpoint with the same prepared messages makes
the condition return False because the append is already present, but the fresh
local uncertainty flag permits `abort`, deleting the file referenced by
snapshot 1; reads then fail with `FileNotFoundError`. Please retain or resolve
uncertain ownership across calls, or verify that the prepared files are not
referenced before cleaning them up. The current invocation's failure to publish
does not prove that these files are uncommitted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]