yugan95 commented on code in PR #9375:
URL: https://github.com/apache/paimon/pull/9375#discussion_r3849416889
##########
paimon-python/pypaimon/write/writer/data_vector_writer.py:
##########
@@ -188,27 +188,32 @@ def _split_data(self, data: pa.RecordBatch) ->
Tuple[pa.RecordBatch, pa.RecordBa
return normal_data, vector_data
def _should_roll_normal(self) -> bool:
- if self.pending_normal_data is None:
+ # Runs on every write, so it answers from the running counts only.
+ if self._normal_buffer.is_empty:
return False
- if self.pending_normal_data.num_rows >= self.target_file_row_num:
+ if self._normal_buffer.num_rows >= self.target_file_row_num:
return True
if self.record_count % self.CHECK_ROLLING_RECORD_CNT != 0:
return False
- return self.pending_normal_data.nbytes > self.target_file_size
+ return self._normal_buffer.nbytes > self.target_file_size
def _current_row_count(self) -> int:
- if self.pending_normal_data is not None:
- return self.pending_normal_data.num_rows
- if self.vector_writer is not None and self.vector_writer.pending_data
is not None:
- return self.vector_writer.pending_data.num_rows
+ if not self._normal_buffer.is_empty:
+ return self._normal_buffer.num_rows
+ if self.vector_writer is not None:
+ # Running count, not a folded buffer: this runs on every write.
+ return self.vector_writer.pending_row_count
return 0
def _close_current_writers(self):
- has_normal = self.pending_normal_data is not None and
self.pending_normal_data.num_rows > 0
+ # Cleared at the end, once the normal file and the vector sidecars have
+ # both landed: a failure in either half has to leave the normal rows
+ # buffered, or a retry would commit sidecar-only metadata.
+ normal_data = self._normal_buffer.materialize()
Review Comment:
Confirmed. A flush now publishes all of its files or none: nothing reaches
`committed_files` until the normal file and every sidecar has landed.
The retry resumes rather than restarts, because the blob and vector writers
drain their own buffers as they write — their half can't be rewritten.
`_pending_normal_meta` remembers the normal file that landed so the retry picks
up where it failed, and `write`/`write_row` are rejected until it does.
`CompositeFlushResumeTest` covers it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
