XiaoHongbo-Hope commented on code in PR #9375:
URL: https://github.com/apache/paimon/pull/9375#discussion_r3843628136
##########
paimon-python/pypaimon/write/writer/data_vector_writer.py:
##########
@@ -188,27 +188,32 @@ def _split_data(self, data: pa.RecordBatch) ->
Tuple[pa.RecordBatch, pa.RecordBa
return normal_data, vector_data
def _should_roll_normal(self) -> bool:
- if self.pending_normal_data is None:
+ # Runs on every write, so it answers from the running counts only.
+ if self._normal_buffer.is_empty:
return False
- if self.pending_normal_data.num_rows >= self.target_file_row_num:
+ if self._normal_buffer.num_rows >= self.target_file_row_num:
return True
if self.record_count % self.CHECK_ROLLING_RECORD_CNT != 0:
return False
- return self.pending_normal_data.nbytes > self.target_file_size
+ return self._normal_buffer.nbytes > self.target_file_size
def _current_row_count(self) -> int:
- if self.pending_normal_data is not None:
- return self.pending_normal_data.num_rows
- if self.vector_writer is not None and self.vector_writer.pending_data
is not None:
- return self.vector_writer.pending_data.num_rows
+ if not self._normal_buffer.is_empty:
+ return self._normal_buffer.num_rows
+ if self.vector_writer is not None:
+ # Running count, not a folded buffer: this runs on every write.
+ return self.vector_writer.pending_row_count
return 0
def _close_current_writers(self):
- has_normal = self.pending_normal_data is not None and
self.pending_normal_data.num_rows > 0
+ # Cleared at the end, once the normal file and the vector sidecars have
+ # both landed: a failure in either half has to leave the normal rows
+ # buffered, or a retry would commit sidecar-only metadata.
+ normal_data = self._normal_buffer.materialize()
Review Comment:
Sidecar failure can duplicate normal data and break normal-sidecar
consistency on retry. Please make the composite flush retry-safe.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
