yugan95 commented on code in PR #9375:
URL: https://github.com/apache/paimon/pull/9375#discussion_r3842329161
##########
paimon-python/pypaimon/write/writer/data_vector_writer.py:
##########
@@ -188,27 +188,29 @@ def _split_data(self, data: pa.RecordBatch) ->
Tuple[pa.RecordBatch, pa.RecordBa
return normal_data, vector_data
def _should_roll_normal(self) -> bool:
- if self.pending_normal_data is None:
+ # Runs on every write, so it answers from the running counts only.
+ if self._normal_buffer.is_empty:
return False
- if self.pending_normal_data.num_rows >= self.target_file_row_num:
+ if self._normal_buffer.num_rows >= self.target_file_row_num:
return True
if self.record_count % self.CHECK_ROLLING_RECORD_CNT != 0:
return False
- return self.pending_normal_data.nbytes > self.target_file_size
+ return self._normal_buffer.nbytes > self.target_file_size
def _current_row_count(self) -> int:
- if self.pending_normal_data is not None:
- return self.pending_normal_data.num_rows
- if self.vector_writer is not None and self.vector_writer.pending_data
is not None:
- return self.vector_writer.pending_data.num_rows
+ if not self._normal_buffer.is_empty:
+ return self._normal_buffer.num_rows
+ if self.vector_writer is not None:
+ # Running count, not a folded buffer: this runs on every write.
+ return self.vector_writer.pending_row_count
return 0
def _close_current_writers(self):
- has_normal = self.pending_normal_data is not None and
self.pending_normal_data.num_rows > 0
+ normal_data = self._normal_buffer.take()
Review Comment:
Fixed the same way: `_close_current_writers` resets at the end, once the
normal file and the sidecars have both landed. Regression test added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
