codope commented on code in PR #411:
URL: https://github.com/apache/hudi-rs/pull/411#discussion_r2289852431
##########
crates/core/src/table/mod.rs:
##########
@@ -520,6 +520,72 @@ impl Table {
})
}
+ /// Get all the changed [FileSlice]s in splits from the table between the
given timestamps.
+ ///
+ /// # Arguments
+ /// * `n` - The number of chunks to split the file slices into.
+ /// * `start_timestamp` - If provided, only file slices that were
changed after this timestamp will be returned.
+ /// * `end_timestamp` - If provided, only file slices that were
changed before or at this timestamp will be returned.
+ ///
+ /// # Notes
+ /// * This API is useful for implementing incremental query with read
parallelism.
+ /// * Uses the same splitting flow as the time-travel API to respect
read parallelism config.
+ pub async fn get_file_slices_splits_between(
+ &self,
+ n: usize,
+ start_timestamp: Option<&str>,
+ end_timestamp: Option<&str>,
+ ) -> Result<Vec<Vec<FileSlice>>> {
+ // If the end timestamp is not provided, use the latest commit
timestamp.
+ let Some(end) =
+ end_timestamp.or_else(||
self.timeline.get_latest_commit_timestamp_as_option())
+ else {
+ // No latest commit timestamp means the table is empty.
+ return Ok(Vec::new());
+ };
+
+ let start = start_timestamp.unwrap_or(EARLIEST_START_TIMESTAMP);
+
+ self.get_file_slices_splits_between_internal(n, start, end)
+ .await
+ }
+
+ /// Same as [Table::get_file_slices_splits_between], but blocking.
+ pub fn get_file_slices_splits_between_blocking(
+ &self,
+ n: usize,
+ start_timestamp: Option<&str>,
+ end_timestamp: Option<&str>,
+ ) -> Result<Vec<Vec<FileSlice>>> {
Review Comment:
Would you like to pick up https://github.com/apache/hudi-rs/issues/367?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
