JeonDaehong commented on code in PR #1454:
URL: https://github.com/apache/paimon-rust/pull/1454#discussion_r4642984337


##########
crates/paimon/src/table/table_commit.rs:
##########
@@ -2700,6 +2718,101 @@ impl TableCommit {
         Ok(Some(entries))
     }
 
+    async fn check_dv_compaction_history(
+        &self,
+        captured: &Snapshot,
+        latest: &Snapshot,
+        entries: &[ManifestEntry],
+    ) -> Result<()> {
+        if captured.id() == latest.id() || entries.is_empty() {
+            return Ok(());
+        }
+
+        fn bucket_key(partition: &[u8], bucket: i32) -> (&[u8], i32) {
+            let partition = if is_empty_partition(partition) {
+                &[]
+            } else {
+                partition
+            };
+            (partition, bucket)
+        }
+
+        let buckets: HashSet<_> = entries
+            .iter()
+            .map(|entry| bucket_key(entry.partition(), entry.bucket()))
+            .collect();
+        let partition_filter =
+            
self.build_entries_partition_filter(&entries.iter().collect::<Vec<_>>())?;
+        let manifest_dir = self.snapshot_manager.manifest_dir();
+        let read_indexes = async |name: Option<&str>| -> Result<_> {
+            let Some(name) = name else {
+                return Ok(HashMap::new());
+            };
+            let indexes = IndexManifest::read(
+                self.snapshot_manager.file_io(),
+                &format!("{manifest_dir}/{name}"),
+            )
+            .await?;
+            Ok(indexes
+                .into_iter()
+                .filter_map(|entry| {
+                    let key = bucket_key(&entry.partition, entry.bucket);
+                    buckets.contains(&key).then(|| {
+                        let identity = (
+                            key.0.to_vec(),
+                            key.1,
+                            entry.kind == FileKind::Add,
+                            entry.index_file.index_type.clone(),
+                            entry.index_file.file_name.clone(),
+                        );
+                        (identity, entry.index_file)
+                    })
+                })
+                .collect::<HashMap<_, _>>())
+        };
+
+        let mut previous = captured.clone();
+        for id in captured.id() + 1..=latest.id() {
+            let snapshot = if id == latest.id() {
+                latest.clone()
+            } else {
+                self.snapshot_manager.get_snapshot(id).await?
+            };
+            if snapshot.commit_user() != self.commit_user

Review Comment:
   Dropping the `commit_user` condition here keeps all of those tests passing, 
so the skip isn't exercised by any of them. Which case is it for? Two 
compactions that share a commit user (for example tasks of one job) on the same 
bucket would not be checked against each other, so a test that pins down why 
it's safe would help.



##########
crates/paimon/src/table/table_commit.rs:
##########
@@ -2700,6 +2718,101 @@ impl TableCommit {
         Ok(Some(entries))
     }
 
+    async fn check_dv_compaction_history(
+        &self,
+        captured: &Snapshot,
+        latest: &Snapshot,
+        entries: &[ManifestEntry],
+    ) -> Result<()> {
+        if captured.id() == latest.id() || entries.is_empty() {
+            return Ok(());
+        }
+
+        fn bucket_key(partition: &[u8], bucket: i32) -> (&[u8], i32) {
+            let partition = if is_empty_partition(partition) {
+                &[]
+            } else {
+                partition
+            };
+            (partition, bucket)
+        }
+
+        let buckets: HashSet<_> = entries
+            .iter()
+            .map(|entry| bucket_key(entry.partition(), entry.bucket()))
+            .collect();
+        let partition_filter =
+            
self.build_entries_partition_filter(&entries.iter().collect::<Vec<_>>())?;
+        let manifest_dir = self.snapshot_manager.manifest_dir();
+        let read_indexes = async |name: Option<&str>| -> Result<_> {
+            let Some(name) = name else {
+                return Ok(HashMap::new());
+            };
+            let indexes = IndexManifest::read(
+                self.snapshot_manager.file_io(),
+                &format!("{manifest_dir}/{name}"),
+            )
+            .await?;
+            Ok(indexes
+                .into_iter()
+                .filter_map(|entry| {
+                    let key = bucket_key(&entry.partition, entry.bucket);
+                    buckets.contains(&key).then(|| {
+                        let identity = (
+                            key.0.to_vec(),
+                            key.1,
+                            entry.kind == FileKind::Add,
+                            entry.index_file.index_type.clone(),
+                            entry.index_file.file_name.clone(),
+                        );
+                        (identity, entry.index_file)
+                    })
+                })
+                .collect::<HashMap<_, _>>())
+        };
+
+        let mut previous = captured.clone();
+        for id in captured.id() + 1..=latest.id() {
+            let snapshot = if id == latest.id() {
+                latest.clone()
+            } else {
+                self.snapshot_manager.get_snapshot(id).await?
+            };
+            if snapshot.commit_user() != self.commit_user
+                && matches!(
+                    snapshot.commit_kind(),
+                    CommitKind::COMPACT | CommitKind::OVERWRITE
+                )
+            {
+                let changes_data = self
+                    .read_delta_entries(partition_filter.as_ref(), &snapshot)
+                    .await?
+                    .iter()
+                    .any(|entry| 
buckets.contains(&bucket_key(entry.partition(), entry.bucket())));
+                // Index manifests contain the full index state. Compare to the
+                // immediate predecessor to detect both additions and removals.
+                let before = previous.index_manifest();
+                let after = snapshot.index_manifest();
+                if changes_data
+                    || (before != after
+                        && read_indexes(before).await? != 
read_indexes(after).await?)

Review Comment:
   `before` is the previous snapshot's index manifest, which the previous 
iteration may already have read as `after`. Keeping the last decoded map would 
avoid reading it twice when several foreign COMPACT or OVERWRITE snapshots 
follow each other.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to