jackylee-ch commented on code in PR #1042:
URL: https://github.com/apache/paimon-rust/pull/1042#discussion_r4444418450


##########
crates/paimon/src/table/data_evolution_reader.rs:
##########
@@ -267,6 +267,160 @@ impl DataEvolutionReader {
         }
     }
 
+    /// Whether a row-id group of change events merges into rows: its files
+    /// are column slices of one row range, and the source plan reads each
+    /// field from a file that holds it, in the one layout the group stores it
+    /// in. Other groups are read file by file.
+    async fn merges_change_events(&self, files: &[DataFileMeta]) -> 
crate::Result<bool> {
+        if !is_data_evolution_column_slice_group(files) {
+            return Ok(false);
+        }
+        // The source plan reads a BLOB field from dedicated files only, or,
+        // for descriptor and view fields, from normal files only, whichever
+        // layout the field had when the files were written.
+        let dedicated_blobs: HashSet<i32> = self
+            .table_fields
+            .iter()
+            .filter(|field| {
+                field.data_type().is_blob_file_field()
+                    && !self.blob_descriptor_fields.contains(field.name())
+                    && !self.blob_view_fields.contains(field.name())
+            })
+            .map(DataField::id)
+            .collect();
+        let (infos, _) = load_file_infos(
+            &self.schema_manager,
+            self.table_schema_id,
+            &self.table_fields,
+            files,
+        )
+        .await?;
+        let mut layouts = Vec::with_capacity(files.len());
+        // Normal files with the fields their write columns name.
+        let mut normal_files = Vec::new();
+        for (file, info) in files.iter().zip(infos) {
+            if is_vector_store_file_name(&file.file_name) {
+                let columns = info.normalized_write_cols.unwrap_or_default();
+                layouts.push((
+                    FieldLayout::Vector(vector_bunch_key(file, &columns)),
+                    info.field_ids,
+                ));
+            } else if is_blob_or_video_file_name(&file.file_name) {
+                layouts.push((FieldLayout::Blob, info.field_ids));
+            } else {
+                normal_files.push((file, info.field_ids));
+            }
+        }
+        // Only the layouts of the table's BLOB and VECTOR fields can decide:
+        // the merge reads any other field from normal files alone, a field
+        // retyped from VECTOR to text too, and does not read a field dropped
+        // since. So normal layouts list only those fields, while dedicated
+        // layouts keep every field, as the source plan rejects a field of two
+        // vector bunches, read or not. A normal file without write columns
+        // leaves out the fields its own schema stores in dedicated files; that
+        // schema is read only where the answer can decide: for a BLOB or
+        // VECTOR field that a dedicated file of the group also holds or that
+        // the table now stores in BLOB files.
+        let maybe_dedicated: HashSet<i32> = self
+            .table_fields
+            .iter()
+            .filter(|field| {
+                field.data_type().is_blob_file_field()
+                    || matches!(field.data_type(), DataType::Vector(_))
+            })
+            .map(DataField::id)
+            .collect();
+        let in_dedicated_files: HashSet<i32> = layouts
+            .iter()
+            .flat_map(|(_, field_ids)| field_ids.iter().copied())
+            .collect();
+        for (file, written) in &normal_files {
+            let decides = file.write_cols.is_none()
+                && written.iter().any(|id| {
+                    maybe_dedicated.contains(id)
+                        && (in_dedicated_files.contains(id) || 
dedicated_blobs.contains(id))
+                });
+            let held = if decides {
+                self.fields_without_write_cols(file).await?

Review Comment:
   Fixed in 5147e44: a file without write columns now lacks only the columns 
its own commit wrote to dedicated files; when that can't decide the merge, the 
group is read file by file. PTAL.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to