kosiew commented on code in PR #26274:
URL: https://github.com/apache/datafusion/pull/26274#discussion_r4627058920


##########
datafusion/physical-plan/src/materialized_cte.rs:
##########
@@ -724,7 +783,11 @@ impl MaterializedCteScanExec {
             id: self.id,
             name: self.name.clone(),
             schema: Arc::clone(&self.schema),
-            statistics: Some(statistics),
+            // The body statistics have a column for each column of the body.
+            statistics: Some(Arc::new(
+                
Arc::unwrap_or_clone(statistics).project(self.projection.as_ref()),

Review Comment:
   Projected scans retain the body's total byte size, which can change join 
selection. Please call `calculate_total_byte_size(&self.schema)` after 
`project` and assert the expected bytes directly in the test.



##########
datafusion/expr/src/logical_plan/materialized_cte.rs:
##########
@@ -218,6 +235,120 @@ impl MaterializedCteScan {
         &self.schema
     }
 
+    /// The indexes of the columns of the CTE body that this scan outputs.
+    /// `None` means all the columns.
+    pub fn projection(&self) -> Option<&[usize]> {
+        self.projection.as_deref()
+    }
+
+    /// Return a scan that outputs only the columns `indices` of this scan.
+    ///
+    /// The indexes refer to the schema of this scan, not to the schema of the
+    /// CTE body.
+    pub fn with_projection(&self, indices: &[usize]) -> Result<Self> {
+        let len = self.schema.fields().len();
+        if let Some(index) = indices.iter().find(|index| **index >= len) {
+            return internal_err!(
+                "Column {index} is not in the scan of the materialized CTE {}, 
which has {len} columns",
+                self.name
+            );
+        }
+        if indices.iter().copied().eq(0..len) {
+            return Ok(self.clone());
+        }
+        let projection = match &self.projection {
+            Some(projection) => indices.iter().map(|index| 
projection[*index]).collect(),
+            None => indices.to_vec(),
+        };
+        Ok(Self {
+            id: self.id,
+            name: self.name.clone(),
+            schema: Arc::new(project_schema(&self.schema, indices)?),
+            projection: Some(projection),
+        })
+    }
+
+    /// Return a scan with the same output, for a CTE body that keeps only its
+    /// columns `kept`, in this order.
+    ///
+    /// Returns an error if this scan reads a column that is not in `kept`.
+    pub fn with_body_columns(&self, kept: &[usize]) -> Result<Self> {
+        let old: Vec<usize> = match &self.projection {
+            Some(projection) => projection.clone(),
+            // The scan reads all the columns, so `kept` must be all of them.
+            None => (0..self.schema.fields().len()).collect(),
+        };
+        let projection = old
+            .iter()
+            .map(|index| {
+                kept.iter().position(|k| k == index).ok_or_else(|| {
+                    internal_datafusion_err!(
+                        "A scan of the materialized CTE {} reads the column 
{index}, which was removed from the CTE",
+                        self.name
+                    )
+                })
+            })
+            .collect::<Result<Vec<_>>>()?;
+        // A scan that reads all the columns in order has no projection.
+        let projection =
+            
(!projection.iter().copied().eq(0..kept.len())).then_some(projection);
+        Ok(Self {
+            id: self.id,
+            name: self.name.clone(),
+            schema: Arc::clone(&self.schema),
+            projection,
+        })
+    }
+
+    /// Return this scan with the fields that it reads from a CTE body with
+    /// the schema `body`. The scan keeps its qualifiers and its projection.
+    pub fn with_body_schema(&self, body: &DFSchema) -> Result<Self> {
+        let body = match &self.projection {
+            Some(projection) => {
+                self.check_body_schema(body)?;

Review Comment:
   `with_body_schema` rejects valid projected-field type changes because 
`check_body_schema` compares old types. Please validate indices and names 
without comparing types, and add tests for type changes and name mismatches.



##########
datafusion/physical-plan/src/materialized_cte.rs:
##########
@@ -746,7 +809,17 @@ impl DisplayAs for MaterializedCteScanExec {
     fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> 
fmt::Result {
         match t {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
-                write!(f, "MaterializedCteScanExec: name={}", self.name)
+                write!(f, "MaterializedCteScanExec: name={}", self.name)?;
+                if self.projection.is_some() {
+                    let names: Vec<&str> = self
+                        .schema
+                        .fields()
+                        .iter()
+                        .map(|field| field.name().as_str())
+                        .collect();
+                    write!(f, ", projection=[{}]", names.join(", "))?;
+                }
+                Ok(())
             }
             DisplayFormatType::TreeRender => write!(f, "name={}", self.name),

Review Comment:
   Could we show the projection in physical `TreeRender`? Currently, projected 
and unprojected scans appear identical, making plans harder to diagnose.



##########
datafusion/physical-plan/src/materialized_cte.rs:
##########
@@ -691,12 +697,64 @@ impl MaterializedCteScanExec {
             name: name.into(),
             schema,
             statistics: None,
+            projection: None,
             buffer: None,
             metrics: ExecutionPlanMetricsSet::new(),
             cache: Arc::new(cache),
         }
     }
 
+    /// Make this scan output only the columns `projection` of the body, in
+    /// this order.
+    ///
+    /// The schema given to [`Self::new`] must then be the schema of these
+    /// columns, not the schema of the body. Returns an error if `projection`
+    /// does not have one index for each column of that schema.
+    pub fn with_body_projection(mut self, projection: Vec<usize>) -> 
Result<Self> {
+        let len = self.schema.fields().len();
+        if projection.len() != len {
+            return internal_err!(
+                "The scan of the materialized CTE {} has {len} columns, but 
its projection has {} indexes",
+                self.name,
+                projection.len()
+            );
+        }
+        self.projection = Some(projection);
+        Ok(self)
+    }
+
+    /// The indexes of the columns of the body that this scan outputs. `None`
+    /// means all the columns.
+    pub fn projection(&self) -> Option<&[usize]> {
+        self.projection.as_deref()
+    }
+
+    /// Check that this scan can read a body with the schema `body`: the
+    /// columns that it reads must exist and have the types of its schema.
+    pub fn check_body_schema(&self, body: &Schema) -> Result<()> {
+        let indices: Vec<usize> = match &self.projection {
+            Some(projection) => projection.clone(),
+            None => (0..body.fields().len()).collect(),
+        };
+        let matches = indices.len() == self.schema.fields().len()
+            && indices
+                .iter()
+                .zip(self.schema.fields())
+                .all(|(index, field)| {
+                    body.fields()
+                        .get(*index)
+                        .is_some_and(|b| b.data_type() == field.data_type())

Review Comment:
   Could we also validate field names to prevent manually constructed scans 
from mislabeling columns? First confirm that valid physical and logical field 
names always match.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to