This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new b6fb298 test: add mixed-format drop column schema evolution coverage
(#363)
b6fb298 is described below
commit b6fb2984939f92d83147a3a035321ff677ee1763
Author: Jiwen liu <[email protected]>
AuthorDate: Fri Jun 12 14:40:44 2026 +0800
test: add mixed-format drop column schema evolution coverage (#363)
---
crates/integration_tests/tests/read_tables.rs | 92 +++++++++++++++++++++++++++
dev/spark/provision.py | 44 +++++++++++++
2 files changed, 136 insertions(+)
diff --git a/crates/integration_tests/tests/read_tables.rs
b/crates/integration_tests/tests/read_tables.rs
index 82a69aa..e5c3b05 100644
--- a/crates/integration_tests/tests/read_tables.rs
+++ b/crates/integration_tests/tests/read_tables.rs
@@ -1583,6 +1583,98 @@ async fn test_read_schema_evolution_rename_column() {
);
}
+/// Test reading a mixed-format table after ALTER TABLE DROP COLUMN.
+/// Old Parquet/ORC data files have the dropped column; new Avro files do not.
+#[tokio::test]
+async fn test_read_mixed_format_schema_evolution_drop_column() {
+ let table_name = "mixed_format_schema_evolution_drop_column";
+ let (plan, batches) = scan_and_read_with_fs_catalog(table_name,
None).await;
+ assert_plan_file_formats(&plan, &["avro", "orc", "parquet"], table_name);
+
+ for batch in &batches {
+ assert!(
+ batch.column_by_name("score").is_none(),
+ "Dropped column 'score' should not appear in output"
+ );
+ }
+
+ let mut rows: Vec<(i32, String)> = Vec::new();
+ for batch in &batches {
+ let id = batch
+ .column_by_name("id")
+ .and_then(|c| c.as_any().downcast_ref::<Int32Array>())
+ .expect("id");
+ let name = batch
+ .column_by_name("name")
+ .and_then(|c| c.as_any().downcast_ref::<StringArray>())
+ .expect("name");
+ for i in 0..batch.num_rows() {
+ rows.push((id.value(i), name.value(i).to_string()));
+ }
+ }
+ rows.sort_by_key(|(id, _)| *id);
+
+ assert_eq!(
+ rows,
+ vec![
+ (1, "parquet-alice".into()),
+ (2, "parquet-bob".into()),
+ (3, "orc-carol".into()),
+ (4, "orc-dave".into()),
+ (5, "avro-eve".into()),
+ (6, "avro-frank".into()),
+ ],
+ "Mixed-format DROP COLUMN should expose only remaining columns from
all file formats"
+ );
+
+ let (_, projected_batches) = scan_and_read_with_fs_catalog(
+ "mixed_format_schema_evolution_drop_column",
+ Some(&["name", "id"]),
+ )
+ .await;
+
+ let mut projected_rows: Vec<(i32, String)> = Vec::new();
+ for batch in &projected_batches {
+ let schema = batch.schema();
+ let field_names: Vec<&str> = schema.fields().iter().map(|f|
f.name().as_str()).collect();
+ assert_eq!(
+ field_names,
+ vec!["name", "id"],
+ "Projection should preserve caller-specified order after DROP
COLUMN"
+ );
+ assert!(
+ batch.column_by_name("score").is_none(),
+ "Dropped column 'score' should not appear in projected output"
+ );
+
+ let name = batch
+ .column_by_name("name")
+ .and_then(|c| c.as_any().downcast_ref::<StringArray>())
+ .expect("projected name");
+ let id = batch
+ .column_by_name("id")
+ .and_then(|c| c.as_any().downcast_ref::<Int32Array>())
+ .expect("projected id");
+ for i in 0..batch.num_rows() {
+ projected_rows.push((id.value(i), name.value(i).to_string()));
+ }
+ }
+ projected_rows.sort_by_key(|(id, _)| *id);
+
+ assert_eq!(
+ projected_rows,
+ vec![
+ (1, "parquet-alice".into()),
+ (2, "parquet-bob".into()),
+ (3, "orc-carol".into()),
+ (4, "orc-dave".into()),
+ (5, "avro-eve".into()),
+ (6, "avro-frank".into()),
+ ],
+ "Projection should read remaining columns across old and new file
schemas"
+ );
+}
+
// ---------------------------------------------------------------------------
// Complex type integration tests
// ---------------------------------------------------------------------------
diff --git a/dev/spark/provision.py b/dev/spark/provision.py
index d56156e..44f981e 100644
--- a/dev/spark/provision.py
+++ b/dev/spark/provision.py
@@ -646,6 +646,50 @@ def main():
"""
)
+ # ===== Mixed-format Schema Evolution: Drop Column =====
+ # Old Parquet/ORC files have (id, name, score); after DROP COLUMN, Avro
files
+ # have only (id, name). Reader should ignore the dropped column in old
files.
+ spark.sql(
+ """
+ CREATE TABLE IF NOT EXISTS mixed_format_schema_evolution_drop_column (
+ id INT,
+ name STRING,
+ score INT
+ ) USING paimon
+ TBLPROPERTIES (
+ 'file.format' = 'parquet'
+ )
+ """
+ )
+ spark.sql(
+ """
+ INSERT INTO mixed_format_schema_evolution_drop_column VALUES
+ (1, 'parquet-alice', 100),
+ (2, 'parquet-bob', 200)
+ """
+ )
+ spark.sql(
+ "ALTER TABLE mixed_format_schema_evolution_drop_column SET
TBLPROPERTIES ('file.format' = 'orc')"
+ )
+ spark.sql(
+ """
+ INSERT INTO mixed_format_schema_evolution_drop_column VALUES
+ (3, 'orc-carol', 300),
+ (4, 'orc-dave', 400)
+ """
+ )
+ spark.sql("ALTER TABLE mixed_format_schema_evolution_drop_column DROP
COLUMN score")
+ spark.sql(
+ "ALTER TABLE mixed_format_schema_evolution_drop_column SET
TBLPROPERTIES ('file.format' = 'avro')"
+ )
+ spark.sql(
+ """
+ INSERT INTO mixed_format_schema_evolution_drop_column VALUES
+ (5, 'avro-eve'),
+ (6, 'avro-frank')
+ """
+ )
+
# ===== Complex Types table: ARRAY, MAP, STRUCT =====
spark.sql(
"""
