szehon-ho commented on code in PR #4560:
URL: https://github.com/apache/iceberg/pull/4560#discussion_r854397205
##########
spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java:
##########
@@ -261,6 +262,118 @@ public void testEntriesMetadataTable() throws
ParseException {
}
}
+ @Test
+ public void testPartitionMetadataTable() throws ParseException {
+ sql("CREATE TABLE %s (id bigint NOT NULL, category string, data string)
USING iceberg " +
+ "TBLPROPERTIES ('commit.manifest-merge.enabled' 'false')", tableName);
+ initTable();
+
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ // verify the metadata tables while the current spec is still unpartitioned
+ Dataset<Row> df = loadMetadataTable(PARTITIONS);
+ Assert.assertTrue("Partition must be skipped",
df.schema().getFieldIndex("partition").isEmpty());
+
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ table.updateSpec()
+ .addField("data")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ // verify the metadata tables after adding the first partition column
+ assertPartitions(
+ ImmutableList.of(row(new Object[]{null}), row("d1"), row("d2")),
+ "STRUCT<data:STRING>",
+ PARTITIONS);
+
+ table.updateSpec()
+ .addField("category")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ // verify the metadata tables after adding the second partition column
+ assertPartitions(ImmutableList.of(
+ row(null, null),
+ row("d1", null),
+ row("d1", "c1"),
+ row("d2", null),
+ row("d2", "c2")),
+ "STRUCT<data:STRING,category:STRING>",
+ PARTITIONS);
+
+ // verify the metadata tables after removing the first partition column
+ table.updateSpec()
+ .removeField("data")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ assertPartitions(
+ ImmutableList.of(
+ row(null, null),
+ row(null, "c1"),
+ row(null, "c2"),
+ row("d1", null),
+ row("d1", "c1"),
+ row("d2", null),
+ row("d2", "c2")),
+ "STRUCT<data:STRING,category:STRING>",
+ PARTITIONS);
+
+ // verify the metadata tables after renaming the remaining partition column
+ table.updateSpec()
+ .renameField("category", "category_another_name")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+
+ assertPartitions(
+ ImmutableList.of(
+ row(null, null),
+ row(null, "c1"),
+ row(null, "c2"),
+ row("d1", null),
+ row("d1", "c1"),
+ row("d2", null),
+ row("d2", "c2")),
+ "STRUCT<data:STRING,category_another_name:STRING>",
+ PARTITIONS);
+
+ // verify the metadata tables after re-adding the first dropped column in
the second location
+ table.updateSpec()
+ .addField("data")
+ .commit();
+ sql("REFRESH TABLE %s", tableName);
+
+ sql("INSERT INTO TABLE %s VALUES (1, 'c1', 'd1')", tableName);
+ sql("INSERT INTO TABLE %s VALUES (2, 'c2', 'd2')", tableName);
+
+ // Re-added partition fields currently not re-associated:
https://github.com/apache/iceberg/issues/4292
+ // In V1, dropped partition fields show separately when field is re-added
+ // In V2, re-added field currently conflicts with its deleted form
+ if (formatVersion == 1) {
Review Comment:
Done, split this into 3 tests
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
