rdblue commented on a change in pull request #1508:
URL: https://github.com/apache/iceberg/pull/1508#discussion_r711780661
##########
File path:
spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java
##########
@@ -1112,6 +1126,239 @@ public void testPartitionsTable() {
}
}
+ @Test
+ public synchronized void testSnapshotReadAfterAddColumn() {
+ TableIdentifier tableIdentifier = TableIdentifier.of("db", "table");
+ Table table = createTable(tableIdentifier, SCHEMA,
PartitionSpec.unpartitioned());
+
+ List<Row> originalRecords = Lists.newArrayList(
+ RowFactory.create(1, "x"),
+ RowFactory.create(2, "y"),
+ RowFactory.create(3, "z"));
+
+ StructType originalSparkSchema = SparkSchemaUtil.convert(SCHEMA);
+ Dataset<Row> inputDf = spark.createDataFrame(originalRecords,
originalSparkSchema);
+ inputDf.select("id", "data").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords = resultDf.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords);
+ Snapshot snapshot1 = table.currentSnapshot();
+
+ table.updateSchema().addColumn("category",
Types.StringType.get()).commit();
+
+ List<Row> newRecords = Lists.newArrayList(
+ RowFactory.create(4, "xy", "B"),
+ RowFactory.create(5, "xyz", "C"));
+
+ StructType newSparkSchema = SparkSchemaUtil.convert(SCHEMA2);
+ Dataset<Row> inputDf2 = spark.createDataFrame(newRecords, newSparkSchema);
+ inputDf2.select("id", "data", "category").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf2 = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords2 = resultDf2.orderBy("id")
+ .collectAsList();
+
+ List<Row> updatedRecords = Lists.newArrayList(
+ RowFactory.create(1, "x", null),
+ RowFactory.create(2, "y", null),
+ RowFactory.create(3, "z", null),
+ RowFactory.create(4, "xy", "B"),
+ RowFactory.create(5, "xyz", "C"));
+ Assert.assertEquals("Records should match", updatedRecords,
actualRecords2);
+
+ Dataset<Row> resultDf3 = spark.read()
+ .format("iceberg")
+ .option(SparkReadOptions.SNAPSHOT_ID, snapshot1.snapshotId())
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords3 = resultDf3.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords3);
+ Assert.assertEquals("Schemas should match", originalSparkSchema,
resultDf3.schema());
+ }
+
+ @Test
+ public synchronized void testSnapshotReadAfterDropColumn() {
+ TableIdentifier tableIdentifier = TableIdentifier.of("db", "table");
+ Table table = createTable(tableIdentifier, SCHEMA2,
PartitionSpec.unpartitioned());
+
+ List<Row> originalRecords = Lists.newArrayList(
+ RowFactory.create(1, "x", "A"),
+ RowFactory.create(2, "y", "A"),
+ RowFactory.create(3, "z", "B"));
+
+ StructType originalSparkSchema = SparkSchemaUtil.convert(SCHEMA2);
+ Dataset<Row> inputDf = spark.createDataFrame(originalRecords,
originalSparkSchema);
+ inputDf.select("id", "data", "category").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords = resultDf.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords);
+
+ long tsBeforeDropColumn = waitUntilAfter(System.currentTimeMillis());
+ table.updateSchema().deleteColumn("data").commit();
+ long tsAfterDropColumn = waitUntilAfter(System.currentTimeMillis());
+
+ List<Row> newRecords = Lists.newArrayList(
+ RowFactory.create(4, "B"),
+ RowFactory.create(5, "C"));
+
+ StructType newSparkSchema = SparkSchemaUtil.convert(SCHEMA3);
+ Dataset<Row> inputDf2 = spark.createDataFrame(newRecords, newSparkSchema);
+ inputDf2.select("id", "category").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf2 = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords2 = resultDf2.orderBy("id")
+ .collectAsList();
+
+ List<Row> updatedRecords = Lists.newArrayList(
+ RowFactory.create(1, "A"),
+ RowFactory.create(2, "A"),
+ RowFactory.create(3, "B"),
+ RowFactory.create(4, "B"),
+ RowFactory.create(5, "C"));
+ Assert.assertEquals("Records should match", updatedRecords,
actualRecords2);
+
+ Dataset<Row> resultDf3 = spark.read()
+ .format("iceberg")
+ .option(SparkReadOptions.AS_OF_TIMESTAMP, tsBeforeDropColumn)
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords3 = resultDf3.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords3);
+ Assert.assertEquals("Schemas should match", originalSparkSchema,
resultDf3.schema());
+
+ Dataset<Row> resultDf4 = spark.read()
+ .format("iceberg")
+ .option(SparkReadOptions.AS_OF_TIMESTAMP, tsAfterDropColumn)
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords4 = resultDf4.orderBy("id")
+ .collectAsList();
+
+ // At tsAfterDropColumn, there has been a schema change, but no new
snapshot,
+ // so the snapshot as of tsAfterDropColumn is the same as that as of
tsBeforeDropColumn.
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords4);
+ Assert.assertEquals("Schemas should match", originalSparkSchema,
resultDf4.schema());
+ }
+
+ @Test
+ public synchronized void testSnapshotReadAfterAddAndDropColumn() {
+ TableIdentifier tableIdentifier = TableIdentifier.of("db", "table");
+ Table table = createTable(tableIdentifier, SCHEMA,
PartitionSpec.unpartitioned());
+
+ List<Row> originalRecords = Lists.newArrayList(
+ RowFactory.create(1, "x"),
+ RowFactory.create(2, "y"),
+ RowFactory.create(3, "z"));
+
+ StructType originalSparkSchema = SparkSchemaUtil.convert(SCHEMA);
+ Dataset<Row> inputDf = spark.createDataFrame(originalRecords,
originalSparkSchema);
+ inputDf.select("id", "data").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords = resultDf.orderBy("id")
+ .collectAsList();
+
+ Assert.assertEquals("Records should match", originalRecords,
actualRecords);
+ Snapshot snapshot1 = table.currentSnapshot();
+
+ table.updateSchema().addColumn("category",
Types.StringType.get()).commit();
+
+ List<Row> newRecords = Lists.newArrayList(
+ RowFactory.create(4, "xy", "B"),
+ RowFactory.create(5, "xyz", "C"));
+
+ StructType sparkSchemaAfterAddColumn = SparkSchemaUtil.convert(SCHEMA2);
+ Dataset<Row> inputDf2 = spark.createDataFrame(newRecords,
sparkSchemaAfterAddColumn);
+ inputDf2.select("id", "data", "category").write()
+ .format("iceberg")
+ .mode(SaveMode.Append)
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+
+ Dataset<Row> resultDf2 = spark.read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier));
+ List<Row> actualRecords2 = resultDf2.orderBy("id")
+ .collectAsList();
+
+ List<Row> updatedRecords = Lists.newArrayList(
+ RowFactory.create(1, "x", null),
+ RowFactory.create(2, "y", null),
+ RowFactory.create(3, "z", null),
+ RowFactory.create(4, "xy", "B"),
+ RowFactory.create(5, "xyz", "C"));
+ Assert.assertEquals("Records should match", updatedRecords,
actualRecords2);
Review comment:
Rather than using names like `actualRecords2`, you can just move the
`collectAsList` into the assertion.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
