This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 22401546e9 Spark: Migrate tests to JUnit5 (#9790)
22401546e9 is described below
commit 22401546e95b4da1ee11457d42d80050572c6c37
Author: Tom Tanaka <[email protected]>
AuthorDate: Tue Feb 27 23:19:02 2024 +0900
Spark: Migrate tests to JUnit5 (#9790)
---
spark/v3.5/build.gradle | 2 +
.../spark/extensions/TestAlterTableSchema.java | 6 +-
.../TestMetaColumnProjectionWithStageScan.java | 28 ++-
.../spark/extensions/TestMetadataTables.java | 214 +++++++++++----------
...toragePartitionedJoinsInRowLevelOperations.java | 57 +++---
.../extensions/TestSystemFunctionPushDownDQL.java | 83 ++++----
.../apache/iceberg/spark/extensions/TestViews.java | 171 ++++++++--------
.../java/org/apache/iceberg/spark/SmokeTest.java | 95 ++++-----
8 files changed, 337 insertions(+), 319 deletions(-)
diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle
index d4cb67a2bb..b29ba6761e 100644
--- a/spark/v3.5/build.gradle
+++ b/spark/v3.5/build.gradle
@@ -239,6 +239,7 @@
project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
integrationImplementation
"org.scala-lang.modules:scala-collection-compat_${scalaVersion}:${libs.versions.scala.collection.compat.get()}"
integrationImplementation
"org.apache.spark:spark-hive_${scalaVersion}:${libs.versions.spark.hive35.get()}"
integrationImplementation libs.junit.vintage.engine
+ integrationImplementation libs.junit.jupiter
integrationImplementation libs.slf4j.simple
integrationImplementation libs.assertj.core
integrationImplementation project(path: ':iceberg-api', configuration:
'testArtifacts')
@@ -288,6 +289,7 @@
project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
}
task integrationTest(type: Test) {
+ useJUnitPlatform()
description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}"
group = "verification"
jvmArgs += project.property('extraJvmArgs')
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
index 9112677652..3b2427eea7 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java
@@ -24,7 +24,6 @@ import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Table;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.assertj.core.api.Assertions;
-import org.junit.Assert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -74,8 +73,9 @@ public class TestAlterTableSchema extends ExtensionsTestBase {
public void testSetInvalidIdentifierFields() {
sql("CREATE TABLE %s (id bigint NOT NULL, id2 bigint) USING iceberg",
tableName);
Table table = validationCatalog.loadTable(tableIdent);
- Assert.assertTrue(
- "Table should start without identifier",
table.schema().identifierFieldIds().isEmpty());
+ assertThat(table.schema().identifierFieldIds())
+ .as("Table should start without identifier")
+ .isEmpty();
Assertions.assertThatThrownBy(
() -> sql("ALTER TABLE %s SET IDENTIFIER FIELDS unknown",
tableName))
.isInstanceOf(IllegalArgumentException.class)
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java
index e9013848cf..b783a006ef 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetaColumnProjectionWithStageScan.java
@@ -18,9 +18,12 @@
*/
package org.apache.iceberg.spark.extensions;
+import static org.assertj.core.api.Assertions.assertThat;
+
import java.util.List;
-import java.util.Map;
import java.util.UUID;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
import org.apache.iceberg.ScanTask;
import org.apache.iceberg.Table;
import org.apache.iceberg.io.CloseableIterable;
@@ -34,19 +37,14 @@ import org.apache.iceberg.spark.source.SimpleRecord;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row;
-import org.assertj.core.api.Assertions;
-import org.junit.After;
-import org.junit.Test;
-import org.junit.runners.Parameterized;
-
-public class TestMetaColumnProjectionWithStageScan extends
SparkExtensionsTestBase {
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
- public TestMetaColumnProjectionWithStageScan(
- String catalogName, String implementation, Map<String, String> config) {
- super(catalogName, implementation, config);
- }
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestMetaColumnProjectionWithStageScan extends ExtensionsTestBase {
- @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1},
config = {2}")
+ @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}")
public static Object[][] parameters() {
return new Object[][] {
{
@@ -57,7 +55,7 @@ public class TestMetaColumnProjectionWithStageScan extends
SparkExtensionsTestBa
};
}
- @After
+ @AfterEach
public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}
@@ -68,7 +66,7 @@ public class TestMetaColumnProjectionWithStageScan extends
SparkExtensionsTestBa
taskSetManager.stageTasks(tab, fileSetID, Lists.newArrayList(tasks));
}
- @Test
+ @TestTemplate
public void testReadStageTableMeta() throws Exception {
sql(
"CREATE TABLE %s (id bigint, data string) USING iceberg TBLPROPERTIES"
@@ -103,7 +101,7 @@ public class TestMetaColumnProjectionWithStageScan extends
SparkExtensionsTestBa
.option(SparkReadOptions.SCAN_TASK_SET_ID, fileSetID)
.load(tableLocation);
- Assertions.assertThat(scanDF2.columns().length).isEqualTo(2);
+ assertThat(scanDF2.columns()).hasSize(2);
}
try (CloseableIterable<ScanTask> tasks = table.newBatchScan().planFiles())
{
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
index 50376589b6..a22cf61ec8 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
@@ -24,7 +24,6 @@ import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.avro.generic.GenericData.Record;
@@ -33,6 +32,7 @@ import org.apache.iceberg.FileContent;
import org.apache.iceberg.HasTableOperations;
import org.apache.iceberg.HistoryEntry;
import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
@@ -53,22 +53,19 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.catalyst.util.DateTimeUtils;
import org.apache.spark.sql.types.StructType;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
-public class TestMetadataTables extends SparkExtensionsTestBase {
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestMetadataTables extends ExtensionsTestBase {
- public TestMetadataTables(String catalogName, String implementation,
Map<String, String> config) {
- super(catalogName, implementation, config);
- }
-
- @After
+ @AfterEach
public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}
- @Test
+ @TestTemplate
public void testUnpartitionedTable() throws Exception {
sql(
"CREATE TABLE %s (id bigint, data string) USING iceberg TBLPROPERTIES"
@@ -92,8 +89,8 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
Table table = Spark3Util.loadIcebergTable(spark, tableName);
List<ManifestFile> expectedDataManifests =
TestHelpers.dataManifests(table);
List<ManifestFile> expectedDeleteManifests =
TestHelpers.deleteManifests(table);
- Assert.assertEquals("Should have 1 data manifest", 1,
expectedDataManifests.size());
- Assert.assertEquals("Should have 1 delete manifest", 1,
expectedDeleteManifests.size());
+ assertThat(expectedDataManifests).as("Should have 1 data
manifest").hasSize(1);
+ assertThat(expectedDeleteManifests).as("Should have 1 delete
manifest").hasSize(1);
Schema entriesTableSchema = Spark3Util.loadIcebergTable(spark, tableName +
".entries").schema();
Schema filesTableSchema = Spark3Util.loadIcebergTable(spark, tableName +
".files").schema();
@@ -101,13 +98,12 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
// check delete files table
Dataset<Row> actualDeleteFilesDs = spark.sql("SELECT * FROM " + tableName
+ ".delete_files");
List<Row> actualDeleteFiles =
TestHelpers.selectNonDerived(actualDeleteFilesDs).collectAsList();
- Assert.assertEquals(
- "Metadata table should return one delete file", 1,
actualDeleteFiles.size());
+ assertThat(actualDeleteFiles).as("Metadata table should return one delete
file").hasSize(1);
List<Record> expectedDeleteFiles =
expectedEntries(
table, FileContent.POSITION_DELETES, entriesTableSchema,
expectedDeleteManifests, null);
- Assert.assertEquals("Should be one delete file manifest entry", 1,
expectedDeleteFiles.size());
+ assertThat(expectedDeleteFiles).as("Should be one delete file manifest
entry").hasSize(1);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDeleteFilesDs),
expectedDeleteFiles.get(0),
@@ -116,11 +112,11 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
// check data files table
Dataset<Row> actualDataFilesDs = spark.sql("SELECT * FROM " + tableName +
".data_files");
List<Row> actualDataFiles =
TestHelpers.selectNonDerived(actualDataFilesDs).collectAsList();
- Assert.assertEquals("Metadata table should return one data file", 1,
actualDataFiles.size());
+ assertThat(actualDataFiles).as("Metadata table should return one data
file").hasSize(1);
List<Record> expectedDataFiles =
expectedEntries(table, FileContent.DATA, entriesTableSchema,
expectedDataManifests, null);
- Assert.assertEquals("Should be one data file manifest entry", 1,
expectedDataFiles.size());
+ assertThat(expectedDataFiles).as("Should be one data file manifest
entry").hasSize(1);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDataFilesDs),
expectedDataFiles.get(0),
@@ -131,19 +127,19 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
spark.sql("SELECT * FROM " + tableName + ".files ORDER BY content");
List<Row> actualFiles =
TestHelpers.selectNonDerived(actualFilesDs).collectAsList();
- Assert.assertEquals("Metadata table should return two files", 2,
actualFiles.size());
+ assertThat(actualFiles).as("Metadata table should return two
files").hasSize(2);
List<Record> expectedFiles =
Stream.concat(expectedDataFiles.stream(), expectedDeleteFiles.stream())
.collect(Collectors.toList());
- Assert.assertEquals("Should have two files manifest entries", 2,
expectedFiles.size());
+ assertThat(expectedFiles).as("Should have two files manifest
entries").hasSize(2);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(0),
actualFiles.get(0));
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(1),
actualFiles.get(1));
}
- @Test
+ @TestTemplate
public void testPartitionedTable() throws Exception {
sql(
"CREATE TABLE %s (id bigint, data string) "
@@ -177,8 +173,8 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<ManifestFile> expectedDataManifests =
TestHelpers.dataManifests(table);
List<ManifestFile> expectedDeleteManifests =
TestHelpers.deleteManifests(table);
- Assert.assertEquals("Should have 2 data manifests", 2,
expectedDataManifests.size());
- Assert.assertEquals("Should have 2 delete manifests", 2,
expectedDeleteManifests.size());
+ assertThat(expectedDataManifests).as("Should have 2 data
manifest").hasSize(2);
+ assertThat(expectedDeleteManifests).as("Should have 2 delete
manifest").hasSize(2);
Schema filesTableSchema =
Spark3Util.loadIcebergTable(spark, tableName +
".delete_files").schema();
@@ -187,15 +183,13 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Record> expectedDeleteFiles =
expectedEntries(
table, FileContent.POSITION_DELETES, entriesTableSchema,
expectedDeleteManifests, "a");
- Assert.assertEquals(
- "Should have one delete file manifest entry", 1,
expectedDeleteFiles.size());
+ assertThat(expectedDeleteFiles).as("Should have one delete file manifest
entry").hasSize(1);
Dataset<Row> actualDeleteFilesDs =
spark.sql("SELECT * FROM " + tableName + ".delete_files " + "WHERE
partition.data='a'");
List<Row> actualDeleteFiles = actualDeleteFilesDs.collectAsList();
- Assert.assertEquals(
- "Metadata table should return one delete file", 1,
actualDeleteFiles.size());
+ assertThat(actualDeleteFiles).as("Metadata table should return one delete
file").hasSize(1);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDeleteFilesDs),
expectedDeleteFiles.get(0),
@@ -204,13 +198,13 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
// Check data files table
List<Record> expectedDataFiles =
expectedEntries(table, FileContent.DATA, entriesTableSchema,
expectedDataManifests, "a");
- Assert.assertEquals("Should have one data file manifest entry", 1,
expectedDataFiles.size());
+ assertThat(expectedDataFiles).as("Should have one data file manifest
entry").hasSize(1);
Dataset<Row> actualDataFilesDs =
spark.sql("SELECT * FROM " + tableName + ".data_files " + "WHERE
partition.data='a'");
List<Row> actualDataFiles =
TestHelpers.selectNonDerived(actualDataFilesDs).collectAsList();
- Assert.assertEquals("Metadata table should return one data file", 1,
actualDataFiles.size());
+ assertThat(actualDataFiles).as("Metadata table should return one data
file").hasSize(1);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDataFilesDs),
expectedDataFiles.get(0),
@@ -218,32 +212,29 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Row> actualPartitionsWithProjection =
spark.sql("SELECT file_count FROM " + tableName + ".partitions
").collectAsList();
- Assert.assertEquals(
- "Metadata table should return two partitions record",
- 2,
- actualPartitionsWithProjection.size());
- for (int i = 0; i < 2; ++i) {
- Assert.assertEquals(1, actualPartitionsWithProjection.get(i).get(0));
- }
+ assertThat(actualPartitionsWithProjection)
+ .as("Metadata table should return two partitions record")
+ .hasSize(2)
+ .containsExactly(RowFactory.create(1), RowFactory.create(1));
// Check files table
List<Record> expectedFiles =
Stream.concat(expectedDataFiles.stream(), expectedDeleteFiles.stream())
.collect(Collectors.toList());
- Assert.assertEquals("Should have two file manifest entries", 2,
expectedFiles.size());
+ assertThat(expectedFiles).as("Should have two file manifest
entries").hasSize(2);
Dataset<Row> actualFilesDs =
spark.sql(
"SELECT * FROM " + tableName + ".files " + "WHERE
partition.data='a' ORDER BY content");
List<Row> actualFiles =
TestHelpers.selectNonDerived(actualFilesDs).collectAsList();
- Assert.assertEquals("Metadata table should return two files", 2,
actualFiles.size());
+ assertThat(actualFiles).as("Metadata table should return two
files").hasSize(2);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(0),
actualFiles.get(0));
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(1),
actualFiles.get(1));
}
- @Test
+ @TestTemplate
public void testAllFilesUnpartitioned() throws Exception {
sql(
"CREATE TABLE %s (id bigint, data string) USING iceberg TBLPROPERTIES"
@@ -267,13 +258,13 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
Table table = Spark3Util.loadIcebergTable(spark, tableName);
List<ManifestFile> expectedDataManifests =
TestHelpers.dataManifests(table);
- Assert.assertEquals("Should have 1 data manifest", 1,
expectedDataManifests.size());
+ assertThat(expectedDataManifests).as("Should have 1 data
manifest").hasSize(1);
List<ManifestFile> expectedDeleteManifests =
TestHelpers.deleteManifests(table);
- Assert.assertEquals("Should have 1 delete manifest", 1,
expectedDeleteManifests.size());
+ assertThat(expectedDeleteManifests).as("Should have 1 delete
manifest").hasSize(1);
// Clear table to test whether 'all_files' can read past files
List<Object[]> results = sql("DELETE FROM %s", tableName);
- Assert.assertEquals("Table should be cleared", 0, results.size());
+ assertThat(results).as("Table should be cleared").isEmpty();
Schema entriesTableSchema = Spark3Util.loadIcebergTable(spark, tableName +
".entries").schema();
Schema filesTableSchema =
@@ -285,8 +276,8 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Record> expectedDataFiles =
expectedEntries(table, FileContent.DATA, entriesTableSchema,
expectedDataManifests, null);
- Assert.assertEquals("Should be one data file manifest entry", 1,
expectedDataFiles.size());
- Assert.assertEquals("Metadata table should return one data file", 1,
actualDataFiles.size());
+ assertThat(expectedDataFiles).as("Should be one data file manifest
entry").hasSize(1);
+ assertThat(actualDataFiles).as("Metadata table should return one data
file").hasSize(1);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDataFilesDs),
expectedDataFiles.get(0),
@@ -299,9 +290,8 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Record> expectedDeleteFiles =
expectedEntries(
table, FileContent.POSITION_DELETES, entriesTableSchema,
expectedDeleteManifests, null);
- Assert.assertEquals("Should be one delete file manifest entry", 1,
expectedDeleteFiles.size());
- Assert.assertEquals(
- "Metadata table should return one delete file", 1,
actualDeleteFiles.size());
+ assertThat(expectedDeleteFiles).as("Should be one delete file manifest
entry").hasSize(1);
+ assertThat(actualDeleteFiles).as("Metadata table should return one delete
file").hasSize(1);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDeleteFilesDs),
expectedDeleteFiles.get(0),
@@ -313,12 +303,12 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Row> actualFiles = actualFilesDs.collectAsList();
List<Record> expectedFiles = ListUtils.union(expectedDataFiles,
expectedDeleteFiles);
expectedFiles.sort(Comparator.comparing(r -> ((Integer)
r.get("content"))));
- Assert.assertEquals("Metadata table should return two files", 2,
actualFiles.size());
+ assertThat(actualFiles).as("Metadata table should return two
files").hasSize(2);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles,
actualFiles);
}
- @Test
+ @TestTemplate
public void testAllFilesPartitioned() throws Exception {
// Create table and insert data
sql(
@@ -350,13 +340,13 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
Table table = Spark3Util.loadIcebergTable(spark, tableName);
List<ManifestFile> expectedDataManifests =
TestHelpers.dataManifests(table);
- Assert.assertEquals("Should have 2 data manifests", 2,
expectedDataManifests.size());
+ assertThat(expectedDataManifests).as("Should have 2 data
manifests").hasSize(2);
List<ManifestFile> expectedDeleteManifests =
TestHelpers.deleteManifests(table);
- Assert.assertEquals("Should have 1 delete manifest", 1,
expectedDeleteManifests.size());
+ assertThat(expectedDeleteManifests).as("Should have 1 delete
manifest").hasSize(1);
// Clear table to test whether 'all_files' can read past files
List<Object[]> results = sql("DELETE FROM %s", tableName);
- Assert.assertEquals("Table should be cleared", 0, results.size());
+ assertThat(results).as("Table should be cleared").isEmpty();
Schema entriesTableSchema = Spark3Util.loadIcebergTable(spark, tableName +
".entries").schema();
Schema filesTableSchema =
@@ -368,8 +358,8 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Row> actualDataFiles =
TestHelpers.selectNonDerived(actualDataFilesDs).collectAsList();
List<Record> expectedDataFiles =
expectedEntries(table, FileContent.DATA, entriesTableSchema,
expectedDataManifests, "a");
- Assert.assertEquals("Should be one data file manifest entry", 1,
expectedDataFiles.size());
- Assert.assertEquals("Metadata table should return one data file", 1,
actualDataFiles.size());
+ assertThat(expectedDataFiles).as("Should be one data file manifest
entry").hasSize(1);
+ assertThat(actualDataFiles).as("Metadata table should return one data
file").hasSize(1);
TestHelpers.assertEqualsSafe(
SparkSchemaUtil.convert(TestHelpers.selectNonDerived(actualDataFilesDs).schema())
.asStruct(),
@@ -384,8 +374,8 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Record> expectedDeleteFiles =
expectedEntries(
table, FileContent.POSITION_DELETES, entriesTableSchema,
expectedDeleteManifests, "a");
- Assert.assertEquals("Should be one data file manifest entry", 1,
expectedDeleteFiles.size());
- Assert.assertEquals("Metadata table should return one data file", 1,
actualDeleteFiles.size());
+ assertThat(expectedDeleteFiles).as("Should be one data file manifest
entry").hasSize(1);
+ assertThat(actualDeleteFiles).as("Metadata table should return one data
file").hasSize(1);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDeleteFilesDs),
@@ -403,12 +393,12 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Record> expectedFiles = ListUtils.union(expectedDataFiles,
expectedDeleteFiles);
expectedFiles.sort(Comparator.comparing(r -> ((Integer)
r.get("content"))));
- Assert.assertEquals("Metadata table should return two files", 2,
actualFiles.size());
+ assertThat(actualFiles).as("Metadata table should return two
files").hasSize(2);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualDataFilesDs), expectedFiles,
actualFiles);
}
- @Test
+ @TestTemplate
public void testMetadataLogEntries() throws Exception {
// Create table and insert data
sql(
@@ -465,8 +455,9 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
sql(
"SELECT * FROM %s.metadata_log_entries WHERE latest_snapshot_id =
%s",
tableName, currentSnapshotId);
- Assert.assertEquals(
- "metadataLogEntries table should return 1 row", 1,
metadataLogWithFilters.size());
+ assertThat(metadataLogWithFilters)
+ .as("metadataLogEntries table should return 1 row")
+ .hasSize(1);
assertEquals(
"Result should match the latest snapshot entry",
ImmutableList.of(
@@ -487,15 +478,16 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
metadataFiles.add(tableMetadata.metadataFileLocation());
List<Object[]> metadataLogWithProjection =
sql("SELECT file FROM %s.metadata_log_entries", tableName);
- Assert.assertEquals(
- "metadataLogEntries table should return 3 rows", 3,
metadataLogWithProjection.size());
+ assertThat(metadataLogWithProjection)
+ .as("metadataLogEntries table should return 3 rows")
+ .hasSize(3);
assertEquals(
"metadataLog entry should be of same file",
metadataFiles.stream().map(this::row).collect(Collectors.toList()),
metadataLogWithProjection);
}
- @Test
+ @TestTemplate
public void testFilesTableTimeTravelWithSchemaEvolution() throws Exception {
// Create table and insert data
sql(
@@ -545,7 +537,7 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
List<Record> expectedFiles =
expectedEntries(table, FileContent.DATA, entriesTableSchema,
expectedDataManifests, null);
- Assert.assertEquals("actualFiles size should be 2", 2, actualFiles.size());
+ assertThat(actualFiles).as("actualFiles size should be 2").hasSize(2);
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(0),
actualFiles.get(0));
@@ -553,13 +545,12 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
TestHelpers.assertEqualsSafe(
TestHelpers.nonDerivedSchema(actualFilesDs), expectedFiles.get(1),
actualFiles.get(1));
- Assert.assertEquals(
- "expectedFiles and actualFiles size should be the same",
- actualFiles.size(),
- expectedFiles.size());
+ assertThat(actualFiles)
+ .as("expectedFiles and actualFiles size should be the same")
+ .hasSameSizeAs(expectedFiles);
}
- @Test
+ @TestTemplate
public void testSnapshotReferencesMetatable() throws Exception {
// Create table and insert data
sql(
@@ -605,43 +596,64 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
.commit();
// Check refs table
List<Row> references = spark.sql("SELECT * FROM " + tableName +
".refs").collectAsList();
- Assert.assertEquals("Refs table should return 3 rows", 3,
references.size());
+ assertThat(references).as("Refs table should return 3 rows").hasSize(3);
List<Row> branches =
spark.sql("SELECT * FROM " + tableName + ".refs WHERE
type='BRANCH'").collectAsList();
- Assert.assertEquals("Refs table should return 2 branches", 2,
branches.size());
+ assertThat(branches).as("Refs table should return 2 branches").hasSize(2);
List<Row> tags =
spark.sql("SELECT * FROM " + tableName + ".refs WHERE
type='TAG'").collectAsList();
- Assert.assertEquals("Refs table should return 1 tag", 1, tags.size());
+ assertThat(tags).as("Refs table should return 1 tag").hasSize(1);
// Check branch entries in refs table
List<Row> mainBranch =
spark
.sql("SELECT * FROM " + tableName + ".refs WHERE name = 'main' AND
type='BRANCH'")
.collectAsList();
- Assert.assertEquals("main", mainBranch.get(0).getAs("name"));
- Assert.assertEquals("BRANCH", mainBranch.get(0).getAs("type"));
- Assert.assertEquals(currentSnapshotId,
mainBranch.get(0).getAs("snapshot_id"));
+ assertThat(mainBranch)
+ .hasSize(1)
+ .containsExactly(RowFactory.create("main", "BRANCH",
currentSnapshotId, null, null, null));
+ assertThat(mainBranch.get(0).schema().fieldNames())
+ .containsExactly(
+ "name",
+ "type",
+ "snapshot_id",
+ "max_reference_age_in_ms",
+ "min_snapshots_to_keep",
+ "max_snapshot_age_in_ms");
List<Row> testBranch =
spark
.sql("SELECT * FROM " + tableName + ".refs WHERE name =
'testBranch' AND type='BRANCH'")
.collectAsList();
- Assert.assertEquals("testBranch", testBranch.get(0).getAs("name"));
- Assert.assertEquals("BRANCH", testBranch.get(0).getAs("type"));
- Assert.assertEquals(currentSnapshotId,
testBranch.get(0).getAs("snapshot_id"));
- Assert.assertEquals(Long.valueOf(10),
testBranch.get(0).getAs("max_reference_age_in_ms"));
- Assert.assertEquals(Integer.valueOf(20),
testBranch.get(0).getAs("min_snapshots_to_keep"));
- Assert.assertEquals(Long.valueOf(30),
testBranch.get(0).getAs("max_snapshot_age_in_ms"));
+ assertThat(testBranch)
+ .hasSize(1)
+ .containsExactly(
+ RowFactory.create("testBranch", "BRANCH", currentSnapshotId, 10L,
20L, 30L));
+ assertThat(testBranch.get(0).schema().fieldNames())
+ .containsExactly(
+ "name",
+ "type",
+ "snapshot_id",
+ "max_reference_age_in_ms",
+ "min_snapshots_to_keep",
+ "max_snapshot_age_in_ms");
// Check tag entries in refs table
List<Row> testTag =
spark
.sql("SELECT * FROM " + tableName + ".refs WHERE name = 'testTag'
AND type='TAG'")
.collectAsList();
- Assert.assertEquals("testTag", testTag.get(0).getAs("name"));
- Assert.assertEquals("TAG", testTag.get(0).getAs("type"));
- Assert.assertEquals(currentSnapshotId,
testTag.get(0).getAs("snapshot_id"));
- Assert.assertEquals(Long.valueOf(50),
testTag.get(0).getAs("max_reference_age_in_ms"));
+ assertThat(testTag)
+ .hasSize(1)
+ .containsExactly(RowFactory.create("testTag", "TAG",
currentSnapshotId, 50L, null, null));
+ assertThat(testTag.get(0).schema().fieldNames())
+ .containsExactly(
+ "name",
+ "type",
+ "snapshot_id",
+ "max_reference_age_in_ms",
+ "min_snapshots_to_keep",
+ "max_snapshot_age_in_ms");
// Check projection in refs table
List<Row> testTagProjection =
@@ -651,12 +663,12 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
+ tableName
+ ".refs where type='TAG'")
.collectAsList();
- Assert.assertEquals("testTag", testTagProjection.get(0).getAs("name"));
- Assert.assertEquals("TAG", testTagProjection.get(0).getAs("type"));
- Assert.assertEquals(currentSnapshotId,
testTagProjection.get(0).getAs("snapshot_id"));
- Assert.assertEquals(
- Long.valueOf(50),
testTagProjection.get(0).getAs("max_reference_age_in_ms"));
- Assert.assertNull(testTagProjection.get(0).getAs("min_snapshots_to_keep"));
+ assertThat(testTagProjection)
+ .hasSize(1)
+ .containsExactly(RowFactory.create("testTag", "TAG",
currentSnapshotId, 50L, null));
+ assertThat(testTagProjection.get(0).schema().fieldNames())
+ .containsExactly(
+ "name", "type", "snapshot_id", "max_reference_age_in_ms",
"min_snapshots_to_keep");
List<Row> mainBranchProjection =
spark
@@ -665,21 +677,23 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
+ tableName
+ ".refs WHERE name = 'main' AND type = 'BRANCH'")
.collectAsList();
- Assert.assertEquals("main", mainBranchProjection.get(0).getAs("name"));
- Assert.assertEquals("BRANCH", mainBranchProjection.get(0).getAs("type"));
+ assertThat(mainBranchProjection)
+ .hasSize(1)
+ .containsExactly(RowFactory.create("main", "BRANCH"));
+
assertThat(mainBranchProjection.get(0).schema().fieldNames()).containsExactly("name",
"type");
List<Row> testBranchProjection =
spark
.sql(
- "SELECT type, name, max_reference_age_in_ms, snapshot_id FROM "
+ "SELECT name, type, snapshot_id, max_reference_age_in_ms FROM "
+ tableName
+ ".refs WHERE name = 'testBranch' AND type = 'BRANCH'")
.collectAsList();
- Assert.assertEquals("testBranch",
testBranchProjection.get(0).getAs("name"));
- Assert.assertEquals("BRANCH", testBranchProjection.get(0).getAs("type"));
- Assert.assertEquals(currentSnapshotId,
testBranchProjection.get(0).getAs("snapshot_id"));
- Assert.assertEquals(
- Long.valueOf(10),
testBranchProjection.get(0).getAs("max_reference_age_in_ms"));
+ assertThat(testBranchProjection)
+ .hasSize(1)
+ .containsExactly(RowFactory.create("testBranch", "BRANCH",
currentSnapshotId, 10L));
+ assertThat(testBranchProjection.get(0).schema().fieldNames())
+ .containsExactly("name", "type", "snapshot_id",
"max_reference_age_in_ms");
}
/**
@@ -724,7 +738,7 @@ public class TestMetadataTables extends
SparkExtensionsTestBase {
return partValue.equals(partition.get(0).toString());
}
- @Test
+ @TestTemplate
public void metadataLogEntriesAfterReplacingTable() throws Exception {
sql(
"CREATE TABLE %s (id bigint, data string) "
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java
index 00130b4d7e..ce609450c0 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestStoragePartitionedJoinsInRowLevelOperations.java
@@ -20,9 +20,12 @@ package org.apache.iceberg.spark.extensions;
import static org.apache.iceberg.RowLevelOperationMode.COPY_ON_WRITE;
import static org.apache.iceberg.RowLevelOperationMode.MERGE_ON_READ;
+import static org.assertj.core.api.Assertions.assertThat;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
import org.apache.iceberg.RowLevelOperationMode;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
@@ -31,13 +34,12 @@ import org.apache.iceberg.spark.SparkCatalogConfig;
import org.apache.iceberg.spark.SparkSQLProperties;
import org.apache.spark.sql.execution.SparkPlan;
import org.apache.spark.sql.internal.SQLConf;
-import org.assertj.core.api.Assertions;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runners.Parameterized;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
-public class TestStoragePartitionedJoinsInRowLevelOperations extends
SparkExtensionsTestBase {
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestStoragePartitionedJoinsInRowLevelOperations extends
ExtensionsTestBase {
private static final String OTHER_TABLE_NAME = "other_table";
@@ -68,7 +70,7 @@ public class TestStoragePartitionedJoinsInRowLevelOperations
extends SparkExtens
SparkSQLProperties.PRESERVE_DATA_GROUPING,
"true");
- @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1},
config = {2}")
+ @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}")
public static Object[][] parameters() {
return new Object[][] {
{
@@ -79,23 +81,18 @@ public class
TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtens
};
}
- public TestStoragePartitionedJoinsInRowLevelOperations(
- String catalogName, String implementation, Map<String, String> config) {
- super(catalogName, implementation, config);
- }
-
- @After
+ @AfterEach
public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
sql("DROP TABLE IF EXISTS %s", tableName(OTHER_TABLE_NAME));
}
- @Test
+ @TestTemplate
public void testCopyOnWriteDeleteWithoutShuffles() {
checkDelete(COPY_ON_WRITE);
}
- @Test
+ @TestTemplate
public void testMergeOnReadDeleteWithoutShuffles() {
checkDelete(MERGE_ON_READ);
}
@@ -139,10 +136,10 @@ public class
TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtens
String planAsString = plan.toString();
if (mode == COPY_ON_WRITE) {
int actualNumShuffles = StringUtils.countMatches(planAsString,
"Exchange");
- Assert.assertEquals("Should be 1 shuffle with SPJ", 1,
actualNumShuffles);
- Assertions.assertThat(planAsString).contains("Exchange
hashpartitioning(_file");
+ assertThat(actualNumShuffles).as("Should be 1 shuffle with
SPJ").isEqualTo(1);
+ assertThat(planAsString).contains("Exchange
hashpartitioning(_file");
} else {
- Assertions.assertThat(planAsString).doesNotContain("Exchange");
+ assertThat(planAsString).doesNotContain("Exchange");
}
});
@@ -158,12 +155,12 @@ public class
TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtens
sql("SELECT * FROM %s ORDER BY id, salary", tableName));
}
- @Test
+ @TestTemplate
public void testCopyOnWriteUpdateWithoutShuffles() {
checkUpdate(COPY_ON_WRITE);
}
- @Test
+ @TestTemplate
public void testMergeOnReadUpdateWithoutShuffles() {
checkUpdate(MERGE_ON_READ);
}
@@ -207,10 +204,10 @@ public class
TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtens
String planAsString = plan.toString();
if (mode == COPY_ON_WRITE) {
int actualNumShuffles = StringUtils.countMatches(planAsString,
"Exchange");
- Assert.assertEquals("Should be 1 shuffle with SPJ", 1,
actualNumShuffles);
- Assertions.assertThat(planAsString).contains("Exchange
hashpartitioning(_file");
+ assertThat(actualNumShuffles).as("Should be 1 shuffle with
SPJ").isEqualTo(1);
+ assertThat(planAsString).contains("Exchange
hashpartitioning(_file");
} else {
- Assertions.assertThat(planAsString).doesNotContain("Exchange");
+ assertThat(planAsString).doesNotContain("Exchange");
}
});
@@ -227,22 +224,22 @@ public class
TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtens
sql("SELECT * FROM %s ORDER BY id, salary", tableName));
}
- @Test
+ @TestTemplate
public void testCopyOnWriteMergeWithoutShuffles() {
checkMerge(COPY_ON_WRITE, false /* with ON predicate */);
}
- @Test
+ @TestTemplate
public void testCopyOnWriteMergeWithoutShufflesWithPredicate() {
checkMerge(COPY_ON_WRITE, true /* with ON predicate */);
}
- @Test
+ @TestTemplate
public void testMergeOnReadMergeWithoutShuffles() {
checkMerge(MERGE_ON_READ, false /* with ON predicate */);
}
- @Test
+ @TestTemplate
public void testMergeOnReadMergeWithoutShufflesWithPredicate() {
checkMerge(MERGE_ON_READ, true /* with ON predicate */);
}
@@ -294,10 +291,10 @@ public class
TestStoragePartitionedJoinsInRowLevelOperations extends SparkExtens
String planAsString = plan.toString();
if (mode == COPY_ON_WRITE) {
int actualNumShuffles = StringUtils.countMatches(planAsString,
"Exchange");
- Assert.assertEquals("Should be 1 shuffle with SPJ", 1,
actualNumShuffles);
- Assertions.assertThat(planAsString).contains("Exchange
hashpartitioning(_file");
+ assertThat(actualNumShuffles).as("Should be 1 shuffle with
SPJ").isEqualTo(1);
+ assertThat(planAsString).contains("Exchange
hashpartitioning(_file");
} else {
- Assertions.assertThat(planAsString).doesNotContain("Exchange");
+ assertThat(planAsString).doesNotContain("Exchange");
}
});
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java
index 7f2857cce0..f6102bab69 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownDQL.java
@@ -37,9 +37,11 @@ import static
org.apache.iceberg.spark.SystemFunctionPushDownHelper.timestampStr
import static
org.apache.iceberg.spark.SystemFunctionPushDownHelper.timestampStrToHourOrdinal;
import static
org.apache.iceberg.spark.SystemFunctionPushDownHelper.timestampStrToMonthOrdinal;
import static
org.apache.iceberg.spark.SystemFunctionPushDownHelper.timestampStrToYearOrdinal;
+import static org.assertj.core.api.Assertions.assertThat;
import java.util.List;
-import java.util.Map;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
import org.apache.iceberg.expressions.ExpressionUtil;
import org.apache.iceberg.spark.SparkCatalogConfig;
import org.apache.iceberg.spark.source.PlanUtils;
@@ -49,19 +51,15 @@ import
org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression;
import org.apache.spark.sql.catalyst.expressions.Expression;
import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke;
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan;
-import org.assertj.core.api.Assertions;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runners.Parameterized;
-
-public class TestSystemFunctionPushDownDQL extends SparkExtensionsTestBase {
- public TestSystemFunctionPushDownDQL(
- String catalogName, String implementation, Map<String, String> config) {
- super(catalogName, implementation, config);
- }
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestSystemFunctionPushDownDQL extends ExtensionsTestBase {
- @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1},
config = {2}")
+ @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}")
public static Object[][] parameters() {
return new Object[][] {
{
@@ -72,23 +70,24 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
};
}
- @Before
+ @BeforeEach
public void before() {
+ super.before();
sql("USE %s", catalogName);
}
- @After
+ @AfterEach
public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}
- @Test
+ @TestTemplate
public void testYearsFunctionOnUnpartitionedTable() {
createUnpartitionedTable(spark, tableName);
testYearsFunction(false);
}
- @Test
+ @TestTemplate
public void testYearsFunctionOnPartitionedTable() {
createPartitionedTable(spark, tableName, "years(ts)");
testYearsFunction(true);
@@ -107,16 +106,16 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
checkPushedFilters(optimizedPlan, equal(year("ts"), targetYears));
List<Object[]> actual = rowsToJava(df.collectAsList());
- Assertions.assertThat(actual.size()).isEqualTo(5);
+ assertThat(actual).hasSize(5);
}
- @Test
+ @TestTemplate
public void testMonthsFunctionOnUnpartitionedTable() {
createUnpartitionedTable(spark, tableName);
testMonthsFunction(false);
}
- @Test
+ @TestTemplate
public void testMonthsFunctionOnPartitionedTable() {
createPartitionedTable(spark, tableName, "months(ts)");
testMonthsFunction(true);
@@ -135,16 +134,16 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
checkPushedFilters(optimizedPlan, greaterThan(month("ts"), targetMonths));
List<Object[]> actual = rowsToJava(df.collectAsList());
- Assertions.assertThat(actual.size()).isEqualTo(5);
+ assertThat(actual).hasSize(5);
}
- @Test
+ @TestTemplate
public void testDaysFunctionOnUnpartitionedTable() {
createUnpartitionedTable(spark, tableName);
testDaysFunction(false);
}
- @Test
+ @TestTemplate
public void testDaysFunctionOnPartitionedTable() {
createPartitionedTable(spark, tableName, "days(ts)");
testDaysFunction(true);
@@ -165,16 +164,16 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
checkPushedFilters(optimizedPlan, lessThan(day("ts"), targetDays));
List<Object[]> actual = rowsToJava(df.collectAsList());
- Assertions.assertThat(actual.size()).isEqualTo(5);
+ assertThat(actual).hasSize(5);
}
- @Test
+ @TestTemplate
public void testHoursFunctionOnUnpartitionedTable() {
createUnpartitionedTable(spark, tableName);
testHoursFunction(false);
}
- @Test
+ @TestTemplate
public void testHoursFunctionOnPartitionedTable() {
createPartitionedTable(spark, tableName, "hours(ts)");
testHoursFunction(true);
@@ -193,16 +192,16 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
checkPushedFilters(optimizedPlan, greaterThanOrEqual(hour("ts"),
targetHours));
List<Object[]> actual = rowsToJava(df.collectAsList());
- Assertions.assertThat(actual.size()).isEqualTo(8);
+ assertThat(actual).hasSize(8);
}
- @Test
+ @TestTemplate
public void testBucketLongFunctionOnUnpartitionedTable() {
createUnpartitionedTable(spark, tableName);
testBucketLongFunction(false);
}
- @Test
+ @TestTemplate
public void testBucketLongFunctionOnPartitionedTable() {
createPartitionedTable(spark, tableName, "bucket(5, id)");
testBucketLongFunction(true);
@@ -221,16 +220,16 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
checkPushedFilters(optimizedPlan, lessThanOrEqual(bucket("id", 5),
target));
List<Object[]> actual = rowsToJava(df.collectAsList());
- Assertions.assertThat(actual.size()).isEqualTo(5);
+ assertThat(actual).hasSize(5);
}
- @Test
+ @TestTemplate
public void testBucketStringFunctionOnUnpartitionedTable() {
createUnpartitionedTable(spark, tableName);
testBucketStringFunction(false);
}
- @Test
+ @TestTemplate
public void testBucketStringFunctionOnPartitionedTable() {
createPartitionedTable(spark, tableName, "bucket(5, data)");
testBucketStringFunction(true);
@@ -249,16 +248,16 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
checkPushedFilters(optimizedPlan, notEqual(bucket("data", 5), target));
List<Object[]> actual = rowsToJava(df.collectAsList());
- Assertions.assertThat(actual.size()).isEqualTo(8);
+ assertThat(actual).hasSize(8);
}
- @Test
+ @TestTemplate
public void testTruncateFunctionOnUnpartitionedTable() {
createUnpartitionedTable(spark, tableName);
testTruncateFunction(false);
}
- @Test
+ @TestTemplate
public void testTruncateFunctionOnPartitionedTable() {
createPartitionedTable(spark, tableName, "truncate(4, data)");
testTruncateFunction(true);
@@ -278,7 +277,7 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
checkPushedFilters(optimizedPlan, equal(truncate("data", 4), target));
List<Object[]> actual = rowsToJava(df.collectAsList());
- Assertions.assertThat(actual.size()).isEqualTo(5);
+ assertThat(actual).hasSize(5);
}
private void checkExpressions(
@@ -286,18 +285,18 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
List<Expression> staticInvokes =
PlanUtils.collectSparkExpressions(
optimizedPlan, expression -> expression instanceof StaticInvoke);
- Assertions.assertThat(staticInvokes).isEmpty();
+ assertThat(staticInvokes).isEmpty();
List<Expression> applyExpressions =
PlanUtils.collectSparkExpressions(
optimizedPlan, expression -> expression instanceof
ApplyFunctionExpression);
if (partitioned) {
- Assertions.assertThat(applyExpressions).isEmpty();
+ assertThat(applyExpressions).isEmpty();
} else {
- Assertions.assertThat(applyExpressions.size()).isEqualTo(1);
+ assertThat(applyExpressions).hasSize(1);
ApplyFunctionExpression expression = (ApplyFunctionExpression)
applyExpressions.get(0);
- Assertions.assertThat(expression.name()).isEqualTo(expectedFunctionName);
+ assertThat(expression.name()).isEqualTo(expectedFunctionName);
}
}
@@ -305,9 +304,9 @@ public class TestSystemFunctionPushDownDQL extends
SparkExtensionsTestBase {
LogicalPlan optimizedPlan, org.apache.iceberg.expressions.Expression
expected) {
List<org.apache.iceberg.expressions.Expression> pushedFilters =
PlanUtils.collectPushDownFilters(optimizedPlan);
- Assertions.assertThat(pushedFilters.size()).isEqualTo(1);
+ assertThat(pushedFilters).hasSize(1);
org.apache.iceberg.expressions.Expression actual = pushedFilters.get(0);
- Assertions.assertThat(ExpressionUtil.equivalent(expected, actual, STRUCT,
true))
+ assertThat(ExpressionUtil.equivalent(expected, actual, STRUCT, true))
.as("Pushed filter should match")
.isTrue();
}
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
index ac02561a3c..786d0bd1c6 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
@@ -23,11 +23,12 @@ import static
org.assertj.core.api.Assertions.assertThatNoException;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.iceberg.IcebergBuild;
+import org.apache.iceberg.ParameterizedTestExtension;
+import org.apache.iceberg.Parameters;
import org.apache.iceberg.Schema;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.catalog.Namespace;
@@ -51,30 +52,32 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
import org.apache.spark.sql.catalyst.catalog.SessionCatalog;
import org.assertj.core.api.InstanceOfAssertFactories;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runners.Parameterized;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
-public class TestViews extends SparkExtensionsTestBase {
+@ExtendWith(ParameterizedTestExtension.class)
+public class TestViews extends ExtensionsTestBase {
private static final Namespace NAMESPACE = Namespace.of("default");
private final String tableName = "table";
- @Before
+ @BeforeEach
public void before() {
+ super.before();
spark.conf().set("spark.sql.defaultCatalog", catalogName);
sql("USE %s", catalogName);
sql("CREATE NAMESPACE IF NOT EXISTS %s", NAMESPACE);
sql("CREATE TABLE %s (id INT, data STRING)", tableName);
}
- @After
+ @AfterEach
public void removeTable() {
sql("USE %s", catalogName);
sql("DROP TABLE IF EXISTS %s", tableName);
}
- @Parameterized.Parameters(name = "catalogName = {0}, implementation = {1},
config = {2}")
+ @Parameters(name = "catalogName = {0}, implementation = {1}, config = {2}")
public static Object[][] parameters() {
return new Object[][] {
{
@@ -85,11 +88,7 @@ public class TestViews extends SparkExtensionsTestBase {
};
}
- public TestViews(String catalog, String implementation, Map<String, String>
properties) {
- super(catalog, implementation, properties);
- }
-
- @Test
+ @TestTemplate
public void readFromView() throws NoSuchTableException {
insertRows(10);
String viewName = "simpleView";
@@ -115,7 +114,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrderElementsOf(expected);
}
- @Test
+ @TestTemplate
public void readFromTrinoView() throws NoSuchTableException {
insertRows(10);
String viewName = "trinoView";
@@ -140,7 +139,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrderElementsOf(expected);
}
- @Test
+ @TestTemplate
public void readFromMultipleViews() throws NoSuchTableException {
insertRows(6);
String viewName = "firstView";
@@ -173,7 +172,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrder(row(4), row(5), row(6));
}
- @Test
+ @TestTemplate
public void readFromViewUsingNonExistingTable() throws NoSuchTableException {
insertRows(10);
String viewName = "viewWithNonExistingTable";
@@ -197,7 +196,7 @@ public class TestViews extends SparkExtensionsTestBase {
catalogName, NAMESPACE));
}
- @Test
+ @TestTemplate
public void readFromViewUsingNonExistingTableColumn() throws
NoSuchTableException {
insertRows(10);
String viewName = "viewWithNonExistingColumn";
@@ -219,7 +218,7 @@ public class TestViews extends SparkExtensionsTestBase {
"A column or function parameter with name `non_existing` cannot be
resolved");
}
- @Test
+ @TestTemplate
public void readFromViewUsingInvalidSQL() throws NoSuchTableException {
insertRows(10);
String viewName = "viewWithInvalidSQL";
@@ -241,7 +240,7 @@ public class TestViews extends SparkExtensionsTestBase {
String.format("The view `%s` cannot be displayed due to invalid
view text", viewName));
}
- @Test
+ @TestTemplate
public void readFromViewWithStaleSchema() throws NoSuchTableException {
insertRows(10);
String viewName = "staleView";
@@ -267,7 +266,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("A column or function parameter with name `data`
cannot be resolved");
}
- @Test
+ @TestTemplate
public void readFromViewHiddenByTempView() throws NoSuchTableException {
insertRows(10);
String viewName = "viewHiddenByTempView";
@@ -294,7 +293,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrderElementsOf(expected);
}
- @Test
+ @TestTemplate
public void readFromViewWithGlobalTempView() throws NoSuchTableException {
insertRows(10);
String viewName = "viewWithGlobalTempView";
@@ -324,7 +323,7 @@ public class TestViews extends SparkExtensionsTestBase {
IntStream.rangeClosed(6,
10).mapToObj(this::row).collect(Collectors.toList()));
}
- @Test
+ @TestTemplate
public void readFromViewReferencingAnotherView() throws NoSuchTableException
{
insertRows(10);
String firstView = "viewBeingReferencedInAnotherView";
@@ -355,7 +354,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactly(row(5));
}
- @Test
+ @TestTemplate
public void readFromViewReferencingTempView() throws NoSuchTableException {
insertRows(10);
String tempView = "tempViewBeingReferencedInAnotherView";
@@ -391,7 +390,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("cannot be found");
}
- @Test
+ @TestTemplate
public void readFromViewReferencingAnotherViewHiddenByTempView() throws
NoSuchTableException {
insertRows(10);
String innerViewName = "inner_view";
@@ -439,7 +438,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrderElementsOf(expectedViewRows);
}
- @Test
+ @TestTemplate
public void readFromViewReferencingGlobalTempView() throws
NoSuchTableException {
insertRows(10);
String globalTempView = "globalTempViewBeingReferenced";
@@ -477,7 +476,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("cannot be found");
}
- @Test
+ @TestTemplate
public void readFromViewReferencingTempFunction() throws
NoSuchTableException {
insertRows(10);
String viewName = viewName("viewReferencingTempFunction");
@@ -510,7 +509,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("cannot be found");
}
- @Test
+ @TestTemplate
public void readFromViewWithCTE() throws NoSuchTableException {
insertRows(10);
String viewName = "viewWithCTE";
@@ -533,7 +532,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(sql("SELECT * FROM %s",
viewName)).hasSize(1).containsExactly(row(10, 1L));
}
- @Test
+ @TestTemplate
public void rewriteFunctionIdentifier() {
String viewName = "rewriteFunctionIdentifier";
String sql = "SELECT iceberg_version() AS version";
@@ -559,7 +558,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactly(row(IcebergBuild.version()));
}
- @Test
+ @TestTemplate
public void builtinFunctionIdentifierNotRewritten() {
String viewName = "builtinFunctionIdentifierNotRewritten";
String sql = "SELECT trim(' abc ') AS result";
@@ -578,7 +577,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(sql("SELECT * FROM %s",
viewName)).hasSize(1).containsExactly(row("abc"));
}
- @Test
+ @TestTemplate
public void rewriteFunctionIdentifierWithNamespace() {
String viewName = "rewriteFunctionIdentifierWithNamespace";
String sql = "SELECT system.bucket(100, 'a') AS bucket_result, 'a' AS
value";
@@ -605,7 +604,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactly(row(50, "a"));
}
- @Test
+ @TestTemplate
public void fullFunctionIdentifier() {
String viewName = "fullFunctionIdentifier";
String sql =
@@ -629,7 +628,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactly(row(50, "a"));
}
- @Test
+ @TestTemplate
public void fullFunctionIdentifierNotRewrittenLoadFailure() {
String viewName = "fullFunctionIdentifierNotRewrittenLoadFailure";
String sql = "SELECT spark_catalog.system.bucket(100, 'a') AS
bucket_result, 'a' AS value";
@@ -674,7 +673,7 @@ public class TestViews extends SparkExtensionsTestBase {
return Spark3Util.loadIcebergCatalog(spark, catalogName);
}
- @Test
+ @TestTemplate
public void renameView() throws NoSuchTableException {
insertRows(10);
String viewName = viewName("originalView");
@@ -700,7 +699,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrderElementsOf(expected);
}
- @Test
+ @TestTemplate
public void renameViewHiddenByTempView() throws NoSuchTableException {
insertRows(10);
String viewName = viewName("originalView");
@@ -739,7 +738,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(viewCatalog.viewExists(TableIdentifier.of(NAMESPACE,
renamedView))).isTrue();
}
- @Test
+ @TestTemplate
public void renameViewToDifferentTargetCatalog() {
String viewName = viewName("originalView");
String renamedView = viewName("renamedView");
@@ -761,14 +760,14 @@ public class TestViews extends SparkExtensionsTestBase {
"Cannot move view between catalogs: from=spark_with_views and
to=spark_catalog");
}
- @Test
+ @TestTemplate
public void renameNonExistingView() {
assertThatThrownBy(() -> sql("ALTER VIEW non_existing RENAME TO target"))
.isInstanceOf(AnalysisException.class)
.hasMessageContaining("The table or view `non_existing` cannot be
found");
}
- @Test
+ @TestTemplate
public void renameViewTargetAlreadyExistsAsView() {
String viewName = viewName("renameViewSource");
String target = viewName("renameViewTarget");
@@ -798,7 +797,7 @@ public class TestViews extends SparkExtensionsTestBase {
String.format("Cannot create view default.%s because it already
exists", target));
}
- @Test
+ @TestTemplate
public void renameViewTargetAlreadyExistsAsTable() {
String viewName = viewName("renameViewSource");
String target = viewName("renameViewTarget");
@@ -821,7 +820,7 @@ public class TestViews extends SparkExtensionsTestBase {
String.format("Cannot create view default.%s because it already
exists", target));
}
- @Test
+ @TestTemplate
public void dropView() {
String viewName = "viewToBeDropped";
String sql = String.format("SELECT id FROM %s", tableName);
@@ -843,14 +842,14 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(viewCatalog.viewExists(identifier)).isFalse();
}
- @Test
+ @TestTemplate
public void dropNonExistingView() {
assertThatThrownBy(() -> sql("DROP VIEW non_existing"))
.isInstanceOf(AnalysisException.class)
.hasMessageContaining("The view %s.%s cannot be found", NAMESPACE,
"non_existing");
}
- @Test
+ @TestTemplate
public void dropViewIfExists() {
String viewName = "viewToBeDropped";
String sql = String.format("SELECT id FROM %s", tableName);
@@ -875,7 +874,7 @@ public class TestViews extends SparkExtensionsTestBase {
}
/** The purpose of this test is mainly to make sure that normal view
deletion isn't messed up */
- @Test
+ @TestTemplate
public void dropGlobalTempView() {
String globalTempView = "globalViewToBeDropped";
sql("CREATE GLOBAL TEMPORARY VIEW %s AS SELECT id FROM %s",
globalTempView, tableName);
@@ -886,7 +885,7 @@ public class TestViews extends SparkExtensionsTestBase {
}
/** The purpose of this test is mainly to make sure that normal view
deletion isn't messed up */
- @Test
+ @TestTemplate
public void dropTempView() {
String tempView = "tempViewToBeDropped";
sql("CREATE TEMPORARY VIEW %s AS SELECT id FROM %s", tempView, tableName);
@@ -897,7 +896,7 @@ public class TestViews extends SparkExtensionsTestBase {
}
/** The purpose of this test is mainly to make sure that normal view
deletion isn't messed up */
- @Test
+ @TestTemplate
public void dropV1View() {
String v1View = "v1ViewToBeDropped";
sql("USE spark_catalog");
@@ -928,7 +927,7 @@ public class TestViews extends SparkExtensionsTestBase {
return viewName + new Random().nextInt(1000000);
}
- @Test
+ @TestTemplate
public void createViewIfNotExists() {
String viewName = "viewThatAlreadyExists";
sql("CREATE VIEW %s AS SELECT id FROM %s", viewName, tableName);
@@ -945,7 +944,7 @@ public class TestViews extends SparkExtensionsTestBase {
() -> sql("CREATE VIEW IF NOT EXISTS %s AS SELECT id FROM %s",
viewName, tableName));
}
- @Test
+ @TestTemplate
public void createOrReplaceView() throws NoSuchTableException {
insertRows(6);
String viewName = viewName("simpleView");
@@ -961,14 +960,14 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrder(row(4), row(5), row(6));
}
- @Test
+ @TestTemplate
public void createViewWithInvalidSQL() {
assertThatThrownBy(() -> sql("CREATE VIEW simpleViewWithInvalidSQL AS
invalid SQL"))
.isInstanceOf(AnalysisException.class)
.hasMessageContaining("Syntax error");
}
- @Test
+ @TestTemplate
public void createViewReferencingTempView() throws NoSuchTableException {
insertRows(10);
String tempView = "temporaryViewBeingReferencedInAnotherView";
@@ -987,7 +986,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(tempView);
}
- @Test
+ @TestTemplate
public void createViewReferencingGlobalTempView() throws
NoSuchTableException {
insertRows(10);
String globalTempView = "globalTemporaryViewBeingReferenced";
@@ -1011,7 +1010,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(String.format("%s.%s", "global_temp",
globalTempView));
}
- @Test
+ @TestTemplate
public void createViewReferencingTempFunction() {
String viewName = viewName("viewReferencingTemporaryFunction");
String functionName = "test_avg_func";
@@ -1030,7 +1029,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(functionName);
}
- @Test
+ @TestTemplate
public void createViewReferencingQualifiedTempFunction() {
String viewName = viewName("viewReferencingTemporaryFunction");
String functionName = "test_avg_func_qualified";
@@ -1061,7 +1060,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(String.format("`%s`.`%s`", NAMESPACE,
functionName));
}
- @Test
+ @TestTemplate
public void createViewUsingNonExistingTable() {
assertThatThrownBy(
() -> sql("CREATE VIEW viewWithNonExistingTable AS SELECT id FROM
non_existing"))
@@ -1069,7 +1068,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("The table or view `non_existing` cannot be
found");
}
- @Test
+ @TestTemplate
public void createViewWithMismatchedColumnCounts() {
String viewName = "viewWithMismatchedColumnCounts";
@@ -1092,7 +1091,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("Data columns: id, data");
}
- @Test
+ @TestTemplate
public void createViewWithColumnAliases() throws NoSuchTableException {
insertRows(6);
String viewName = "viewWithColumnAliases";
@@ -1128,7 +1127,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrder(row(1), row(2), row(3));
}
- @Test
+ @TestTemplate
public void createViewWithDuplicateColumnNames() {
assertThatThrownBy(
() ->
@@ -1139,7 +1138,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("The column `new_id` already exists");
}
- @Test
+ @TestTemplate
public void createViewWithDuplicateQueryColumnNames() throws
NoSuchTableException {
insertRows(3);
String viewName = "viewWithDuplicateQueryColumnNames";
@@ -1157,7 +1156,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactlyInAnyOrder(row(1, 1), row(2, 2), row(3, 3));
}
- @Test
+ @TestTemplate
public void createViewWithCTE() throws NoSuchTableException {
insertRows(10);
String viewName = "simpleViewWithCTE";
@@ -1172,7 +1171,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(sql("SELECT * FROM %s",
viewName)).hasSize(1).containsExactly(row(10, 1L));
}
- @Test
+ @TestTemplate
public void createViewWithConflictingNamesForCTEAndTempView() throws
NoSuchTableException {
insertRows(10);
String viewName = "viewWithConflictingNamesForCTEAndTempView";
@@ -1191,7 +1190,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(sql("SELECT * FROM %s",
viewName)).hasSize(1).containsExactly(row(10, 1L));
}
- @Test
+ @TestTemplate
public void createViewWithCTEReferencingTempView() {
String viewName = "viewWithCTEReferencingTempView";
String tempViewInCTE = "tempViewInCTE";
@@ -1211,7 +1210,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(tempViewInCTE);
}
- @Test
+ @TestTemplate
public void createViewWithCTEReferencingTempFunction() {
String viewName = "viewWithCTEReferencingTempFunction";
String functionName = "avg_function_in_cte";
@@ -1233,7 +1232,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(functionName);
}
- @Test
+ @TestTemplate
public void createViewWithNonExistingQueryColumn() {
assertThatThrownBy(
() ->
@@ -1245,7 +1244,7 @@ public class TestViews extends SparkExtensionsTestBase {
"A column or function parameter with name `non_existing` cannot be
resolved");
}
- @Test
+ @TestTemplate
public void createViewWithSubqueryExpressionUsingTempView() {
String viewName = "viewWithSubqueryExpression";
String tempView = "simpleTempView";
@@ -1262,7 +1261,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(tempView);
}
- @Test
+ @TestTemplate
public void createViewWithSubqueryExpressionUsingGlobalTempView() {
String viewName = "simpleViewWithSubqueryExpression";
String globalTempView = "simpleGlobalTempView";
@@ -1283,7 +1282,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(String.format("%s.%s", "global_temp",
globalTempView));
}
- @Test
+ @TestTemplate
public void createViewWithSubqueryExpressionUsingTempFunction() {
String viewName = viewName("viewWithSubqueryExpression");
String functionName = "avg_function_in_subquery";
@@ -1304,7 +1303,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining(functionName);
}
- @Test
+ @TestTemplate
public void createViewWithSubqueryExpressionInFilterThatIsRewritten()
throws NoSuchTableException {
insertRows(5);
@@ -1329,7 +1328,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactly(row(5));
}
- @Test
+ @TestTemplate
public void createViewWithSubqueryExpressionInQueryThatIsRewritten() throws
NoSuchTableException {
insertRows(3);
String viewName = viewName("viewWithSubqueryExpression");
@@ -1354,7 +1353,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactly(row(3), row(3), row(3));
}
- @Test
+ @TestTemplate
public void describeView() {
String viewName = "describeView";
@@ -1363,7 +1362,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsExactly(row("id", "int", ""), row("data", "string", ""));
}
- @Test
+ @TestTemplate
public void describeExtendedView() {
String viewName = "describeExtendedView";
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3",
tableName);
@@ -1388,7 +1387,7 @@ public class TestViews extends SparkExtensionsTestBase {
""));
}
- @Test
+ @TestTemplate
public void showViewProperties() {
String viewName = "showViewProps";
@@ -1399,7 +1398,7 @@ public class TestViews extends SparkExtensionsTestBase {
.contains(row("key1", "val1"), row("key2", "val2"));
}
- @Test
+ @TestTemplate
public void showViewPropertiesByKey() {
String viewName = "showViewPropsByKey";
@@ -1418,7 +1417,7 @@ public class TestViews extends SparkExtensionsTestBase {
catalogName, NAMESPACE, viewName)));
}
- @Test
+ @TestTemplate
public void showViews() throws NoSuchTableException {
insertRows(6);
String sql = String.format("SELECT * from %s", tableName);
@@ -1466,7 +1465,7 @@ public class TestViews extends SparkExtensionsTestBase {
row("global_temp", "globalviewforlisting", true), tempView);
}
- @Test
+ @TestTemplate
public void showViewsWithCurrentNamespace() {
String namespaceOne = "show_views_ns1";
String namespaceTwo = "show_views_ns2";
@@ -1497,7 +1496,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(sql("SHOW VIEWS LIKE
'viewTwo*'")).contains(v2).doesNotContain(v1);
}
- @Test
+ @TestTemplate
public void showCreateSimpleView() {
String viewName = "showCreateSimpleView";
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3",
tableName);
@@ -1518,7 +1517,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(sql("SHOW CREATE TABLE %s",
viewName)).containsExactly(row(expected));
}
- @Test
+ @TestTemplate
public void showCreateComplexView() {
String viewName = "showCreateComplexView";
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3",
tableName);
@@ -1545,7 +1544,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(sql("SHOW CREATE TABLE %s",
viewName)).containsExactly(row(expected));
}
- @Test
+ @TestTemplate
public void alterViewSetProperties() {
String viewName = "viewWithSetProperties";
@@ -1567,7 +1566,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsEntry("comment", "view comment");
}
- @Test
+ @TestTemplate
public void alterViewSetReservedProperties() {
String viewName = "viewWithSetReservedProperties";
@@ -1598,7 +1597,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("Cannot set reserved property:
'spark.query-column-names'");
}
- @Test
+ @TestTemplate
public void alterViewUnsetProperties() {
String viewName = "viewWithUnsetProperties";
sql("CREATE VIEW %s AS SELECT id FROM %s WHERE id <= 3", viewName,
tableName);
@@ -1619,7 +1618,7 @@ public class TestViews extends SparkExtensionsTestBase {
.containsEntry("comment", "view comment");
}
- @Test
+ @TestTemplate
public void alterViewUnsetUnknownProperty() {
String viewName = "viewWithUnsetUnknownProp";
sql("CREATE VIEW %s AS SELECT id FROM %s WHERE id <= 3", viewName,
tableName);
@@ -1633,7 +1632,7 @@ public class TestViews extends SparkExtensionsTestBase {
() -> sql("ALTER VIEW %s UNSET TBLPROPERTIES IF EXISTS
('unknown-key')", viewName));
}
- @Test
+ @TestTemplate
public void alterViewUnsetReservedProperties() {
String viewName = "viewWithUnsetReservedProperties";
@@ -1669,7 +1668,7 @@ public class TestViews extends SparkExtensionsTestBase {
.hasMessageContaining("Cannot unset reserved property:
'spark.query-column-names'");
}
- @Test
+ @TestTemplate
public void createOrReplaceViewWithColumnAliases() throws
NoSuchTableException {
insertRows(6);
String viewName = viewName("viewWithColumnAliases");
@@ -1715,7 +1714,7 @@ public class TestViews extends SparkExtensionsTestBase {
assertThat(second.doc()).isEqualTo("new ID");
}
- @Test
+ @TestTemplate
public void alterViewIsNotSupported() throws NoSuchTableException {
insertRows(6);
String viewName = "alteredView";
@@ -1733,7 +1732,7 @@ public class TestViews extends SparkExtensionsTestBase {
"ALTER VIEW <viewName> AS is not supported. Use CREATE OR REPLACE
VIEW instead");
}
- @Test
+ @TestTemplate
public void createOrReplaceViewKeepsViewHistory() {
String viewName = viewName("viewWithHistoryAfterReplace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3",
tableName);
@@ -1772,7 +1771,7 @@ public class TestViews extends SparkExtensionsTestBase {
.asStruct());
}
- @Test
+ @TestTemplate
public void replacingTrinoViewShouldFail() {
String viewName = viewName("trinoView");
String sql = String.format("SELECT id FROM %s", tableName);
@@ -1795,7 +1794,7 @@ public class TestViews extends SparkExtensionsTestBase {
+ "New dialects: [spark]");
}
- @Test
+ @TestTemplate
public void replacingTrinoAndSparkViewShouldFail() {
String viewName = viewName("trinoAndSparkView");
String sql = String.format("SELECT id FROM %s", tableName);
@@ -1819,7 +1818,7 @@ public class TestViews extends SparkExtensionsTestBase {
+ "New dialects: [spark]");
}
- @Test
+ @TestTemplate
public void replacingViewWithDialectDropAllowed() {
String viewName = viewName("trinoView");
String sql = String.format("SELECT id FROM %s", tableName);
diff --git
a/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java
b/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java
index 25d7e74715..207fca3cc2 100644
---
a/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java
+++
b/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java
@@ -19,22 +19,19 @@
package org.apache.iceberg.spark;
import java.io.IOException;
-import java.util.Map;
+import java.nio.file.Files;
+import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Table;
import org.apache.iceberg.catalog.TableIdentifier;
-import org.apache.iceberg.spark.extensions.SparkExtensionsTestBase;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class SmokeTest extends SparkExtensionsTestBase {
-
- public SmokeTest(String catalogName, String implementation, Map<String,
String> config) {
- super(catalogName, implementation, config);
- }
-
- @Before
+import org.apache.iceberg.spark.extensions.ExtensionsTestBase;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(ParameterizedTestExtension.class)
+public class SmokeTest extends ExtensionsTestBase {
+ @AfterEach
public void dropTable() {
sql("DROP TABLE IF EXISTS %s", tableName);
}
@@ -42,30 +39,32 @@ public class SmokeTest extends SparkExtensionsTestBase {
// Run through our Doc's Getting Started Example
// TODO Update doc example so that it can actually be run, modifications
were required for this
// test suite to run
- @Test
+ @TestTemplate
public void testGettingStarted() throws IOException {
// Creating a table
sql("CREATE TABLE %s (id bigint, data string) USING iceberg", tableName);
// Writing
sql("INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", tableName);
- Assert.assertEquals(
- "Should have inserted 3 rows", 3L, scalarSql("SELECT COUNT(*) FROM
%s", tableName));
+ Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s", tableName))
+ .as("Should have inserted 3 rows")
+ .isEqualTo(3L);
sql("DROP TABLE IF EXISTS source PURGE");
sql(
"CREATE TABLE source (id bigint, data string) USING parquet LOCATION
'%s'",
- temp.newFolder());
+ Files.createTempDirectory(temp, "junit"));
sql("INSERT INTO source VALUES (10, 'd'), (11, 'ee')");
sql("INSERT INTO %s SELECT id, data FROM source WHERE length(data) = 1",
tableName);
- Assert.assertEquals(
- "Table should now have 4 rows", 4L, scalarSql("SELECT COUNT(*) FROM
%s", tableName));
+ Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s", tableName))
+ .as("Table should now have 4 rows")
+ .isEqualTo(4L);
sql("DROP TABLE IF EXISTS updates PURGE");
sql(
"CREATE TABLE updates (id bigint, data string) USING parquet LOCATION
'%s'",
- temp.newFolder());
+ Files.createTempDirectory(temp, "junit"));
sql("INSERT INTO updates VALUES (1, 'x'), (2, 'x'), (4, 'z')");
sql(
@@ -73,31 +72,31 @@ public class SmokeTest extends SparkExtensionsTestBase {
+ "WHEN MATCHED THEN UPDATE SET t.data = u.data\n"
+ "WHEN NOT MATCHED THEN INSERT *",
tableName);
- Assert.assertEquals(
- "Table should now have 5 rows", 5L, scalarSql("SELECT COUNT(*) FROM
%s", tableName));
- Assert.assertEquals(
- "Record 1 should now have data x",
- "x",
- scalarSql("SELECT data FROM %s WHERE id = 1", tableName));
+ Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s", tableName))
+ .as("Table should now have 5 rows")
+ .isEqualTo(5L);
+ Assertions.assertThat(scalarSql("SELECT data FROM %s WHERE id = 1",
tableName))
+ .as("Record 1 should now have data x")
+ .isEqualTo("x");
// Reading
- Assert.assertEquals(
- "There should be 2 records with data x",
- 2L,
- scalarSql("SELECT count(1) as count FROM %s WHERE data = 'x' GROUP BY
data ", tableName));
+ Assertions.assertThat(
+ scalarSql(
+ "SELECT count(1) as count FROM %s WHERE data = 'x' GROUP BY
data ", tableName))
+ .as("There should be 2 records with data x")
+ .isEqualTo(2L);
// Not supported because of Spark limitation
if (!catalogName.equals("spark_catalog")) {
- Assert.assertEquals(
- "There should be 3 snapshots",
- 3L,
- scalarSql("SELECT COUNT(*) FROM %s.snapshots", tableName));
+ Assertions.assertThat(scalarSql("SELECT COUNT(*) FROM %s.snapshots",
tableName))
+ .as("There should be 3 snapshots")
+ .isEqualTo(3L);
}
}
// From Spark DDL Docs section
- @Test
- public void testAlterTable() throws NoSuchTableException {
+ @TestTemplate
+ public void testAlterTable() {
sql(
"CREATE TABLE %s (category int, id bigint, data string, ts timestamp)
USING iceberg",
tableName);
@@ -108,7 +107,9 @@ public class SmokeTest extends SparkExtensionsTestBase {
sql("ALTER TABLE %s ADD PARTITION FIELD years(ts)", tableName);
sql("ALTER TABLE %s ADD PARTITION FIELD bucket(16, category) AS shard",
tableName);
table = getTable();
- Assert.assertEquals("Table should have 4 partition fields", 4,
table.spec().fields().size());
+ Assertions.assertThat(table.spec().fields())
+ .as("Table should have 4 partition fields")
+ .hasSize(4);
// Drop Examples
sql("ALTER TABLE %s DROP PARTITION FIELD bucket(16, id)", tableName);
@@ -117,17 +118,21 @@ public class SmokeTest extends SparkExtensionsTestBase {
sql("ALTER TABLE %s DROP PARTITION FIELD shard", tableName);
table = getTable();
- Assert.assertTrue("Table should be unpartitioned",
table.spec().isUnpartitioned());
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should be unpartitioned")
+ .isTrue();
// Sort order examples
sql("ALTER TABLE %s WRITE ORDERED BY category, id", tableName);
sql("ALTER TABLE %s WRITE ORDERED BY category ASC, id DESC", tableName);
sql("ALTER TABLE %s WRITE ORDERED BY category ASC NULLS LAST, id DESC
NULLS FIRST", tableName);
table = getTable();
- Assert.assertEquals("Table should be sorted on 2 fields", 2,
table.sortOrder().fields().size());
+ Assertions.assertThat(table.sortOrder().fields())
+ .as("Table should be sorted on 2 fields")
+ .hasSize(2);
}
- @Test
+ @TestTemplate
public void testCreateTable() {
sql("DROP TABLE IF EXISTS %s", tableName("first"));
sql("DROP TABLE IF EXISTS %s", tableName("second"));
@@ -150,7 +155,9 @@ public class SmokeTest extends SparkExtensionsTestBase {
+ "PARTITIONED BY (category)",
tableName("second"));
Table second = getTable("second");
- Assert.assertEquals("Should be partitioned on 1 column", 1,
second.spec().fields().size());
+ Assertions.assertThat(second.spec().fields())
+ .as("Should be partitioned on 1 column")
+ .hasSize(1);
sql(
"CREATE TABLE %s (\n"
@@ -162,7 +169,9 @@ public class SmokeTest extends SparkExtensionsTestBase {
+ "PARTITIONED BY (bucket(16, id), days(ts), category)",
tableName("third"));
Table third = getTable("third");
- Assert.assertEquals("Should be partitioned on 3 columns", 3,
third.spec().fields().size());
+ Assertions.assertThat(third.spec().fields())
+ .as("Should be partitioned on 3 columns")
+ .hasSize(3);
}
private Table getTable(String name) {
