This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new c0bed74d34 Spark: Correct partition transform functions to match Spec
(#8192)
c0bed74d34 is described below
commit c0bed74d345cc275427d476b2e842880ee33678a
Author: Chris Lettieri <[email protected]>
AuthorDate: Mon Sep 25 08:57:12 2023 -0400
Spark: Correct partition transform functions to match Spec (#8192)
---
.../extensions/TestAlterTablePartitionFields.java | 77 ++++++++++++++++++++++
.../java/org/apache/iceberg/spark/Spark3Util.java | 8 +++
.../apache/iceberg/spark/sql/TestCreateTable.java | 20 ++++++
.../extensions/TestAlterTablePartitionFields.java | 77 ++++++++++++++++++++++
.../java/org/apache/iceberg/spark/Spark3Util.java | 8 +++
.../apache/iceberg/spark/sql/TestCreateTable.java | 20 ++++++
.../extensions/TestAlterTablePartitionFields.java | 77 ++++++++++++++++++++++
.../java/org/apache/iceberg/spark/Spark3Util.java | 8 +++
.../apache/iceberg/spark/sql/TestCreateTable.java | 20 ++++++
.../extensions/TestAlterTablePartitionFields.java | 77 ++++++++++++++++++++++
.../java/org/apache/iceberg/spark/Spark3Util.java | 8 +++
.../apache/iceberg/spark/sql/TestCreateTable.java | 20 ++++++
.../extensions/TestAlterTablePartitionFields.java | 77 ++++++++++++++++++++++
.../java/org/apache/iceberg/spark/Spark3Util.java | 8 +++
.../apache/iceberg/spark/sql/TestCreateTable.java | 20 ++++++
15 files changed, 525 insertions(+)
diff --git
a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
index 5102da8442..5589b1b05c 100644
---
a/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
+++
b/spark/v3.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -27,6 +27,7 @@ import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.connector.catalog.CatalogManager;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.assertj.core.api.Assertions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -179,6 +180,82 @@ public class TestAlterTablePartitionFields extends
SparkExtensionsTestBase {
Assert.assertEquals("Should have new spec field", expected, table.spec());
}
+ @Test
+ public void testAddYearPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddMonthPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddDayPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddHourPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
@Test
public void testAddNamedPartition() {
createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
diff --git
a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 2bfd0aaf8d..3e0452d94a 100644
--- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -374,14 +374,18 @@ public class Spark3Util {
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
return org.apache.iceberg.expressions.Expressions.bucket(colName,
findWidth(transform));
+ case "year":
case "years":
return org.apache.iceberg.expressions.Expressions.year(colName);
+ case "month":
case "months":
return org.apache.iceberg.expressions.Expressions.month(colName);
case "date":
+ case "day":
case "days":
return org.apache.iceberg.expressions.Expressions.day(colName);
case "date_hour":
+ case "hour":
case "hours":
return org.apache.iceberg.expressions.Expressions.hour(colName);
case "truncate":
@@ -417,17 +421,21 @@ public class Spark3Util {
case "bucket":
builder.bucket(colName, findWidth(transform));
break;
+ case "year":
case "years":
builder.year(colName);
break;
+ case "month":
case "months":
builder.month(colName);
break;
case "date":
+ case "day":
case "days":
builder.day(colName);
break;
case "date_hour":
+ case "hour":
case "hours":
builder.hour(colName);
break;
diff --git
a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 1411c83ddc..a6256afcdf 100644
---
a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++
b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -63,6 +63,26 @@ public class TestCreateTable extends SparkCatalogTestBase {
Assert.assertTrue("Table should already exist",
validationCatalog.tableExists(tableIdent));
}
+ @Test
+ public void testTransformSingularForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hour(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
+ @Test
+ public void testTransformPluralForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hours(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
diff --git
a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
index 9b4bd12ec1..2ecf6b0c4c 100644
---
a/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
+++
b/spark/v3.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -27,6 +27,7 @@ import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.connector.catalog.CatalogManager;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.assertj.core.api.Assertions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -179,6 +180,82 @@ public class TestAlterTablePartitionFields extends
SparkExtensionsTestBase {
Assert.assertEquals("Should have new spec field", expected, table.spec());
}
+ @Test
+ public void testAddYearPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddMonthPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddDayPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddHourPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
@Test
public void testAddNamedPartition() {
createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
diff --git
a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index cdd8a45af6..c5c7ee4d53 100644
--- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -342,14 +342,18 @@ public class Spark3Util {
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
return org.apache.iceberg.expressions.Expressions.bucket(colName,
findWidth(transform));
+ case "year":
case "years":
return org.apache.iceberg.expressions.Expressions.year(colName);
+ case "month":
case "months":
return org.apache.iceberg.expressions.Expressions.month(colName);
case "date":
+ case "day":
case "days":
return org.apache.iceberg.expressions.Expressions.day(colName);
case "date_hour":
+ case "hour":
case "hours":
return org.apache.iceberg.expressions.Expressions.hour(colName);
case "truncate":
@@ -399,17 +403,21 @@ public class Spark3Util {
case "bucket":
builder.bucket(colName, findWidth(transform));
break;
+ case "year":
case "years":
builder.year(colName);
break;
+ case "month":
case "months":
builder.month(colName);
break;
case "date":
+ case "day":
case "days":
builder.day(colName);
break;
case "date_hour":
+ case "hour":
case "hours":
builder.hour(colName);
break;
diff --git
a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 1411c83ddc..a6256afcdf 100644
---
a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++
b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -63,6 +63,26 @@ public class TestCreateTable extends SparkCatalogTestBase {
Assert.assertTrue("Table should already exist",
validationCatalog.tableExists(tableIdent));
}
+ @Test
+ public void testTransformSingularForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hour(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
+ @Test
+ public void testTransformPluralForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hours(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
diff --git
a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
index a43f2a041b..0e978e52e5 100644
---
a/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
+++
b/spark/v3.3/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -27,6 +27,7 @@ import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.connector.catalog.CatalogManager;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.assertj.core.api.Assertions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -179,6 +180,82 @@ public class TestAlterTablePartitionFields extends
SparkExtensionsTestBase {
Assert.assertEquals("Should have new spec field", expected, table.spec());
}
+ @Test
+ public void testAddYearPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddMonthPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddDayPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddHourPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
@Test
public void testAddNamedPartition() {
createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
diff --git
a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 23a53ea9e8..d7717e2bfd 100644
--- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -359,14 +359,18 @@ public class Spark3Util {
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
return org.apache.iceberg.expressions.Expressions.bucket(colName,
findWidth(transform));
+ case "year":
case "years":
return org.apache.iceberg.expressions.Expressions.year(colName);
+ case "month":
case "months":
return org.apache.iceberg.expressions.Expressions.month(colName);
case "date":
+ case "day":
case "days":
return org.apache.iceberg.expressions.Expressions.day(colName);
case "date_hour":
+ case "hour":
case "hours":
return org.apache.iceberg.expressions.Expressions.hour(colName);
case "truncate":
@@ -416,17 +420,21 @@ public class Spark3Util {
case "bucket":
builder.bucket(colName, findWidth(transform));
break;
+ case "year":
case "years":
builder.year(colName);
break;
+ case "month":
case "months":
builder.month(colName);
break;
case "date":
+ case "day":
case "days":
builder.day(colName);
break;
case "date_hour":
+ case "hour":
case "hours":
builder.hour(colName);
break;
diff --git
a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 1411c83ddc..a6256afcdf 100644
---
a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++
b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -63,6 +63,26 @@ public class TestCreateTable extends SparkCatalogTestBase {
Assert.assertTrue("Table should already exist",
validationCatalog.tableExists(tableIdent));
}
+ @Test
+ public void testTransformSingularForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hour(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
+ @Test
+ public void testTransformPluralForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hours(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
diff --git
a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
index a43f2a041b..0e978e52e5 100644
---
a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
+++
b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -27,6 +27,7 @@ import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.connector.catalog.CatalogManager;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.assertj.core.api.Assertions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -179,6 +180,82 @@ public class TestAlterTablePartitionFields extends
SparkExtensionsTestBase {
Assert.assertEquals("Should have new spec field", expected, table.spec());
}
+ @Test
+ public void testAddYearPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddMonthPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddDayPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddHourPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
@Test
public void testAddNamedPartition() {
createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index bbd7986b26..62301e9676 100644
--- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -368,14 +368,18 @@ public class Spark3Util {
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
return org.apache.iceberg.expressions.Expressions.bucket(colName,
findWidth(transform));
+ case "year":
case "years":
return org.apache.iceberg.expressions.Expressions.year(colName);
+ case "month":
case "months":
return org.apache.iceberg.expressions.Expressions.month(colName);
case "date":
+ case "day":
case "days":
return org.apache.iceberg.expressions.Expressions.day(colName);
case "date_hour":
+ case "hour":
case "hours":
return org.apache.iceberg.expressions.Expressions.hour(colName);
case "truncate":
@@ -425,17 +429,21 @@ public class Spark3Util {
case "bucket":
builder.bucket(colName, findWidth(transform));
break;
+ case "year":
case "years":
builder.year(colName);
break;
+ case "month":
case "months":
builder.month(colName);
break;
case "date":
+ case "day":
case "days":
builder.day(colName);
break;
case "date_hour":
+ case "hour":
case "hours":
builder.hour(colName);
break;
diff --git
a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 01b8b99062..ecfd6759b9 100644
---
a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++
b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -63,6 +63,26 @@ public class TestCreateTable extends SparkCatalogTestBase {
Assert.assertTrue("Table should already exist",
validationCatalog.tableExists(tableIdent));
}
+ @Test
+ public void testTransformSingularForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hour(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
+ @Test
+ public void testTransformPluralForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hours(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
index a43f2a041b..0e978e52e5 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java
@@ -27,6 +27,7 @@ import org.apache.iceberg.spark.source.SparkTable;
import org.apache.spark.sql.connector.catalog.CatalogManager;
import org.apache.spark.sql.connector.catalog.Identifier;
import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.assertj.core.api.Assertions;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -179,6 +180,82 @@ public class TestAlterTablePartitionFields extends
SparkExtensionsTestBase {
Assert.assertEquals("Should have new spec field", expected, table.spec());
}
+ @Test
+ public void testAddYearPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD year(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).year("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddMonthPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD month(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).month("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddDayPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD day(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).day("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
+ @Test
+ public void testAddHourPartition() {
+ createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
+ Table table = validationCatalog.loadTable(tableIdent);
+
+ Assertions.assertThat(table.spec().isUnpartitioned())
+ .as("Table should start unpartitioned")
+ .isTrue();
+
+ sql("ALTER TABLE %s ADD PARTITION FIELD hour(ts)", tableName);
+
+ table.refresh();
+
+ PartitionSpec expected =
+
PartitionSpec.builderFor(table.schema()).withSpecId(1).hour("ts").build();
+
+ Assertions.assertThat(table.spec()).as("Should have new spec
field").isEqualTo(expected);
+ }
+
@Test
public void testAddNamedPartition() {
createTable("id bigint NOT NULL, category string, ts timestamp, data
string");
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 3611dd7960..cfcc3941c7 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -368,14 +368,18 @@ public class Spark3Util {
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
return org.apache.iceberg.expressions.Expressions.bucket(colName,
findWidth(transform));
+ case "year":
case "years":
return org.apache.iceberg.expressions.Expressions.year(colName);
+ case "month":
case "months":
return org.apache.iceberg.expressions.Expressions.month(colName);
case "date":
+ case "day":
case "days":
return org.apache.iceberg.expressions.Expressions.day(colName);
case "date_hour":
+ case "hour":
case "hours":
return org.apache.iceberg.expressions.Expressions.hour(colName);
case "truncate":
@@ -425,17 +429,21 @@ public class Spark3Util {
case "bucket":
builder.bucket(colName, findWidth(transform));
break;
+ case "year":
case "years":
builder.year(colName);
break;
+ case "month":
case "months":
builder.month(colName);
break;
case "date":
+ case "day":
case "days":
builder.day(colName);
break;
case "date_hour":
+ case "hour":
case "hours":
builder.hour(colName);
break;
diff --git
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 01b8b99062..ecfd6759b9 100644
---
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -63,6 +63,26 @@ public class TestCreateTable extends SparkCatalogTestBase {
Assert.assertTrue("Table should already exist",
validationCatalog.tableExists(tableIdent));
}
+ @Test
+ public void testTransformSingularForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hour(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
+ @Test
+ public void testTransformPluralForm() {
+ Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
+ sql(
+ "CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) "
+ + "USING iceberg partitioned by (hours(ts))",
+ tableName);
+ Assert.assertTrue("Table should exist",
validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist",
validationCatalog.tableExists(tableIdent));
