This is an automated email from the ASF dual-hosted git repository. maxgekk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 72fa176 [SPARK-36948][SQL][TESTS] Check CREATE TABLE with ANSI intervals using Hive external catalog and Parquet 72fa176 is described below commit 72fa176cae40a8a25ba72e1e5ba3a49928efebe1 Author: Max Gekk <max.g...@gmail.com> AuthorDate: Fri Oct 8 10:47:31 2021 +0300 [SPARK-36948][SQL][TESTS] Check CREATE TABLE with ANSI intervals using Hive external catalog and Parquet ### What changes were proposed in this pull request? In the PR, I propose new test to check: 1. CREATE TABLE with ANSI interval columns 2. INSERT INTO the table ANSI interval values 3. SELECT the table with ANSI interval columns Since Hive Metastore/Parquet Serde doesn't support interval types natively, Spark fallbacks to its specific format while saving the schema to Hive external catalog, and outputs the warning: ``` 20:10:52.797 WARN org.apache.spark.sql.hive.test.TestHiveExternalCatalog: Could not persist `default`.`tbl_with_ansi_intervals` in a Hive compatible way. Persisting it into Hive metastore in Spark SQL specific format. org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.IllegalArgumentException: Error: type expected at the position 0 of 'interval year to month:interval day to second' but 'interval year to month' is found. at org.apache.hadoop.hive.ql.metadata.Hive.createTable(Hive.java:869) ``` ### Why are the changes needed? To improve test coverage. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? By running new test: ``` $ ./build/sbt -Phive-2.3 "test:testOnly *HiveParquetSuite" ``` Closes #34215 from MaxGekk/create-table-ansi-intervals-2. Authored-by: Max Gekk <max.g...@gmail.com> Signed-off-by: Max Gekk <max.g...@gmail.com> --- .../org/apache/spark/sql/hive/HiveParquetSuite.scala | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala index 8940ab4..ae7ca38 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql.hive +import java.time.{Duration, Period} +import java.time.temporal.ChronoUnit + import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.execution.datasources.parquet.ParquetTest import org.apache.spark.sql.hive.test.TestHiveSingleton @@ -123,4 +126,21 @@ class HiveParquetSuite extends QueryTest with ParquetTest with TestHiveSingleton assert(msg.contains("cannot resolve 'c3' given input columns")) } } + + test("SPARK-36948: Create a table with ANSI intervals using Hive external catalog") { + val tbl = "tbl_with_ansi_intervals" + withTable(tbl) { + sql(s"CREATE TABLE $tbl (ym INTERVAL YEAR TO MONTH, dt INTERVAL DAY TO SECOND) USING PARQUET") + sql( + s"""INSERT INTO $tbl VALUES ( + | INTERVAL '1-1' YEAR TO MONTH, + | INTERVAL '1 02:03:04.123456' DAY TO SECOND)""".stripMargin) + checkAnswer( + sql(s"SELECT * FROM $tbl"), + Row( + Period.ofYears(1).plusMonths(1), + Duration.ofDays(1).plusHours(2).plusMinutes(3).plusSeconds(4) + .plus(123456, ChronoUnit.MICROS))) + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org