This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new ebb5cff64 fix: add TimestampLTZ-as-NTZ correctness tests and
compatibility docs (#4220)
ebb5cff64 is described below
commit ebb5cff64c045210b184efa586c51963f286db06
Author: Andy Grove <[email protected]>
AuthorDate: Wed May 13 06:23:07 2026 -0600
fix: add TimestampLTZ-as-NTZ correctness tests and compatibility docs
(#4220)
---
.github/workflows/pr_build_linux.yml | 1 +
.github/workflows/pr_build_macos.yml | 1 +
dev/diffs/3.4.3.diff | 2 +-
dev/diffs/3.5.8.diff | 2 +-
dev/diffs/4.0.2.diff | 22 ++---
dev/diffs/4.1.1.diff | 34 +++----
.../latest/compatibility/spark-versions.md | 10 ++
.../parquet/ParquetTimestampLtzAsNtzSuite.scala | 101 +++++++++++++++++++++
8 files changed, 133 insertions(+), 40 deletions(-)
diff --git a/.github/workflows/pr_build_linux.yml
b/.github/workflows/pr_build_linux.yml
index 78b9481cb..d00abea96 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -329,6 +329,7 @@ jobs:
org.apache.comet.parquet.ParquetReadV1Suite
org.apache.comet.parquet.ParquetReadV2Suite
org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite
+ org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
org.apache.spark.sql.comet.ParquetEncryptionITCase
diff --git a/.github/workflows/pr_build_macos.yml
b/.github/workflows/pr_build_macos.yml
index e77cc8f72..c2c47beea 100644
--- a/.github/workflows/pr_build_macos.yml
+++ b/.github/workflows/pr_build_macos.yml
@@ -177,6 +177,7 @@ jobs:
org.apache.comet.parquet.ParquetReadV1Suite
org.apache.comet.parquet.ParquetReadV2Suite
org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite
+ org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
org.apache.spark.sql.comet.ParquetEncryptionITCase
diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff
index c01fbaa4e..ebe53f49d 100644
--- a/dev/diffs/3.4.3.diff
+++ b/dev/diffs/3.4.3.diff
@@ -2170,7 +2170,7 @@ index 29cb224c878..ee5a87fa200 100644
- test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ") {
+ test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ",
-+
IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720";))
{
++
IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4219";))
{
val data = (1 to 1000).map { i =>
val ts = new java.sql.Timestamp(i)
Row(ts)
diff --git a/dev/diffs/3.5.8.diff b/dev/diffs/3.5.8.diff
index f7021829d..76ed210d3 100644
--- a/dev/diffs/3.5.8.diff
+++ b/dev/diffs/3.5.8.diff
@@ -2137,7 +2137,7 @@ index f6472ba3d9d..5ea2d938664 100644
- test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ") {
+ test("SPARK-36182: can't read TimestampLTZ as TimestampNTZ",
-+
IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720";))
{
++
IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/4219";))
{
val data = (1 to 1000).map { i =>
val ts = new java.sql.Timestamp(i)
Row(ts)
diff --git a/dev/diffs/4.0.2.diff b/dev/diffs/4.0.2.diff
index e38e97557..34deaa582 100644
--- a/dev/diffs/4.0.2.diff
+++ b/dev/diffs/4.0.2.diff
@@ -2729,7 +2729,7 @@ index 4474ec1fd42..05fa0257c82 100644
checkAnswer(
// "fruit" column in this file is encoded using
DELTA_LENGTH_BYTE_ARRAY.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index bba71f1c48d..35247c13ad9 100644
+index bba71f1c48d..5a111a937a9 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -27,6 +27,7 @@ import org.apache.parquet.hadoop.ParquetOutputFormat
@@ -2740,17 +2740,7 @@ index bba71f1c48d..35247c13ad9 100644
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow
import org.apache.spark.sql.catalyst.util.ArrayData
-@@ -185,7 +186,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
- }
- }
-
-- test("SPARK-47447: read TimestampLTZ as TimestampNTZ") {
-+ test("SPARK-47447: read TimestampLTZ as TimestampNTZ",
-+
IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720";))
{
- val providedSchema = StructType(Seq(StructField("time", TimestampNTZType,
false)))
-
- Seq("INT96", "TIMESTAMP_MICROS", "TIMESTAMP_MILLIS").foreach { tsType =>
-@@ -996,7 +998,11 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -996,7 +997,11 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
Seq(Some("A"), Some("A"), None).toDF().repartition(1)
.write.parquet(path.getAbsolutePath)
val df = spark.read.parquet(path.getAbsolutePath)
@@ -2763,7 +2753,7 @@ index bba71f1c48d..35247c13ad9 100644
}
}
}
-@@ -1042,7 +1048,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1042,7 +1047,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
testMigration(fromTsType = "TIMESTAMP_MICROS", toTsType = "INT96")
}
@@ -2773,7 +2763,7 @@ index bba71f1c48d..35247c13ad9 100644
def readParquet(schema: String, path: File): DataFrame = {
spark.read.schema(schema).parquet(path.toString)
}
-@@ -1060,7 +1067,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1060,7 +1066,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
checkAnswer(readParquet(schema2, path), df)
}
@@ -2783,7 +2773,7 @@ index bba71f1c48d..35247c13ad9 100644
val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
checkAnswer(readParquet(schema1, path), df)
val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
-@@ -1084,7 +1092,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1084,7 +1091,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c,
CAST('1.2' AS BINARY) d")
df.write.parquet(path.toString)
@@ -2793,7 +2783,7 @@ index bba71f1c48d..35247c13ad9 100644
checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00"))
checkAnswer(readParquet("a DECIMAL(11, 2)", path), sql("SELECT 1.00"))
checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null))
-@@ -1131,7 +1140,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1131,7 +1139,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
}
}
diff --git a/dev/diffs/4.1.1.diff b/dev/diffs/4.1.1.diff
index 72b875a6d..f685e6146 100644
--- a/dev/diffs/4.1.1.diff
+++ b/dev/diffs/4.1.1.diff
@@ -39,7 +39,7 @@ index 6df8bc85b51..dabb75e2b75 100644
withSpark(sc) { sc =>
TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
diff --git a/pom.xml b/pom.xml
-index dc757d78812..18841e95f3d 100644
+index dc757d78812..10f7b202e71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -152,6 +152,8 @@
@@ -2685,7 +2685,7 @@ index cd6f41b4ef4..4b6a17344bc 100644
ParquetOutputFormat.WRITER_VERSION ->
ParquetProperties.WriterVersion.PARQUET_2_0.toString
)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
-index 6b73cc8618d..624694916fb 100644
+index 6b73cc8618d..81a58f43784 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -38,6 +38,7 @@ import org.apache.parquet.schema.MessageType
@@ -2880,7 +2880,7 @@ index 3072657a095..6b5b9103363 100644
checkAnswer(
// "fruit" column in this file is encoded using
DELTA_LENGTH_BYTE_ARRAY.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
-index c530dc0d3df..418d5ea4b4d 100644
+index c530dc0d3df..7e1dd663873 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -27,6 +27,7 @@ import org.apache.parquet.hadoop.ParquetOutputFormat
@@ -2891,17 +2891,7 @@ index c530dc0d3df..418d5ea4b4d 100644
import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier}
import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow
import org.apache.spark.sql.catalyst.util.ArrayData
-@@ -185,7 +186,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
- }
- }
-
-- test("SPARK-47447: read TimestampLTZ as TimestampNTZ") {
-+ test("SPARK-47447: read TimestampLTZ as TimestampNTZ",
-+
IgnoreCometNativeDataFusion("https://github.com/apache/datafusion-comet/issues/3720";))
{
- val providedSchema = StructType(Seq(StructField("time", TimestampNTZType,
false)))
-
- Seq("INT96", "TIMESTAMP_MICROS", "TIMESTAMP_MILLIS").foreach { tsType =>
-@@ -996,7 +998,11 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -996,7 +997,11 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
Seq(Some("A"), Some("A"), None).toDF().repartition(1)
.write.parquet(path.getAbsolutePath)
val df = spark.read.parquet(path.getAbsolutePath)
@@ -2914,7 +2904,7 @@ index c530dc0d3df..418d5ea4b4d 100644
}
}
}
-@@ -1042,7 +1048,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1042,7 +1047,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
testMigration(fromTsType = "TIMESTAMP_MICROS", toTsType = "INT96")
}
@@ -2924,7 +2914,7 @@ index c530dc0d3df..418d5ea4b4d 100644
def readParquet(schema: String, path: File): DataFrame = {
spark.read.schema(schema).parquet(path.toString)
}
-@@ -1060,7 +1067,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1060,7 +1066,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
checkAnswer(readParquet(schema2, path), df)
}
@@ -2934,7 +2924,7 @@ index c530dc0d3df..418d5ea4b4d 100644
val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)"
checkAnswer(readParquet(schema1, path), df)
val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)"
-@@ -1084,7 +1092,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1084,7 +1091,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c,
CAST('1.2' AS BINARY) d")
df.write.parquet(path.toString)
@@ -2944,7 +2934,7 @@ index c530dc0d3df..418d5ea4b4d 100644
checkAnswer(readParquet("a DECIMAL(3, 2)", path), sql("SELECT 1.00"))
checkAnswer(readParquet("a DECIMAL(11, 2)", path), sql("SELECT 1.00"))
checkAnswer(readParquet("b DECIMAL(3, 2)", path), Row(null))
-@@ -1131,7 +1140,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
+@@ -1131,7 +1139,8 @@ abstract class ParquetQuerySuite extends QueryTest with
ParquetTest with SharedS
}
}
@@ -3841,7 +3831,7 @@ index 89f65562295..166a11a9079 100644
val tblTargetName = "tbl_target"
val tblSourceQualified = s"default.$tblSourceName"
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
-index f0f3f94b811..f77b54dcef9 100644
+index f0f3f94b811..be5e113c3ed 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
@@ -27,13 +27,14 @@ import scala.jdk.CollectionConverters._
@@ -3919,7 +3909,7 @@ index f0f3f94b811..f77b54dcef9 100644
super.withSQLConf(pairs: _*)(f)
}
-@@ -451,6 +497,8 @@ private[sql] trait SQLTestUtilsBase
+@@ -451,6 +488,8 @@ private[sql] trait SQLTestUtilsBase
val schema = df.schema
val withoutFilters = df.queryExecution.executedPlan.transform {
case FilterExec(_, child) => child
@@ -3929,7 +3919,7 @@ index f0f3f94b811..f77b54dcef9 100644
spark.internalCreateDataFrame(withoutFilters.execute(), schema)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
-index 245219c1756..a611836f086 100644
+index 245219c1756..b566f970ccd 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
@@ -75,6 +75,21 @@ trait SharedSparkSessionBase
@@ -4072,7 +4062,7 @@ index b67370f6eb9..746b3974b29 100644
override def beforeEach(): Unit = {
super.beforeEach()
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-index a394d0b7393..a4bc3d3fd8e 100644
+index a394d0b7393..3e1f0404a37 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -53,24 +53,34 @@ object TestHive
diff --git a/docs/source/user-guide/latest/compatibility/spark-versions.md
b/docs/source/user-guide/latest/compatibility/spark-versions.md
index 663be37a2..8c17fd6b1 100644
--- a/docs/source/user-guide/latest/compatibility/spark-versions.md
+++ b/docs/source/user-guide/latest/compatibility/spark-versions.md
@@ -34,6 +34,11 @@ Spark 3.4.3 is supported with Java 11/17 and Scala 2.12/2.13.
(SPARK-36182), but Comet's `native_datafusion` scan silently returns the raw
UTC value instead.
See [Parquet Compatibility](scans.md#native_datafusion-limitations) for
details.
+- **Unsupported Parquet type conversions**: Spark 3.4 raises schema
incompatibility errors for
+ certain type mismatches (e.g., reading INT32 as BIGINT, decimal precision
changes), but Comet's
+ `native_datafusion` scan may not detect these and could return unexpected
values.
+ See [Parquet Compatibility](scans.md#native_datafusion-limitations) for
details.
+
## Spark 3.5
Spark 3.5.8 is supported with Java 11/17 and Scala 2.12/2.13.
@@ -44,6 +49,11 @@ Spark 3.5.8 is supported with Java 11/17 and Scala 2.12/2.13.
(SPARK-36182), but Comet's `native_datafusion` scan silently returns the raw
UTC value instead.
See [Parquet Compatibility](scans.md#native_datafusion-limitations) for
details.
+- **Unsupported Parquet type conversions**: Spark 3.5 raises schema
incompatibility errors for
+ certain type mismatches (e.g., reading INT32 as BIGINT, decimal precision
changes), but Comet's
+ `native_datafusion` scan may not detect these and could return unexpected
values.
+ See [Parquet Compatibility](scans.md#native_datafusion-limitations) for
details.
+
## Spark 4.0
Spark 4.0.2 is supported with Java 17 and Scala 2.13.
diff --git
a/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala
b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala
new file mode 100644
index 000000000..cd7d2a627
--- /dev/null
+++
b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.comet.parquet
+
+import java.sql.Timestamp
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.CometTestBase
+import org.apache.spark.sql.internal.SQLConf
+
+import org.apache.comet.CometConf
+import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus
+
+/**
+ * Tests for reading Parquet TimestampLTZ columns as TimestampNTZ.
+ *
+ * Prior to Spark 4.0, Spark raises an error (SPARK-36182) when asked to read
TimestampLTZ as
+ * TimestampNTZ. Comet should match this behavior. In Spark 4.0+, this read is
permitted
+ * (SPARK-47447) and Comet should produce matching results.
+ *
+ * See https://github.com/apache/datafusion-comet/issues/4219
+ */
+class ParquetTimestampLtzAsNtzSuite extends CometTestBase {
+ import testImplicits._
+
+ private val tsTypes = Seq("INT96", "TIMESTAMP_MICROS", "TIMESTAMP_MILLIS")
+
+ tsTypes.foreach { tsType =>
+ test(s"read TimestampLTZ ($tsType) as TimestampNTZ throws pre-Spark 4") {
+ assume(!isSpark40Plus, "Spark 4.0+ allows reading TimestampLTZ as
TimestampNTZ")
+
+ val scanImpl = CometConf.COMET_NATIVE_SCAN_IMPL.get()
+ assume(
+ scanImpl != CometConf.SCAN_AUTO && scanImpl !=
CometConf.SCAN_NATIVE_DATAFUSION,
+ s"https://github.com/apache/datafusion-comet/issues/4219 ($scanImpl
scan does not " +
+ "reject TimestampLTZ read as TimestampNTZ)")
+
+ val sessionTz = "America/Los_Angeles"
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> sessionTz,
+ SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> tsType,
+ SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+ Seq(Timestamp.valueOf("2020-01-01
12:00:00")).toDF("ts").write.parquet(path)
+
+ // Spark refuses to read TimestampLTZ as TimestampNTZ (SPARK-36182)
+ withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
+ intercept[SparkException] {
+ spark.read.schema("ts timestamp_ntz").parquet(path).collect()
+ }
+ }
+
+ // Comet should also refuse
+ intercept[SparkException] {
+ spark.read.schema("ts timestamp_ntz").parquet(path).collect()
+ }
+ }
+ }
+ }
+ }
+
+ tsTypes.foreach { tsType =>
+ test(s"read TimestampLTZ ($tsType) as TimestampNTZ matches Spark") {
+ assume(isSpark40Plus, "Spark 4.0+ allows reading TimestampLTZ as
TimestampNTZ")
+ val sessionTz = "America/Los_Angeles"
+
+ withSQLConf(
+ SQLConf.SESSION_LOCAL_TIMEZONE.key -> sessionTz,
+ SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> tsType,
+ SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+ Seq(Timestamp.valueOf("2020-01-01
12:00:00")).toDF("ts").write.parquet(path)
+
+ withSQLConf(CometConf.COMET_NATIVE_SCAN_IMPL.key ->
CometConf.SCAN_NATIVE_DATAFUSION) {
+ checkSparkAnswerAndOperator(spark.read.schema("ts
timestamp_ntz").parquet(path))
+ }
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
