cloud-fan commented on a change in pull request #33344:
URL: https://github.com/apache/spark/pull/33344#discussion_r670645552
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
##########
@@ -366,31 +368,44 @@ abstract class ParquetPartitionDiscoverySuite
s"hdfs://host:9000/path2"),
PartitionSpec.emptySpec)
- // The cases below check the resolution for type conflicts.
- val t1 = Timestamp.valueOf("2014-01-01 00:00:00.0").getTime * 1000
- val t2 = Timestamp.valueOf("2014-01-01 00:01:00.0").getTime * 1000
- // Values in column 'a' are inferred as null, date and timestamp each, and
timestamp is set
- // as a common type.
- // Values in column 'b' are inferred as integer, decimal(22, 0) and null,
and decimal(22, 0)
- // is set as a common type.
- check(Seq(
- s"hdfs://host:9000/path/a=$defaultPartitionName/b=0",
- s"hdfs://host:9000/path/a=2014-01-01/b=${Long.MaxValue}111",
- s"hdfs://host:9000/path/a=2014-01-01
00%3A01%3A00.0/b=$defaultPartitionName"),
- PartitionSpec(
- StructType(Seq(
- StructField("a", TimestampType),
- StructField("b", DecimalType(22, 0)))),
- Seq(
- Partition(
- InternalRow(null, Decimal(0)),
- s"hdfs://host:9000/path/a=$defaultPartitionName/b=0"),
- Partition(
- InternalRow(t1, Decimal(s"${Long.MaxValue}111")),
- s"hdfs://host:9000/path/a=2014-01-01/b=${Long.MaxValue}111"),
- Partition(
- InternalRow(t2, null),
- s"hdfs://host:9000/path/a=2014-01-01
00%3A01%3A00.0/b=$defaultPartitionName"))))
+ // The inferred timestmap type is consistent with the value of
`SQLConf.TIMESTAMP_TYPE`
+ Seq(TimestampTypes.TIMESTAMP_LTZ, TimestampTypes.TIMESTAMP_NTZ).foreach {
tsType =>
+ withSQLConf(SQLConf.TIMESTAMP_TYPE.key -> tsType.toString) {
+ // The cases below check the resolution for type conflicts.
+ val t1 = if (tsType == TimestampTypes.TIMESTAMP_LTZ) {
+ Timestamp.valueOf("2014-01-01 00:00:00.0").getTime * 1000
+ } else {
+ localDateTimeToMicros(LocalDateTime.parse("2014-01-01T00:00:00"))
+ }
+ val t2 = if (tsType == TimestampTypes.TIMESTAMP_LTZ) {
+ Timestamp.valueOf("2014-01-01 00:01:00.0").getTime * 1000
+ } else {
+ localDateTimeToMicros(LocalDateTime.parse("2014-01-01T00:01:00"))
+ }
+ // Values in column 'a' are inferred as null, date and timestamp each,
and timestamp is set
+ // as a common type.
+ // Values in column 'b' are inferred as integer, decimal(22, 0) and
null, and decimal(22, 0)
+ // is set as a common type.
+ check(Seq(
+ s"hdfs://host:9000/path/a=$defaultPartitionName/b=0",
+ s"hdfs://host:9000/path/a=2014-01-01/b=${Long.MaxValue}111",
+ s"hdfs://host:9000/path/a=2014-01-01
00%3A01%3A00.0/b=$defaultPartitionName"),
+ PartitionSpec(
+ StructType(Seq(
+ StructField("a", SQLConf.get.timestampType),
Review comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
