Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/19769#discussion_r151634925
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
---
@@ -87,4 +95,107 @@ class ParquetInteroperabilitySuite extends
ParquetCompatibilityTest with SharedS
Row(Seq(2, 3))))
}
}
+
+ test("parquet timestamp conversion") {
+ // Make a table with one parquet file written by impala, and one
parquet file written by spark.
+ // We should only adjust the timestamps in the impala file, and only
if the conf is set
+ val impalaFile = "test-data/impala_timestamp.parq"
+
+ // here are the timestamps in the impala file, as they were saved by
impala
+ val impalaFileData =
+ Seq(
+ "2001-01-01 01:01:01",
+ "2002-02-02 02:02:02",
+ "2003-03-03 03:03:03"
+ ).map { s => java.sql.Timestamp.valueOf(s) }
+ val impalaPath =
Thread.currentThread().getContextClassLoader.getResource(impalaFile)
+ .toURI.getPath
+ withTempPath { tableDir =>
+ val ts = Seq(
+ "2004-04-04 04:04:04",
+ "2005-05-05 05:05:05",
+ "2006-06-06 06:06:06"
+ ).map { s => java.sql.Timestamp.valueOf(s) }
+ import testImplicits._
+ // match the column names of the file from impala
+ val df =
spark.createDataset(ts).toDF().repartition(1).withColumnRenamed("value", "ts")
+ df.write.parquet(tableDir.getAbsolutePath)
+ FileUtils.copyFile(new File(impalaPath), new File(tableDir,
"part-00001.parq"))
+
+ Seq(false, true).foreach { int96TimestampConversion =>
+ Seq(false, true).foreach { vectorized =>
+ withSQLConf(
+ (SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION.key,
int96TimestampConversion.toString()),
+ (SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key,
vectorized.toString())
--- End diff --
to be future proof, let's explicitly set
`PARQUET_OUTPUT_TIMESTAMP_TYPE=INT96`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
