Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/19769#discussion_r151575898
--- Diff:
sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
---
@@ -105,13 +108,23 @@
*/
private static final MemoryMode DEFAULT_MEMORY_MODE = MemoryMode.ON_HEAP;
+ private TimeZone convertTz = null;
+
/**
* Implementation of RecordReader API.
*/
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext
taskAttemptContext)
throws IOException, InterruptedException,
UnsupportedOperationException {
super.initialize(inputSplit, taskAttemptContext);
+ Configuration hadoopConf = taskAttemptContext.getConfiguration();
+ // this is *not* the raw value in the sql conf -- we've already looked
at the createdBy field
+ // for this particular file and adjusted that value accordingly by
this point.
+ boolean doTsConversion =
+
Boolean.valueOf(hadoopConf.get(SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION().key()));
--- End diff --
tiny nit: double spaces
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
