MaxGekk commented on a change in pull request #34973:
URL: https://github.com/apache/spark/pull/34973#discussion_r775826435
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
##########
@@ -464,30 +485,43 @@ object RebaseDateTime {
final val lastSwitchJulianTs: Long = getLastSwitchTs(julianGregRebaseMap)
/**
- * An optimized version of [[rebaseJulianToGregorianMicros(ZoneId, Long)]].
This method leverages
- * the pre-calculated rebasing maps to save calculation. If the rebasing map
doesn't contain
- * information about the current JVM system time zone or `micros` is related
to Before Common Era,
- * the function falls back to the regular unoptimized version.
- *
- * Note: The function assumes that the input micros was derived from a local
timestamp
- * at the default system JVM time zone in the Julian calendar.
+ * An optimized version of [[rebaseJulianToGregorianMicros(TimeZone,
Long)]]. This method
+ * leverages the pre-calculated rebasing maps to save calculation. If the
rebasing map doesn't
+ * contain information about the given time zone `timeZoneId` or `micros` is
related to Before
+ * Common Era, the function falls back to the regular unoptimized version.
*
+ * @param timeZoneId A string identifier of a time zone.
* @param micros The number of microseconds since the epoch
'1970-01-01T00:00:00Z'
* in the Julian calendar. It can be negative.
* @return The rebased microseconds since the epoch in Proleptic Gregorian
calendar.
*/
- def rebaseJulianToGregorianMicros(micros: Long): Long = {
+ def rebaseJulianToGregorianMicros(timeZoneId: String, micros: Long): Long = {
if (micros >= lastSwitchJulianTs) {
micros
} else {
- val timeZone = TimeZone.getDefault
- val tzId = timeZone.getID
- val rebaseRecord = julianGregRebaseMap.getOrNull(tzId)
+ val rebaseRecord = julianGregRebaseMap.getOrNull(timeZoneId)
if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
- rebaseJulianToGregorianMicros(timeZone, micros)
+ rebaseJulianToGregorianMicros(TimeZone.getTimeZone(timeZoneId), micros)
} else {
rebaseMicros(rebaseRecord, micros)
}
}
}
+
+ /**
+ * An optimized version of [[rebaseJulianToGregorianMicros(TimeZone,
Long)]]. This method
+ * leverages the pre-calculated rebasing maps to save calculation. If the
rebasing map doesn't
+ * contain information about the current JVM system time zone or `micros` is
related to Before
+ * Common Era, the function falls back to the regular unoptimized version.
+ *
+ * Note: The function assumes that the input micros was derived from a local
timestamp
+ * at the default system JVM time zone in the Julian calendar.
+ *
+ * @param micros The number of microseconds since the epoch
'1970-01-01T00:00:00Z'
+ * in the Julian calendar. It can be negative.
+ * @return The rebased microseconds since the epoch in Proleptic Gregorian
calendar.
+ */
+ def rebaseJulianToGregorianMicros(micros: Long): Long = {
Review comment:
> ParquetFilters.timestampToMicros <-- NEED TO RE-CHECK THIS ONE
This is fine too since the rebasing happens locally on the same JVM. Also
users can enable Java 8 for filters, see
https://github.com/apache/spark/pull/28696
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
