[
https://issues.apache.org/jira/browse/SPARK-31328?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Wenchen Fan resolved SPARK-31328.
---------------------------------
Resolution: Fixed
Issue resolved by pull request 28101
[https://github.com/apache/spark/pull/28101]
> Incorrect timestamps rebasing on autumn daylight saving time
> ------------------------------------------------------------
>
> Key: SPARK-31328
> URL: https://issues.apache.org/jira/browse/SPARK-31328
> Project: Spark
> Issue Type: Sub-task
> Components: SQL
> Affects Versions: 3.0.0
> Reporter: Maxim Gekk
> Assignee: Maxim Gekk
> Priority: Major
> Fix For: 3.0.0
>
>
> Run the following code in the *America/Los_Angeles* time zone:
> {code:scala}
> test("rebasing differences") {
> withDefaultTimeZone(getZoneId("America/Los_Angeles")) {
> val start = instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
> .atZone(getZoneId("America/Los_Angeles"))
> .toInstant)
> val end = instantToMicros(LocalDateTime.of(2030, 1, 1, 0, 0, 0)
> .atZone(getZoneId("America/Los_Angeles"))
> .toInstant)
> var micros = start
> var diff = Long.MaxValue
> var counter = 0
> while (micros < end) {
> val rebased = rebaseGregorianToJulianMicros(micros)
> val curDiff = rebased - micros
> if (curDiff != diff) {
> counter += 1
> diff = curDiff
> val ldt =
> microsToInstant(micros).atZone(getZoneId("America/Los_Angeles")).toLocalDateTime
> println(s"local date-time = $ldt diff = ${diff / MICROS_PER_MINUTE}
> minutes")
> }
> micros += 30 * MICROS_PER_MINUTE
> }
> println(s"counter = $counter")
> }
> }
> {code}
> The rebased and original micros must be the same after 1883-11-18 because the
> standard zone offset and DST offset are the same in Proleptic Gregorian
> calendar and in the hybrid calendar (Julian+Gregorian) but actually there are
> differences of 60 minutes:
> {code:java}
> local date-time = 0001-01-01T00:00 diff = -2872 minutes
> local date-time = 0100-03-01T00:00 diff = -1432 minutes
> local date-time = 0200-03-01T00:00 diff = 7 minutes
> local date-time = 0300-03-01T00:00 diff = 1447 minutes
> local date-time = 0500-03-01T00:00 diff = 2887 minutes
> local date-time = 0600-03-01T00:00 diff = 4327 minutes
> local date-time = 0700-03-01T00:00 diff = 5767 minutes
> local date-time = 0900-03-01T00:00 diff = 7207 minutes
> local date-time = 1000-03-01T00:00 diff = 8647 minutes
> local date-time = 1100-03-01T00:00 diff = 10087 minutes
> local date-time = 1300-03-01T00:00 diff = 11527 minutes
> local date-time = 1400-03-01T00:00 diff = 12967 minutes
> local date-time = 1500-03-01T00:00 diff = 14407 minutes
> local date-time = 1582-10-15T00:00 diff = 7 minutes
> local date-time = 1883-11-18T12:22:58 diff = 0 minutes
> local date-time = 1918-10-27T01:22:58 diff = 60 minutes
> local date-time = 1918-10-27T01:22:58 diff = 0 minutes
> local date-time = 1919-10-26T01:22:58 diff = 60 minutes
> local date-time = 1919-10-26T01:22:58 diff = 0 minutes
> local date-time = 1945-09-30T01:22:58 diff = 60 minutes
> local date-time = 1945-09-30T01:22:58 diff = 0 minutes
> local date-time = 1949-01-01T01:22:58 diff = 60 minutes
> local date-time = 1949-01-01T01:22:58 diff = 0 minutes
> local date-time = 1950-09-24T01:22:58 diff = 60 minutes
> local date-time = 1950-09-24T01:22:58 diff = 0 minutes
> local date-time = 1951-09-30T01:22:58 diff = 60 minutes
> local date-time = 1951-09-30T01:22:58 diff = 0 minutes
> local date-time = 1952-09-28T01:22:58 diff = 60 minutes
> local date-time = 1952-09-28T01:22:58 diff = 0 minutes
> local date-time = 1953-09-27T01:22:58 diff = 60 minutes
> local date-time = 1953-09-27T01:22:58 diff = 0 minutes
> local date-time = 1954-09-26T01:22:58 diff = 60 minutes
> local date-time = 1954-09-26T01:22:58 diff = 0 minutes
> local date-time = 1955-09-25T01:22:58 diff = 60 minutes
> local date-time = 1955-09-25T01:22:58 diff = 0 minutes
> local date-time = 1956-09-30T01:22:58 diff = 60 minutes
> local date-time = 1956-09-30T01:22:58 diff = 0 minutes
> local date-time = 1957-09-29T01:22:58 diff = 60 minutes
> local date-time = 1957-09-29T01:22:58 diff = 0 minutes
> local date-time = 1958-09-28T01:22:58 diff = 60 minutes
> local date-time = 1958-09-28T01:22:58 diff = 0 minutes
> local date-time = 1959-09-27T01:22:58 diff = 60 minutes
> local date-time = 1959-09-27T01:22:58 diff = 0 minutes
> local date-time = 1960-09-25T01:22:58 diff = 60 minutes
> local date-time = 1960-09-25T01:22:58 diff = 0 minutes
> local date-time = 1961-09-24T01:22:58 diff = 60 minutes
> local date-time = 1961-09-24T01:22:58 diff = 0 minutes
> local date-time = 1962-10-28T01:22:58 diff = 60 minutes
> local date-time = 1962-10-28T01:22:58 diff = 0 minutes
> local date-time = 1963-10-27T01:22:58 diff = 60 minutes
> local date-time = 1963-10-27T01:22:58 diff = 0 minutes
> local date-time = 1964-10-25T01:22:58 diff = 60 minutes
> local date-time = 1964-10-25T01:22:58 diff = 0 minutes
> local date-time = 1965-10-31T01:22:58 diff = 60 minutes
> local date-time = 1965-10-31T01:22:58 diff = 0 minutes
> local date-time = 1966-10-30T01:22:58 diff = 60 minutes
> local date-time = 1966-10-30T01:22:58 diff = 0 minutes
> local date-time = 1967-10-29T01:22:58 diff = 60 minutes
> local date-time = 1967-10-29T01:22:58 diff = 0 minutes
> local date-time = 1968-10-27T01:22:58 diff = 60 minutes
> local date-time = 1968-10-27T01:22:58 diff = 0 minutes
> local date-time = 1969-10-26T01:22:58 diff = 60 minutes
> local date-time = 1969-10-26T01:22:58 diff = 0 minutes
> local date-time = 1970-10-25T01:22:58 diff = 60 minutes
> local date-time = 1970-10-25T01:22:58 diff = 0 minutes
> local date-time = 1971-10-31T01:22:58 diff = 60 minutes
> local date-time = 1971-10-31T01:22:58 diff = 0 minutes
> local date-time = 1972-10-29T01:22:58 diff = 60 minutes
> local date-time = 1972-10-29T01:22:58 diff = 0 minutes
> local date-time = 1973-10-28T01:22:58 diff = 60 minutes
> local date-time = 1973-10-28T01:22:58 diff = 0 minutes
> local date-time = 1974-10-27T01:22:58 diff = 60 minutes
> local date-time = 1974-10-27T01:22:58 diff = 0 minutes
> local date-time = 1975-10-26T01:22:58 diff = 60 minutes
> local date-time = 1975-10-26T01:22:58 diff = 0 minutes
> local date-time = 1976-10-31T01:22:58 diff = 60 minutes
> local date-time = 1976-10-31T01:22:58 diff = 0 minutes
> local date-time = 1977-10-30T01:22:58 diff = 60 minutes
> local date-time = 1977-10-30T01:22:58 diff = 0 minutes
> local date-time = 1978-10-29T01:22:58 diff = 60 minutes
> local date-time = 1978-10-29T01:22:58 diff = 0 minutes
> local date-time = 1979-10-28T01:22:58 diff = 60 minutes
> local date-time = 1979-10-28T01:22:58 diff = 0 minutes
> local date-time = 1980-10-26T01:22:58 diff = 60 minutes
> local date-time = 1980-10-26T01:22:58 diff = 0 minutes
> local date-time = 1981-10-25T01:22:58 diff = 60 minutes
> local date-time = 1981-10-25T01:22:58 diff = 0 minutes
> local date-time = 1982-10-31T01:22:58 diff = 60 minutes
> local date-time = 1982-10-31T01:22:58 diff = 0 minutes
> local date-time = 1983-10-30T01:22:58 diff = 60 minutes
> local date-time = 1983-10-30T01:22:58 diff = 0 minutes
> local date-time = 1984-10-28T01:22:58 diff = 60 minutes
> local date-time = 1984-10-28T01:22:58 diff = 0 minutes
> local date-time = 1985-10-27T01:22:58 diff = 60 minutes
> local date-time = 1985-10-27T01:22:58 diff = 0 minutes
> local date-time = 1986-10-26T01:22:58 diff = 60 minutes
> local date-time = 1986-10-26T01:22:58 diff = 0 minutes
> local date-time = 1987-10-25T01:22:58 diff = 60 minutes
> local date-time = 1987-10-25T01:22:58 diff = 0 minutes
> local date-time = 1988-10-30T01:22:58 diff = 60 minutes
> local date-time = 1988-10-30T01:22:58 diff = 0 minutes
> local date-time = 1989-10-29T01:22:58 diff = 60 minutes
> local date-time = 1989-10-29T01:22:58 diff = 0 minutes
> local date-time = 1990-10-28T01:22:58 diff = 60 minutes
> local date-time = 1990-10-28T01:22:58 diff = 0 minutes
> local date-time = 1991-10-27T01:22:58 diff = 60 minutes
> local date-time = 1991-10-27T01:22:58 diff = 0 minutes
> local date-time = 1992-10-25T01:22:58 diff = 60 minutes
> local date-time = 1992-10-25T01:22:58 diff = 0 minutes
> local date-time = 1993-10-31T01:22:58 diff = 60 minutes
> local date-time = 1993-10-31T01:22:58 diff = 0 minutes
> local date-time = 1994-10-30T01:22:58 diff = 60 minutes
> local date-time = 1994-10-30T01:22:58 diff = 0 minutes
> local date-time = 1995-10-29T01:22:58 diff = 60 minutes
> local date-time = 1995-10-29T01:22:58 diff = 0 minutes
> local date-time = 1996-10-27T01:22:58 diff = 60 minutes
> local date-time = 1996-10-27T01:22:58 diff = 0 minutes
> local date-time = 1997-10-26T01:22:58 diff = 60 minutes
> local date-time = 1997-10-26T01:22:58 diff = 0 minutes
> local date-time = 1998-10-25T01:22:58 diff = 60 minutes
> local date-time = 1998-10-25T01:22:58 diff = 0 minutes
> local date-time = 1999-10-31T01:22:58 diff = 60 minutes
> local date-time = 1999-10-31T01:22:58 diff = 0 minutes
> local date-time = 2000-10-29T01:22:58 diff = 60 minutes
> local date-time = 2000-10-29T01:22:58 diff = 0 minutes
> local date-time = 2001-10-28T01:22:58 diff = 60 minutes
> local date-time = 2001-10-28T01:22:58 diff = 0 minutes
> local date-time = 2002-10-27T01:22:58 diff = 60 minutes
> local date-time = 2002-10-27T01:22:58 diff = 0 minutes
> local date-time = 2003-10-26T01:22:58 diff = 60 minutes
> local date-time = 2003-10-26T01:22:58 diff = 0 minutes
> local date-time = 2004-10-31T01:22:58 diff = 60 minutes
> local date-time = 2004-10-31T01:22:58 diff = 0 minutes
> local date-time = 2005-10-30T01:22:58 diff = 60 minutes
> local date-time = 2005-10-30T01:22:58 diff = 0 minutes
> local date-time = 2006-10-29T01:22:58 diff = 60 minutes
> local date-time = 2006-10-29T01:22:58 diff = 0 minutes
> local date-time = 2007-11-04T01:22:58 diff = 60 minutes
> local date-time = 2007-11-04T01:22:58 diff = 0 minutes
> local date-time = 2008-11-02T01:22:58 diff = 60 minutes
> local date-time = 2008-11-02T01:22:58 diff = 0 minutes
> local date-time = 2009-11-01T01:22:58 diff = 60 minutes
> local date-time = 2009-11-01T01:22:58 diff = 0 minutes
> local date-time = 2010-11-07T01:22:58 diff = 60 minutes
> local date-time = 2010-11-07T01:22:58 diff = 0 minutes
> local date-time = 2011-11-06T01:22:58 diff = 60 minutes
> local date-time = 2011-11-06T01:22:58 diff = 0 minutes
> local date-time = 2012-11-04T01:22:58 diff = 60 minutes
> local date-time = 2012-11-04T01:22:58 diff = 0 minutes
> local date-time = 2013-11-03T01:22:58 diff = 60 minutes
> local date-time = 2013-11-03T01:22:58 diff = 0 minutes
> local date-time = 2014-11-02T01:22:58 diff = 60 minutes
> local date-time = 2014-11-02T01:22:58 diff = 0 minutes
> local date-time = 2015-11-01T01:22:58 diff = 60 minutes
> local date-time = 2015-11-01T01:22:58 diff = 0 minutes
> local date-time = 2016-11-06T01:22:58 diff = 60 minutes
> local date-time = 2016-11-06T01:22:58 diff = 0 minutes
> local date-time = 2017-11-05T01:22:58 diff = 60 minutes
> local date-time = 2017-11-05T01:22:58 diff = 0 minutes
> local date-time = 2018-11-04T01:22:58 diff = 60 minutes
> local date-time = 2018-11-04T01:22:58 diff = 0 minutes
> local date-time = 2019-11-03T01:22:58 diff = 60 minutes
> local date-time = 2019-11-03T01:22:58 diff = 0 minutes
> local date-time = 2020-11-01T01:22:58 diff = 60 minutes
> local date-time = 2020-11-01T01:22:58 diff = 0 minutes
> local date-time = 2021-11-07T01:22:58 diff = 60 minutes
> local date-time = 2021-11-07T01:22:58 diff = 0 minutes
> local date-time = 2022-11-06T01:22:58 diff = 60 minutes
> local date-time = 2022-11-06T01:22:58 diff = 0 minutes
> local date-time = 2023-11-05T01:22:58 diff = 60 minutes
> local date-time = 2023-11-05T01:22:58 diff = 0 minutes
> local date-time = 2024-11-03T01:22:58 diff = 60 minutes
> local date-time = 2024-11-03T01:22:58 diff = 0 minutes
> local date-time = 2025-11-02T01:22:58 diff = 60 minutes
> local date-time = 2025-11-02T01:22:58 diff = 0 minutes
> local date-time = 2026-11-01T01:22:58 diff = 60 minutes
> local date-time = 2026-11-01T01:22:58 diff = 0 minutes
> local date-time = 2027-11-07T01:22:58 diff = 60 minutes
> local date-time = 2027-11-07T01:22:58 diff = 0 minutes
> local date-time = 2028-11-05T01:22:58 diff = 60 minutes
> local date-time = 2028-11-05T01:22:58 diff = 0 minutes
> local date-time = 2029-11-04T01:22:58 diff = 60 minutes
> local date-time = 2029-11-04T01:22:58 diff = 0 minutes
> counter = 183
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
