This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/branch-1.5 by this push:
new 4a54b18 [KYUUBI #2075] Using thread-safe FastDateFormat instead of
SimpleDateFormat
4a54b18 is described below
commit 4a54b18d1636a09e62413a46fb6aae79cb3fd35d
Author: Fei Wang <[email protected]>
AuthorDate: Wed Mar 9 17:50:29 2022 +0800
[KYUUBI #2075] Using thread-safe FastDateFormat instead of SimpleDateFormat
### _Why are the changes needed?_
We met below issue when fetching result.
```
Caused by: java.lang.RuntimeException:
java.lang.ArrayIndexOutOfBoundsException:46
7022 at
sun.util.calendar.BaseCalendar.getCalendarDateFromFixedDate(BaseCalendar.java:453)
7023 at
java.util.GregorianCalendar.computeFields(GregorianCalendar.java:2397)
7024 at
java.util.GregorianCalendar.computeFields(GregorianCalendar.java:2312)
7025 at java.util.Calendar.setTimeInMillis(Calendar.java:1804)
7026 at java.util.Calendar.setTime(Calendar.java:1770)
7027 at java.text.SimpleDateFormat.format(SimpleDateFormat.java:943)
7028 at java.text.SimpleDateFormat.format(SimpleDateFormat.java:936)
7029 at java.text.DateFormat.format(DateFormat.java:345)
7030 at
org.apache.kyuubi.schema.RowSet$.toHiveString(RowSet.scala:245)
7031 at
org.apache.kyuubi.schema.RowSet$.$anonfun$toTColumn$3(RowSet.scala:120)
7032 at scala.collection.immutable.List.map(List.scala:290)
7033 at org.apache.kyuubi.schema.RowSet$.toTColumn(RowSet.scala:115)
7034 at
org.apache.kyuubi.schema.RowSet$.$anonfun$toColumnBasedSet$1(RowSet.scala:65)
7035 at
org.apache.kyuubi.schema.RowSet$.$anonfun$toColumnBasedSet$1$adapted(RowSet.scala:64)
7036 at scala.collection.immutable.List.foreach(List.scala:392)
7037 at
org.apache.kyuubi.schema.RowSet$.toColumnBasedSet(RowSet.scala:64)
7038 at org.apache.kyuubi.schema.RowSet$.toTRowSet(RowSet.scala:47)
7039 at
org.apache.kyuubi.engine.spark.operation.SparkOperation.getNextRowSet(SparkOperation.scala:183)
7040 at
org.apache.kyuubi.operation.OperationManager.getOperationNextRowSet(OperationManager.scala:116)
7041 at
org.apache.kyuubi.session.AbstractSession.fetchResults(AbstractSession.scala:197)
7042 at
org.apache.kyuubi.service.AbstractBackendService.fetchResults(AbstractBackendService.scala:169)
7043 at
org.apache.kyuubi.service.ThriftBinaryFrontendService.FetchResults(ThriftBinaryFrontendService.scala:505)
```
The root cause is that the date time formatter used to convert the result
to rowSet is not thread-safe.
In this pr, we use thread-safe FastDateFormat instead of SimpleDateFormat.
### _How was this patch tested?_
Existing UT.
Closes #2075 from turboFei/thread_safe_fat_stimeformatter.
Closes #2075
44ae8fd1 [Fei Wang] Using thread-safe FastDateFormat instead of
SimpleDateFormat
Authored-by: Fei Wang <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
(cherry picked from commit 5a64e1242eabf9480d2898842c1e50c03093e430)
Signed-off-by: Kent Yao <[email protected]>
---
.../apache/kyuubi/engine/flink/schema/RowSet.scala | 12 +++----
.../apache/kyuubi/engine/spark/schema/RowSet.scala | 8 ++---
.../scala/org/apache/kyuubi/util/RowSetUtils.scala | 38 ++++++++++++++++++----
3 files changed, 41 insertions(+), 17 deletions(-)
diff --git
a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala
b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala
index 8b0db93..a3640ba 100644
---
a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala
+++
b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala
@@ -34,7 +34,7 @@ import org.apache.flink.types.Row
import org.apache.hive.service.rpc.thrift._
import org.apache.kyuubi.engine.flink.result.ResultSet
-import org.apache.kyuubi.util.RowSetUtils.{dateFormatter, timestampFormatter}
+import org.apache.kyuubi.util.RowSetUtils._
object RowSet {
@@ -289,19 +289,19 @@ object RowSet {
"null"
case (d: Int, _: DateType) =>
- dateFormatter.format(LocalDate.ofEpochDay(d))
+ formatLocalDate(LocalDate.ofEpochDay(d))
case (ld: LocalDate, _: DateType) =>
- dateFormatter.format(ld)
+ formatLocalDate(ld)
case (d: Date, _: DateType) =>
- dateFormatter.format(d.toInstant)
+ formatInstant(d.toInstant)
case (ldt: LocalDateTime, _: TimestampType) =>
- timestampFormatter.format(ldt)
+ formatLocalDateTime(ldt)
case (ts: Timestamp, _: TimestampType) =>
- timestampFormatter.format(ts.toInstant)
+ formatInstant(ts.toInstant)
case (decimal: java.math.BigDecimal, _: DecimalType) =>
decimal.toPlainString
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
index 47407d4..8d2fe8d 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
@@ -212,16 +212,16 @@ object RowSet {
"null"
case (d: Date, DateType) =>
- simpleDateFormatter.format(d)
+ formatDate(d)
case (ld: LocalDate, DateType) =>
- dateFormatter.format(ld)
+ formatLocalDate(ld)
case (t: Timestamp, TimestampType) =>
- simpleTimestampFormatter.format(t)
+ formatTimestamp(t)
case (i: Instant, TimestampType) =>
- timestampFormatter.withZone(timeZone).format(i)
+ formatInstant(i, Option(timeZone))
case (bin: Array[Byte], BinaryType) =>
new String(bin, StandardCharsets.UTF_8)
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala
index 3c1cb83..38db271 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala
@@ -18,40 +18,64 @@
package org.apache.kyuubi.util
import java.nio.ByteBuffer
-import java.text.SimpleDateFormat
+import java.sql.Timestamp
+import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
import java.time.chrono.IsoChronology
import java.time.format.DateTimeFormatter
import java.time.format.DateTimeFormatterBuilder
import java.time.temporal.ChronoField
-import java.util.Locale
+import java.util.{Date, Locale}
import scala.language.implicitConversions
+import org.apache.commons.lang3.time.FastDateFormat
+
private[kyuubi] object RowSetUtils {
- lazy val dateFormatter = {
+ private lazy val dateFormatter = {
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd")
.toFormatter(Locale.US)
.withChronology(IsoChronology.INSTANCE)
}
- lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
+ private lazy val legacyDateFormatter =
FastDateFormat.getInstance("yyyy-MM-dd", Locale.US)
- lazy val timestampFormatter: DateTimeFormatter = {
+ private lazy val timestampFormatter: DateTimeFormatter = {
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
.toFormatter(Locale.US)
.withChronology(IsoChronology.INSTANCE)
}
- lazy val simpleTimestampFormatter = {
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
+ private lazy val legacyTimestampFormatter = {
+ FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
}
private def createDateTimeFormatterBuilder(): DateTimeFormatterBuilder = {
new DateTimeFormatterBuilder().parseCaseInsensitive()
}
+ def formatDate(d: Date): String = {
+ legacyDateFormatter.format(d)
+ }
+
+ def formatLocalDate(ld: LocalDate): String = {
+ dateFormatter.format(ld)
+ }
+
+ def formatLocalDateTime(ldt: LocalDateTime): String = {
+ timestampFormatter.format(ldt)
+ }
+
+ def formatInstant(i: Instant, timeZone: Option[ZoneId] = None): String = {
+ timeZone.map(timestampFormatter.withZone(_).format(i))
+ .getOrElse(timestampFormatter.format(i))
+ }
+
+ def formatTimestamp(t: Timestamp): String = {
+ legacyTimestampFormatter.format(t)
+ }
+
implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
ByteBuffer.wrap(bitSet.toByteArray)
}
