This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9eae2a8e99b5 [SPARK-59546][SQL] Avoid per-row exceptions when
inferring TimeType in JSON, CSV, and XML
9eae2a8e99b5 is described below
commit 9eae2a8e99b50d10b318aba79ec0a16c142e6e13
Author: david-mollitor-db <[email protected]>
AuthorDate: Fri Oct 30 16:57:34 2026 -0700
[SPARK-59546][SQL] Avoid per-row exceptions when inferring TimeType in
JSON, CSV, and XML
### What changes were proposed in this pull request?
JSON, CSV, and XML schema inference probe whether a string value is a
`TIME` by calling the
throwing `TimeFormatter.parse` inside `allCatch opt`:
```scala
if (isTimeTypeEnabled && (allCatch opt
timeFormatter.parse(field)).isDefined) {
TimeType(TimeType.DEFAULT_PRECISION)
}
```
For the default (pattern-less) formatter, `parse` calls
`SparkDateTimeUtils.stringToTimeAnsi`, which
throws a `SparkDateTimeException` (filling in a stack trace) for every
value that is not a time.
This PR adds a non-throwing `TimeFormatter.parseOptional`, mirroring
`TimestampFormatter`:
- trait default: `try Some(parse(s)) catch { case _: Exception => None }`;
- `Iso8601TimeFormatter`: an exception-free implementation using
`DateTimeFormatter.parseUnresolved`
with a `ParsePosition`, requiring the whole input to be consumed (as in
`Iso8601TimestampFormatter`);
- `DefaultTimeFormatter`: delegates to `SparkDateTimeUtils.stringToTime`
(already `Option[Long]`).
and switches the `TimeType` inference probe in `JsonInferSchema`,
`CSVInferSchema`, and
`XmlInferSchema` to `timeFormatter.parseOptional(field).isDefined`.
### Why are the changes needed?
Inferring a non-time column throws and catches one exception per row. This
is exactly the cost that
SPARK-39193 / SPARK-39280 / SPARK-39281 removed for timestamp and date
inference by adding
`parseOptional` (that work reported exception handling as more than 90% of
type-inference time).
`TimeType` was added later and never got a `parseOptional`, so it is the
only datetime formatter
still using the throwing probe — the timestamp probe immediately below the
time probe already uses
`parseOptional`.
Profiling `JsonBenchmark` under JFR (`settings=profile`):
- allocation samples through `SparkDateTimeUtils.stringToTimeAnsi` (the
throwing time probe):
~64,400 → 0;
- `java.lang.Throwable.fillInStackTrace`: ~1.7% of sampled allocation →
absent;
- total sampled allocation: about 23% lower.
No new allocation is introduced: the default inference path (no custom
`timeFormat`) uses
`stringToTime` and allocates no `ParsePosition`; the `Iso8601` path
allocates only a small
`ParsePosition` in place of the former exception and stack trace.
### Does this PR introduce _any_ user-facing change?
No. `parseOptional` accepts exactly what `parse` accepts (full-string
consumption), so inference
results are unchanged.
### How was this patch tested?
`TimeFormatterSuite` gains a `parseOptional` test (valid input returns
`Some`, invalid returns
`None`, prefix/partial match returns `None`). Existing inference tests pass
unchanged:
```
build/sbt 'catalyst/testOnly *TimeFormatterSuite *CSVInferSchemaSuite'
build/sbt 'sql/testOnly *JsonSuite *JsonV1Suite *JsonV2Suite *CSVSuite
*CSVv1Suite *CSVv2Suite *XmlSuite'
```
All 902 tests pass.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.8
Closes #58834 from david-mollitor-db/SPARK-59546.
Authored-by: david-mollitor-db <[email protected]>
Signed-off-by: Holden Karau <[email protected]>
---
.../spark/sql/catalyst/util/TimeFormatter.scala | 30 ++++++++++++++++++++++
.../spark/sql/catalyst/csv/CSVInferSchema.scala | 2 +-
.../spark/sql/catalyst/json/JsonInferSchema.scala | 2 +-
.../spark/sql/catalyst/xml/XmlInferSchema.scala | 2 +-
.../sql/catalyst/util/TimeFormatterSuite.scala | 17 ++++++++++++
5 files changed, 50 insertions(+), 3 deletions(-)
diff --git
a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimeFormatter.scala
b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimeFormatter.scala
index a8c5225353d7..2a6008766fba 100644
---
a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimeFormatter.scala
+++
b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/TimeFormatter.scala
@@ -17,16 +17,29 @@
package org.apache.spark.sql.catalyst.util
+import java.text.ParsePosition
import java.time.LocalTime
import java.time.format.DateTimeFormatter
import java.util.Locale
+import scala.util.control.NonFatal
+
import org.apache.spark.sql.catalyst.util.SparkDateTimeUtils._
import org.apache.spark.unsafe.types.UTF8String
sealed trait TimeFormatter extends Serializable {
def parse(s: String): Long // returns nanoseconds since midnight
+ // A non-throwing variant of `parse`, returning None instead of throwing
when the input cannot be
+ // parsed. Used by schema inference to probe the time type without paying an
exception per
+ // non-time value; subclasses override it with a more efficient,
exception-free implementation.
+ def parseOptional(s: String): Option[Long] =
+ try {
+ Some(parse(s))
+ } catch {
+ case _: Exception => None
+ }
+
def format(localTime: LocalTime): String
// Converts nanoseconds since the midnight to time string
def format(nanos: Long): String
@@ -50,6 +63,20 @@ class Iso8601TimeFormatter(pattern: String, locale: Locale,
isParsing: Boolean)
localTimeToNanos(localTime)
}
+ override def parseOptional(s: String): Option[Long] = {
+ try {
+ val parsePosition = new ParsePosition(0)
+ val parsed = formatter.parseUnresolved(s, parsePosition)
+ if (parsed != null && s.length == parsePosition.getIndex) {
+ Some(localTimeToNanos(toLocalTime(parsed)))
+ } else {
+ None
+ }
+ } catch {
+ case NonFatal(_) => None
+ }
+ }
+
override def format(localTime: LocalTime): String = {
localTime.format(formatter)
}
@@ -99,6 +126,9 @@ class DefaultTimeFormatter(locale: Locale, isParsing:
Boolean)
override def parse(s: String): Long = {
SparkDateTimeUtils.stringToTimeAnsi(UTF8String.fromString(s))
}
+
+ override def parseOptional(s: String): Option[Long] =
+ SparkDateTimeUtils.stringToTime(UTF8String.fromString(s))
}
object TimeFormatter {
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index 48871764aa45..301e501bf188 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -208,7 +208,7 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
}
private def tryParseTime(field: String): DataType = {
- if (isTimeTypeEnabled && (allCatch opt
timeFormatter.parse(field)).isDefined) {
+ if (isTimeTypeEnabled && timeFormatter.parseOptional(field).isDefined) {
TimeType(TimeType.DEFAULT_PRECISION)
} else {
tryParseDate(field)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index 9614747ffa70..bfc0c24d347c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -180,7 +180,7 @@ class JsonInferSchema(private val options: JSONOptions)
extends Serializable wit
decimalTry.get
} else if (options.inferTimestamp) {
if (isTimeTypeEnabled &&
- (allCatch opt timeFormatter.parse(field)).isDefined) {
+ timeFormatter.parseOptional(field).isDefined) {
TimeType(TimeType.DEFAULT_PRECISION)
// For text-based format, it's ambiguous to infer a timestamp string
without
// timezone, as it can be both TIMESTAMP LTZ and NTZ. To avoid
behavior changes
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
index 3c1334c4c215..23cdadeb64c3 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala
@@ -782,7 +782,7 @@ class XmlInferSchema(private val options: XmlOptions,
private val caseSensitive:
}
private def isTime(value: String): Boolean = {
- (allCatch opt timeFormatter.parse(value)).isDefined
+ timeFormatter.parseOptional(value).isDefined
}
/**
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimeFormatterSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimeFormatterSuite.scala
index e7e7136202e9..370e8d68ebc7 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimeFormatterSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimeFormatterSuite.scala
@@ -207,4 +207,21 @@ class TimeFormatterSuite extends SparkFunSuite with
SQLHelper {
"targetType" -> "\"TIME(6)\"",
"ansiConfig" -> "\"spark.sql.ansi.enabled\""))
}
+
+ test("parseOptional returns None instead of throwing on unparseable times") {
+ // The default (pattern-less) formatter delegates to stringToTime.
+ val defaultFormatter = new DefaultTimeFormatter(
+ locale = DateFormatter.defaultLocale,
+ isParsing = true)
+
assert(defaultFormatter.parseOptional("01:02:03").contains(defaultFormatter.parse("01:02:03")))
+ assert(defaultFormatter.parseOptional("x123").isEmpty)
+
+ // The Iso8601 formatter uses parseUnresolved and must consume the whole
input.
+ val isoFormatter = new Iso8601TimeFormatter("HH:mm:ss",
DateFormatter.defaultLocale,
+ isParsing = true)
+
assert(isoFormatter.parseOptional("01:02:03").contains(isoFormatter.parse("01:02:03")))
+ assert(isoFormatter.parseOptional("abc").isEmpty)
+ // A prefix/partial match must be rejected (full-string consumption).
+ assert(isoFormatter.parseOptional("01:02:03xxx").isEmpty)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]