uros-b commented on code in PR #56557:
URL: https://github.com/apache/spark/pull/56557#discussion_r3431857451
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala:
##########
@@ -196,13 +196,34 @@ object OrcUtils extends Logging {
requiredSchema: StructType,
orcSchema: TypeDescription,
conf: Configuration): Option[(Array[Int], Boolean)] = {
+ def isOrcTimestamp(dt: DataType): Boolean = dt match {
+ case TimestampType | TimestampNTZType | _: AnyTimestampNanoType => true
+ case _ => false
+ }
+
+ // The ORC reader does not coerce between timestamp families/precisions,
except between
+ // nanos timestamps of the same kind (NTZ or LTZ), which share an ORC
physical category and
+ // only differ by the precision applied on read. Any other mismatch (zone
or micros<->nanos)
+ // would otherwise fail obscurely, so reject it with a clear error.
+ def timestampReadCompatible(orcType: DataType, dataType: DataType):
Boolean =
+ (orcType, dataType) match {
+ case _ if orcType == dataType => true
+ case (_: TimestampNTZNanosType, _: TimestampNTZNanosType) => true
+ case (_: TimestampLTZNanosType, _: TimestampLTZNanosType) => true
+ case _ => false
+ }
+
def checkTimestampCompatibility(orcCatalystSchema: StructType, dataSchema:
StructType): Unit = {
orcCatalystSchema.fields.map(_.dataType).zip(dataSchema.fields.map(_.dataType)).foreach
{
case (TimestampType, TimestampNTZType) =>
throw
QueryExecutionErrors.cannotConvertOrcTimestampToTimestampNTZError()
case (TimestampNTZType, TimestampType) =>
throw
QueryExecutionErrors.cannotConvertOrcTimestampNTZToTimestampLTZError()
case (t1: StructType, t2: StructType) =>
checkTimestampCompatibility(t1, t2)
Review Comment:
One question regarding pre-existing behaviour: why do we not recurse into
ArrayType/MapType too? Does the checker look inside containers where timestamps
are nested inside array or map?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
