This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 0d66f0649240 [SPARK-58166][SQL] Replace TypeTag on PhysicalDataType
with ClassTag
0d66f0649240 is described below
commit 0d66f0649240a0b8170ec7d309ce048fd91341ab
Author: Emil Ejbyfeldt <[email protected]>
AuthorDate: Sun Aug 23 13:41:05 2026 +0000
[SPARK-58166][SQL] Replace TypeTag on PhysicalDataType with ClassTag
### What changes were proposed in this pull request?
Replace the TypeTag in PhysicalDataType with a ClassTag.
### Why are the changes needed?
Because of this issue https://github.com/scala/scala3/issues/25896 the
change top make it possible to use Spark from Scala 3.8.0 or later.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests ensure no regressions. That it makes the code more
compatible with Scala 3.8 was done by running a local build against the tests
in https://github.com/unum-io/tyda (a Scala 3 library that can use Spark as a
backend).
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #57296 from eejbyfeldt/SPARK-58166.
Authored-by: Emil Ejbyfeldt <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
(cherry picked from commit 9da9f8d673914d1648514f59d85e3adafb300d1a)
Signed-off-by: Uros Bojanic <[email protected]>
---
.../expressions/collectionOperations.scala | 2 +-
.../sql/catalyst/types/PhysicalDataType.scala | 46 +++++++++++-----------
2 files changed, 24 insertions(+), 24 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 9120cfa5f275..8561ce3aae8e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -3631,7 +3631,7 @@ case class Sequence(
val physicalDataType = PhysicalDataType(iType)
type T = physicalDataType.InternalType
val integral = PhysicalIntegralType.integral(iType)
- val ct =
ClassTag[T](physicalDataType.tag.mirror.runtimeClass(physicalDataType.tag.tpe))
+ val ct = physicalDataType.tag
new IntegralSequenceImpl[T](iType)(ct,
integral.asInstanceOf[Integral[T]])
case TimestampType | TimestampNTZType =>
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
index 9acbf4be66f9..d77da0b159af 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/types/PhysicalDataType.scala
@@ -17,8 +17,8 @@
package org.apache.spark.sql.catalyst.types
-import scala.reflect.runtime.universe.TypeTag
-import scala.reflect.runtime.universe.typeTag
+import scala.reflect.ClassTag
+import scala.reflect.classTag
import org.apache.spark.sql.catalyst.expressions.{Ascending, BoundReference,
InterpretedOrdering, SortOrder}
import org.apache.spark.sql.catalyst.types.ops.TypeOps
@@ -31,7 +31,7 @@ import org.apache.spark.util.ArrayImplicits._
sealed abstract class PhysicalDataType {
private[sql] type InternalType
private[sql] def ordering: Ordering[InternalType]
- private[sql] val tag: TypeTag[InternalType]
+ private[sql] val tag: ClassTag[InternalType]
}
object PhysicalDataType {
@@ -133,7 +133,7 @@ class PhysicalBinaryType() extends PhysicalDataType {
(x: Array[Byte], y: Array[Byte]) => ByteArray.compareBinary(x, y)
private[sql] type InternalType = Array[Byte]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
case object PhysicalBinaryType extends PhysicalBinaryType
@@ -143,14 +143,14 @@ class PhysicalBooleanType extends PhysicalDataType with
PhysicalPrimitiveType {
// Defined with a private constructor so the companion object is the only
possible instantiation.
private[sql] type InternalType = Boolean
private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
case object PhysicalBooleanType extends PhysicalBooleanType
class PhysicalByteType() extends PhysicalIntegralType with
PhysicalPrimitiveType {
private[sql] type InternalType = Byte
private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Byte]]
override private[sql] val exactNumeric = ByteExactNumeric
private[sql] val integral = implicitly[Integral[Byte]]
@@ -162,7 +162,7 @@ class PhysicalCalendarIntervalType() extends
PhysicalDataType {
throw QueryExecutionErrors.orderedOperationUnsupportedByDataTypeError(
"PhysicalCalendarIntervalType")
override private[sql] type InternalType = Any
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
case object PhysicalCalendarIntervalType extends PhysicalCalendarIntervalType
@@ -180,7 +180,7 @@ case object PhysicalCalendarIntervalType extends
PhysicalCalendarIntervalType
class PhysicalTimestampNTZNanosType() extends PhysicalDataType {
override private[sql] type InternalType = TimestampNanosVal
override private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
case object PhysicalTimestampNTZNanosType extends PhysicalTimestampNTZNanosType
@@ -198,14 +198,14 @@ case object PhysicalTimestampNTZNanosType extends
PhysicalTimestampNTZNanosType
class PhysicalTimestampLTZNanosType() extends PhysicalDataType {
override private[sql] type InternalType = TimestampNanosVal
override private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
case object PhysicalTimestampLTZNanosType extends PhysicalTimestampLTZNanosType
case class PhysicalDecimalType(precision: Int, scale: Int) extends
PhysicalFractionalType {
private[sql] type InternalType = Decimal
private[sql] val ordering = Decimal.DecimalIsFractional
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] val numeric = Decimal.DecimalIsFractional
override private[sql] def exactNumeric = DecimalExactNumeric
private[sql] val fractional = Decimal.DecimalIsFractional
@@ -225,7 +225,7 @@ class PhysicalDoubleType() extends PhysicalFractionalType
with PhysicalPrimitive
private[sql] type InternalType = Double
private[sql] val ordering =
(x: Double, y: Double) => SQLOrderingUtil.compareDoubles(x, y)
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Double]]
override private[sql] def exactNumeric = DoubleExactNumeric
private[sql] val fractional = implicitly[Fractional[Double]]
@@ -240,7 +240,7 @@ class PhysicalFloatType() extends PhysicalFractionalType
with PhysicalPrimitiveT
private[sql] type InternalType = Float
private[sql] val ordering =
(x: Float, y: Float) => SQLOrderingUtil.compareFloats(x, y)
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Float]]
override private[sql] def exactNumeric = FloatExactNumeric
private[sql] val fractional = implicitly[Fractional[Float]]
@@ -254,7 +254,7 @@ class PhysicalIntegerType() extends PhysicalIntegralType
with PhysicalPrimitiveT
// Defined with a private constructor so the companion object is the only
possible instantiation.
private[sql] type InternalType = Int
private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Int]]
override private[sql] val exactNumeric = IntegerExactNumeric
private[sql] val integral = implicitly[Integral[Int]]
@@ -267,7 +267,7 @@ class PhysicalLongType() extends PhysicalIntegralType with
PhysicalPrimitiveType
// Defined with a private constructor so the companion object is the only
possible instantiation.
private[sql] type InternalType = Long
private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Long]]
override private[sql] val exactNumeric = LongExactNumeric
private[sql] val integral = implicitly[Integral[Long]]
@@ -279,7 +279,7 @@ case class PhysicalMapType(keyType: DataType, valueType:
DataType, valueContains
// maps are not orderable, we use `ordering` just to support group by queries
override private[sql] def ordering = interpretedOrdering
override private[sql] type InternalType = MapData
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
@transient
private[sql] lazy val interpretedOrdering: Ordering[MapData] = new
Ordering[MapData] {
@@ -348,14 +348,14 @@ class PhysicalNullType() extends PhysicalDataType with
PhysicalPrimitiveType {
override private[sql] def ordering =
implicitly[Ordering[Unit]].asInstanceOf[Ordering[Any]]
override private[sql] type InternalType = Any
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
case object PhysicalNullType extends PhysicalNullType
class PhysicalShortType() extends PhysicalIntegralType with
PhysicalPrimitiveType {
private[sql] type InternalType = Short
private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Short]]
override private[sql] val exactNumeric = ShortExactNumeric
private[sql] val integral = implicitly[Integral[Short]]
@@ -368,7 +368,7 @@ case class PhysicalStringType(collationId: Int) extends
PhysicalDataType {
// Defined with a private constructor so the companion object is the only
possible instantiation.
private[sql] type InternalType = UTF8String
private[sql] val ordering =
CollationFactory.fetchCollation(collationId).comparator.compare(_, _)
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
object PhysicalStringType {
def apply(collationId: Int): PhysicalStringType = new
PhysicalStringType(collationId)
@@ -378,7 +378,7 @@ case class PhysicalArrayType(
elementType: DataType, containsNull: Boolean) extends PhysicalDataType {
override private[sql] type InternalType = ArrayData
override private[sql] def ordering = interpretedOrdering
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
@transient
private[sql] lazy val interpretedOrdering: Ordering[ArrayData] = new
Ordering[ArrayData] {
@@ -425,7 +425,7 @@ case class PhysicalStructType(fields: Array[StructField])
extends PhysicalDataTy
override private[sql] type InternalType = Any
override private[sql] def ordering =
forSchema(this.fields.map(_.dataType).toImmutableArraySeq).asInstanceOf[Ordering[InternalType]]
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
private[sql] def forSchema(dataTypes: Seq[DataType]): InterpretedOrdering = {
new InterpretedOrdering(dataTypes.zipWithIndex.map {
@@ -436,7 +436,7 @@ case class PhysicalStructType(fields: Array[StructField])
extends PhysicalDataTy
class PhysicalVariantType extends PhysicalDataType {
private[sql] type InternalType = VariantVal
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
// TODO(SPARK-45891): Support comparison for the Variant type.
override private[sql] def ordering =
@@ -451,7 +451,7 @@ object UninitializedPhysicalType extends PhysicalDataType {
throw QueryExecutionErrors.orderedOperationUnsupportedByDataTypeError(
"UninitializedPhysicalType")
override private[sql] type InternalType = Any
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
// Physical type for opaque, variable-length byte payloads that are addressed
as a zero-copy
@@ -466,7 +466,7 @@ object UninitializedPhysicalType extends PhysicalDataType {
class PhysicalBinaryViewType extends PhysicalDataType {
private[sql] val ordering = (x: BinaryView, y: BinaryView) => x.compareTo(y)
private[sql] type InternalType = BinaryView
- @transient private[sql] lazy val tag = typeTag[InternalType]
+ @transient private[sql] lazy val tag = classTag[InternalType]
}
case object PhysicalBinaryViewType extends PhysicalBinaryViewType
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
