This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new e32d0e3bdd2b [SPARK-54427][SQL] Allow ColumnarRow to call `copy` with
variant types
e32d0e3bdd2b is described below
commit e32d0e3bdd2b1eb99017dadabbcf73ec7cbdbb51
Author: Richard Chen <[email protected]>
AuthorDate: Fri Nov 21 13:54:02 2025 -0800
[SPARK-54427][SQL] Allow ColumnarRow to call `copy` with variant types
### What changes were proposed in this pull request?
add a `Variant` matching `if` statement when trying to copy columnar row.
### Why are the changes needed?
currently, trying to copy a columnar row with a variant type will crash the
query.
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
added UT
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #53137 from richardc-db/fix_columnar_row_variant_copy.
Authored-by: Richard Chen <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 0d99d6d852cd859aaac8cbc0211093ccc9f98287)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../apache/spark/sql/vectorized/ColumnarRow.java | 2 ++
.../execution/vectorized/ColumnVectorSuite.scala | 39 +++++++++++++++++++++-
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
index d9e65afe1cb0..656c5f8a8f30 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarRow.java
@@ -91,6 +91,8 @@ public final class ColumnarRow extends InternalRow {
row.update(i, getArray(i).copy());
} else if (pdt instanceof PhysicalMapType) {
row.update(i, getMap(i).copy());
+ } else if (pdt instanceof PhysicalVariantType) {
+ row.update(i, getVariant(i));
} else {
throw new RuntimeException("Not implemented. " + dt);
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
index 966e892dc404..3be6c14eb4d3 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala
@@ -26,7 +26,7 @@ import
org.apache.spark.sql.execution.columnar.compression.ColumnBuilderHelper
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
import org.apache.spark.sql.vectorized.ColumnarArray
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.unsafe.types.{UTF8String, VariantVal}
import org.apache.spark.util.ArrayImplicits._
class ColumnVectorSuite extends SparkFunSuite with SQLHelper {
@@ -166,6 +166,43 @@ class ColumnVectorSuite extends SparkFunSuite with
SQLHelper {
}
}
+ testVectors("variant", 3, new StructType().add("v", VariantType)) {
structVector =>
+ val variantCol = structVector.getChild(0)
+ val valueChild = variantCol.getChild(0)
+ val metadataChild = variantCol.getChild(1)
+
+ variantCol.putNotNull(0)
+ valueChild.appendByteArray(Array[Byte](1, 2, 3), 0, 3)
+ metadataChild.appendByteArray(Array[Byte](10, 11), 0, 2)
+
+ variantCol.putNotNull(1)
+ valueChild.appendByteArray(Array[Byte](4, 5), 0, 2)
+ metadataChild.appendByteArray(Array[Byte](12, 13, 14), 0, 3)
+
+ variantCol.putNull(2)
+ valueChild.appendNull()
+ metadataChild.appendNull()
+
+ (0 until 3).foreach { i =>
+ val row = structVector.getStruct(i)
+ val rowCopy = row.copy()
+
+ if (i < 2) {
+ assert(!row.isNullAt(0))
+ assert(!rowCopy.isNullAt(0))
+
+ val originalVariant = row.get(0, VariantType).asInstanceOf[VariantVal]
+ val copiedVariant = rowCopy.get(0,
VariantType).asInstanceOf[VariantVal]
+
+ assert(java.util.Arrays.equals(originalVariant.getValue,
copiedVariant.getValue))
+ assert(java.util.Arrays.equals(originalVariant.getMetadata,
copiedVariant.getMetadata))
+ } else {
+ assert(row.isNullAt(0))
+ assert(rowCopy.isNullAt(0))
+ }
+ }
+ }
+
testVectors("float", 10, FloatType) { testVector =>
(0 until 10).foreach { i =>
testVector.appendFloat(i.toFloat)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
