This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8049f3a533b2 [SPARK-58061][SQL] Add `variant_array_append` expression
8049f3a533b2 is described below
commit 8049f3a533b25e80d752a9595bbc7d4d9927d201
Author: bojana-db <[email protected]>
AuthorDate: Thu Jul 16 21:22:45 2026 +0800
[SPARK-58061][SQL] Add `variant_array_append` expression
### What changes were proposed in this pull request?
Adds the SQL function `variant_array_append(v, path, val)`, which appends a
value to the array in a Variant value at a single JSONPath location.
Details:
- Appends `val` as a new last element of the array at `path` (e.g. `$.a`,
`$[0]`, `$.a[1]`); the root path `$` targets the whole Variant, which must
itself be an array;
- A missing key or out-of-range array index along the path leaves `v`
unchanged;
- Any NULL argument returns NULL;
- The value may be any expression castable to variant (primitives, arrays,
or another variant; structs and maps are rejected). An array value is appended
as a single nested element rather than flattened, and a variant null appends a
null element;
- `VARIANT_PATH_TYPE_MISMATCH` is raised when a path segment is applied to
a value of an incompatible type or when the target is not an array; a malformed
path is rejected with `INVALID_VARIANT_PATH`, and results exceeding the size
limit raise `VARIANT_SIZE_LIMIT`.
### Why are the changes needed?
Without `variant_array_append`, appending to an array in a variant means
converting it to another datatype (e.g. array), mutating, and converting back.
### Does this PR introduce _any_ user-facing change?
Yes, a new SQL function (and Scala/Python `functions` API).
### How was this patch tested?
Unit tests.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code with Claude Opus 4.8
Closes #57156 from bojana-db/variant-array-insert.
Authored-by: bojana-db <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../apache/spark/types/variant/VariantBuilder.java | 83 +++++++++++++
.../source/reference/pyspark.sql/functions.rst | 1 +
python/pyspark/sql/connect/functions/builtin.py | 10 ++
python/pyspark/sql/functions/__init__.py | 1 +
python/pyspark/sql/functions/builtin.py | 58 +++++++++
python/pyspark/sql/tests/test_functions.py | 20 +++-
.../scala/org/apache/spark/sql/functions.scala | 37 ++++++
.../sql/catalyst/analysis/FunctionRegistry.scala | 1 +
.../variant/VariantExpressionEvalUtils.scala | 49 +++++++-
.../expressions/variant/variantExpressions.scala | 131 +++++++++++++++++++++
.../variant/VariantExpressionSuite.scala | 107 +++++++++++++++++
.../apache/spark/sql/PlanGenerationTestSuite.scala | 4 +
.../function_variant_array_append.explain | 2 +
.../queries/function_variant_array_append.json | 125 ++++++++++++++++++++
.../function_variant_array_append.proto.bin | Bin 0 -> 1157 bytes
.../sql-functions/sql-expression-schema.md | 3 +-
.../scala/org/apache/spark/sql/VariantSuite.scala | 60 ++++++++++
17 files changed, 686 insertions(+), 6 deletions(-)
diff --git
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
index fa23d610f239..296bc339ee08 100644
---
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
+++
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantBuilder.java
@@ -160,6 +160,17 @@ public class VariantBuilder {
return builder.result();
}
+ // Return a new variant with `val` appended to the array at `segments`,
where an empty `segments`
+ // refers to the whole variant. If the path does not match (missing key or
out-of-range index), a
+ // semantically equivalent variant is returned. A target that is not an
array, or a segment
+ // applied to an incompatible container type, throws
VariantPathTypeMismatchException, which the
+ // caller maps to VARIANT_PATH_TYPE_MISMATCH.
+ public static Variant arrayAppendAtPath(Variant v, PathSegment[] segments,
Variant val) {
+ VariantBuilder builder = new VariantBuilder(false);
+ builder.appendWithArrayAppendImpl(v.value, v.metadata, v.pos, segments, 0,
val);
+ return builder.result();
+ }
+
// Build the variant metadata from `dictionaryKeys` and return the variant
result.
public Variant result() {
int numKeys = dictionaryKeys.size();
@@ -779,6 +790,78 @@ public class VariantBuilder {
}
}
+ private void appendWithArrayAppendImpl(
+ byte[] value, byte[] metadata, int pos, PathSegment[] segments, int
depth, Variant val) {
+ checkIndex(pos, value.length);
+ int basicType = value[pos] & BASIC_TYPE_MASK;
+ if (depth == segments.length) {
+ // Reached the target: it must be an array, to which `val` is appended.
+ if (basicType != ARRAY) {
+ throw new VariantPathTypeMismatchException(depth);
+ }
+ handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ ArrayList<Integer> offsets = new ArrayList<>(size + 1);
+ int start = writePos;
+ for (int i = 0; i < size; ++i) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i,
offsetSize);
+ offsets.add(writePos - start);
+ appendVariantImpl(value, metadata, dataStart + offset);
+ }
+ offsets.add(writePos - start);
+ appendVariant(val);
+ finishWritingArray(start, offsets);
+ return null;
+ });
+ return;
+ }
+ PathSegment seg = segments[depth];
+ if (seg instanceof ObjectKeySegment && basicType == OBJECT) {
+ String key = ((ObjectKeySegment) seg).key;
+ handleObject(value, pos, (size, idSize, offsetSize, idStart,
offsetStart, dataStart) -> {
+ ArrayList<FieldEntry> fields = new ArrayList<>(size);
+ int start = writePos;
+ for (int i = 0; i < size; ++i) {
+ int id = readUnsigned(value, idStart + idSize * i, idSize);
+ int offset = readUnsigned(value, offsetStart + offsetSize * i,
offsetSize);
+ int elementPos = dataStart + offset;
+ String fieldKey = getMetadataKey(metadata, id);
+ int newId = addKey(fieldKey);
+ fields.add(new FieldEntry(fieldKey, newId, writePos - start));
+ if (fieldKey.equals(key)) {
+ appendWithArrayAppendImpl(value, metadata, elementPos, segments,
depth + 1, val);
+ } else {
+ appendVariantImpl(value, metadata, elementPos);
+ }
+ }
+ // A missing key is a no-op: the fields copied above already reproduce
the input object.
+ finishWritingObject(start, fields);
+ return null;
+ });
+ } else if (seg instanceof ArrayIndexSegment && basicType == ARRAY) {
+ int index = ((ArrayIndexSegment) seg).index;
+ handleArray(value, pos, (size, offsetSize, offsetStart, dataStart) -> {
+ ArrayList<Integer> offsets = new ArrayList<>(size);
+ int start = writePos;
+ for (int i = 0; i < size; ++i) {
+ int offset = readUnsigned(value, offsetStart + offsetSize * i,
offsetSize);
+ int elementPos = dataStart + offset;
+ offsets.add(writePos - start);
+ if (i == index) {
+ appendWithArrayAppendImpl(value, metadata, elementPos, segments,
depth + 1, val);
+ } else {
+ appendVariantImpl(value, metadata, elementPos);
+ }
+ }
+ // An out-of-range index is a no-op: the elements copied above
reproduce the input array.
+ finishWritingArray(start, offsets);
+ return null;
+ });
+ } else {
+ // The segment kind does not match the container at this path prefix.
+ throw new VariantPathTypeMismatchException(depth);
+ }
+ }
+
// Build a fresh chain of containers for `segments[depth..]`, terminating in
`val`. Used to
// materialize missing intermediate path segments during insertion. The kind
of each segment
// decides the container created: an object-key segment creates a
single-field object, while an
diff --git a/python/docs/source/reference/pyspark.sql/functions.rst
b/python/docs/source/reference/pyspark.sql/functions.rst
index 0f34809c0067..2ccfde351d87 100644
--- a/python/docs/source/reference/pyspark.sql/functions.rst
+++ b/python/docs/source/reference/pyspark.sql/functions.rst
@@ -602,6 +602,7 @@ VARIANT Functions
schema_of_variant
schema_of_variant_agg
try_variant_get
+ variant_array_append
variant_delete
variant_get
variant_insert
diff --git a/python/pyspark/sql/connect/functions/builtin.py
b/python/pyspark/sql/connect/functions/builtin.py
index 1b88329583e2..1caa3a9bf7f2 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -2252,6 +2252,16 @@ def variant_set(
variant_set.__doc__ = pysparkfuncs.variant_set.__doc__
+def variant_array_append(
+ v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName"
+) -> Column:
+ path_col = path if isinstance(path, Column) else lit(path)
+ return _invoke_function("variant_array_append", _to_col(v), path_col,
_to_col(value))
+
+
+variant_array_append.__doc__ = pysparkfuncs.variant_array_append.__doc__
+
+
def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str)
-> Column:
assert isinstance(path, (Column, str))
if isinstance(path, str):
diff --git a/python/pyspark/sql/functions/__init__.py
b/python/pyspark/sql/functions/__init__.py
index cc1433bbb916..b3da0adee990 100644
--- a/python/pyspark/sql/functions/__init__.py
+++ b/python/pyspark/sql/functions/__init__.py
@@ -480,6 +480,7 @@ __all__ = [ # noqa: F405
"schema_of_variant",
"schema_of_variant_agg",
"try_variant_get",
+ "variant_array_append",
"variant_delete",
"variant_get",
"variant_insert",
diff --git a/python/pyspark/sql/functions/builtin.py
b/python/pyspark/sql/functions/builtin.py
index cee894d4ed55..2c2345c1a58a 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -21911,6 +21911,64 @@ def variant_set(
)
+@_try_remote_functions
+def variant_array_append(
+ v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName"
+) -> Column:
+ """
+ Appends a value to the array in a variant at the given JSONPath location.
Returns the variant
+ unchanged if a path key or index is absent, throws an error if a path
segment hits a value of
+ an incompatible type or the target is not an array, and returns NULL if
any argument is NULL.
+
+ .. versionadded:: 4.3.0
+
+ Parameters
+ ----------
+ v : :class:`~pyspark.sql.Column` or str
+ a variant column or column name
+ path : :class:`~pyspark.sql.Column` or str
+ the JSONPath target array. A `str` is a literal path; a
:class:`~pyspark.sql.Column`
+ supplies the path at runtime. A valid path should start with `$` and
is followed by zero or
+ more segments like `[123]`, `.name`, `['name']`, or `["name"]`.
+ value : :class:`~pyspark.sql.Column` or str
+ the value to append. Any expression castable to variant.
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ a variant column with `value` appended to the array at `path`
+
+ Examples
+ --------
+ >>> from pyspark.sql.functions import lit, parse_json, to_json,
variant_array_append
+ >>> df = spark.createDataFrame([{
+ ... 'json': '''[[1, 2], 5]''',
+ ... 'path': '$[0]'
+ ... }])
+ >>> v = parse_json(df.json)
+ >>> df.select(to_json(variant_array_append(v, "$",
lit(3))).alias("r")).collect()
+ [Row(r='[[1,2],5,3]')]
+ >>> df.select(to_json(variant_array_append(v, "$[5]",
lit(3))).alias("r")).collect()
+ [Row(r='[[1,2],5]')]
+ >>> df.select(to_json(variant_array_append(v, df.path,
lit(9))).alias("r")).collect()
+ [Row(r='[[1,2,9],5]')]
+ >>> nested = variant_array_append(v, "$", parse_json(lit('[4, 5]')))
+ >>> df.select(to_json(nested).alias("r")).collect()
+ [Row(r='[[1,2],5,[4,5]]')]
+ >>> df.select(variant_array_append(v, "$", lit(None)).alias("r")).collect()
+ [Row(r=None)]
+ """
+ from pyspark.sql.classic.column import _to_java_column
+
+ path_col = path if isinstance(path, Column) else lit(path)
+ return _invoke_function(
+ "variant_array_append",
+ _to_java_column(v),
+ _to_java_column(path_col),
+ _to_java_column(value),
+ )
+
+
@_try_remote_functions
def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str)
-> Column:
"""
diff --git a/python/pyspark/sql/tests/test_functions.py
b/python/pyspark/sql/tests/test_functions.py
index f168b30c48ca..6367ebfdd9bf 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -3508,8 +3508,14 @@ class FunctionsTestsMixin:
def test_variant_expressions(self):
df = self.spark.createDataFrame(
[
- Row(json="""{ "a" : 1 }""", path="$.a", newpath="$.z"),
- Row(json="""{ "b" : 2 }""", path="$.b", newpath="$.z"),
+ Row(json="""{ "a" : 1 }""", path="$.a", newpath="$.z",
arr="[1, 2]", arrpath="$"),
+ Row(
+ json="""{ "b" : 2 }""",
+ path="$.b",
+ newpath="$.z",
+ arr="[[3], 4]",
+ arrpath="$[0]",
+ ),
]
)
v = F.parse_json(df.json)
@@ -3553,6 +3559,16 @@ class FunctionsTestsMixin:
df.select(F.to_json(F.variant_set(v, df.newpath, F.lit(9)))),
['{"a":1,"z":9}', '{"b":2,"z":9}'],
)
+ arr = F.parse_json(df.arr)
+ check(
+ df.select(F.to_json(F.variant_array_append(arr, "$", F.lit(9)))),
+ ["[1,2,9]", "[[3],4,9]"],
+ )
+ check(df.select(F.to_json(F.variant_array_append(arr, "$",
F.lit(None)))), [None, None])
+ check(
+ df.select(F.to_json(F.variant_array_append(arr, df.arrpath,
F.lit(9)))),
+ ["[1,2,9]", "[[3,9],4]"],
+ )
check(df.select(F.schema_of_variant(v)), ["OBJECT<a: BIGINT>",
"OBJECT<b: BIGINT>"])
check(df.select(F.schema_of_variant_agg(v)), ["OBJECT<a: BIGINT, b:
BIGINT>"])
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 71c77152beb0..f602c64a6248 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -9987,6 +9987,43 @@ object functions {
def variant_set(v: Column, path: String, value: Column, createIfMissing:
Boolean): Column =
Column.fn("variant_set", v, lit(path), value, lit(createIfMissing))
+ /**
+ * Appends a value to the array in a variant at the given JSONPath location.
Returns the variant
+ * unchanged if a path key or index is absent, throws an error if a path
segment hits a value of
+ * an incompatible type or the target is not an array, and returns NULL if
any argument is NULL.
+ *
+ * @param v
+ * a variant column.
+ * @param path
+ * the column containing the JSONPath string identifying the target array.
A valid path should
+ * start with `$` and is followed by zero or more segments like `[123]`,
`.name`, `['name']`,
+ * or `["name"]`.
+ * @param value
+ * the value to append. Any expression castable to variant.
+ * @group variant_funcs
+ * @since 4.3.0
+ */
+ def variant_array_append(v: Column, path: Column, value: Column): Column =
+ Column.fn("variant_array_append", v, path, value)
+
+ /**
+ * Appends a value to the array in a variant at the given JSONPath location.
Returns the variant
+ * unchanged if a path key or index is absent, throws an error if a path
segment hits a value of
+ * an incompatible type or the target is not an array, and returns NULL if
any argument is NULL.
+ *
+ * @param v
+ * a variant column.
+ * @param path
+ * the JSONPath identifying the target array. A valid path should start
with `$` and is
+ * followed by zero or more segments like `[123]`, `.name`, `['name']`, or
`["name"]`.
+ * @param value
+ * the value to append. Any expression castable to variant.
+ * @group variant_funcs
+ * @since 4.3.0
+ */
+ def variant_array_append(v: Column, path: String, value: Column): Column =
+ Column.fn("variant_array_append", v, lit(path), value)
+
/**
* Extracts a sub-variant from `v` according to `path` string, and then cast
the sub-variant to
* `targetType`. Returns null if the path does not exist. Throws an
exception if the cast fails.
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 9f818c109a59..a9be95d7b951 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -989,6 +989,7 @@ object FunctionRegistry {
expressionBuilder("variant_insert", VariantInsertExpressionBuilder),
expressionBuilder("try_variant_insert", TryVariantInsertExpressionBuilder),
expressionBuilder("variant_set", VariantSetExpressionBuilder),
+ expression[VariantArrayAppend]("variant_array_append"),
// Spatial
expression[ST_AsBinary]("st_asbinary"),
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala
index 47c555c1e691..ba5d72c56474 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala
@@ -78,13 +78,17 @@ object VariantExpressionEvalUtils {
/**
* Parse a JSONPath for a variant manipulation function. Throws
`INVALID_VARIANT_PATH` on a
- * malformed path or on the empty (root `$`) path.
+ * malformed path, or on the empty (root `$`) path unless `allowRoot` is set
(as
+ * `variant_array_append` does, since `$` refers to the root array there).
*/
- def parseVariantPath(pathValue: String, functionName: String):
Array[VariantPathSegment] = {
+ def parseVariantPath(
+ pathValue: String,
+ functionName: String,
+ allowRoot: Boolean = false): Array[VariantPathSegment] = {
val parsed = VariantPathParser.parse(pathValue).getOrElse {
throw QueryExecutionErrors.invalidVariantPath(pathValue, functionName)
}
- if (parsed.isEmpty) {
+ if (!allowRoot && parsed.isEmpty) {
throw QueryExecutionErrors.invalidVariantPath(pathValue, functionName)
}
parsed
@@ -216,6 +220,45 @@ object VariantExpressionEvalUtils {
setAtPath(input, javaSegments, pathStr, value, valueDataType,
createIfMissing, functionName)
}
+ /**
+ * Append `value` to the array in `input` at `javaSegments`. `path` is the
source string used in
+ * error messages. `value` is cast to a variant first; a size overflow maps
to
+ * `VARIANT_SIZE_LIMIT` and a target that is not an array (or an
incompatible path segment) maps
+ * to `VARIANT_PATH_TYPE_MISMATCH`.
+ */
+ def arrayAppendAtPath(
+ input: VariantVal,
+ javaSegments: Array[VariantBuilder.PathSegment],
+ path: String,
+ value: Any,
+ valueDataType: DataType,
+ functionName: String): VariantVal = {
+ val v = new Variant(input.getValue, input.getMetadata)
+ try {
+ val valVal = castToVariant(value, valueDataType)
+ val valVariant = new Variant(valVal.getValue, valVal.getMetadata)
+ val out = VariantBuilder.arrayAppendAtPath(v, javaSegments, valVariant)
+ new VariantVal(out.getValue, out.getMetadata)
+ } catch {
+ case e: VariantPathTypeMismatchException =>
+ throw QueryExecutionErrors.variantPathTypeMismatch(
+ path, renderVariantPath(javaSegments.take(e.depth)), functionName)
+ case _: VariantSizeLimitException =>
+ throw
QueryExecutionErrors.variantSizeLimitError(VariantUtil.SIZE_LIMIT, functionName)
+ }
+ }
+
+ def arrayAppendAtPath(
+ input: VariantVal,
+ path: UTF8String,
+ value: Any,
+ valueDataType: DataType,
+ functionName: String): VariantVal = {
+ val pathStr = path.toString
+ val javaSegments = toJavaSegments(parseVariantPath(pathStr, functionName,
allowRoot = true))
+ arrayAppendAtPath(input, javaSegments, pathStr, value, valueDataType,
functionName)
+ }
+
/** Cast a Spark value from `dataType` into the variant type. */
def castToVariant(input: Any, dataType: DataType): VariantVal = {
// Enforce strict check because it is illegal for input struct/map/variant
to contain duplicate
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index 41552cb2d383..add9ef79207e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -1182,6 +1182,137 @@ object VariantSetExpressionBuilder extends
ExpressionBuilder {
}
}
+// scalastyle:off line.size.limit
+@ExpressionDescription(
+ usage = "_FUNC_(v, path, val) - Appends a value to the array in a variant at
the given JSONPath " +
+ "location. Returns the variant unchanged if a path key or index is absent.
Throws an error " +
+ "if a path segment hits a value of an incompatible type or the target is
not an array. " +
+ "Returns NULL if any argument is NULL.",
+ arguments = """
+ Arguments:
+ * v - A variant value to mutate.
+ * path - A string expression evaluating to a JSONPath identifying the
target array. A valid
+ path should start with `$` and is followed by zero or more segments
like `[123]`, `.name`,
+ `['name']`, or `["name"]`.
+ * val - Any expression castable to variant.
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_(parse_json('[1, 2, 3]'), '$', 4);
+ [1,2,3,4]
+ > SELECT _FUNC_(parse_json('{"a": [1, 2]}'), '$.a', 3);
+ {"a":[1,2,3]}
+ > SELECT _FUNC_(parse_json('[1]'), '$', parse_json('[2, 3]'));
+ [1,[2,3]]
+ > SELECT _FUNC_(parse_json('{"a": 1}'), '$.missing', 2);
+ {"a":1}
+ > SELECT _FUNC_(parse_json('[1, 2]'), '$', parse_json('null'));
+ [1,2,null]
+ > SELECT _FUNC_(parse_json('[1, 2]'), '$', null);
+ NULL
+ """,
+ since = "4.3.0",
+ group = "variant_funcs"
+)
+// scalastyle:on line.size.limit
+case class VariantArrayAppend(input: Expression, path: Expression, value:
Expression)
+ extends TernaryExpression
+ with ExpectsInputTypes
+ with QueryErrorsBase {
+
+ override def first: Expression = input
+ override def second: Expression = path
+ override def third: Expression = value
+
+ override def nullIntolerant: Boolean = true
+
+ override def dataType: DataType = VariantType
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(VariantType, StringTypeWithCollation(supportsTrimCollation = true),
AnyDataType)
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ val result = super.checkInputDataTypes()
+ if (result.isFailure) {
+ result
+ } else if (value.dataType == NullType) {
+ TypeCheckResult.TypeCheckSuccess
+ } else if (!VariantGet.checkDataType(value.dataType, allowStructsAndMaps =
false)) {
+ DataTypeMismatch(
+ errorSubClass = "CAST_WITHOUT_SUGGESTION",
+ messageParameters =
+ Map("srcType" -> toSQLType(value.dataType), "targetType" ->
toSQLType(VariantType)))
+ } else {
+ TypeCheckResult.TypeCheckSuccess
+ }
+ }
+
+ // When the path is a foldable expression, parse it once at planning time
and cache it. `None`
+ // means the path is dynamic (or a foldable NULL, which makes the whole
expression NULL
+ // and is never evaluated).
+ @transient private lazy val foldablePath:
Option[VariantArrayAppend.ParsedAppendPath] = {
+ if (path.foldable) {
+ val p = path.eval()
+ if (p == null) {
+ None
+ } else {
+ val s = p.asInstanceOf[UTF8String].toString
+ Some(VariantArrayAppend.ParsedAppendPath(
+ VariantExpressionEvalUtils.parseVariantPath(s, prettyName, allowRoot
= true), s))
+ }
+ } else {
+ None
+ }
+ }
+
+ override protected def nullSafeEval(v: Any, p: Any, valValue: Any): Any = {
+ val inputVariant = v.asInstanceOf[VariantVal]
+ foldablePath match {
+ case Some(parsed) =>
+ VariantExpressionEvalUtils.arrayAppendAtPath(
+ inputVariant, parsed.javaSegments, parsed.pathStr, valValue,
value.dataType, prettyName)
+ case None =>
+ VariantExpressionEvalUtils.arrayAppendAtPath(
+ inputVariant, p.asInstanceOf[UTF8String], valValue, value.dataType,
prettyName)
+ }
+ }
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode = {
+ val cls = VariantExpressionEvalUtils.getClass.getName.stripSuffix("$")
+ nullSafeCodeGen(ctx, ev, (vVal, pVal, valVal) => {
+ val fromArg = ctx.addReferenceObj("from", value.dataType)
+ foldablePath match {
+ case Some(parsed) =>
+ val parsedArg = ctx.addReferenceObj("appendPath", parsed)
+ s"""
+ |${ev.value} = $cls.arrayAppendAtPath(
+ | $vVal, $parsedArg.javaSegments(), $parsedArg.pathStr(),
$valVal, $fromArg,
+ | "$prettyName");
+ """.stripMargin
+ case None =>
+ s"""
+ |${ev.value} = $cls.arrayAppendAtPath($vVal, $pVal, $valVal,
$fromArg, "$prettyName");
+ """.stripMargin
+ }
+ })
+ }
+
+ override def prettyName: String = "variant_array_append"
+
+ override protected def withNewChildrenInternal(
+ newFirst: Expression, newSecond: Expression, newThird: Expression):
VariantArrayAppend =
+ copy(input = newFirst, path = newSecond, value = newThird)
+}
+
+object VariantArrayAppend {
+ // Caches a foldable path. `VariantBuilder.PathSegment` is not
`Serializable`, so the Java form is
+ // `@transient` and re-derived once per executor task after deserialization.
`pathStr` is the
+ // source string, retained for error messages.
+ case class ParsedAppendPath(segments: Array[VariantPathSegment], pathStr:
String) {
+ @transient lazy val javaSegments: Array[VariantBuilder.PathSegment] =
+ VariantExpressionEvalUtils.toJavaSegments(segments)
+ }
+}
+
case class VariantExplode(child: Expression) extends UnaryExpression with
Generator
with ExpectsInputTypes {
override def inputTypes: Seq[AbstractDataType] = Seq(VariantType)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index 1f872d7f17ef..c5488e1f52ff 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -1654,4 +1654,111 @@ class VariantExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper {
"VARIANT_SIZE_LIMIT",
Map("sizeLimit" -> "16.0 MiB", "functionName" -> "`variant_set`"))
}
+
+ test("variant_array_append") {
+ def checkAppend(input: String, path: String, value: Expression, expected:
String): Unit = {
+ val expr = VariantArrayAppend(
+ Literal(parseJson(input)), Literal.create(path, StringType), value)
+ checkEvaluation(
+ ResolveTimeZone.resolveTimeZones(Cast(expr, StringType)),
+ expected)
+ }
+
+ // Append to the root array, and to nested arrays reached through object
keys / array indices.
+ checkAppend("[1, 2, 3]", "$", Literal(4), "[1,2,3,4]")
+ checkAppend("[]", "$", Literal("a"), """["a"]""")
+ checkAppend("""{"a": [1, 2]}""", "$.a", Literal(3), """{"a":[1,2,3]}""")
+ checkAppend("""{"a": [1], "b": 2}""", "$.a", Literal(9),
"""{"a":[1,9],"b":2}""")
+ checkAppend("""{"a": [[0], [1, 2]]}""", "$.a[1]", Literal(9),
"""{"a":[[0],[1,2,9]]}""")
+
+ // Non-numeric scalar values (boolean, floating point).
+ checkAppend("[1]", "$", Literal(true), "[1,true]")
+ checkAppend("[1]", "$", Literal(2.5), "[1,2.5]")
+
+ // An array value is cast to a variant and appended as a single element.
+ checkAppend("[1]", "$", Literal.create(Array(2, 3),
ArrayType(IntegerType)), "[1,[2,3]]")
+ checkAppend("[1, 2]", "$", Literal(parseJson("null")), "[1,2,null]")
+ // Strings are stored verbatim; use parse_json for structured JSON.
+ checkAppend("[]", "$", Literal("""{"x":1}"""), """["{\"x\":1}"]""")
+ checkAppend("[]", "$", Literal(parseJson("""{"x":1}""")), """[{"x":1}]""")
+
+ // scalastyle:off nonascii
+ checkAppend("""{"你好": [1]}""", """$['你好']""", Literal(2),
"""{"你好":[1,2]}""")
+ checkAppend("[]", "$", Literal("café"), """["café"]""")
+ // scalastyle:on nonascii
+
+ // A missing key or out-of-range index leaves the variant unchanged.
+ checkAppend("""{"a": [1]}""", "$.missing", Literal(2), """{"a":[1]}""")
+ checkAppend("[[1]]", "$[5]", Literal(2), "[[1]]")
+
+ // NULL-intolerant: any NULL argument yields NULL.
+ checkEvaluation(
+ VariantArrayAppend(Literal.create(null, VariantType), Literal("$"),
Literal(1)),
+ null)
+ checkAppend("[1]", null, Literal(1), null)
+ checkAppend("[1]", "$", Literal.create(null, VariantType), null)
+ checkAppend("[1]", "$", Literal.create(null, NullType), null)
+
+ // Dynamic (non-foldable) path.
+ val dynamic = VariantArrayAppend(
+ Literal(parseJson("""{"a": [1, 2]}""")),
+ BoundReference(0, StringType, nullable = true),
+ Literal(3))
+ checkEvaluation(
+ ResolveTimeZone.resolveTimeZones(Cast(dynamic, StringType)),
+ """{"a":[1,2,3]}""",
+ InternalRow(UTF8String.fromString("$.a")))
+
+ // The target is not an array (a scalar leaf, or an object at the root).
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("""{"a": 1}""")), Literal("$.a"),
Literal(2)),
+ "VARIANT_PATH_TYPE_MISMATCH",
+ Map("path" -> "$.a", "failedAt" -> "$.a", "functionName" ->
"`variant_array_append`"))
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("""{"a": 1}""")), Literal("$"),
Literal(2)),
+ "VARIANT_PATH_TYPE_MISMATCH",
+ Map("path" -> "$", "failedAt" -> "$", "functionName" ->
"`variant_array_append`"))
+
+ // A segment applied to an incompatible container: descending into a
scalar, an array index on
+ // an object, or an object key on an array.
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("""{"a": 1}""")), Literal("$.a.b"),
Literal(2)),
+ "VARIANT_PATH_TYPE_MISMATCH",
+ Map("path" -> "$.a.b", "failedAt" -> "$.a", "functionName" ->
"`variant_array_append`"))
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("""{"a": 1}""")), Literal("$[0]"),
Literal(2)),
+ "VARIANT_PATH_TYPE_MISMATCH",
+ Map("path" -> "$[0]", "failedAt" -> "$", "functionName" ->
"`variant_array_append`"))
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("[1, 2]")), Literal("$.a"),
Literal(2)),
+ "VARIANT_PATH_TYPE_MISMATCH",
+ Map("path" -> "$.a", "failedAt" -> "$", "functionName" ->
"`variant_array_append`"))
+
+ // Structs and maps are rejected at analysis.
+ Seq(
+ Literal.create(null, MapType(StringType, IntegerType)),
+ Literal.create(null, StructType(Seq(StructField("x", IntegerType))))
+ ).foreach { v =>
+ assert(
+ VariantArrayAppend(Literal(parseJson("[]")), Literal("$"),
v).checkInputDataTypes()
+ .isFailure)
+ }
+
+ // The root `$` is a valid target here, but a malformed path is still
rejected.
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("[]")), Literal("abc"), Literal(1)),
+ "INVALID_VARIANT_PATH",
+ Map("path" -> "abc", "functionName" -> "`variant_array_append`"))
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("[]")), Literal(""), Literal(1)),
+ "INVALID_VARIANT_PATH",
+ Map("path" -> "", "functionName" -> "`variant_array_append`"))
+
+ // Appending a value that overflows the variant size limit surfaces as
VARIANT_SIZE_LIMIT.
+ val tooBig = "x".repeat(16 * 1024 * 1024)
+ checkErrorInExpression[SparkRuntimeException](
+ VariantArrayAppend(Literal(parseJson("[]")), Literal("$"),
Literal(tooBig)),
+ "VARIANT_SIZE_LIMIT",
+ Map("sizeLimit" -> "16.0 MiB", "functionName" ->
"`variant_array_append`"))
+ }
}
diff --git
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index 747e348891bc..77ec3eacc510 100644
---
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -2779,6 +2779,10 @@ class PlanGenerationTestSuite extends ConnectFunSuite
with Logging {
fn.variant_set(fn.parse_json(fn.col("g")), "$.a", fn.lit(1), false)
}
+ functionTest("variant_array_append") {
+ fn.variant_array_append(fn.parse_json(fn.col("g")), "$.a", fn.lit(1))
+ }
+
functionTest("variant_get") {
fn.variant_get(fn.parse_json(fn.col("g")), "$", "int")
}
diff --git
a/sql/connect/common/src/test/resources/query-tests/explain-results/function_variant_array_append.explain
b/sql/connect/common/src/test/resources/query-tests/explain-results/function_variant_array_append.explain
new file mode 100644
index 000000000000..4ebc4a11af67
--- /dev/null
+++
b/sql/connect/common/src/test/resources/query-tests/explain-results/function_variant_array_append.explain
@@ -0,0 +1,2 @@
+Project
[variant_array_append(static_invoke(VariantExpressionEvalUtils.parseJson(g#0,
false, true, true)), $.a, 1) AS variant_array_append(parse_json(g), $.a, 1)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/sql/connect/common/src/test/resources/query-tests/queries/function_variant_array_append.json
b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_array_append.json
new file mode 100644
index 000000000000..73ee8457b781
--- /dev/null
+++
b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_array_append.json
@@ -0,0 +1,125 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "variant_array_append",
+ "arguments": [{
+ "unresolvedFunction": {
+ "functionName": "parse_json",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "g"
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "col",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass":
"org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }],
+ "isInternal": false
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "parse_json",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass":
"org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }, {
+ "literal": {
+ "string": "$.a"
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "variant_array_append",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass":
"org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }, {
+ "literal": {
+ "integer": 1
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "lit",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass":
"org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }],
+ "isInternal": false
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "variant_array_append",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/sql/connect/common/src/test/resources/query-tests/queries/function_variant_array_append.proto.bin
b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_array_append.proto.bin
new file mode 100644
index 000000000000..ebdc87f5fa7f
Binary files /dev/null and
b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_array_append.proto.bin
differ
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index d999d9c96e79..481c17e12486 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -556,6 +556,7 @@
|
org.apache.spark.sql.catalyst.expressions.variant.TryParseJsonExpressionBuilder
| try_parse_json | SELECT try_parse_json('{"a":1,"b":0.8}') |
struct<try_parse_json({"a":1,"b":0.8}):variant> |
|
org.apache.spark.sql.catalyst.expressions.variant.TryVariantGetExpressionBuilder
| try_variant_get | SELECT try_variant_get(parse_json('{"a": 1}'), '$.a',
'int') | struct<try_variant_get(parse_json({"a": 1}), $.a):int> |
|
org.apache.spark.sql.catalyst.expressions.variant.TryVariantInsertExpressionBuilder
| try_variant_insert | SELECT try_variant_insert(parse_json('{"a": 1}'),
'$.b', 2) | struct<try_variant_insert(parse_json({"a": 1}), $.b, 2):variant> |
+| org.apache.spark.sql.catalyst.expressions.variant.VariantArrayAppend |
variant_array_append | SELECT variant_array_append(parse_json('[1, 2, 3]'),
'$', 4) | struct<variant_array_append(parse_json([1, 2, 3]), $, 4):variant> |
| org.apache.spark.sql.catalyst.expressions.variant.VariantDelete |
variant_delete | SELECT variant_delete(parse_json('{"a": 1, "b": 2, "c": 3,
"items": [1, 2, 3]}'), NULL, '$.a', '$.c') |
struct<variant_delete(parse_json({"a": 1, "b": 2, "c": 3, "items": [1, 2, 3]}),
NULL, $.a, $.c):variant> |
|
org.apache.spark.sql.catalyst.expressions.variant.VariantGetExpressionBuilder |
variant_get | SELECT variant_get(parse_json('{"a": 1}'), '$.a', 'int') |
struct<variant_get(parse_json({"a": 1}), $.a):int> |
|
org.apache.spark.sql.catalyst.expressions.variant.VariantInsertExpressionBuilder
| variant_insert | SELECT variant_insert(parse_json('{"a": 1}'), '$.b', 2) |
struct<variant_insert(parse_json({"a": 1}), $.b, 2):variant> |
@@ -568,4 +569,4 @@
| org.apache.spark.sql.catalyst.expressions.xml.XPathList | xpath | SELECT
xpath('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()') |
struct<xpath(<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>,
a/b/text()):array<string>> |
| org.apache.spark.sql.catalyst.expressions.xml.XPathLong | xpath_long |
SELECT xpath_long('<a><b>1</b><b>2</b></a>', 'sum(a/b)') |
struct<xpath_long(<a><b>1</b><b>2</b></a>, sum(a/b)):bigint> |
| org.apache.spark.sql.catalyst.expressions.xml.XPathShort | xpath_short |
SELECT xpath_short('<a><b>1</b><b>2</b></a>', 'sum(a/b)') |
struct<xpath_short(<a><b>1</b><b>2</b></a>, sum(a/b)):smallint> |
-| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string |
SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') |
struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
\ No newline at end of file
+| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string |
SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') |
struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
index 300479cf5d49..02f56c6d0892 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
@@ -521,6 +521,66 @@ class VariantSuite extends SharedSparkSession with
ExpressionEvalHelper {
}
}
+ test("variant_array_append with literal arguments") {
+ def rows(results: Any*): Seq[Row] = results.map(Row(_))
+
+ // A basic invocation, exercising the SQL parse/registration path end to
end.
+ checkAnswer(
+ sql("SELECT to_json(variant_array_append(parse_json('{\"a\": [1, 2]}'),
'$.a', 3))"),
+ rows("""{"a":[1,2,3]}"""))
+ // The root `$` targets the whole variant.
+ checkAnswer(
+ sql("SELECT to_json(variant_array_append(parse_json('[1, 2]'), '$',
3))"),
+ rows("[1,2,3]"))
+
+ // NULL-intolerant.
+ checkAnswer(
+ sql("SELECT to_json(variant_array_append(parse_json('[1]'), '$',
NULL))"),
+ rows(null))
+
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ sql("SELECT variant_array_append(parse_json('{\"a\": 1}'), '$.a',
2)").collect()
+ },
+ condition = "VARIANT_PATH_TYPE_MISMATCH",
+ parameters = Map(
+ "path" -> "$.a", "failedAt" -> "$.a", "functionName" ->
toSQLId("variant_array_append")))
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ sql("SELECT variant_array_append(parse_json('[]'), 'bad',
1)").collect()
+ },
+ condition = "INVALID_VARIANT_PATH",
+ parameters = Map("path" -> "bad", "functionName" ->
toSQLId("variant_array_append")))
+
+ // A raw struct/map value is not castable to variant and is rejected at
analysis.
+ assert(intercept[AnalysisException] {
+ sql("SELECT variant_array_append(parse_json('[]'), '$',
named_struct('x', 1))")
+ }.getCondition == "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION")
+ }
+
+ test("variant_array_append with dynamic arguments") {
+ def rows(results: Any*): Seq[Row] = results.map(Row(_))
+ Seq("CODEGEN_ONLY", "NO_CODEGEN").foreach { codegenMode =>
+ withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) {
+ val df = Seq(
+ ("""{"a": [1, 2]}""", "$.a", 3),
+ ("""[1, 2]""", "$", 9),
+ (null, "$", 2)
+ ).toDF("json", "path", "val")
+ val v = parse_json(col("json"))
+ val out = df.select(to_json(variant_array_append(v, col("path"),
col("val"))).alias("r"))
+ checkAnswer(out, rows("""{"a":[1,2,3]}""", "[1,2,9]", null))
+
+ // String-path overload of the DataFrame API (root array).
+ val arrDf = Seq(("[1, 2]", 3), (null, 3)).toDF("json", "val")
+ val arrV = parse_json(arrDf("json"))
+ checkAnswer(
+ arrDf.select(to_json(variant_array_append(arrV, "$",
col("val"))).alias("r")),
+ rows("[1,2,3]", null))
+ }
+ }
+ }
+
test("round trip tests") {
withSQLConf(SQLConf.VARIANT_INFER_SHREDDING_SCHEMA.key -> "false") {
val rand = new Random(42)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
