This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9e14596f47ba [SPARK-58788][SQL] Add the TRIM_ARRAY array function
9e14596f47ba is described below
commit 9e14596f47ba7b7909bcc0961fb27b72a7c3ffc1
Author: Vinod KC <[email protected]>
AuthorDate: Fri Aug 21 08:44:09 2026 +0800
[SPARK-58788][SQL] Add the TRIM_ARRAY array function
### What changes were proposed in this pull request?
This PR adds the ANSI SQL `TRIM_ARRAY(array, n)` function, which returns
the given array with the last n elements removed (i.e. it keeps the first
cardinality - n elements). Per the SQL standard, it raises an error when n is
negative or greater than the array cardinality.
```
SELECT trim_array(array(1, 2, 3, 4, 5), 2); -- [1, 2, 3]
SELECT trim_array(array('a', 'b', 'c'), 0); -- [a, b, c]
SELECT trim_array(array(1, 2, 3), 3); -- []
SELECT trim_array(array(1, 2, 3), -1); -- error
SELECT trim_array(array(1, 2, 3), 4); -- error (n > cardinality)
```
`TRIM_ARRAY` is a standard SQL function (present since SQL:2011) . It is
distinct from `slice`: `trim_array` drops a count of elements from the end and
enforces ANSI bounds semantics (erroring when n > cardinality), whereas `slice`
performs general start/length subsetting and silently clamps.
### Why are the changes needed?
The function is part of the SQL standard and is supported by PostgreSQL,
IBM DB2, Trino / Presto. Today users hand-roll `slice(a, 1, size(a) - n)`,
which has an off-by-one hazard and, unlike the standard, silently clamps (or
raises an unrelated "unexpected length" error) instead of failing cleanly when
n exceeds the array length.
### Does this PR introduce _any_ user-facing change?
Yes. It adds a new built-in function trim_array, available through SQL, the
Scala DataFrame API , PySpark (Classic and Spark Connect).
### How was this patch tested?
Added tests in `CollectionExpressionsSuite`, `PlanGenerationTestSuite`,
`FunctionTestSuite`
### Was this patch authored or co-authored using generative AI tooling?
Yes. Generated-by: Claude Code
Closes #58017 from vinodkc/fix-spark-trim-array.
Authored-by: Vinod KC <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../src/main/resources/error/error-conditions.json | 5 ++
.../source/reference/pyspark.sql/functions.rst | 1 +
python/pyspark/sql/connect/functions/builtin.py | 22 +++++
python/pyspark/sql/functions/__init__.py | 1 +
python/pyspark/sql/functions/builtin.py | 56 ++++++++++++
.../scala/org/apache/spark/sql/functions.scala | 34 +++++++
.../sql/catalyst/analysis/FunctionRegistry.scala | 1 +
.../expressions/collectionOperations.scala | 85 ++++++++++++++++++
.../spark/sql/errors/QueryExecutionErrors.scala | 11 +++
.../expressions/CollectionExpressionsSuite.scala | 39 ++++++++
.../apache/spark/sql/PlanGenerationTestSuite.scala | 4 +
.../spark/sql/connect/FunctionTestSuite.scala | 1 +
.../explain-results/function_trim_array.explain | 2 +
.../query-tests/queries/function_trim_array.json | 81 +++++++++++++++++
.../queries/function_trim_array.proto.bin | Bin 0 -> 736 bytes
.../sql-functions/sql-expression-schema.md | 1 +
.../sql-tests/analyzer-results/array.sql.out | 70 +++++++++++++++
.../analyzer-results/nonansi/array.sql.out | 70 +++++++++++++++
.../src/test/resources/sql-tests/inputs/array.sql | 14 +++
.../test/resources/sql-tests/results/array.sql.out | 100 +++++++++++++++++++++
.../sql-tests/results/nonansi/array.sql.out | 100 +++++++++++++++++++++
.../apache/spark/sql/CharVarcharTestSuite.scala | 2 +-
22 files changed, 699 insertions(+), 1 deletion(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json
b/common/utils/src/main/resources/error/error-conditions.json
index df8c07f38cb3..59da38fdc8dd 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -4846,6 +4846,11 @@
"expects one of the units 'HOUR', 'MINUTE', 'SECOND', 'MILLISECOND',
'MICROSECOND', but got '<invalidValue>'."
]
},
+ "TRIM_ARRAY_LENGTH" : {
+ "message" : [
+ "Expects a value between 0 and the array cardinality
(<numElements>), but got <length>."
+ ]
+ },
"ZERO_INDEX" : {
"message" : [
"expects %1$, %2$ and so on, but got %0$."
diff --git a/python/docs/source/reference/pyspark.sql/functions.rst
b/python/docs/source/reference/pyspark.sql/functions.rst
index 9ac05bd3b0ed..661fc086c4e5 100644
--- a/python/docs/source/reference/pyspark.sql/functions.rst
+++ b/python/docs/source/reference/pyspark.sql/functions.rst
@@ -413,6 +413,7 @@ Array Functions
shuffle
slice
sort_array
+ trim_array
Struct Functions
diff --git a/python/pyspark/sql/connect/functions/builtin.py
b/python/pyspark/sql/connect/functions/builtin.py
index 91fadcf056ab..2e696f344b12 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -2520,6 +2520,28 @@ def slice(
slice.__doc__ = pysparkfuncs.slice.__doc__
+def trim_array(x: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column:
+ n = _enum_to_value(n)
+ if isinstance(n, (Column, str)):
+ _n = n
+ elif isinstance(n, int):
+ _n = lit(n)
+ else:
+ raise PySparkTypeError(
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "n",
+ "arg_type": type(n).__name__,
+ },
+ )
+
+ return _invoke_function_over_columns("trim_array", x, _n)
+
+
+trim_array.__doc__ = pysparkfuncs.trim_array.__doc__
+
+
def sort_array(col: "ColumnOrName", asc: bool = True) -> Column:
return _invoke_function("sort_array", _to_col(col), lit(asc))
diff --git a/python/pyspark/sql/functions/__init__.py
b/python/pyspark/sql/functions/__init__.py
index a5a95196c1ad..a3c0b7016942 100644
--- a/python/pyspark/sql/functions/__init__.py
+++ b/python/pyspark/sql/functions/__init__.py
@@ -344,6 +344,7 @@ __all__ = [ # noqa: F405
"shuffle",
"slice",
"sort_array",
+ "trim_array",
# Struct Functions
"named_struct",
"struct",
diff --git a/python/pyspark/sql/functions/builtin.py
b/python/pyspark/sql/functions/builtin.py
index 9413f68726c7..1d2c6355e9e2 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -20454,6 +20454,62 @@ def slice(
return _invoke_function_over_columns("slice", x, start, length)
+@_try_remote_functions
+def trim_array(x: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column:
+ """
+ Array function: Returns the given array column with the last ``n``
elements removed.
+ Raises an error if ``n`` is negative or greater than the number of
elements in the array.
+
+ .. versionadded:: 4.4.0
+
+ Parameters
+ ----------
+ x : :class:`~pyspark.sql.Column` or str
+ Input array column or column name to be trimmed.
+ A column that evaluates to an array.
+ n : :class:`~pyspark.sql.Column`, str, or int
+ The number of elements to remove from the end of the array. Must be
between 0 and
+ the number of elements in the array (inclusive).
+ A column that evaluates to an integer.
+
+ Returns
+ -------
+ :class:`~pyspark.sql.Column`
+ A new Column object of Array type, where each value is the
corresponding input array
+ with its last ``n`` elements removed.
+ Returns a column that evaluates to an array.
+
+ Examples
+ --------
+ Example 1: Basic usage of the trim_array function.
+
+ >>> from pyspark.sql import functions as sf
+ >>> df = spark.createDataFrame([([1, 2, 3, 4, 5],), ([4, 5],)], ['x'])
+ >>> df.select(sf.trim_array(df.x, 2)).show()
+ +----------------+
+ |trim_array(x, 2)|
+ +----------------+
+ | [1, 2, 3]|
+ | []|
+ +----------------+
+
+ Example 2: trim_array function with a column input for n.
+
+ >>> from pyspark.sql import functions as sf
+ >>> df = spark.createDataFrame([([1, 2, 3, 4, 5], 1), ([4, 5], 0)], ['x',
'n'])
+ >>> df.select(sf.trim_array(df.x, df.n)).show()
+ +----------------+
+ |trim_array(x, n)|
+ +----------------+
+ | [1, 2, 3, 4]|
+ | [4, 5]|
+ +----------------+
+ """
+ n = _enum_to_value(n)
+ n = lit(n) if isinstance(n, int) else n
+ return _invoke_function_over_columns("trim_array", x, n)
+
+
@_try_remote_functions
def array_join(
col: "ColumnOrName", delimiter: str, null_replacement: Optional[str] = None
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index 636782a1333d..3ac36551b0ed 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -13234,6 +13234,40 @@ object functions {
def slice(x: Column, start: Column, length: Column): Column =
Column.fn("slice", x, start, length)
+ /**
+ * Returns the given array `x` with the last `n` elements removed. Raises an
error if `n` is
+ * negative or greater than the number of elements in the array.
+ *
+ * @param x
+ * the array column to be trimmed. A column that evaluates to an array.
+ * @param n
+ * the number of elements to remove from the end of the array. Must be
between 0 and the
+ * number of elements in the array (inclusive).
+ *
+ * @group array_funcs
+ * @since 4.4.0
+ * @return
+ * Returns a column that evaluates to an array.
+ */
+ def trim_array(x: Column, n: Int): Column = trim_array(x, lit(n))
+
+ /**
+ * Returns the given array `x` with the last `n` elements removed. Raises an
error if `n` is
+ * negative or greater than the number of elements in the array.
+ *
+ * @param x
+ * the array column to be trimmed. A column that evaluates to an array.
+ * @param n
+ * the number of elements to remove from the end of the array. Must be
between 0 and the
+ * number of elements in the array (inclusive).
+ *
+ * @group array_funcs
+ * @since 4.4.0
+ * @return
+ * Returns a column that evaluates to an array.
+ */
+ def trim_array(x: Column, n: Column): Column = Column.fn("trim_array", x, n)
+
/**
* Concatenates the elements of `column` using the `delimiter`. Null values
are replaced with
* `nullReplacement`.
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 30b65c7ee608..a23e70188622 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -813,6 +813,7 @@ object FunctionRegistry {
expression[MapConcat]("map_concat"),
expression[Size]("size"),
expression[Slice]("slice"),
+ expression[TrimArray]("trim_array"),
expression[Size]("cardinality", true, Some("2.4.0")),
expression[ArraysZip]("arrays_zip"),
expression[SortArray]("sort_array"),
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 4565c63e1922..0d598aecfe8e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -2230,6 +2230,91 @@ case class Slice(x: Expression, start: Expression,
length: Expression)
copy(x = newFirst, start = newSecond, length = newThird)
}
+/**
+ * Removes the last `n` elements from the given array, per the ANSI SQL
`TRIM_ARRAY` function.
+ */
+@ExpressionDescription(
+ usage = """
+ _FUNC_(array, n) - Returns the given array with the last `n` elements
removed. Raises an error
+ if `n` is negative or greater than the number of elements in the
array.""",
+ arguments = """
+ Arguments:
+ * array - the array to trim.
+ * n - the number of elements to remove from the end of the array. Must
be between 0 and the
+ number of elements in the array (inclusive).
+ """,
+ examples = """
+ Examples:
+ > SELECT _FUNC_(array(1, 2, 3, 4, 5), 2);
+ [1,2,3]
+ > SELECT _FUNC_(array('a', 'b', 'c'), 0);
+ ["a","b","c"]
+ > SELECT _FUNC_(array(1, 2, 3), 3);
+ []
+ """,
+ group = "array_funcs",
+ since = "4.4.0")
+case class TrimArray(left: Expression, right: Expression)
+ extends BinaryExpression with ImplicitCastInputTypes {
+ override def nullIntolerant: Boolean = true
+
+ override def prettyName: String = "trim_array"
+
+ override def dataType: DataType = left.dataType
+
+ private def resultArrayElementNullable =
dataType.asInstanceOf[ArrayType].containsNull
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, IntegerType)
+
+ @transient private lazy val elementType: DataType =
+ left.dataType.asInstanceOf[ArrayType].elementType
+
+ override def nullSafeEval(arrayVal: Any, nVal: Any): Any = {
+ val arr = arrayVal.asInstanceOf[ArrayData]
+ val n = nVal.asInstanceOf[Int]
+ val numElements = arr.numElements()
+ if (n < 0 || n > numElements) {
+ throw
QueryExecutionErrors.invalidElementCountForTrimArrayError(prettyName,
numElements, n)
+ }
+ val retainCount = numElements - n
+ val values = new Array[Any](retainCount)
+ for (i <- 0 until retainCount) {
+ if (!arr.isNullAt(i)) values(i) = arr.get(i, elementType)
+ }
+ new GenericArrayData(values)
+ }
+
+ override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ nullSafeCodeGen(ctx, ev, (array, n) => {
+ val numElements = ctx.freshName("numElements")
+ val resLength = ctx.freshName("resLength")
+ val values = ctx.freshName("values")
+ val i = ctx.freshName("i")
+ val allocation = CodeGenerator.createArrayData(
+ values, elementType, resLength, s" $prettyName failed.")
+ val assignment = CodeGenerator.createArrayAssignment(
+ values, elementType, array, i, i, resultArrayElementNullable)
+ s"""
+ |${CodeGenerator.JAVA_INT} $numElements = $array.numElements();
+ |if ($n < 0 || $n > $numElements) {
+ | throw QueryExecutionErrors.invalidElementCountForTrimArrayError(
+ | "$prettyName", $numElements, $n);
+ |}
+ |${CodeGenerator.JAVA_INT} $resLength = $numElements - $n;
+ |$allocation
+ |for (int $i = 0; $i < $resLength; $i ++) {
+ | $assignment
+ |}
+ |${ev.value} = $values;
+ """.stripMargin
+ })
+ }
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression, newRight: Expression): TrimArray =
+ copy(left = newLeft, right = newRight)
+}
+
/**
* Creates a String containing all the elements of the input array separated
by the delimiter.
*/
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index 191b7880a97c..a17b1ca7e9bb 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -1475,6 +1475,17 @@ private[sql] object QueryExecutionErrors extends
QueryErrorsBase with ExecutionE
"functionName" -> toSQLId(prettyName)))
}
+ def invalidElementCountForTrimArrayError(
+ prettyName: String, numElements: Int, length: Int):
SparkRuntimeException = {
+ new SparkRuntimeException(
+ errorClass = "INVALID_PARAMETER_VALUE.TRIM_ARRAY_LENGTH",
+ messageParameters = Map(
+ "parameter" -> toSQLId("n"),
+ "functionName" -> toSQLId(prettyName),
+ "numElements" -> numElements.toString,
+ "length" -> length.toString))
+ }
+
def invalidIndexOfZeroError(context: QueryContext): RuntimeException = {
new SparkRuntimeException(
errorClass = "INVALID_INDEX_OF_ZERO",
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 2ea0f3d8cdf4..f26285fb2d16 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -946,6 +946,45 @@ class CollectionExpressionsSuite
checkEvaluation(Slice(a3, Literal(2), Literal(3)), Seq(2, null, 4))
}
+ test("TrimArray") {
+ val a0 = Literal.create(Seq(1, 2, 3, 4, 5), ArrayType(IntegerType))
+ val a1 = Literal.create(Seq[String]("a", "b", "c"), ArrayType(StringType))
+ val a2 = Literal.create(Seq[String]("a", null, "b"), ArrayType(StringType,
containsNull = true))
+ val a3 = Literal.create(Seq.empty[Int], ArrayType(IntegerType))
+
+ // n between 0 and cardinality removes the last n elements.
+ checkEvaluation(TrimArray(a0, Literal(0)), Seq(1, 2, 3, 4, 5))
+ checkEvaluation(TrimArray(a0, Literal(2)), Seq(1, 2, 3))
+ checkEvaluation(TrimArray(a0, Literal(5)), Seq.empty[Int])
+ checkEvaluation(TrimArray(a1, Literal(1)), Seq("a", "b"))
+ checkEvaluation(TrimArray(a2, Literal(1)), Seq("a", null))
+ checkEvaluation(TrimArray(a3, Literal(0)), Seq.empty[Int])
+
+ // NULL array or NULL n yields NULL.
+ checkEvaluation(TrimArray(Literal.create(null, ArrayType(IntegerType)),
Literal(1)), null)
+ checkEvaluation(TrimArray(a0, Literal.create(null, IntegerType)), null)
+
+ // n < 0 and n > cardinality are rejected.
+ checkErrorInExpression[SparkRuntimeException](
+ expression = TrimArray(a0, Literal(-1)),
+ condition = "INVALID_PARAMETER_VALUE.TRIM_ARRAY_LENGTH",
+ parameters = Map(
+ "parameter" -> toSQLId("n"),
+ "functionName" -> toSQLId("trim_array"),
+ "numElements" -> "5",
+ "length" -> (-1).toString
+ ))
+ checkErrorInExpression[SparkRuntimeException](
+ expression = TrimArray(a0, Literal(6)),
+ condition = "INVALID_PARAMETER_VALUE.TRIM_ARRAY_LENGTH",
+ parameters = Map(
+ "parameter" -> toSQLId("n"),
+ "functionName" -> toSQLId("trim_array"),
+ "numElements" -> "5",
+ "length" -> 6.toString
+ ))
+ }
+
test("ArrayJoin") {
def testArrays(
arrays: Seq[Expression],
diff --git
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index 88f887d302a5..0c6ff268aaf2 100644
---
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -2593,6 +2593,10 @@ class PlanGenerationTestSuite extends ConnectFunSuite
with Logging {
fn.slice(fn.col("e"), 0, 5)
}
+ functionTest("trim_array") {
+ fn.trim_array(fn.col("e"), 2)
+ }
+
functionTest("array_join") {
fn.array_join(fn.col("e"), ";")
}
diff --git
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/FunctionTestSuite.scala
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/FunctionTestSuite.scala
index 8c5fc2b2b8ec..ec191e4707f2 100644
---
a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/FunctionTestSuite.scala
+++
b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/FunctionTestSuite.scala
@@ -180,6 +180,7 @@ class FunctionTestSuite extends ConnectFunSuite {
window(a, "10 seconds"))
testEquals("session_window", session_window(a, "1 second"),
session_window(a, lit("1 second")))
testEquals("slice", slice(a, 1, 2), slice(a, lit(1), lit(2)))
+ testEquals("trim_array", trim_array(a, 2), trim_array(a, lit(2)))
testEquals("bucket", bucket(lit(3), a), bucket(3, a))
testEquals(
"lag",
diff --git
a/sql/connect/common/src/test/resources/query-tests/explain-results/function_trim_array.explain
b/sql/connect/common/src/test/resources/query-tests/explain-results/function_trim_array.explain
new file mode 100644
index 000000000000..e4f13ce71a19
--- /dev/null
+++
b/sql/connect/common/src/test/resources/query-tests/explain-results/function_trim_array.explain
@@ -0,0 +1,2 @@
+Project [trim_array(e#0, 2) AS trim_array(e, 2)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git
a/sql/connect/common/src/test/resources/query-tests/queries/function_trim_array.json
b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_array.json
new file mode 100644
index 000000000000..1cd8f3d1b876
--- /dev/null
+++
b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_array.json
@@ -0,0 +1,81 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema":
"struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "trim_array",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "e"
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "col",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass":
"org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }, {
+ "literal": {
+ "integer": 2
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "trim_array",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass":
"org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }],
+ "isInternal": false
+ },
+ "common": {
+ "origin": {
+ "jvmOrigin": {
+ "stackTrace": [{
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.functions$",
+ "methodName": "trim_array",
+ "fileName": "functions.scala"
+ }, {
+ "classLoaderName": "app",
+ "declaringClass": "org.apache.spark.sql.PlanGenerationTestSuite",
+ "methodName": "~~trimmed~anonfun~~",
+ "fileName": "PlanGenerationTestSuite.scala"
+ }]
+ }
+ }
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git
a/sql/connect/common/src/test/resources/query-tests/queries/function_trim_array.proto.bin
b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_array.proto.bin
new file mode 100644
index 000000000000..d32a2256dad8
Binary files /dev/null and
b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_array.proto.bin
differ
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 2db516884258..e0d9862bc22a 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -394,6 +394,7 @@
| org.apache.spark.sql.catalyst.expressions.ToUnixTimestamp |
to_unix_timestamp | SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd') |
struct<to_unix_timestamp(2016-04-08, yyyy-MM-dd):bigint> |
| org.apache.spark.sql.catalyst.expressions.TransformKeys | transform_keys |
SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v)
-> k + 1) | struct<transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2,
3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(),
namedlambdavariable())):map<int,int>> |
| org.apache.spark.sql.catalyst.expressions.TransformValues | transform_values
| SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k,
v) -> v + 1) | struct<transform_values(map_from_arrays(array(1, 2, 3), array(1,
2, 3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(),
namedlambdavariable())):map<int,int>> |
+| org.apache.spark.sql.catalyst.expressions.TrimArray | trim_array | SELECT
trim_array(array(1, 2, 3, 4, 5), 2) | struct<trim_array(array(1, 2, 3, 4, 5),
2):array<int>> |
| org.apache.spark.sql.catalyst.expressions.TruncDate | trunc | SELECT
trunc('2019-08-04', 'week') | struct<trunc(2019-08-04, week):date> |
| org.apache.spark.sql.catalyst.expressions.TruncTimestamp | date_trunc |
SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359') | struct<date_trunc(YEAR,
2015-03-05T09:32:05.359):timestamp> |
| org.apache.spark.sql.catalyst.expressions.Truncate | truncate | SELECT
truncate(1234.5678, 2) | struct<truncate(1234.5678, 2):decimal(7,2)> |
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
index ede479100dcb..aa2795c16f85 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/array.sql.out
@@ -951,3 +951,73 @@ select array_distinct(array(0.0, -0.0, -0.0,
DOUBLE("NaN"), DOUBLE("NaN")))
-- !query analysis
Project [array_distinct(array(cast(0.0 as double), cast(0.0 as double),
cast(0.0 as double), cast(NaN as double), cast(NaN as double))) AS
array_distinct(array(0.0, 0.0, 0.0, NaN, NaN))#x]
+- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 0)
+-- !query analysis
+Project [trim_array(array(1, 2, 3, 4, 5), 0) AS trim_array(array(1, 2, 3, 4,
5), 0)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 2)
+-- !query analysis
+Project [trim_array(array(1, 2, 3, 4, 5), 2) AS trim_array(array(1, 2, 3, 4,
5), 2)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 5)
+-- !query analysis
+Project [trim_array(array(1, 2, 3, 4, 5), 5) AS trim_array(array(1, 2, 3, 4,
5), 5)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array('a', 'b', 'c'), 1)
+-- !query analysis
+Project [trim_array(array(a, b, c), 1) AS trim_array(array(a, b, c), 1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, null, 4), 1)
+-- !query analysis
+Project [trim_array(array(1, 2, cast(null as int), 4), 1) AS
trim_array(array(1, 2, NULL, 4), 1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(), 0)
+-- !query analysis
+Project [trim_array(array(), 0) AS trim_array(array(), 0)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3), -1)
+-- !query analysis
+Project [trim_array(array(1, 2, 3), -1) AS trim_array(array(1, 2, 3), -1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3), 4)
+-- !query analysis
+Project [trim_array(array(1, 2, 3), 4) AS trim_array(array(1, 2, 3), 4)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(CAST(null AS ARRAY<INT>), 1)
+-- !query analysis
+Project [trim_array(cast(null as array<int>), 1) AS trim_array(NULL, 1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3), CAST(null AS INT))
+-- !query analysis
+Project [trim_array(array(1, 2, 3), cast(null as int)) AS trim_array(array(1,
2, 3), CAST(NULL AS INT))#x]
++- OneRowRelation
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
index 5f2ea9b475ea..1e5f15dff829 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/array.sql.out
@@ -951,3 +951,73 @@ select array_distinct(array(0.0, -0.0, -0.0,
DOUBLE("NaN"), DOUBLE("NaN")))
-- !query analysis
Project [array_distinct(array(cast(0.0 as double), cast(0.0 as double),
cast(0.0 as double), cast(NaN as double), cast(NaN as double))) AS
array_distinct(array(0.0, 0.0, 0.0, NaN, NaN))#x]
+- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 0)
+-- !query analysis
+Project [trim_array(array(1, 2, 3, 4, 5), 0) AS trim_array(array(1, 2, 3, 4,
5), 0)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 2)
+-- !query analysis
+Project [trim_array(array(1, 2, 3, 4, 5), 2) AS trim_array(array(1, 2, 3, 4,
5), 2)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 5)
+-- !query analysis
+Project [trim_array(array(1, 2, 3, 4, 5), 5) AS trim_array(array(1, 2, 3, 4,
5), 5)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array('a', 'b', 'c'), 1)
+-- !query analysis
+Project [trim_array(array(a, b, c), 1) AS trim_array(array(a, b, c), 1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, null, 4), 1)
+-- !query analysis
+Project [trim_array(array(1, 2, cast(null as int), 4), 1) AS
trim_array(array(1, 2, NULL, 4), 1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(), 0)
+-- !query analysis
+Project [trim_array(array(), 0) AS trim_array(array(), 0)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3), -1)
+-- !query analysis
+Project [trim_array(array(1, 2, 3), -1) AS trim_array(array(1, 2, 3), -1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3), 4)
+-- !query analysis
+Project [trim_array(array(1, 2, 3), 4) AS trim_array(array(1, 2, 3), 4)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(CAST(null AS ARRAY<INT>), 1)
+-- !query analysis
+Project [trim_array(cast(null as array<int>), 1) AS trim_array(NULL, 1)#x]
++- OneRowRelation
+
+
+-- !query
+select trim_array(array(1, 2, 3), CAST(null AS INT))
+-- !query analysis
+Project [trim_array(array(1, 2, 3), cast(null as int)) AS trim_array(array(1,
2, 3), CAST(NULL AS INT))#x]
++- OneRowRelation
diff --git a/sql/core/src/test/resources/sql-tests/inputs/array.sql
b/sql/core/src/test/resources/sql-tests/inputs/array.sql
index 27fb3b3d0777..cf642cbbfb10 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/array.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/array.sql
@@ -192,3 +192,17 @@ select array_prepend(array(CAST(NULL AS String)),
CAST(NULL AS String));
-- SPARK-45599: Confirm 0.0, -0.0, and NaN are handled appropriately.
select array_union(array(0.0, -0.0, DOUBLE("NaN")), array(0.0, -0.0,
DOUBLE("NaN")));
select array_distinct(array(0.0, -0.0, -0.0, DOUBLE("NaN"), DOUBLE("NaN")));
+
+-- function trim_array
+select trim_array(array(1, 2, 3, 4, 5), 0);
+select trim_array(array(1, 2, 3, 4, 5), 2);
+select trim_array(array(1, 2, 3, 4, 5), 5);
+select trim_array(array('a', 'b', 'c'), 1);
+select trim_array(array(1, 2, null, 4), 1);
+select trim_array(array(), 0);
+-- trim_array errors: n negative or greater than the array cardinality
+select trim_array(array(1, 2, 3), -1);
+select trim_array(array(1, 2, 3), 4);
+-- trim_array with NULL array or NULL n returns NULL
+select trim_array(CAST(null AS ARRAY<INT>), 1);
+select trim_array(array(1, 2, 3), CAST(null AS INT));
diff --git a/sql/core/src/test/resources/sql-tests/results/array.sql.out
b/sql/core/src/test/resources/sql-tests/results/array.sql.out
index 2113ff5ee491..7de11e3adeef 100644
--- a/sql/core/src/test/resources/sql-tests/results/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/array.sql.out
@@ -1152,3 +1152,103 @@ select array_distinct(array(0.0, -0.0, -0.0,
DOUBLE("NaN"), DOUBLE("NaN")))
struct<array_distinct(array(0.0, 0.0, 0.0, NaN, NaN)):array<double>>
-- !query output
[0.0,NaN]
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 0)
+-- !query schema
+struct<trim_array(array(1, 2, 3, 4, 5), 0):array<int>>
+-- !query output
+[1,2,3,4,5]
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 2)
+-- !query schema
+struct<trim_array(array(1, 2, 3, 4, 5), 2):array<int>>
+-- !query output
+[1,2,3]
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 5)
+-- !query schema
+struct<trim_array(array(1, 2, 3, 4, 5), 5):array<int>>
+-- !query output
+[]
+
+
+-- !query
+select trim_array(array('a', 'b', 'c'), 1)
+-- !query schema
+struct<trim_array(array(a, b, c), 1):array<string>>
+-- !query output
+["a","b"]
+
+
+-- !query
+select trim_array(array(1, 2, null, 4), 1)
+-- !query schema
+struct<trim_array(array(1, 2, NULL, 4), 1):array<int>>
+-- !query output
+[1,2,null]
+
+
+-- !query
+select trim_array(array(), 0)
+-- !query schema
+struct<trim_array(array(), 0):array<void>>
+-- !query output
+[]
+
+
+-- !query
+select trim_array(array(1, 2, 3), -1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkRuntimeException
+{
+ "errorClass" : "INVALID_PARAMETER_VALUE.TRIM_ARRAY_LENGTH",
+ "sqlState" : "22023",
+ "messageParameters" : {
+ "functionName" : "`trim_array`",
+ "length" : "-1",
+ "numElements" : "3",
+ "parameter" : "`n`"
+ }
+}
+
+
+-- !query
+select trim_array(array(1, 2, 3), 4)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkRuntimeException
+{
+ "errorClass" : "INVALID_PARAMETER_VALUE.TRIM_ARRAY_LENGTH",
+ "sqlState" : "22023",
+ "messageParameters" : {
+ "functionName" : "`trim_array`",
+ "length" : "4",
+ "numElements" : "3",
+ "parameter" : "`n`"
+ }
+}
+
+
+-- !query
+select trim_array(CAST(null AS ARRAY<INT>), 1)
+-- !query schema
+struct<trim_array(NULL, 1):array<int>>
+-- !query output
+NULL
+
+
+-- !query
+select trim_array(array(1, 2, 3), CAST(null AS INT))
+-- !query schema
+struct<trim_array(array(1, 2, 3), CAST(NULL AS INT)):array<int>>
+-- !query output
+NULL
diff --git
a/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
b/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
index 3bd7671ae386..b5c3ee79929b 100644
--- a/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/nonansi/array.sql.out
@@ -1010,3 +1010,103 @@ select array_distinct(array(0.0, -0.0, -0.0,
DOUBLE("NaN"), DOUBLE("NaN")))
struct<array_distinct(array(0.0, 0.0, 0.0, NaN, NaN)):array<double>>
-- !query output
[0.0,NaN]
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 0)
+-- !query schema
+struct<trim_array(array(1, 2, 3, 4, 5), 0):array<int>>
+-- !query output
+[1,2,3,4,5]
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 2)
+-- !query schema
+struct<trim_array(array(1, 2, 3, 4, 5), 2):array<int>>
+-- !query output
+[1,2,3]
+
+
+-- !query
+select trim_array(array(1, 2, 3, 4, 5), 5)
+-- !query schema
+struct<trim_array(array(1, 2, 3, 4, 5), 5):array<int>>
+-- !query output
+[]
+
+
+-- !query
+select trim_array(array('a', 'b', 'c'), 1)
+-- !query schema
+struct<trim_array(array(a, b, c), 1):array<string>>
+-- !query output
+["a","b"]
+
+
+-- !query
+select trim_array(array(1, 2, null, 4), 1)
+-- !query schema
+struct<trim_array(array(1, 2, NULL, 4), 1):array<int>>
+-- !query output
+[1,2,null]
+
+
+-- !query
+select trim_array(array(), 0)
+-- !query schema
+struct<trim_array(array(), 0):array<void>>
+-- !query output
+[]
+
+
+-- !query
+select trim_array(array(1, 2, 3), -1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkRuntimeException
+{
+ "errorClass" : "INVALID_PARAMETER_VALUE.TRIM_ARRAY_LENGTH",
+ "sqlState" : "22023",
+ "messageParameters" : {
+ "functionName" : "`trim_array`",
+ "length" : "-1",
+ "numElements" : "3",
+ "parameter" : "`n`"
+ }
+}
+
+
+-- !query
+select trim_array(array(1, 2, 3), 4)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkRuntimeException
+{
+ "errorClass" : "INVALID_PARAMETER_VALUE.TRIM_ARRAY_LENGTH",
+ "sqlState" : "22023",
+ "messageParameters" : {
+ "functionName" : "`trim_array`",
+ "length" : "4",
+ "numElements" : "3",
+ "parameter" : "`n`"
+ }
+}
+
+
+-- !query
+select trim_array(CAST(null AS ARRAY<INT>), 1)
+-- !query schema
+struct<trim_array(NULL, 1):array<int>>
+-- !query output
+NULL
+
+
+-- !query
+select trim_array(array(1, 2, 3), CAST(null AS INT))
+-- !query schema
+struct<trim_array(array(1, 2, 3), CAST(NULL AS INT)):array<int>>
+-- !query output
+NULL
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
index f2905da9a2d8..0c8008b5ddbf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala
@@ -1227,7 +1227,7 @@ class BasicCharVarcharTestSuite extends
SharedSparkSession {
"coalesce", "collect_list", "collect_set", "collect_union", "concat",
"explode",
"explode_outer", "first", "first_value", "get", "greatest", "ifnull",
"last", "last_value",
"least", "map", "max", "max_by", "measure", "min", "min_by", "mode",
"named_struct", "nullif",
- "nullifzero", "nvl", "reverse", "shuffle", "sort_array", "struct", "when")
+ "nullifzero", "nvl", "reverse", "shuffle", "sort_array", "struct",
"trim_array", "when")
test("SPARK-58794: inventoried shapes do not leak CHAR/VARCHAR under
standardSemantics") {
val argumentShapes = Seq(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
