This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new ff022e54c2c [DOCS] Update concat and concat_ws documentation to point out unexpected behavior ff022e54c2c is described below commit ff022e54c2caedb164ce943dc628fcf607eddcdd Author: Christopher Watford <132389385+watford...@users.noreply.github.com> AuthorDate: Mon Jul 31 10:11:28 2023 +0800 [DOCS] Update concat and concat_ws documentation to point out unexpected behavior Add documentation covering unexpected behavior of concat and concat_ws with respect to null values. ### What changes were proposed in this pull request? Adds additional documentation to `concat` and `concat_ws`. ### Why are the changes needed? The behavior of `concat` and `concat_ws` were unexpected w.r.t. null values and the documentation did not help make their behavior clear. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Closes #42153 from watfordkcf/patch-1. Lead-authored-by: Christopher Watford <132389385+watford...@users.noreply.github.com> Co-authored-by: Christopher Watford <christopher.watf...@kcftech.com> Signed-off-by: Kent Yao <y...@apache.org> --- .../apache/spark/sql/catalyst/expressions/stringExpressions.scala | 6 +++++- sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 03596ac40b1..46f8e1a9d67 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -53,13 +53,17 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String} */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`.", + usage = "_FUNC_(sep[, str | array(str)]+) - Returns the concatenation of the strings separated by `sep`, skipping null values.", examples = """ Examples: > SELECT _FUNC_(' ', 'Spark', 'SQL'); Spark SQL > SELECT _FUNC_('s'); + > SELECT _FUNC_('/', 'foo', null, 'bar'); + foo/bar + > SELECT _FUNC_(null, 'Spark', 'SQL'); + NULL """, since = "1.5.0", group = "string_funcs") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index ca5e4422ca9..e7e8b945d91 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -3822,6 +3822,8 @@ object functions { * Concatenates multiple input string columns together into a single string column, * using the given separator. * + * @note Input strings which are null are skipped. + * * @group string_funcs * @since 1.5.0 */ @@ -6019,6 +6021,8 @@ object functions { * Concatenates multiple input columns together into a single column. * The function works with strings, binary and compatible array columns. * + * @note Returns null if any of the input columns are null. + * * @group collection_funcs * @since 1.5.0 */ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org