Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/21011#discussion_r181363454
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
---
@@ -287,3 +288,173 @@ case class ArrayContains(left: Expression, right:
Expression)
override def prettyName: String = "array_contains"
}
+
+/**
+ * Creates a String containing all the elements of the input array
separated by the delimiter.
+ */
+@ExpressionDescription(
+ usage = """
+ _FUNC_(array, delimiter[, nullReplacement]) - Concatenates the
elements of the given array
+ using the delimiter and an optional string to replace nulls. If no
value is set for
+ nullReplacement, any null value is filtered.""",
+ examples = """
+ Examples:
+ > SELECT _FUNC_(array('hello', 'world'), ' ');
+ hello world
+ > SELECT _FUNC_(array('hello', null ,'world'), ' ');
+ hello world
+ > SELECT _FUNC_(array('hello', null ,'world'), ' ', ',');
+ hello , world
+ """, since = "2.4.0")
+case class ArrayJoin(
+ array: Expression,
+ delimiter: Expression,
+ nullReplacement: Option[Expression]) extends Expression with
ExpectsInputTypes {
+
+ def this(array: Expression, delimiter: Expression) = this(array,
delimiter, None)
+
+ def this(array: Expression, delimiter: Expression, nullReplacement:
Expression) =
+ this(array, delimiter, Some(nullReplacement))
+
+ override def inputTypes: Seq[AbstractDataType] = if
(nullReplacement.isDefined) {
+ Seq(ArrayType(StringType), StringType, StringType)
+ } else {
+ Seq(ArrayType(StringType), StringType)
+ }
+
+ override def children: Seq[Expression] = if (nullReplacement.isDefined) {
+ Seq(array, delimiter, nullReplacement.get)
+ } else {
+ Seq(array, delimiter)
+ }
+
+ override def nullable: Boolean = children.exists(_.nullable)
+
+ override def foldable: Boolean = children.forall(_.foldable)
+
+ override def eval(input: InternalRow): Any = {
+ val arrayEval = array.eval(input)
+ if (arrayEval == null) return null
+ val delimiterEval = delimiter.eval(input)
+ if (delimiterEval == null) return null
+ val nullReplacementEval = nullReplacement.map(_.eval(input))
+ if (nullReplacementEval.contains(null)) return null
+
+
+ val buffer = new UTF8StringBuilder()
+ var firstItem = true
+ val nullHandling = nullReplacementEval match {
+ case Some(rep) => (prependDelimiter: Boolean) => {
+ if (!prependDelimiter) {
+ buffer.append(delimiterEval.asInstanceOf[UTF8String])
+ }
+ buffer.append(rep.asInstanceOf[UTF8String])
+ true
+ }
+ case None => (_: Boolean) => false
+ }
+ arrayEval.asInstanceOf[ArrayData].foreach(StringType, (_, item) => {
+ if (item == null) {
+ if (nullHandling(firstItem)) {
+ firstItem = false
+ }
+ } else {
+ if (!firstItem) {
+ buffer.append(delimiterEval.asInstanceOf[UTF8String])
+ }
+ buffer.append(item.asInstanceOf[UTF8String])
+ firstItem = false
+ }
+ })
+ buffer.build()
+ }
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode):
ExprCode = {
+ val code = nullReplacement match {
+ case Some(replacement) =>
+ val replacementGen = replacement.genCode(ctx)
+ val nullHandling = (buffer: String, delimiter: String, firstItem:
String) => {
+ s"""
+ |if (!$firstItem) {
+ | $buffer.append($delimiter);
+ |}
+ |$buffer.append(${replacementGen.value});
+ |$firstItem = false;
+ """.stripMargin
+ }
+ val execCode = if (replacement.nullable) {
+ ctx.nullSafeExec(replacement.nullable, replacementGen.isNull) {
+ genCodeForArrayAndDelimiter(ctx, ev, nullHandling)
+ }
+ } else {
+ genCodeForArrayAndDelimiter(ctx, ev, nullHandling)
+ }
+ s"""
+ |${replacementGen.code}
+ |$execCode
+ """.stripMargin
+ case None => genCodeForArrayAndDelimiter(ctx, ev,
+ (_: String, _: String, _: String) => "// nulls are ignored")
+ }
+ if (nullable) {
+ ev.copy(
+ s"""
+ |boolean ${ev.isNull} = true;
+ |UTF8String ${ev.value} = null;
+ |$code
+ """.stripMargin)
+ } else {
+ ev.copy(s"""
+ |boolean ${ev.isNull} = false;
--- End diff --
nit: I guess we can remove this?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
