Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/20858#discussion_r218677837
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
---
@@ -665,3 +667,219 @@ case class ElementAt(left: Expression, right:
Expression) extends GetMapValueUti
override def prettyName: String = "element_at"
}
+
+/**
+ * Concatenates multiple input columns together into a single column.
+ * The function works with strings, binary and compatible array columns.
+ */
+@ExpressionDescription(
+ usage = "_FUNC_(col1, col2, ..., colN) - Returns the concatenation of
col1, col2, ..., colN.",
+ examples = """
+ Examples:
+ > SELECT _FUNC_('Spark', 'SQL');
+ SparkSQL
+ > SELECT _FUNC_(array(1, 2, 3), array(4, 5), array(6));
+ | [1,2,3,4,5,6]
+ """)
+case class Concat(children: Seq[Expression]) extends Expression {
+
+ private val MAX_ARRAY_LENGTH: Int =
ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH
+
+ val allowedTypes = Seq(StringType, BinaryType, ArrayType)
+
+ override def checkInputDataTypes(): TypeCheckResult = {
+ if (children.isEmpty) {
+ TypeCheckResult.TypeCheckSuccess
+ } else {
+ val childTypes = children.map(_.dataType)
+ if (childTypes.exists(tpe =>
!allowedTypes.exists(_.acceptsType(tpe)))) {
+ return TypeCheckResult.TypeCheckFailure(
+ s"input to function $prettyName should have been StringType,
BinaryType or ArrayType," +
+ s" but it's " + childTypes.map(_.simpleString).mkString("[",
", ", "]"))
+ }
+ TypeUtils.checkForSameTypeInputExpr(childTypes, s"function
$prettyName")
+ }
+ }
+
+ override def dataType: DataType =
children.map(_.dataType).headOption.getOrElse(StringType)
+
+ lazy val javaType: String = CodeGenerator.javaType(dataType)
+
+ override def nullable: Boolean = children.exists(_.nullable)
+
+ override def foldable: Boolean = children.forall(_.foldable)
+
+ override def eval(input: InternalRow): Any = dataType match {
--- End diff --
so this pattern match will probably cause significant regression in the
interpreted (non-codegen) mode, due to the way scala pattern matching is
implemented.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
