Github user kiszk commented on a diff in the pull request:

    https://github.com/apache/spark/pull/21050#discussion_r189153390
  
    --- Diff: 
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
 ---
    @@ -1882,3 +1882,141 @@ case class ArrayRepeat(left: Expression, right: 
Expression)
       }
     
     }
    +
    +/**
    + * Removes duplicate values from the array.
    + */
    +@ExpressionDescription(
    +  usage = "_FUNC_(array) - Removes duplicate values from the array.",
    +  examples = """
    +    Examples:
    +      > SELECT _FUNC_(array(1, 2, 3, null, 3));
    +       [1,2,3,null]
    +  """, since = "2.4.0")
    +case class ArrayDistinct(child: Expression)
    +  extends UnaryExpression with ExpectsInputTypes {
    +
    +  override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType)
    +
    +  override def dataType: DataType = child.dataType
    +
    +  lazy val elementType: DataType = 
dataType.asInstanceOf[ArrayType].elementType
    +
    +  override def nullSafeEval(array: Any): Any = {
    +    val elementType = child.dataType.asInstanceOf[ArrayType].elementType
    +    val data = 
array.asInstanceOf[ArrayData].toArray[AnyRef](elementType).distinct
    +    new GenericArrayData(data.asInstanceOf[Array[Any]])
    +  }
    +
    +  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
    +    nullSafeCodeGen(ctx, ev, (array) => {
    +      val i = ctx.freshName("i")
    +      val j = ctx.freshName("j")
    +      val hs = ctx.freshName("hs")
    +      val distinctArrayLen = ctx.freshName("distinctArrayLen")
    +      val getValue = CodeGenerator.getValue(array, elementType, i)
    +      val openHashSet = classOf[OpenHashSet[_]].getName
    +      val classTag = s"scala.reflect.ClassTag$$.MODULE$$.Object()"
    +      s"""
    +         |int $distinctArrayLen = 0;
    +         |$openHashSet $hs = new $openHashSet($classTag);
    +         |for (int $i = 0; $i < $array.numElements(); $i++) {
    +         |  if ($array.isNullAt($i)) {
    +         |    int $j;
    +         |    for ($j = 0; $j < $i; $j ++) {
    +         |      if ($array.isNullAt($j))
    +         |        break;
    +         |    }
    +         |    if ($i == $j) {
    +         |      $distinctArrayLen = $distinctArrayLen + 1;
    +         |    }
    +         |  }
    +         |  else {
    +         |    if (!($hs.contains($getValue))) {
    +         |      $hs.add($getValue);
    +         |      $distinctArrayLen = $distinctArrayLen + 1;
    +         |    }
    +         |  }
    +         |}
    +         |${genCodeForResult(ctx, ev, array, distinctArrayLen)}
    +      """.stripMargin
    +    })
    +  }
    +
    +  def genCodeForResult(
    +      ctx: CodegenContext,
    +      ev: ExprCode,
    +      inputArray: String,
    +      newArraySize: String): String = {
    +    val distinctArr = ctx.freshName("distinctArray")
    +    val hs = ctx.freshName("hs")
    +    val openHashSet = classOf[OpenHashSet[_]].getName
    +    val i = ctx.freshName("i")
    +    val j = ctx.freshName("j")
    +    val pos = ctx.freshName("pos")
    +    val genericArrayData = classOf[GenericArrayData].getName
    +    val getValue = CodeGenerator.getValue(inputArray, elementType, i)
    +
    +    if (!CodeGenerator.isPrimitiveType(elementType)) {
    +      val arrayClass = classOf[GenericArrayData].getName
    +      val classTag = s"scala.reflect.ClassTag$$.MODULE$$.Object()"
    +      s"""
    +         |Object[] $distinctArr = new Object[$newArraySize];
    +         |int $pos = 0;
    +         |$openHashSet $hs = new $openHashSet($classTag);
    +         |for (int $i = 0; $i < $inputArray.numElements(); $i++) {
    +         |  if ($inputArray.isNullAt($i)) {
    +         |    int $j;
    --- End diff --
    
    ditto


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

Reply via email to