Repository: spark Updated Branches: refs/heads/master 46d2d2c74 -> 72a23a6c4
[SPARK-25772][SQL][FOLLOWUP] remove GetArrayFromMap ## What changes were proposed in this pull request? In https://github.com/apache/spark/pull/22745 we introduced the `GetArrayFromMap` expression. Later on I realized this is duplicated as we already have `MapKeys` and `MapValues`. This PR removes `GetArrayFromMap` ## How was this patch tested? existing tests Closes #22825 from cloud-fan/minor. Authored-by: Wenchen Fan <wenc...@databricks.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/72a23a6c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/72a23a6c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/72a23a6c Branch: refs/heads/master Commit: 72a23a6c43fe1b5a6583ea6b35b4fbb08474abbe Parents: 46d2d2c Author: Wenchen Fan <wenc...@databricks.com> Authored: Fri Oct 26 10:19:35 2018 +0800 Committer: Wenchen Fan <wenc...@databricks.com> Committed: Fri Oct 26 10:19:35 2018 +0800 ---------------------------------------------------------------------- .../spark/sql/catalyst/JavaTypeInference.scala | 6 +- .../catalyst/expressions/objects/objects.scala | 75 -------------------- 2 files changed, 3 insertions(+), 78 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/72a23a6c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala index f32e080..8ef8b2b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala @@ -26,7 +26,7 @@ import scala.language.existentials import com.google.common.reflect.TypeToken -import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedAttribute, UnresolvedExtractValue} +import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedExtractValue} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.objects._ import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils, GenericArrayData} @@ -280,7 +280,7 @@ object JavaTypeInference { Invoke( UnresolvedMapObjects( p => deserializerFor(keyType, p), - GetKeyArrayFromMap(path)), + MapKeys(path)), "array", ObjectType(classOf[Array[Any]])) @@ -288,7 +288,7 @@ object JavaTypeInference { Invoke( UnresolvedMapObjects( p => deserializerFor(valueType, p), - GetValueArrayFromMap(path)), + MapValues(path)), "array", ObjectType(classOf[Array[Any]])) http://git-wip-us.apache.org/repos/asf/spark/blob/72a23a6c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index 5bfa485..b6f9b47 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -1788,78 +1788,3 @@ case class ValidateExternalType(child: Expression, expected: DataType) ev.copy(code = code, isNull = input.isNull) } } - -object GetKeyArrayFromMap { - - /** - * Construct an instance of GetArrayFromMap case class - * extracting a key array from a Map expression. - * - * @param child a Map expression to extract a key array from - */ - def apply(child: Expression): Expression = { - GetArrayFromMap( - child, - "keyArray", - _.keyArray(), - { case MapType(kt, _, _) => kt }) - } -} - -object GetValueArrayFromMap { - - /** - * Construct an instance of GetArrayFromMap case class - * extracting a value array from a Map expression. - * - * @param child a Map expression to extract a value array from - */ - def apply(child: Expression): Expression = { - GetArrayFromMap( - child, - "valueArray", - _.valueArray(), - { case MapType(_, vt, _) => vt }) - } -} - -/** - * Extracts a key/value array from a Map expression. - * - * @param child a Map expression to extract an array from - * @param functionName name of the function that is invoked to extract an array - * @param arrayGetter function extracting `ArrayData` from `MapData` - * @param elementTypeGetter function extracting array element `DataType` from `MapType` - */ -case class GetArrayFromMap private( - child: Expression, - functionName: String, - arrayGetter: MapData => ArrayData, - elementTypeGetter: MapType => DataType) extends UnaryExpression with NonSQLExpression { - - private lazy val encodedFunctionName: String = TermName(functionName).encodedName.toString - - lazy val dataType: DataType = { - val mt: MapType = child.dataType.asInstanceOf[MapType] - ArrayType(elementTypeGetter(mt)) - } - - override def checkInputDataTypes(): TypeCheckResult = { - if (child.dataType.isInstanceOf[MapType]) { - TypeCheckResult.TypeCheckSuccess - } else { - TypeCheckResult.TypeCheckFailure( - s"Can't extract array from $child: need map type but got ${child.dataType.catalogString}") - } - } - - override def nullSafeEval(input: Any): Any = { - arrayGetter(input.asInstanceOf[MapData]) - } - - override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - defineCodeGen(ctx, ev, childValue => s"$childValue.$encodedFunctionName()") - } - - override def toString: String = s"$child.$functionName" -} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
