Repository: spark Updated Branches: refs/heads/branch-2.0 fb1c69714 -> 88481ea21
Revert "[SPARK-17061][SPARK-17093][SQL] MapObjects` should make copies of unsafe-backed data" This reverts commit fb1c697143a5bb2df69d9f2c9cbddc4eb526f047. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/88481ea2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/88481ea2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/88481ea2 Branch: refs/heads/branch-2.0 Commit: 88481ea2169e0813cfc326eb1440ddaaf3110f4a Parents: fb1c697 Author: Herman van Hovell <[email protected]> Authored: Thu Aug 25 11:48:13 2016 +0200 Committer: Herman van Hovell <[email protected]> Committed: Thu Aug 25 11:48:13 2016 +0200 ---------------------------------------------------------------------- .../maven_app_core/src/main/java/SimpleApp.java | 41 -------------------- .../catalyst/expressions/objects/objects.scala | 12 +----- .../expressions/ExpressionEvalHelper.scala | 2 +- 3 files changed, 2 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/88481ea2/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java ---------------------------------------------------------------------- diff --git a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java index 059e51b..5217689 100644 --- a/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java +++ b/dev/audit-release/maven_app_core/src/main/java/SimpleApp.java @@ -18,20 +18,11 @@ import org.apache.spark.api.java.*; import org.apache.spark.api.java.function.Function; -<<<<<<< HEAD:dev/audit-release/maven_app_core/src/main/java/SimpleApp.java public class SimpleApp { public static void main(String[] args) { String logFile = "input.txt"; JavaSparkContext sc = new JavaSparkContext("local", "Simple App"); JavaRDD<String> logData = sc.textFile(logFile).cache(); -======= -import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.catalyst.expressions.objects.Invoke -import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData} -import org.apache.spark.sql.types.{IntegerType, ObjectType} ->>>>>>> e0b20f9... [SPARK-17061][SPARK-17093][SQL] MapObjects` should make copies of unsafe-backed data:sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala long numAs = logData.filter(new Function<String, Boolean>() { public Boolean call(String s) { return s.contains("a"); } @@ -48,36 +39,4 @@ import org.apache.spark.sql.types.{IntegerType, ObjectType} System.out.println("Test succeeded"); sc.stop(); } - - test("MapObjects should make copies of unsafe-backed data") { - // test UnsafeRow-backed data - val structEncoder = ExpressionEncoder[Array[Tuple2[java.lang.Integer, java.lang.Integer]]] - val structInputRow = InternalRow.fromSeq(Seq(Array((1, 2), (3, 4)))) - val structExpected = new GenericArrayData( - Array(InternalRow.fromSeq(Seq(1, 2)), InternalRow.fromSeq(Seq(3, 4)))) - checkEvalutionWithUnsafeProjection( - structEncoder.serializer.head, structExpected, structInputRow) - - // test UnsafeArray-backed data - val arrayEncoder = ExpressionEncoder[Array[Array[Int]]] - val arrayInputRow = InternalRow.fromSeq(Seq(Array(Array(1, 2), Array(3, 4)))) - val arrayExpected = new GenericArrayData( - Array(new GenericArrayData(Array(1, 2)), new GenericArrayData(Array(3, 4)))) - checkEvalutionWithUnsafeProjection( - arrayEncoder.serializer.head, arrayExpected, arrayInputRow) - - // test UnsafeMap-backed data - val mapEncoder = ExpressionEncoder[Array[Map[Int, Int]]] - val mapInputRow = InternalRow.fromSeq(Seq(Array( - Map(1 -> 100, 2 -> 200), Map(3 -> 300, 4 -> 400)))) - val mapExpected = new GenericArrayData(Seq( - new ArrayBasedMapData( - new GenericArrayData(Array(1, 2)), - new GenericArrayData(Array(100, 200))), - new ArrayBasedMapData( - new GenericArrayData(Array(3, 4)), - new GenericArrayData(Array(300, 400))))) - checkEvalutionWithUnsafeProjection( - mapEncoder.serializer.head, mapExpected, mapInputRow) - } } http://git-wip-us.apache.org/repos/asf/spark/blob/88481ea2/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index 1cdda53..37ec1a6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -481,16 +481,6 @@ case class MapObjects private( s"$seq == null ? $array[$loopIndex] : $seq.apply($loopIndex)" } - // Make a copy of the data if it's unsafe-backed - def makeCopyIfInstanceOf(clazz: Class[_ <: Any], value: String) = - s"$value instanceof ${clazz.getSimpleName}? ${value}.copy() : $value" - val genFunctionValue = lambdaFunction.dataType match { - case StructType(_) => makeCopyIfInstanceOf(classOf[UnsafeRow], genFunction.value) - case ArrayType(_, _) => makeCopyIfInstanceOf(classOf[UnsafeArrayData], genFunction.value) - case MapType(_, _, _) => makeCopyIfInstanceOf(classOf[UnsafeMapData], genFunction.value) - case _ => genFunction.value - } - val loopNullCheck = inputDataType match { case _: ArrayType => s"$loopIsNull = ${genInputData.value}.isNullAt($loopIndex);" // The element of primitive array will never be null. @@ -518,7 +508,7 @@ case class MapObjects private( if (${genFunction.isNull}) { $convertedArray[$loopIndex] = null; } else { - $convertedArray[$loopIndex] = $genFunctionValue; + $convertedArray[$loopIndex] = ${genFunction.value}; } $loopIndex += 1; http://git-wip-us.apache.org/repos/asf/spark/blob/88481ea2/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala index 668543a..d6a9672 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala @@ -136,7 +136,7 @@ trait ExpressionEvalHelper extends GeneratorDrivenPropertyChecks { // some expression is reusing variable names across different instances. // This behavior is tested in ExpressionEvalHelperSuite. val plan = generateProject( - UnsafeProjection.create( + GenerateUnsafeProjection.generate( Alias(expression, s"Optimized($expression)1")() :: Alias(expression, s"Optimized($expression)2")() :: Nil), expression) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
