Repository: spark Updated Branches: refs/heads/master 69e5282d3 -> fd52a747f
[SPARK-19810][SPARK-19810][MINOR][FOLLOW-UP] Follow-ups from to remove Scala 2.10 ## What changes were proposed in this pull request? Follow up to a few comments on https://github.com/apache/spark/pull/17150#issuecomment-315020196 that couldn't be addressed before it was merged. ## How was this patch tested? Existing tests. Author: Sean Owen <[email protected]> Closes #18646 from srowen/SPARK-19810.2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fd52a747 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fd52a747 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fd52a747 Branch: refs/heads/master Commit: fd52a747fd7f5a0f9657b4fa16c797916aef9a95 Parents: 69e5282 Author: Sean Owen <[email protected]> Authored: Mon Jul 17 09:22:42 2017 +0800 Committer: Wenchen Fan <[email protected]> Committed: Mon Jul 17 09:22:42 2017 +0800 ---------------------------------------------------------------------- .../sql/catalyst/ScalaReflectionSuite.scala | 39 -------------------- .../execution/python/ExtractPythonUDFs.scala | 2 +- 2 files changed, 1 insertion(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/fd52a747/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala index ff2414b..a5b9855 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala @@ -17,17 +17,13 @@ package org.apache.spark.sql.catalyst -import java.net.URLClassLoader import java.sql.{Date, Timestamp} -import scala.reflect.runtime.universe.typeOf - import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.expressions.{BoundReference, Literal, SpecificInternalRow} import org.apache.spark.sql.catalyst.expressions.objects.NewInstance import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String -import org.apache.spark.util.Utils case class PrimitiveData( intField: Int, @@ -339,39 +335,4 @@ class ScalaReflectionSuite extends SparkFunSuite { assert(linkedHashMapDeserializer.dataType == ObjectType(classOf[LHMap[_, _]])) } - private val dataTypeForComplexData = dataTypeFor[ComplexData] - private val typeOfComplexData = typeOf[ComplexData] - - Seq( - ("mirror", () => mirror), - ("dataTypeFor", () => dataTypeFor[ComplexData]), - ("constructorFor", () => deserializerFor[ComplexData]), - ("extractorsFor", { - val inputObject = BoundReference(0, dataTypeForComplexData, nullable = false) - () => serializerFor[ComplexData](inputObject) - }), - ("getConstructorParameters(cls)", () => getConstructorParameters(classOf[ComplexData])), - ("getConstructorParameterNames", () => getConstructorParameterNames(classOf[ComplexData])), - ("getClassFromType", () => getClassFromType(typeOfComplexData)), - ("schemaFor", () => schemaFor[ComplexData]), - ("localTypeOf", () => localTypeOf[ComplexData]), - ("getClassNameFromType", () => getClassNameFromType(typeOfComplexData)), - ("getParameterTypes", () => getParameterTypes(() => ())), - ("getConstructorParameters(tpe)", () => getClassNameFromType(typeOfComplexData))).foreach { - case (name, exec) => - test(s"SPARK-13640: thread safety of ${name}") { - (0 until 100).foreach { _ => - val loader = new URLClassLoader(Array.empty, Utils.getContextOrSparkClassLoader) - (0 until 10).par.foreach { _ => - val cl = Thread.currentThread.getContextClassLoader - try { - Thread.currentThread.setContextClassLoader(loader) - exec() - } finally { - Thread.currentThread.setContextClassLoader(cl) - } - } - } - } - } } http://git-wip-us.apache.org/repos/asf/spark/blob/fd52a747/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala index 2b3c5f0..1400452 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala @@ -133,7 +133,7 @@ object ExtractPythonUDFs extends Rule[SparkPlan] with PredicateHelper { val validUdfs = udfs.filter { udf => // Check to make sure that the UDF can be evaluated with only the input of this child. udf.references.subsetOf(child.outputSet) - }.toArray + } if (validUdfs.nonEmpty) { val resultAttrs = udfs.zipWithIndex.map { case (u, i) => AttributeReference(s"pythonUDF$i", u.dataType)() --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
