Repository: spark Updated Branches: refs/heads/master 26cd2ca04 -> e26dac5fe
http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index 4568b67..d802557 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -65,9 +65,9 @@ class DataFrameAggregateSuite extends QueryTest with SharedSQLContext { checkAnswer( decimalData.groupBy("a").agg(sum("b")), - Seq(Row(new java.math.BigDecimal(1.0), new java.math.BigDecimal(3.0)), - Row(new java.math.BigDecimal(2.0), new java.math.BigDecimal(3.0)), - Row(new java.math.BigDecimal(3.0), new java.math.BigDecimal(3.0))) + Seq(Row(new java.math.BigDecimal(1), new java.math.BigDecimal(3)), + Row(new java.math.BigDecimal(2), new java.math.BigDecimal(3)), + Row(new java.math.BigDecimal(3), new java.math.BigDecimal(3))) ) val decimalDataWithNulls = spark.sparkContext.parallelize( @@ -80,10 +80,10 @@ class DataFrameAggregateSuite extends QueryTest with SharedSQLContext { DecimalData(null, 2) :: Nil).toDF() checkAnswer( decimalDataWithNulls.groupBy("a").agg(sum("b")), - Seq(Row(new java.math.BigDecimal(1.0), new java.math.BigDecimal(1.0)), - Row(new java.math.BigDecimal(2.0), new java.math.BigDecimal(1.0)), - Row(new java.math.BigDecimal(3.0), new java.math.BigDecimal(3.0)), - Row(null, new java.math.BigDecimal(2.0))) + Seq(Row(new java.math.BigDecimal(1), new java.math.BigDecimal(1)), + Row(new java.math.BigDecimal(2), new java.math.BigDecimal(1)), + Row(new java.math.BigDecimal(3), new java.math.BigDecimal(3)), + Row(null, new java.math.BigDecimal(2))) ) } @@ -259,19 +259,19 @@ class DataFrameAggregateSuite extends QueryTest with SharedSQLContext { checkAnswer( decimalData.agg(avg('a)), - Row(new java.math.BigDecimal(2.0))) + Row(new java.math.BigDecimal(2))) checkAnswer( decimalData.agg(avg('a), sumDistinct('a)), // non-partial - Row(new java.math.BigDecimal(2.0), new java.math.BigDecimal(6)) :: Nil) + Row(new java.math.BigDecimal(2), new java.math.BigDecimal(6)) :: Nil) checkAnswer( decimalData.agg(avg('a cast DecimalType(10, 2))), - Row(new java.math.BigDecimal(2.0))) + Row(new java.math.BigDecimal(2))) // non-partial checkAnswer( decimalData.agg(avg('a cast DecimalType(10, 2)), sumDistinct('a cast DecimalType(10, 2))), - Row(new java.math.BigDecimal(2.0), new java.math.BigDecimal(6)) :: Nil) + Row(new java.math.BigDecimal(2), new java.math.BigDecimal(6)) :: Nil) } test("null average") { @@ -520,9 +520,9 @@ class DataFrameAggregateSuite extends QueryTest with SharedSQLContext { test("SQL decimal test (used for catching certain decimal handling bugs in aggregates)") { checkAnswer( decimalData.groupBy('a cast DecimalType(10, 2)).agg(avg('b cast DecimalType(10, 2))), - Seq(Row(new java.math.BigDecimal(1.0), new java.math.BigDecimal(1.5)), - Row(new java.math.BigDecimal(2.0), new java.math.BigDecimal(1.5)), - Row(new java.math.BigDecimal(3.0), new java.math.BigDecimal(1.5)))) + Seq(Row(new java.math.BigDecimal(1), new java.math.BigDecimal("1.5")), + Row(new java.math.BigDecimal(2), new java.math.BigDecimal("1.5")), + Row(new java.math.BigDecimal(3), new java.math.BigDecimal("1.5")))) } test("SPARK-17616: distinct aggregate combined with a non-partial aggregate") { http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 3f3a622..7c50072 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -1167,7 +1167,7 @@ class DataFrameSuite extends QueryTest with SharedSQLContext { } test("SPARK-6899: type should match when using codegen") { - checkAnswer(decimalData.agg(avg('a)), Row(new java.math.BigDecimal(2.0))) + checkAnswer(decimalData.agg(avg('a)), Row(new java.math.BigDecimal(2))) } test("SPARK-7133: Implement struct, array, and map field accessor") { @@ -1971,7 +1971,7 @@ class DataFrameSuite extends QueryTest with SharedSQLContext { test("SPARK-19691 Calculating percentile of decimal column fails with ClassCastException") { val df = spark.range(1).selectExpr("CAST(id as DECIMAL) as x").selectExpr("percentile(x, 0.5)") - checkAnswer(df, Row(BigDecimal(0.0)) :: Nil) + checkAnswer(df, Row(BigDecimal(0)) :: Nil) } test("SPARK-19893: cannot run set operations with map type") { http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index c78ec6d..e95f6db 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -1546,10 +1546,10 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { Seq(Row(d))) checkAnswer( df.selectExpr("b * a + b"), - Seq(Row(BigDecimal(2.12321)))) + Seq(Row(BigDecimal("2.12321")))) checkAnswer( df.selectExpr("b * a - b"), - Seq(Row(BigDecimal(0.12321)))) + Seq(Row(BigDecimal("0.12321")))) checkAnswer( df.selectExpr("b * a * b"), Seq(Row(d))) http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index bcc2351..a12efc8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala @@ -387,7 +387,7 @@ class StringFunctionsSuite extends QueryTest with SharedSQLContext { Row("6.4817")) checkAnswer( - df.select(format_number(lit(BigDecimal(7.128381)), 4)), // not convert anything + df.select(format_number(lit(BigDecimal("7.128381")), 4)), // not convert anything Row("7.1284")) intercept[AnalysisException] { http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala index 1cde137..80e5dd1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala @@ -826,7 +826,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { StructField("b", DecimalType(2, 2), true):: Nil) assert(expectedSchema === jsonDF.schema) - checkAnswer(jsonDF, Row(1.0E-39D, BigDecimal(0.01))) + checkAnswer(jsonDF, Row(1.0E-39D, BigDecimal("0.01"))) val mergedJsonDF = spark.read .option("prefersDecimal", "true") @@ -839,7 +839,7 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData { assert(expectedMergedSchema === mergedJsonDF.schema) checkAnswer( mergedJsonDF, - Row(1.0E-39D, BigDecimal(0.01)) :: + Row(1.0E-39D, BigDecimal("0.01")) :: Row(1.0E38D, BigDecimal("92233720368547758070")) :: Nil ) } http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala index b4f3de9..84b34d5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala @@ -676,7 +676,7 @@ class ParquetPartitionDiscoverySuite extends QueryTest with ParquetTest with Sha 1.5.toFloat, 4.5, new java.math.BigDecimal(new BigInteger("212500"), 5), - new java.math.BigDecimal(2.125), + new java.math.BigDecimal("2.125"), java.sql.Date.valueOf("2015-05-23"), new Timestamp(0), "This is a string, /[]?=:", http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index 2a522a1..be6339f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -245,7 +245,7 @@ private[spark] object HiveUtils extends Logging { val loader = new IsolatedClientLoader( version = IsolatedClientLoader.hiveVersion(hiveExecutionVersion), sparkConf = conf, - execJars = Seq(), + execJars = Seq.empty, hadoopConf = hadoopConf, config = newTemporaryConfiguration(useInMemoryDerby = true), isolationOn = false, http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala index 16c1103..f238b9a 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala @@ -162,8 +162,8 @@ class HadoopTableReader( if (!sparkSession.sessionState.conf.verifyPartitionPath) { partitionToDeserializer } else { - var existPathSet = collection.mutable.Set[String]() - var pathPatternSet = collection.mutable.Set[String]() + val existPathSet = collection.mutable.Set[String]() + val pathPatternSet = collection.mutable.Set[String]() partitionToDeserializer.filter { case (partition, partDeserializer) => def updateExistPathSetByPathPattern(pathPatternStr: String) { @@ -181,8 +181,8 @@ class HadoopTableReader( } val partPath = partition.getDataLocation - val partNum = Utilities.getPartitionDesc(partition).getPartSpec.size(); - var pathPatternStr = getPathPatternByPath(partNum, partPath) + val partNum = Utilities.getPartitionDesc(partition).getPartSpec.size() + val pathPatternStr = getPathPatternByPath(partNum, partPath) if (!pathPatternSet.contains(pathPatternStr)) { pathPatternSet += pathPatternStr updateExistPathSetByPathPattern(pathPatternStr) http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index be024ad..bde9a81 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -391,7 +391,7 @@ private[hive] class HiveClientImpl( val sortColumnNames = if (allAscendingSorted) { sortColumnOrders.map(_.getCol) } else { - Seq() + Seq.empty } Option(BucketSpec(h.getNumBuckets, h.getBucketCols.asScala, sortColumnNames)) } else { http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala index 3de1f4a..11fd8c5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala @@ -90,7 +90,7 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors { Literal(0.asInstanceOf[Double]) :: Literal("0") :: Literal(java.sql.Date.valueOf("2014-09-23")) :: - Literal(Decimal(BigDecimal(123.123))) :: + Literal(Decimal(BigDecimal("123.123"))) :: Literal(new java.sql.Timestamp(123123)) :: Literal(Array[Byte](1, 2, 3)) :: Literal.create(Seq[Int](1, 2, 3), ArrayType(IntegerType)) :: http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala index 6a2c23a..3eedcf7 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala @@ -21,12 +21,9 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hive.conf.HiveConf import org.scalatest.BeforeAndAfterAll -import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.catalog._ -import org.apache.spark.sql.catalyst.expressions.{And, AttributeReference, EmptyRow, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, InSet, LessThan, LessThanOrEqual, Like, Literal, Or} +import org.apache.spark.sql.catalyst.expressions.{EmptyRow, Expression, In, InSet} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.hive.HiveUtils -import org.apache.spark.sql.types.{ByteType, IntegerType, StringType} // TODO: Refactor this to `HivePartitionFilteringSuite` class HiveClientSuite(version: String) @@ -146,7 +143,7 @@ class HiveClientSuite(version: String) 0 to 23, "aa" :: "ab" :: "ba" :: "bb" :: Nil, { case expr @ In(v, list) if expr.inSetConvertible => - InSet(v, Set() ++ list.map(_.eval(EmptyRow))) + InSet(v, list.map(_.eval(EmptyRow)).toSet) }) } @@ -165,7 +162,7 @@ class HiveClientSuite(version: String) 0 to 23, "ab" :: "ba" :: Nil, { case expr @ In(v, list) if expr.inSetConvertible => - InSet(v, Set() ++ list.map(_.eval(EmptyRow))) + InSet(v, list.map(_.eval(EmptyRow)).toSet) }) } http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala index a34f6c7..f3b4ff2 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala @@ -458,7 +458,7 @@ class StreamingContext private[streaming] ( queue: Queue[RDD[T]], oneAtATime: Boolean = true ): InputDStream[T] = { - queueStream(queue, oneAtATime, sc.makeRDD(Seq[T](), 1)) + queueStream(queue, oneAtATime, sc.makeRDD(Seq.empty[T], 1)) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala index a0a40fc..4a0ec31 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/api/java/JavaDStreamLike.scala @@ -153,7 +153,7 @@ trait JavaDStreamLike[T, This <: JavaDStreamLike[T, This, R], R <: JavaRDDLike[T def context(): StreamingContext = dstream.context /** Return a new DStream by applying a function to all elements of this DStream. */ - def map[R](f: JFunction[T, R]): JavaDStream[R] = { + def map[U](f: JFunction[T, U]): JavaDStream[U] = { new JavaDStream(dstream.map(f)(fakeClassTag))(fakeClassTag) } http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/streaming/src/main/scala/org/apache/spark/streaming/dstream/StateDStream.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/StateDStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/StateDStream.scala index 5bf1dab..d1a5e91 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/StateDStream.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/StateDStream.scala @@ -76,7 +76,7 @@ class StateDStream[K: ClassTag, V: ClassTag, S: ClassTag]( // Re-apply the update function to the old state RDD val updateFuncLocal = updateFunc val finalFunc = (iterator: Iterator[(K, S)]) => { - val i = iterator.map(t => (t._1, Seq[V](), Option(t._2))) + val i = iterator.map(t => (t._1, Seq.empty[V], Option(t._2))) updateFuncLocal(validTime, i) } val stateRDD = prevStateRDD.mapPartitions(finalFunc, preservePartitioning) http://git-wip-us.apache.org/repos/asf/spark/blob/e26dac5f/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala ---------------------------------------------------------------------- diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala index 4089366..eb9996e 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/util/RawTextHelper.scala @@ -63,7 +63,6 @@ object RawTextHelper { var i = 0 var len = 0 - var done = false var value: (String, Long) = null var swap: (String, Long) = null var count = 0 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
