spark git commit: [MINOR][SQL] remove unuse import class

[srowen]

Repository: spark
Updated Branches:
  refs/heads/master 4bab8f599 -> dc74c0e67


[MINOR][SQL] remove unuse import class

## What changes were proposed in this pull request?

this PR describe remove the import class that are unused.

## How was this patch tested?

N/A

Author: caoxuewen 

Closes #19131 from heary-cao/unuse_import.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dc74c0e6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dc74c0e6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dc74c0e6

Branch: refs/heads/master
Commit: dc74c0e67d96ed22210916b73dd896137f54bc37
Parents: 4bab8f5
Author: caoxuewen 
Authored: Mon Sep 11 10:09:20 2017 +0100
Committer: Sean Owen 
Committed: Mon Sep 11 10:09:20 2017 +0100

--
 .../scala/org/apache/spark/broadcast/TorrentBroadcast.scala| 2 +-
 .../main/scala/org/apache/spark/deploy/DependencyUtils.scala   | 2 --
 .../apache/spark/executor/CoarseGrainedExecutorBackend.scala   | 2 --
 .../org/apache/spark/network/netty/NettyBlockRpcServer.scala   | 2 +-
 .../scala/org/apache/spark/scheduler/ShuffleMapStage.scala | 2 +-
 .../scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala   | 1 -
 .../scala/org/apache/spark/security/CryptoStreamUtils.scala| 2 +-
 .../org/apache/spark/storage/BlockManagerManagedBuffer.scala   | 1 -
 core/src/main/scala/org/apache/spark/storage/DiskStore.scala   | 6 ++
 core/src/main/scala/org/apache/spark/ui/JettyUtils.scala   | 1 -
 .../scala/org/apache/spark/ui/scope/RDDOperationGraph.scala| 2 +-
 core/src/main/scala/org/apache/spark/util/AccumulatorV2.scala  | 3 ---
 .../scala/org/apache/spark/util/ByteBufferInputStream.scala| 2 --
 .../apache/spark/util/collection/ExternalAppendOnlyMap.scala   | 1 -
 .../spark/streaming/kafka010/DirectKafkaInputDStream.scala | 4 +---
 .../src/main/scala/org/apache/spark/ml/attribute/package.scala | 1 -
 .../org/apache/spark/ml/classification/GBTClassifier.scala | 1 -
 .../scala/org/apache/spark/ml/classification/LinearSVC.scala   | 2 +-
 .../ml/classification/MultilayerPerceptronClassifier.scala | 1 -
 .../scala/org/apache/spark/ml/classification/NaiveBayes.scala  | 1 -
 mllib/src/main/scala/org/apache/spark/ml/feature/package.scala | 1 -
 .../scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala | 2 +-
 .../scala/org/apache/spark/ml/optim/loss/RDDLossFunction.scala | 1 -
 .../src/main/scala/org/apache/spark/sql/DataFrameReader.scala  | 1 -
 .../scala/org/apache/spark/sql/execution/SparkPlanInfo.scala   | 1 -
 .../apache/spark/sql/execution/basicPhysicalOperators.scala| 3 +--
 .../scala/org/apache/spark/sql/execution/command/cache.scala   | 1 -
 .../apache/spark/sql/execution/exchange/ShuffleExchange.scala  | 2 +-
 .../spark/sql/execution/joins/BroadcastHashJoinExec.scala  | 2 +-
 .../org/apache/spark/sql/execution/joins/HashedRelation.scala  | 2 +-
 .../spark/sql/execution/joins/ShuffledHashJoinExec.scala   | 2 +-
 .../spark/sql/execution/streaming/FileStreamSinkLog.scala  | 1 -
 .../scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala   | 1 -
 .../scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala| 2 +-
 .../scala/org/apache/spark/sql/hive/MultiDatabaseSuite.scala   | 2 --
 35 files changed, 16 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/dc74c0e6/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
--
diff --git 
a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala 
b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index 039df75..67e993c 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -30,7 +30,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.io.CompressionCodec
 import org.apache.spark.serializer.Serializer
 import org.apache.spark.storage._
-import org.apache.spark.util.{ByteBufferInputStream, Utils}
+import org.apache.spark.util.Utils
 import org.apache.spark.util.io.{ChunkedByteBuffer, 
ChunkedByteBufferOutputStream}
 
 /**

http://git-wip-us.apache.org/repos/asf/spark/blob/dc74c0e6/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala
--
diff --git a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala 
b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala
index db92a8f..51c3d9b 100644
--- a/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/DependencyUtils.scala
@@ -19,8 +19,6 

spark git commit: [SPARK-21856] Add probability and rawPrediction to MLPC for Python

[yliang]

Repository: spark
Updated Branches:
  refs/heads/master 828fab035 -> 4bab8f599


[SPARK-21856] Add probability and rawPrediction to MLPC for Python

Probability and rawPrediction has been added to MultilayerPerceptronClassifier 
for Python

Add unit test.

Author: Chunsheng Ji 

Closes #19172 from chunshengji/SPARK-21856.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4bab8f59
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4bab8f59
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4bab8f59

Branch: refs/heads/master
Commit: 4bab8f5996d94a468a40fde2961ebebafc393508
Parents: 828fab0
Author: Chunsheng Ji 
Authored: Mon Sep 11 16:52:48 2017 +0800
Committer: Yanbo Liang 
Committed: Mon Sep 11 16:52:48 2017 +0800

--
 python/pyspark/ml/classification.py | 15 ++-
 python/pyspark/ml/tests.py  | 20 
 2 files changed, 30 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/4bab8f59/python/pyspark/ml/classification.py
--
diff --git a/python/pyspark/ml/classification.py 
b/python/pyspark/ml/classification.py
index f0f42a3..aa747f3 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -1356,7 +1356,8 @@ class NaiveBayesModel(JavaModel, JavaClassificationModel, 
JavaMLWritable, JavaML
 @inherit_doc
 class MultilayerPerceptronClassifier(JavaEstimator, HasFeaturesCol, 
HasLabelCol, HasPredictionCol,
  HasMaxIter, HasTol, HasSeed, HasStepSize, 
HasSolver,
- JavaMLWritable, JavaMLReadable):
+ JavaMLWritable, JavaMLReadable, 
HasProbabilityCol,
+ HasRawPredictionCol):
 """
 Classifier trainer based on the Multilayer Perceptron.
 Each layer has sigmoid activation function, output layer has softmax.
@@ -1425,11 +1426,13 @@ class MultilayerPerceptronClassifier(JavaEstimator, 
HasFeaturesCol, HasLabelCol,
 @keyword_only
 def __init__(self, featuresCol="features", labelCol="label", 
predictionCol="prediction",
  maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, 
stepSize=0.03,
- solver="l-bfgs", initialWeights=None):
+ solver="l-bfgs", initialWeights=None, 
probabilityCol="probability",
+ rawPredicitionCol="rawPrediction"):
 """
 __init__(self, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
  maxIter=100, tol=1e-6, seed=None, layers=None, blockSize=128, 
stepSize=0.03, \
- solver="l-bfgs", initialWeights=None)
+ solver="l-bfgs", initialWeights=None, 
probabilityCol="probability", \
+ rawPredicitionCol="rawPrediction")
 """
 super(MultilayerPerceptronClassifier, self).__init__()
 self._java_obj = self._new_java_obj(
@@ -1442,11 +1445,13 @@ class MultilayerPerceptronClassifier(JavaEstimator, 
HasFeaturesCol, HasLabelCol,
 @since("1.6.0")
 def setParams(self, featuresCol="features", labelCol="label", 
predictionCol="prediction",
   maxIter=100, tol=1e-6, seed=None, layers=None, 
blockSize=128, stepSize=0.03,
-  solver="l-bfgs", initialWeights=None):
+  solver="l-bfgs", initialWeights=None, 
probabilityCol="probability",
+  rawPredicitionCol="rawPrediction"):
 """
 setParams(self, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
   maxIter=100, tol=1e-6, seed=None, layers=None, 
blockSize=128, stepSize=0.03, \
-  solver="l-bfgs", initialWeights=None)
+  solver="l-bfgs", initialWeights=None, 
probabilityCol="probability", \
+  rawPredicitionCol="rawPrediction"):
 Sets params for MultilayerPerceptronClassifier.
 """
 kwargs = self._input_kwargs

http://git-wip-us.apache.org/repos/asf/spark/blob/4bab8f59/python/pyspark/ml/tests.py
--
diff --git a/python/pyspark/ml/tests.py b/python/pyspark/ml/tests.py
index 509698f..15d6c76 100755
--- a/python/pyspark/ml/tests.py
+++ b/python/pyspark/ml/tests.py
@@ -1655,6 +1655,26 @@ class LogisticRegressionTest(SparkSessionTestCase):
 np.allclose(model.interceptVector.toArray(), [-0.9057, -1.1392, 
-0.0033], atol=1E-4))
 
 
+class MultilayerPerceptronClassifierTest(SparkSessionTestCase):
+
+def test_raw_and_probability_prediction(self):
+
+data_path =