Repository: spark
Updated Branches:
refs/heads/master 034ae305c -> bbbdaa82a
[SPARK-26105][PYTHON] Clean unittest2 imports up that were added for Python 2.6
before
## What changes were proposed in this pull request?
Currently, some of PySpark tests sill assume the tests could be ran in Python
2.6 by importing `unittest2`. For instance:
```python
if sys.version_info[:2] <= (2, 6):
try:
import unittest2 as unittest
except ImportError:
sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
sys.exit(1)
else:
import unittest
```
While I am here, I removed some of unused imports and reordered imports per PEP
8.
We officially dropped Python 2.6 support a while ago and started to discuss
about Python 2 drop. It's better to remove them out.
## How was this patch tested?
Manually tests, and existing tests via Jenkins.
Closes #23077 from HyukjinKwon/SPARK-26105.
Lead-authored-by: hyukjinkwon <[email protected]>
Co-authored-by: Bryan Cutler <[email protected]>
Signed-off-by: hyukjinkwon <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bbbdaa82
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bbbdaa82
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bbbdaa82
Branch: refs/heads/master
Commit: bbbdaa82a4f4fc7a84be6641518264d9bb7bde2b
Parents: 034ae30
Author: hyukjinkwon <[email protected]>
Authored: Mon Nov 19 09:22:32 2018 +0800
Committer: hyukjinkwon <[email protected]>
Committed: Mon Nov 19 09:22:32 2018 +0800
----------------------------------------------------------------------
python/pyspark/ml/tests/test_algorithms.py | 11 +----------
python/pyspark/ml/tests/test_base.py | 10 +---------
python/pyspark/ml/tests/test_evaluation.py | 10 +---------
python/pyspark/ml/tests/test_feature.py | 9 +--------
python/pyspark/ml/tests/test_image.py | 10 +---------
python/pyspark/ml/tests/test_linalg.py | 12 ++----------
python/pyspark/ml/tests/test_param.py | 16 +++++-----------
python/pyspark/ml/tests/test_persistence.py | 10 +---------
python/pyspark/ml/tests/test_pipeline.py | 10 +---------
python/pyspark/ml/tests/test_stat.py | 10 +---------
python/pyspark/ml/tests/test_training_summary.py | 9 +--------
python/pyspark/ml/tests/test_tuning.py | 10 +---------
python/pyspark/ml/tests/test_wrapper.py | 10 +---------
python/pyspark/mllib/tests/test_algorithms.py | 13 +------------
python/pyspark/mllib/tests/test_feature.py | 11 +----------
python/pyspark/mllib/tests/test_linalg.py | 11 +----------
python/pyspark/mllib/tests/test_stat.py | 11 +----------
.../mllib/tests/test_streaming_algorithms.py | 11 +----------
python/pyspark/mllib/tests/test_util.py | 15 ++-------------
python/pyspark/testing/mllibutils.py | 11 +----------
20 files changed, 26 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_algorithms.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_algorithms.py
b/python/pyspark/ml/tests/test_algorithms.py
index 1a72e12..516bb56 100644
--- a/python/pyspark/ml/tests/test_algorithms.py
+++ b/python/pyspark/ml/tests/test_algorithms.py
@@ -16,17 +16,8 @@
#
from shutil import rmtree
-import sys
import tempfile
-
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
import numpy as np
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_base.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_base.py
b/python/pyspark/ml/tests/test_base.py
index 59c45f6..31e3deb 100644
--- a/python/pyspark/ml/tests/test_base.py
+++ b/python/pyspark/ml/tests/test_base.py
@@ -15,15 +15,7 @@
# limitations under the License.
#
-import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
from pyspark.sql.types import DoubleType, IntegerType
from pyspark.testing.mlutils import MockDataset, MockEstimator,
MockUnaryTransformer, \
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_evaluation.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_evaluation.py
b/python/pyspark/ml/tests/test_evaluation.py
index 6c3e5c6..5438455 100644
--- a/python/pyspark/ml/tests/test_evaluation.py
+++ b/python/pyspark/ml/tests/test_evaluation.py
@@ -15,15 +15,7 @@
# limitations under the License.
#
-import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
import numpy as np
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_feature.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_feature.py
b/python/pyspark/ml/tests/test_feature.py
index 23f66e7..325feab 100644
--- a/python/pyspark/ml/tests/test_feature.py
+++ b/python/pyspark/ml/tests/test_feature.py
@@ -17,14 +17,7 @@
#
import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
if sys.version > '3':
basestring = str
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_image.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_image.py
b/python/pyspark/ml/tests/test_image.py
index dcc7a32..4c280a4 100644
--- a/python/pyspark/ml/tests/test_image.py
+++ b/python/pyspark/ml/tests/test_image.py
@@ -14,15 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
import py4j
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_linalg.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_linalg.py
b/python/pyspark/ml/tests/test_linalg.py
index 76e5386..71cad5d 100644
--- a/python/pyspark/ml/tests/test_linalg.py
+++ b/python/pyspark/ml/tests/test_linalg.py
@@ -15,17 +15,9 @@
# limitations under the License.
#
-import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
-
+import unittest
import array as pyarray
+
from numpy import arange, array, array_equal, inf, ones, tile, zeros
from pyspark.ml.linalg import DenseMatrix, DenseVector, MatrixUDT,
SparseMatrix, SparseVector, \
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_param.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_param.py
b/python/pyspark/ml/tests/test_param.py
index 1f36d45..17c1b0b 100644
--- a/python/pyspark/ml/tests/test_param.py
+++ b/python/pyspark/ml/tests/test_param.py
@@ -19,17 +19,7 @@
import inspect
import sys
import array as pyarray
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
-
-if sys.version > '3':
- xrange = range
+import unittest
import numpy as np
@@ -45,6 +35,10 @@ from pyspark.ml.wrapper import JavaParams
from pyspark.testing.mlutils import check_params, PySparkTestCase,
SparkSessionTestCase
+if sys.version > '3':
+ xrange = range
+
+
class ParamTypeConversionTests(PySparkTestCase):
"""
Test that param type conversion happens.
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_persistence.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_persistence.py
b/python/pyspark/ml/tests/test_persistence.py
index b5a2e16..34d6870 100644
--- a/python/pyspark/ml/tests/test_persistence.py
+++ b/python/pyspark/ml/tests/test_persistence.py
@@ -17,16 +17,8 @@
import json
from shutil import rmtree
-import sys
import tempfile
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
from pyspark.ml import Transformer
from pyspark.ml.classification import DecisionTreeClassifier,
LogisticRegression, OneVsRest, \
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_pipeline.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_pipeline.py
b/python/pyspark/ml/tests/test_pipeline.py
index 31ef02c..9e3e6c4 100644
--- a/python/pyspark/ml/tests/test_pipeline.py
+++ b/python/pyspark/ml/tests/test_pipeline.py
@@ -14,15 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
from pyspark.ml.pipeline import Pipeline
from pyspark.testing.mlutils import MockDataset, MockEstimator,
MockTransformer, PySparkTestCase
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_stat.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_stat.py
b/python/pyspark/ml/tests/test_stat.py
index bdc4853..11aaf2e 100644
--- a/python/pyspark/ml/tests/test_stat.py
+++ b/python/pyspark/ml/tests/test_stat.py
@@ -15,15 +15,7 @@
# limitations under the License.
#
-import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
from pyspark.ml.linalg import Vectors
from pyspark.ml.stat import ChiSquareTest
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_training_summary.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_training_summary.py
b/python/pyspark/ml/tests/test_training_summary.py
index d5464f7..8575111 100644
--- a/python/pyspark/ml/tests/test_training_summary.py
+++ b/python/pyspark/ml/tests/test_training_summary.py
@@ -16,14 +16,7 @@
#
import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
if sys.version > '3':
basestring = str
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_tuning.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_tuning.py
b/python/pyspark/ml/tests/test_tuning.py
index af00d1d..39bb921 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -15,16 +15,8 @@
# limitations under the License.
#
-import sys
import tempfile
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
from pyspark.ml import Estimator, Model
from pyspark.ml.classification import LogisticRegression,
LogisticRegressionModel, OneVsRest
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/ml/tests/test_wrapper.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests/test_wrapper.py
b/python/pyspark/ml/tests/test_wrapper.py
index 4326d8e..ae672a0 100644
--- a/python/pyspark/ml/tests/test_wrapper.py
+++ b/python/pyspark/ml/tests/test_wrapper.py
@@ -15,15 +15,7 @@
# limitations under the License.
#
-import sys
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
import py4j
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/mllib/tests/test_algorithms.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/tests/test_algorithms.py
b/python/pyspark/mllib/tests/test_algorithms.py
index 8a34541..cc3b64b 100644
--- a/python/pyspark/mllib/tests/test_algorithms.py
+++ b/python/pyspark/mllib/tests/test_algorithms.py
@@ -16,27 +16,16 @@
#
import os
-import sys
import tempfile
from shutil import rmtree
+import unittest
from numpy import array, array_equal
-
from py4j.protocol import Py4JJavaError
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
-
from pyspark.mllib.fpm import FPGrowth
from pyspark.mllib.recommendation import Rating
from pyspark.mllib.regression import LabeledPoint
-from pyspark.sql.utils import IllegalArgumentException
from pyspark.testing.mllibutils import make_serializer, MLlibTestCase
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/mllib/tests/test_feature.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/tests/test_feature.py
b/python/pyspark/mllib/tests/test_feature.py
index 48ed810..3da841c 100644
--- a/python/pyspark/mllib/tests/test_feature.py
+++ b/python/pyspark/mllib/tests/test_feature.py
@@ -15,20 +15,11 @@
# limitations under the License.
#
-import sys
from math import sqrt
+import unittest
from numpy import array, random, exp, abs, tile
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
-
from pyspark.mllib.linalg import Vector, SparseVector, DenseVector, VectorUDT,
Vectors
from pyspark.mllib.linalg.distributed import RowMatrix
from pyspark.mllib.feature import HashingTF, IDF, StandardScaler,
ElementwiseProduct, Word2Vec
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/mllib/tests/test_linalg.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/tests/test_linalg.py
b/python/pyspark/mllib/tests/test_linalg.py
index 550e32a..d0ebd9b 100644
--- a/python/pyspark/mllib/tests/test_linalg.py
+++ b/python/pyspark/mllib/tests/test_linalg.py
@@ -17,18 +17,9 @@
import sys
import array as pyarray
+import unittest
from numpy import array, array_equal, zeros, arange, tile, ones, inf
-from numpy import sum as array_sum
-
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
import pyspark.ml.linalg as newlinalg
from pyspark.mllib.linalg import Vector, SparseVector, DenseVector, VectorUDT,
_convert_to_vector, \
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/mllib/tests/test_stat.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/tests/test_stat.py
b/python/pyspark/mllib/tests/test_stat.py
index 5e74087..f23ae29 100644
--- a/python/pyspark/mllib/tests/test_stat.py
+++ b/python/pyspark/mllib/tests/test_stat.py
@@ -15,20 +15,11 @@
# limitations under the License.
#
-import sys
import array as pyarray
+import unittest
from numpy import array
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
-
from pyspark.mllib.linalg import Vector, SparseVector, DenseVector, VectorUDT,
_convert_to_vector, \
DenseMatrix, SparseMatrix, Vectors, Matrices, MatrixUDT
from pyspark.mllib.random import RandomRDDs
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/mllib/tests/test_streaming_algorithms.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py
b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index ba95855..4bc8904 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -15,21 +15,12 @@
# limitations under the License.
#
-import sys
from time import time, sleep
+import unittest
from numpy import array, random, exp, dot, all, mean, abs
from numpy import sum as array_sum
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
-
from pyspark import SparkContext
from pyspark.mllib.clustering import StreamingKMeans, StreamingKMeansModel
from pyspark.mllib.classification import StreamingLogisticRegressionWithSGD
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/mllib/tests/test_util.py
----------------------------------------------------------------------
diff --git a/python/pyspark/mllib/tests/test_util.py
b/python/pyspark/mllib/tests/test_util.py
index c924eba..e957162 100644
--- a/python/pyspark/mllib/tests/test_util.py
+++ b/python/pyspark/mllib/tests/test_util.py
@@ -16,25 +16,14 @@
#
import os
-import sys
import tempfile
-
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
from pyspark.mllib.common import _to_java_object_rdd
from pyspark.mllib.util import LinearDataGenerator
from pyspark.mllib.util import MLUtils
-from pyspark.mllib.linalg import SparseVector, DenseVector, SparseMatrix,
Vectors
+from pyspark.mllib.linalg import SparseVector, DenseVector, Vectors
from pyspark.mllib.random import RandomRDDs
-from pyspark.mllib.regression import LabeledPoint
-from pyspark.mllib.stat import Statistics
from pyspark.testing.mllibutils import MLlibTestCase
http://git-wip-us.apache.org/repos/asf/spark/blob/bbbdaa82/python/pyspark/testing/mllibutils.py
----------------------------------------------------------------------
diff --git a/python/pyspark/testing/mllibutils.py
b/python/pyspark/testing/mllibutils.py
index 9248182..25f1bba 100644
--- a/python/pyspark/testing/mllibutils.py
+++ b/python/pyspark/testing/mllibutils.py
@@ -15,16 +15,7 @@
# limitations under the License.
#
-import sys
-
-if sys.version_info[:2] <= (2, 6):
- try:
- import unittest2 as unittest
- except ImportError:
- sys.stderr.write('Please install unittest2 to test with Python 2.6 or
earlier')
- sys.exit(1)
-else:
- import unittest
+import unittest
from pyspark import SparkContext
from pyspark.serializers import PickleSerializer
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
