Repository: spark Updated Branches: refs/heads/master 83af297ac -> d6ae7d463
[SPARK-14665][ML][PYTHON] Fixed bug with StopWordsRemover default stopwords ## What changes were proposed in this pull request? The default stopwords were a Java object. They are no longer. ## How was this patch tested? Unit test which failed before the fix Author: Joseph K. Bradley <[email protected]> Closes #12422 from jkbradley/pyspark-stopwords. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d6ae7d46 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d6ae7d46 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d6ae7d46 Branch: refs/heads/master Commit: d6ae7d4637d23c57c4eeab79d1177216f380ec9c Parents: 83af297 Author: Joseph K. Bradley <[email protected]> Authored: Fri Apr 15 11:50:21 2016 -0700 Committer: Joseph K. Bradley <[email protected]> Committed: Fri Apr 15 11:50:21 2016 -0700 ---------------------------------------------------------------------- python/pyspark/ml/feature.py | 2 +- python/pyspark/ml/tests.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/d6ae7d46/python/pyspark/ml/feature.py ---------------------------------------------------------------------- diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py index 809a513..0d8ef12 100644 --- a/python/pyspark/ml/feature.py +++ b/python/pyspark/ml/feature.py @@ -1765,7 +1765,7 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadabl self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StopWordsRemover", self.uid) stopWordsObj = _jvm().org.apache.spark.ml.feature.StopWords - defaultStopWords = stopWordsObj.English() + defaultStopWords = list(stopWordsObj.English()) self._setDefault(stopWords=defaultStopWords, caseSensitive=False) kwargs = self.__init__._input_kwargs self.setParams(**kwargs) http://git-wip-us.apache.org/repos/asf/spark/blob/d6ae7d46/python/pyspark/ml/tests.py ---------------------------------------------------------------------- diff --git a/python/pyspark/ml/tests.py b/python/pyspark/ml/tests.py index 86c0254..85ad949 100644 --- a/python/pyspark/ml/tests.py +++ b/python/pyspark/ml/tests.py @@ -22,6 +22,7 @@ import array import sys if sys.version > '3': xrange = range + basestring = str try: import xmlrunner @@ -398,6 +399,8 @@ class FeatureTests(PySparkTestCase): self.assertEqual(stopWordRemover.getInputCol(), "input") transformedDF = stopWordRemover.transform(dataset) self.assertEqual(transformedDF.head().output, ["panda"]) + self.assertEqual(type(stopWordRemover.getStopWords()), list) + self.assertTrue(isinstance(stopWordRemover.getStopWords()[0], basestring)) # Custom stopwords = ["panda"] stopWordRemover.setStopWords(stopwords) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
