Github user mengxr commented on a diff in the pull request:
https://github.com/apache/spark/pull/8561#discussion_r38991787
--- Diff: python/pyspark/ml/feature.py ---
@@ -167,6 +168,133 @@ def getSplits(self):
@inherit_doc
+class CountVectorizer(JavaEstimator, HasInputCol, HasOutputCol):
+ """
+ Extracts a vocabulary from document collections and generates a
[[CountVectorizerModel]].
+ >>> df = sqlContext.createDataFrame(
+ ... [(0, ["a", "b", "c"]), (1, ["a", "b", "b", "c", "a"])],
+ ... ["label", "raw"])
+ >>> cv = CountVectorizer(inputCol="raw", outputCol="vectors")
+ >>> model = cv.fit(df)
+ >>> model.transform(df).show(truncate=False)
+ +-----+---------------+-------------------------+
+ |label|raw |vectors |
+ +-----+---------------+-------------------------+
+ |0 |[a, b, c] |(3,[0,1,2],[1.0,1.0,1.0])|
+ |1 |[a, b, b, c, a]|(3,[0,1,2],[2.0,2.0,1.0])|
+ +-----+---------------+-------------------------+
+ ...
+ """
+
+ # a placeholder to make it appear in the generated doc
+ minTF = Param(
+ Params._dummy(), "minTF", "Filter to ignore rare words in" +
+ " a document. For each document, terms with frequency/count less
than the given" +
+ " threshold are ignored. If this is an integer >= 1, then this
specifies a count (of" +
+ " times the term must appear in the document); if this is a double
in [0,1), then this " +
+ "specifies a fraction (out of the document's token count). Note
that the parameter is " +
+ "only used in transform of CountVectorizerModel and does not
affect fitting.")
+ minDF = Param(
+ Params._dummy(), "minDF", "Specifies the minimum number of" +
+ " different documents a term must appear in to be included in the
vocabulary." +
+ " If this is an integer >= 1, this specifies the number of
documents the term must" +
+ " appear in; if this is a double in [0,1), then this specifies the
fraction of documents.")
+ vocabSize = Param(
+ Params._dummy(), "vocabSize", "max size of the vocabulary")
+
+ @keyword_only
+ def __init__(self, minTF=None, minDF=None, vocabSize=None,
inputCol=None, outputCol=None):
+ """
+ __init__(self, minTF=None, minDF=None, vocabSize=None,
inputCol=None, outputCol=None)
+ """
+ super(CountVectorizer, self).__init__()
+ self._java_obj =
self._new_java_obj("org.apache.spark.ml.feature.CountVectorizer",
+ self.uid)
+ minTF = Param(
+ self, "minTF", "Filter to ignore rare words in" +
+ " a document. For each document, terms with frequency/count
less than the given" +
+ " threshold are ignored. If this is an integer >= 1, then this
specifies a count (of" +
+ " times the term must appear in the document); if this is a
double in [0,1), then " +
+ "this specifies a fraction (out of the document's token
count). Note that the " +
+ "parameter is only used in transform of CountVectorizerModel
and does not affect" +
+ "fitting.")
+ minDF = Param(
+ self, "minDF", "Specifies the minimum number of" +
+ " different documents a term must appear in to be included in
the vocabulary." +
+ " If this is an integer >= 1, this specifies the number of
documents the term must" +
+ " appear in; if this is a double in [0,1), then this specifies
the fraction of " +
+ "documents.")
+ vocabSize = Param(
+ self, "vocabSize", "max size of the vocabulary")
+ kwargs = self.__init__._input_kwargs
--- End diff --
and `_setDefault` here.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
