Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/16715#discussion_r99928739
--- Diff: python/pyspark/ml/feature.py ---
@@ -755,6 +951,102 @@ def maxAbs(self):
@inherit_doc
+class MinHashLSH(JavaEstimator, LSHParams, HasInputCol, HasOutputCol,
HasSeed,
+ JavaMLReadable, JavaMLWritable):
+
+ """
+ .. note:: Experimental
+
+ LSH class for Jaccard distance.
+ The input can be dense or sparse vectors, but it is more efficient if
it is sparse.
+ For example, `Vectors.sparse(10, Array[(2, 1.0), (3, 1.0), (5, 1.0)])`
+ means there are 10 elements in the space. This set contains elem 2,
elem 3 and elem 5.
+ Also, any input vector must have at least 1 non-zero indices, and all
non-zero values
+ are treated as binary "1" values.
+
+ .. seealso:: `MinHash <https://en.wikipedia.org/wiki/MinHash>`_
+
+ >>> from pyspark.ml.linalg import Vectors
+ >>> data = [(Vectors.sparse(6, [0, 1, 2], [1.0, 1.0, 1.0]),),
+ ... (Vectors.sparse(6, [2, 3, 4], [1.0, 1.0, 1.0]),),
+ ... (Vectors.sparse(6, [0, 2, 4], [1.0, 1.0, 1.0]),)]
+ >>> df = spark.createDataFrame(data, ["keys"])
+ >>> mh = MinHashLSH(inputCol="keys", outputCol="values", seed=12345)
+ >>> model = mh.fit(df)
+ >>> model.transform(df).head()
+ Row(keys=SparseVector(6, {0: 1.0, 1: 1.0, 2: 1.0}),
values=[DenseVector([-1638925712.0])])
+ >>> data2 = [(Vectors.sparse(6, [1, 3, 5], [1.0, 1.0, 1.0]),),
+ ... (Vectors.sparse(6, [2, 3, 5], [1.0, 1.0, 1.0]),),
+ ... (Vectors.sparse(6, [1, 2, 4], [1.0, 1.0, 1.0]),)]
+ >>> df2 = spark.createDataFrame(data2, ["keys"])
+ >>> key = Vectors.sparse(6, [1], [1.0])
+ >>> model.approxNearestNeighbors(df2, key,
1).select("distCol").head()[0]
+ 0.66666...
+ >>> model.approxSimilarityJoin(df, df2,
1.0).select("distCol").head()[0]
+ 0.5
+ >>> mhPath = temp_path + "/mh"
+ >>> mh.save(mhPath)
+ >>> mh2 = MinHashLSH.load(mhPath)
+ >>> mh2.getOutputCol() == mh.getOutputCol()
+ True
+ >>> modelPath = temp_path + "/mh-model"
+ >>> model.save(modelPath)
+ >>> model2 = MinHashLSHModel.load(modelPath)
+
+ .. versionadded:: 2.2.0
+ """
+
+ @keyword_only
+ def __init__(self, inputCol=None, outputCol=None, seed=None,
numHashTables=1):
+ """
+ __init__(self, inputCol=None, outputCol=None, seed=None,
numHashTables=1)
+ """
+ super(MinHashLSH, self).__init__()
+ self._java_obj =
self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid)
+ self._setDefault(numHashTables=1)
+ kwargs = self.__init__._input_kwargs
+ self.setParams(**kwargs)
+
+ @keyword_only
+ @since("2.2.0")
+ def setParams(self, inputCol=None, outputCol=None, seed=None,
numHashTables=1):
+ """
+ setParams(self, inputCol=None, outputCol=None, seed=None,
numHashTables=1)
+ Sets params for this MinHashLSH.
+ """
+ kwargs = self.setParams._input_kwargs
+ return self._set(**kwargs)
+
+ def _create_model(self, java_model):
+ return MinHashLSHModel(java_model)
+
+
+class MinHashLSHModel(JavaModel, LSHModel, JavaMLReadable, JavaMLWritable):
+ """
+ .. note:: Experimental
+
+ Model produced by :py:class:`MinHashLSH`, where where multiple hash
functions are stored. Each
+ hash function is picked from the following family of hash functions,
where :math:`a_i` and
+ :math:`b_i` are randomly chosen integers less than prime:
+ :math:`h_i(x) = ((x \cdot a_i + b_i) \mod prime)` This hash family is
approximately min-wise
+ independent according to the reference.
+
+ .. seealso:: Tom Bohman, Colin Cooper, and Alan Frieze. "Min-wise
independent linear \
+ permutations." Electronic Journal of Combinatorics 7 (2000): R26.
+
+ .. versionadded:: 2.2.0
+ """
+
+ @property
+ @since("2.2.0")
+ def randCoefficients(self):
--- End diff --
This should not be exposed, since it's private in Scala. Also, `Array[(Int,
Int)]` does not serialize to Python.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
