Github user sethah commented on a diff in the pull request:
https://github.com/apache/spark/pull/12660#discussion_r61000897
--- Diff: python/pyspark/ml/tests.py ---
@@ -929,6 +932,50 @@ def test_apply_binary_term_freqs(self):
": expected " + str(expected[i]) + ",
got " + str(features[i]))
+class ALSTest(PySparkTestCase):
+
+ def test_storage_levels(self):
+ sqlContext = SQLContext(self.sc)
+ df = sqlContext.createDataFrame(
+ [(0, 0, 4.0), (0, 1, 2.0), (1, 1, 3.0), (1, 2, 4.0), (2, 1,
1.0), (2, 2, 5.0)],
+ ["user", "item", "rating"])
+ als = ALS().setMaxIter(1)
+ # test default
+ als.fit(df)
+ item_factors = [(r[0], r[1]) for r in
self.sc._jsc.getPersistentRDDs().items()
+ if r[1].name() == "itemFactors"][0][1]
+ java_storage_level = item_factors.getStorageLevel()
+ self.assertEqual(java_storage_level.useDisk(), True)
+ self.assertEqual(java_storage_level.useMemory(), True)
+ self.assertEqual(java_storage_level.useOffHeap(), False)
+ self.assertEqual(java_storage_level.deserialized(), True)
+ self.assertEqual(java_storage_level.replication(), 1)
+ # set intermediate and final RDD storage level to non-default
values
+ als.setIntermediateRDDStorageLevel("MEMORY_ONLY")
+ als.setFinalRDDStorageLevel("MEMORY_ONLY_2")
+ als.fit(df)
+ item_factors2 = [(r[0], r[1]) for r in
self.sc._jsc.getPersistentRDDs().items()
+ if r[1].name() == "itemFactors" and r[0] !=
item_factors.id()][0][1]
+ java_storage_level2 = item_factors2.getStorageLevel()
+ self.assertEqual(java_storage_level2.useDisk(), False)
+ self.assertEqual(java_storage_level2.useMemory(), True)
+ self.assertEqual(java_storage_level2.useOffHeap(), False)
+ self.assertEqual(java_storage_level2.deserialized(), True)
+ self.assertEqual(java_storage_level2.replication(), 2)
+
+ def test_invalid(self):
--- End diff --
This test seems redundant to me. Since you already checked that validation
on the Scala side, there should be no need to test it again here, since Python
simply passes the params to java and calls the same validation.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
