Updated Branches: refs/heads/master fad6aacfb -> cad3002fe
Fix SPARK-1034: Py4JException on PySpark Cartesian Result Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0035dbbc Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0035dbbc Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0035dbbc Branch: refs/heads/master Commit: 0035dbbc8125af94ae27cb6f10e87aa6f5a078b1 Parents: a2b47da Author: Josh Rosen <joshro...@apache.org> Authored: Thu Jan 23 13:05:59 2014 -0800 Committer: Josh Rosen <joshro...@apache.org> Committed: Thu Jan 23 13:05:59 2014 -0800 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/api/java/JavaPairRDD.scala | 3 +-- python/pyspark/tests.py | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0035dbbc/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala index 0fb7e19..f430a33 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala @@ -49,8 +49,7 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])(implicit val kClassTag: ClassTag[K override def wrapRDD(rdd: RDD[(K, V)]): JavaPairRDD[K, V] = JavaPairRDD.fromRDD(rdd) - override val classTag: ClassTag[(K, V)] = - implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[Tuple2[K, V]]] + override val classTag: ClassTag[(K, V)] = rdd.elementClassTag import JavaPairRDD._ http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0035dbbc/python/pyspark/tests.py ---------------------------------------------------------------------- diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py index 7acb6ea..05a9f7f 100644 --- a/python/pyspark/tests.py +++ b/python/pyspark/tests.py @@ -152,6 +152,13 @@ class TestRDDFunctions(PySparkTestCase): raw_contents = ''.join(input(glob(tempFile.name + "/part-0000*"))) self.assertEqual(x, unicode(raw_contents.strip(), "utf-8")) + def test_transforming_cartesian_result(self): + # Regression test for SPARK-1034 + rdd1 = self.sc.parallelize([1, 2]) + rdd2 = self.sc.parallelize([3, 4]) + cart = rdd1.cartesian(rdd2) + result = cart.map(lambda (x, y): x + y).collect() + class TestIO(PySparkTestCase):
