Github user ahirreddy commented on a diff in the pull request:
https://github.com/apache/spark/pull/363#discussion_r11560795
--- Diff: python/pyspark/rdd.py ---
@@ -1387,6 +1387,95 @@ def _jrdd(self):
def _is_pipelinable(self):
return not (self.is_cached or self.is_checkpointed)
+class Row(dict):
+ """
+ An extended L{dict} that takes a L{dict} in its constructor, and
exposes those items as fields.
+
+ >>> r = Row({"hello" : "world", "foo" : "bar"})
+ >>> r.hello
+ 'world'
+ >>> r.foo
+ 'bar'
+ """
+
+ def __init__(self, d):
+ d.update(self.__dict__)
+ self.__dict__ = d
+ dict.__init__(self, d)
+
+class SchemaRDD(RDD):
+ """
+ An RDD of Row objects that has an associated schema. The underlying
JVM object is a SchemaRDD,
+ not a PythonRDD, so we can utilize the relational query api exposed by
SparkSQL.
+
+ For normal L{RDD} operations (map, count, etc.) the L{SchemaRDD} is
not operated on directly, as
+ it's underlying implementation is a RDD composed of Java objects.
Instead it is converted to a
+ PythonRDD in the JVM, on which Python operations can be done.
+ """
+
+ def __init__(self, jschema_rdd, sql_ctx):
+ self.sql_ctx = sql_ctx
+ self._sc = sql_ctx._sc
+ self._jschema_rdd = jschema_rdd
+
+ self.is_cached = False
+ self.is_checkpointed = False
+ self.ctx = self.sql_ctx._sc
+ self._jrdd_deserializer = self.ctx.serializer
+
+ @property
+ def _jrdd(self):
+ """
+ Lazy evaluation of PythonRDD object. Only done when a user calls
methods defined by the
+ L{RDD} super class (map, count, etc.).
+ """
+ return self.toPython()._jrdd
--- End diff --
It get's recomputed every time. I'll add some code, so first time it's
accessed I'll store the value, so subsequent calls won't compute.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
