Github user BryanCutler commented on a diff in the pull request:
https://github.com/apache/spark/pull/14959#discussion_r80565092
--- Diff: python/pyspark/conf.py ---
@@ -149,35 +161,53 @@ def setAll(self, pairs):
:param pairs: list of key-value pairs to set
"""
for (k, v) in pairs:
- self._jconf.set(k, v)
+ self.set(k, v)
return self
def get(self, key, defaultValue=None):
"""Get the configured value for some key, or return a default
otherwise."""
if defaultValue is None: # Py4J doesn't call the right get() if
we pass None
- if not self._jconf.contains(key):
- return None
- return self._jconf.get(key)
+ if self._jconf:
+ if not self._jconf.contains(key):
+ return None
+ return self._jconf.get(key)
+ else:
+ if key not in self._conf:
+ return None
+ return self._conf[key]
else:
- return self._jconf.get(key, defaultValue)
+ if self._jconf:
+ return self._jconf.get(key, defaultValue)
+ else:
+ return self._conf.get(key, defaultValue)
def getAll(self):
"""Get all values as a list of key-value pairs."""
pairs = []
- for elem in self._jconf.getAll():
- pairs.append((elem._1(), elem._2()))
+ if self._jconf:
+ for elem in self._jconf.getAll():
+ pairs.append((elem._1(), elem._2()))
+ else:
+ for k, v in self._conf.items():
+ pairs.append((k, v))
return pairs
def contains(self, key):
"""Does this configuration contain a given key?"""
- return self._jconf.contains(key)
+ if self._jconf:
+ return self._jconf.contains(key)
+ else:
+ return key in self._conf
def toDebugString(self):
"""
Returns a printable version of the configuration, as a list of
key=value pairs, one per line.
"""
- return self._jconf.toDebugString()
+ if self._jconf:
+ return self._jconf.toDebugString()
+ else:
+ return '\n'.join('%s=%s' % (k, v) for k, v in
self._conf.items())
--- End diff --
maybe add a unit test to make sure the 2 ways to make a debug string are
the same?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
