Repository: spark
Updated Branches:
refs/heads/master 8a1efe307 -> 61487b308
[SPARK-23706][PYTHON] spark.conf.get(value, default=None) should produce None
in PySpark
## What changes were proposed in this pull request?
Scala:
```
scala> spark.conf.get("hey", null)
res1: String = null
```
```
scala> spark.conf.get("spark.sql.sources.partitionOverwriteMode", null)
res2: String = null
```
Python:
**Before**
```
>>> spark.conf.get("hey", None)
...
py4j.protocol.Py4JJavaError: An error occurred while calling o30.get.
: java.util.NoSuchElementException: hey
...
```
```
>>> spark.conf.get("spark.sql.sources.partitionOverwriteMode", None)
u'STATIC'
```
**After**
```
>>> spark.conf.get("hey", None) is None
True
```
```
>>> spark.conf.get("spark.sql.sources.partitionOverwriteMode", None) is None
True
```
*Note that this PR preserves the case below:
```
>>> spark.conf.get("spark.sql.sources.partitionOverwriteMode")
u'STATIC'
```
## How was this patch tested?
Manually tested and unit tests were added.
Author: hyukjinkwon <[email protected]>
Closes #20841 from HyukjinKwon/spark-conf-get.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/61487b30
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/61487b30
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/61487b30
Branch: refs/heads/master
Commit: 61487b308b0169e3108c2ad31674a0c80b8ac5f3
Parents: 8a1efe3
Author: hyukjinkwon <[email protected]>
Authored: Sun Mar 18 20:24:14 2018 +0900
Committer: hyukjinkwon <[email protected]>
Committed: Sun Mar 18 20:24:14 2018 +0900
----------------------------------------------------------------------
python/pyspark/sql/conf.py | 9 +++++----
python/pyspark/sql/context.py | 8 ++++----
python/pyspark/sql/tests.py | 11 +++++++++++
3 files changed, 20 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/61487b30/python/pyspark/sql/conf.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py
index d929834..b82224b 100644
--- a/python/pyspark/sql/conf.py
+++ b/python/pyspark/sql/conf.py
@@ -17,7 +17,7 @@
import sys
-from pyspark import since
+from pyspark import since, _NoValue
from pyspark.rdd import ignore_unicode_prefix
@@ -39,15 +39,16 @@ class RuntimeConfig(object):
@ignore_unicode_prefix
@since(2.0)
- def get(self, key, default=None):
+ def get(self, key, default=_NoValue):
"""Returns the value of Spark runtime configuration property for the
given key,
assuming it is set.
"""
self._checkType(key, "key")
- if default is None:
+ if default is _NoValue:
return self._jconf.get(key)
else:
- self._checkType(default, "default")
+ if default is not None:
+ self._checkType(default, "default")
return self._jconf.get(key, default)
@ignore_unicode_prefix
http://git-wip-us.apache.org/repos/asf/spark/blob/61487b30/python/pyspark/sql/context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 6cb9039..e9ec7ba 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -22,7 +22,7 @@ import warnings
if sys.version >= '3':
basestring = unicode = str
-from pyspark import since
+from pyspark import since, _NoValue
from pyspark.rdd import ignore_unicode_prefix
from pyspark.sql.session import _monkey_patch_RDD, SparkSession
from pyspark.sql.dataframe import DataFrame
@@ -124,11 +124,11 @@ class SQLContext(object):
@ignore_unicode_prefix
@since(1.3)
- def getConf(self, key, defaultValue=None):
+ def getConf(self, key, defaultValue=_NoValue):
"""Returns the value of Spark SQL configuration property for the given
key.
- If the key is not set and defaultValue is not None, return
- defaultValue. If the key is not set and defaultValue is None, return
+ If the key is not set and defaultValue is set, return
+ defaultValue. If the key is not set and defaultValue is not set, return
the system default value.
>>> sqlContext.getConf("spark.sql.shuffle.partitions")
http://git-wip-us.apache.org/repos/asf/spark/blob/61487b30/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 480815d..a0d547a 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -2504,6 +2504,17 @@ class SQLTests(ReusedSQLTestCase):
spark.conf.unset("bogo")
self.assertEqual(spark.conf.get("bogo", "colombia"), "colombia")
+ self.assertEqual(spark.conf.get("hyukjin", None), None)
+
+ # This returns 'STATIC' because it's the default value of
+ # 'spark.sql.sources.partitionOverwriteMode', and `defaultValue` in
+ # `spark.conf.get` is unset.
+
self.assertEqual(spark.conf.get("spark.sql.sources.partitionOverwriteMode"),
"STATIC")
+
+ # This returns None because 'spark.sql.sources.partitionOverwriteMode'
is unset, but
+ # `defaultValue` in `spark.conf.get` is set to None.
+
self.assertEqual(spark.conf.get("spark.sql.sources.partitionOverwriteMode",
None), None)
+
def test_current_database(self):
spark = self.spark
spark.catalog._reset()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
