Repository: spark
Updated Branches:
refs/heads/branch-2.3 6937571ab -> 80e79430f
[SPARK-23706][PYTHON] spark.conf.get(value, default=None) should produce None
in PySpark
Scala:
```
scala> spark.conf.get("hey", null)
res1: String = null
```
```
scala> spark.conf.get("spark.sql.sources.partitionOverwriteMode", null)
res2: String = null
```
Python:
**Before**
```
>>> spark.conf.get("hey", None)
...
py4j.protocol.Py4JJavaError: An error occurred while calling o30.get.
: java.util.NoSuchElementException: hey
...
```
```
>>> spark.conf.get("spark.sql.sources.partitionOverwriteMode", None)
u'STATIC'
```
**After**
```
>>> spark.conf.get("hey", None) is None
True
```
```
>>> spark.conf.get("spark.sql.sources.partitionOverwriteMode", None) is None
True
```
*Note that this PR preserves the case below:
```
>>> spark.conf.get("spark.sql.sources.partitionOverwriteMode")
u'STATIC'
```
Manually tested and unit tests were added.
Author: hyukjinkwon <[email protected]>
Closes #20841 from HyukjinKwon/spark-conf-get.
(cherry picked from commit 61487b308b0169e3108c2ad31674a0c80b8ac5f3)
Signed-off-by: hyukjinkwon <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/80e79430
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/80e79430
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/80e79430
Branch: refs/heads/branch-2.3
Commit: 80e79430ff33dca9a3641bf0a882d79b343a3f25
Parents: 6937571
Author: hyukjinkwon <[email protected]>
Authored: Sun Mar 18 20:24:14 2018 +0900
Committer: hyukjinkwon <[email protected]>
Committed: Sun Mar 18 20:26:43 2018 +0900
----------------------------------------------------------------------
python/pyspark/sql/conf.py | 11 +++++++----
python/pyspark/sql/context.py | 8 ++++----
python/pyspark/sql/tests.py | 11 +++++++++++
3 files changed, 22 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/80e79430/python/pyspark/sql/conf.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py
index 792c420..d62ab91 100644
--- a/python/pyspark/sql/conf.py
+++ b/python/pyspark/sql/conf.py
@@ -15,7 +15,9 @@
# limitations under the License.
#
-from pyspark import since
+import sys
+
+from pyspark import since, _NoValue
from pyspark.rdd import ignore_unicode_prefix
@@ -37,15 +39,16 @@ class RuntimeConfig(object):
@ignore_unicode_prefix
@since(2.0)
- def get(self, key, default=None):
+ def get(self, key, default=_NoValue):
"""Returns the value of Spark runtime configuration property for the
given key,
assuming it is set.
"""
self._checkType(key, "key")
- if default is None:
+ if default is _NoValue:
return self._jconf.get(key)
else:
- self._checkType(default, "default")
+ if default is not None:
+ self._checkType(default, "default")
return self._jconf.get(key, default)
@ignore_unicode_prefix
http://git-wip-us.apache.org/repos/asf/spark/blob/80e79430/python/pyspark/sql/context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index cc1cd1a..673415f 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -22,7 +22,7 @@ import warnings
if sys.version >= '3':
basestring = unicode = str
-from pyspark import since
+from pyspark import since, _NoValue
from pyspark.rdd import ignore_unicode_prefix
from pyspark.sql.session import _monkey_patch_RDD, SparkSession
from pyspark.sql.dataframe import DataFrame
@@ -124,11 +124,11 @@ class SQLContext(object):
@ignore_unicode_prefix
@since(1.3)
- def getConf(self, key, defaultValue=None):
+ def getConf(self, key, defaultValue=_NoValue):
"""Returns the value of Spark SQL configuration property for the given
key.
- If the key is not set and defaultValue is not None, return
- defaultValue. If the key is not set and defaultValue is None, return
+ If the key is not set and defaultValue is set, return
+ defaultValue. If the key is not set and defaultValue is not set, return
the system default value.
>>> sqlContext.getConf("spark.sql.shuffle.partitions")
http://git-wip-us.apache.org/repos/asf/spark/blob/80e79430/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 4c51f71..82c5500 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -2452,6 +2452,17 @@ class SQLTests(ReusedSQLTestCase):
spark.conf.unset("bogo")
self.assertEqual(spark.conf.get("bogo", "colombia"), "colombia")
+ self.assertEqual(spark.conf.get("hyukjin", None), None)
+
+ # This returns 'STATIC' because it's the default value of
+ # 'spark.sql.sources.partitionOverwriteMode', and `defaultValue` in
+ # `spark.conf.get` is unset.
+
self.assertEqual(spark.conf.get("spark.sql.sources.partitionOverwriteMode"),
"STATIC")
+
+ # This returns None because 'spark.sql.sources.partitionOverwriteMode'
is unset, but
+ # `defaultValue` in `spark.conf.get` is set to None.
+
self.assertEqual(spark.conf.get("spark.sql.sources.partitionOverwriteMode",
None), None)
+
def test_current_database(self):
spark = self.spark
spark.catalog._reset()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
