Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/21553#discussion_r198341997
--- Diff: python/pyspark/sql/conf.py ---
@@ -64,6 +64,96 @@ def _checkType(self, obj, identifier):
(identifier, obj, type(obj).__name__))
+class ConfigEntry(object):
+ """An entry contains all meta information for a configuration"""
+
+ def __init__(self, confKey):
+ """Create a new ConfigEntry with config key"""
+ self.confKey = confKey
+ self.converter = None
+ self.default = _NoValue
+
+ def boolConf(self):
+ """Designate current config entry is boolean config"""
+ self.converter = lambda x: str(x).lower() == "true"
+ return self
+
+ def intConf(self):
+ """Designate current config entry is integer config"""
+ self.converter = lambda x: int(x)
+ return self
+
+ def stringConf(self):
+ """Designate current config entry is string config"""
+ self.converter = lambda x: str(x)
+ return self
+
+ def withDefault(self, default):
+ """Give a default value for current config entry, the default
value will be set
+ to _NoValue when its absent"""
+ self.default = default
+ return self
+
+ def read(self, ctx):
+ """Read value from this config entry through sql context"""
+ return self.converter(ctx.getConf(self.confKey, self.default))
+
+class SQLConf(object):
+ """A class that enables the getting of SQL config parameters in
pyspark"""
+
+ REPL_EAGER_EVAL_ENABLED =
ConfigEntry("spark.sql.repl.eagerEval.enabled")\
+ .boolConf()\
+ .withDefault("false")
+
+ REPL_EAGER_EVAL_MAX_NUM_ROWS =
ConfigEntry("spark.sql.repl.eagerEval.maxNumRows")\
+ .intConf()\
+ .withDefault("20")
+
+ REPL_EAGER_EVAL_TRUNCATE =
ConfigEntry("spark.sql.repl.eagerEval.truncate")\
+ .intConf()\
+ .withDefault("20")
+
+ PANDAS_RESPECT_SESSION_LOCAL_TIMEZONE = \
+ ConfigEntry("spark.sql.execution.pandas.respectSessionTimeZone")\
+ .boolConf()
+
+ SESSION_LOCAL_TIMEZONE = ConfigEntry("spark.sql.session.timeZone")\
+ .stringConf()
+
+ ARROW_EXECUTION_ENABLED =
ConfigEntry("spark.sql.execution.arrow.enabled")\
--- End diff --
This duplicates the key. I think current way duplicates a lot of codes in
Scala side.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
