LIVY-287. Add Deprecation to Livy Configurations and update naming. (#300) Updates to Livy configurations
- Added config deprecation with alternatives to ClientConf, HTTPConf, RSCConf, and LivyConf. - Added framework for deprecation without alternatives when the need arises. - Updated naming conventions in code and templates to use - instead of _ or camelCase and deprecated previous configs. - Updated TestClientConf and added a new test. Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/221aa9cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/221aa9cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/221aa9cf Branch: refs/heads/master Commit: 221aa9cf8f33b48fd9c792f67c61b78708bfae99 Parents: 0de0e28 Author: Alex Bozarth <ajboz...@us.ibm.com> Authored: Wed Apr 5 15:56:48 2017 -0700 Committer: Alex Man <alex-the-...@users.noreply.github.com> Committed: Wed Apr 5 15:56:48 2017 -0700 ---------------------------------------------------------------------- client-common/pom.xml | 5 ++ .../cloudera/livy/client/common/ClientConf.java | 61 ++++++++++++- .../livy/client/common/TestClientConf.java | 75 +++++++++++++++- .../com/cloudera/livy/client/http/HttpConf.java | 59 ++++++++++++- conf/livy-client.conf.template | 18 ++-- conf/livy.conf.template | 10 +-- conf/spark-blacklist.conf.template | 2 +- .../livy/test/framework/MiniCluster.scala | 4 +- .../livy/repl/SparkContextInitializer.scala | 4 +- .../java/com/cloudera/livy/rsc/RSCConf.java | 91 ++++++++++++++++---- .../main/scala/com/cloudera/livy/LivyConf.scala | 65 +++++++++++--- .../com/cloudera/livy/server/LivyServer.scala | 2 +- .../server/interactive/InteractiveSession.scala | 2 +- 13 files changed, 344 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/client-common/pom.xml ---------------------------------------------------------------------- diff --git a/client-common/pom.xml b/client-common/pom.xml index 9cc713c..e41aaae 100644 --- a/client-common/pom.xml +++ b/client-common/pom.xml @@ -45,5 +45,10 @@ <artifactId>jackson-databind</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>provided</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java ---------------------------------------------------------------------- diff --git a/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java b/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java index 59cce37..4eb7929 100644 --- a/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java +++ b/client-common/src/main/java/com/cloudera/livy/client/common/ClientConf.java @@ -28,6 +28,9 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.cloudera.livy.annotations.Private; /** @@ -37,6 +40,8 @@ import com.cloudera.livy.annotations.Private; public abstract class ClientConf<T extends ClientConf> implements Iterable<Map.Entry<String, String>> { + protected Logger LOG = LoggerFactory.getLogger(getClass()); + public static interface ConfEntry { /** The key in the configuration file. */ @@ -71,24 +76,37 @@ public abstract class ClientConf<T extends ClientConf> this.config = new ConcurrentHashMap<>(); if (config != null) { for (String key : config.stringPropertyNames()) { + logDeprecationWarning(key); this.config.put(key, config.getProperty(key)); } } } public String get(String key) { - return config.get(key); + String val = config.get(key); + if (val != null) { + return val; + } + DeprecatedConf depConf = getConfigsWithAlternatives().get(key); + if (depConf != null) { + return config.get(depConf.key()); + } else { + return val; + } } @SuppressWarnings("unchecked") public T set(String key, String value) { + logDeprecationWarning(key); config.put(key, value); return (T) this; } @SuppressWarnings("unchecked") public T setIfMissing(String key, String value) { - config.putIfAbsent(key, value); + if (config.putIfAbsent(key, value) == null) { + logDeprecationWarning(key); + } return (T) this; } @@ -163,6 +181,7 @@ public abstract class ClientConf<T extends ClientConf> if (value == null) { config.remove(e.key()); } else { + logDeprecationWarning(e.key()); config.put(e.key(), value.toString()); } return (T) this; @@ -176,7 +195,7 @@ public abstract class ClientConf<T extends ClientConf> private String get(ConfEntry e, Class<?> requestedType) { check(getType(e.dflt()).equals(requestedType), "Invalid type conversion requested for %s.", e.key()); - return config.get(e.key()); + return this.get(e.key()); } private boolean typesMatch(Object test, Object expected) { @@ -193,4 +212,40 @@ public abstract class ClientConf<T extends ClientConf> } } + /** Logs a warning message if the given config key is deprecated. */ + private void logDeprecationWarning(String key) { + DeprecatedConf altConfs = getConfigsWithAlternatives().get(key); + if (altConfs != null) { + LOG.warn("The configuration key " + altConfs.key() + " has been deprecated as of Livy " + + altConfs.version() + " and may be removed in the future. Please use the new key " + + key + " instead."); + return; + } + + DeprecatedConf depConfs = getDeprecatedConfigs().get(key); + if (depConfs != null) { + LOG.warn("The configuration key " + depConfs.key() + " has been deprecated as of Livy " + + depConfs.version() + " and may be removed in the future. " + + depConfs.deprecationMessage()); + } + } + + /** Maps valid key to DeprecatedConf with the deprecated key. */ + protected abstract Map<String, DeprecatedConf> getConfigsWithAlternatives(); + + /** Maps deprecated key to DeprecatedConf with the same key. */ + protected abstract Map<String, DeprecatedConf> getDeprecatedConfigs(); + + public static interface DeprecatedConf { + + /** The key in the configuration file. */ + String key(); + + /** The Livy version in which the key was deprecated. */ + String version(); + + /** Message to include in the deprecation warning for configs without alternatives */ + String deprecationMessage(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java ---------------------------------------------------------------------- diff --git a/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java b/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java index c8bd9c9..afb7798 100644 --- a/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java +++ b/client-common/src/test/java/com/cloudera/livy/client/common/TestClientConf.java @@ -18,6 +18,8 @@ package com.cloudera.livy.client.common; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -122,6 +124,31 @@ public class TestClientConf { conf.getTimeAsMs(TestConf.Entry.TIME_NO_DEFAULT); } + + @Test + public void testDeprecation() { + TestConf conf = new TestConf(null); + + assertNull(conf.get("depKey")); + assertNull(conf.get("dep_alt")); + assertNull(conf.get("new-key")); + assertEquals("value", conf.get(TestConf.Entry.NEW_CONF)); + + TestConf depProps = new TestConf(null); + depProps.set("depKey", "dep-val"); + depProps.set("dep_alt", "alt-val"); + conf.setAll(depProps); + assertEquals("dep-val", conf.get("depKey")); + assertEquals("alt-val", conf.get("dep_alt")); + assertEquals("alt-val", conf.get(TestConf.Entry.NEW_CONF)); + assertEquals("alt-val", conf.get("new-key")); + + conf.set("new-key", "new-val"); + assertEquals("new-val", conf.get(TestConf.Entry.NEW_CONF)); + assertEquals("alt-val", conf.get("dep_alt")); + assertEquals("new-val", conf.get("new-key")); + } + private static class TestConf extends ClientConf<TestConf> { static enum Entry implements ConfEntry { @@ -131,7 +158,8 @@ public class TestClientConf { INT("int", 42), LONG("long", 84L), TIME("time", "168ms"), - TIME_NO_DEFAULT("time2", null); + TIME_NO_DEFAULT("time2", null), + NEW_CONF("new-key", "value"); private final String key; private final Object dflt; @@ -153,6 +181,51 @@ public class TestClientConf { super(p); } + private static final Map<String, DeprecatedConf> configsWithAlternatives + = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{ + put(TestConf.Entry.NEW_CONF.key, DepConf.DEP_WITH_ALT); + }}); + + private static final Map<String, DeprecatedConf> deprecatedConfigs + = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{ + put(DepConf.DEP_NO_ALT.key, DepConf.DEP_NO_ALT); + }}); + + protected Map<String, DeprecatedConf> getConfigsWithAlternatives() { + return configsWithAlternatives; + } + + protected Map<String, DeprecatedConf> getDeprecatedConfigs() { + return deprecatedConfigs; + } + + static enum DepConf implements DeprecatedConf { + DEP_WITH_ALT("dep_alt", "0.4"), + DEP_NO_ALT("depKey", "1.0"); + + private final String key; + private final String version; + private final String deprecationMessage; + + private DepConf(String key, String version) { + this(key, version, ""); + } + + private DepConf(String key, String version, String deprecationMessage) { + this.key = key; + this.version = version; + this.deprecationMessage = deprecationMessage; + } + + @Override + public String key() { return key; } + + @Override + public String version() { return version; } + + @Override + public String deprecationMessage() { return deprecationMessage; } + } } } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java ---------------------------------------------------------------------- diff --git a/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java b/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java index 24c191c..2ae25e5 100644 --- a/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java +++ b/client-http/src/main/java/com/cloudera/livy/client/http/HttpConf.java @@ -18,19 +18,24 @@ package com.cloudera.livy.client.http; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import com.cloudera.livy.client.common.ClientConf; class HttpConf extends ClientConf<HttpConf> { + private static final String HTTP_CONF_PREFIX = "livy.client.http."; + static enum Entry implements ConfEntry { CONNECTION_TIMEOUT("connection.timeout", "10s"), CONNECTION_IDLE_TIMEOUT("connection.idle.timeout", "10m"), SOCKET_TIMEOUT("connection.socket.timeout", "5m"), - JOB_INITIAL_POLL_INTERVAL("job.initial_poll_interval", "100ms"), - JOB_MAX_POLL_INTERVAL("job.max_poll_interval", "5s"), + JOB_INITIAL_POLL_INTERVAL("job.initial-poll-interval", "100ms"), + JOB_MAX_POLL_INTERVAL("job.max-poll-interval", "5s"), CONTENT_COMPRESS_ENABLE("content.compress.enable", true), @@ -44,7 +49,7 @@ class HttpConf extends ClientConf<HttpConf> { private final Object dflt; private Entry(String key, Object dflt) { - this.key = "livy.client.http." + key; + this.key = HTTP_CONF_PREFIX + key; this.dflt = dflt; } @@ -80,4 +85,52 @@ class HttpConf extends ClientConf<HttpConf> { boolean isSpnegoEnabled() { return getBoolean(Entry.SPNEGO_ENABLED); } + + private static final Map<String, DeprecatedConf> configsWithAlternatives + = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{ + put(HttpConf.Entry.JOB_INITIAL_POLL_INTERVAL.key, DepConf.JOB_INITIAL_POLL_INTERVAL); + put(HttpConf.Entry.JOB_MAX_POLL_INTERVAL.key, DepConf.JOB_MAX_POLL_INTERVAL); + }}); + + // Maps deprecated key to DeprecatedConf with the same key. + // There are no deprecated configs without alternatives currently. + private static final Map<String, DeprecatedConf> deprecatedConfigs + = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>()); + + protected Map<String, DeprecatedConf> getConfigsWithAlternatives() { + return configsWithAlternatives; + } + + protected Map<String, DeprecatedConf> getDeprecatedConfigs() { + return deprecatedConfigs; + } + + static enum DepConf implements DeprecatedConf { + JOB_INITIAL_POLL_INTERVAL("job.initial_poll_interval", "0.4"), + JOB_MAX_POLL_INTERVAL("job.max_poll_interval", "0.4"); + + private final String key; + private final String version; + private final String deprecationMessage; + + private DepConf(String key, String version) { + this(key, version, ""); + } + + private DepConf(String key, String version, String deprecationMessage) { + this.key = HTTP_CONF_PREFIX + key; + this.version = version; + this.deprecationMessage = deprecationMessage; + } + + @Override + public String key() { return key; } + + @Override + public String version() { return version; } + + @Override + public String deprecationMessage() { return deprecationMessage; } + } + } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/conf/livy-client.conf.template ---------------------------------------------------------------------- diff --git a/conf/livy-client.conf.template b/conf/livy-client.conf.template index c0adb30..2a92b57 100644 --- a/conf/livy-client.conf.template +++ b/conf/livy-client.conf.template @@ -22,9 +22,9 @@ # livy.client.http.connection.idle.timeout = 10m # Initial interval before polling for Job results -# livy.client.http.job.initial_poll_interval = 100ms +# livy.client.http.job.initial-poll-interval = 100ms # Maximum interval between successive polls -# livy.client.http.job.max_poll_interval = 5s +# livy.client.http.job.max-poll-interval = 5s # # Configurations for Livy RSCClient @@ -37,10 +37,10 @@ # livy.rsc.client.auth.secret = # Timeout when stopping a rsc client -# livy.rsc.client.shutdown_timeout = 10s +# livy.rsc.client.shutdown-timeout = 10s # Class of the rsc driver to use -# livy.rsc.driver_class = +# livy.rsc.driver-class = # The kind of rsc session. Examples: pyspark or sparkr # livy.rsc.session.kind = @@ -58,10 +58,10 @@ # livy.rsc.launcher.port = -1 # How long will the RSC wait for a connection for a Livy server before shutting itself down. -# livy.rsc.server.idle_timeout = 10m +# livy.rsc.server.idle-timeout = 10m # The user that should be impersonated when requesting a Livy session -# livy.rsc.proxy_user = +# livy.rsc.proxy-user = # Host or IP adress of the rpc server # livy.rsc.rpc.server.address = @@ -78,9 +78,9 @@ # livy.rsc.rpc.sasl.qop = # Time between status checks for cancelled a Job -# livy.rsc.job_cancel.trigger_interval = 100ms +# livy.rsc.job-cancel.trigger-interval = 100ms # Time before a cancelled a Job is forced into a Cancelled state -# livy.rsc.job_cancel.timeout = 30s +# livy.rsc.job-cancel.timeout = 30s # Number of statements kept in driver's memory -# livy.rsc.retained_statements = 100 \ No newline at end of file +# livy.rsc.retained-statements = 100 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/conf/livy.conf.template ---------------------------------------------------------------------- diff --git a/conf/livy.conf.template b/conf/livy.conf.template index 8c5fe03..dd2f054 100644 --- a/conf/livy.conf.template +++ b/conf/livy.conf.template @@ -17,7 +17,7 @@ # livy.spark.master = local # What spark deploy mode Livy sessions should use. -# livy.spark.deployMode = +# livy.spark.deploy-mode = # Enabled to check whether timeout Livy sessions should be stopped. # livy.server.session.timeout-check = true @@ -58,11 +58,11 @@ # Whether to enable csrf protection, by default it is false. If it is enabled, client should add # http-header "X-Requested-By" in request if the http method is POST/DELETE/PUT/PATCH. -# livy.server.csrf_protection.enabled = +# livy.server.csrf-protection.enabled = # Whether to enable HiveContext in livy interpreter, if it is true hive-site.xml will be detected # on user request and then livy server classpath automatically. -# livy.repl.enableHiveContext = +# livy.repl.enable-hive-context = # Recovery mode of Livy. Possible values: # off: Default. Turn off recovery. Every time Livy shuts down, it stops and forgets all sessions. @@ -88,9 +88,9 @@ # When the cluster is busy, we may fail to launch yarn app in app-lookup-timeout, then it would # cause session leakage, so we need to check session leakage. # How long to check livy session leakage -# livy.server.yarn.app-leakage.check_timeout = 600s +# livy.server.yarn.app-leakage.check-timeout = 600s # how often to check livy session leakage -# livy.server.yarn.app-leakage.check_interval = 60s +# livy.server.yarn.app-leakage.check-interval = 60s # How often Livy polls YARN to refresh YARN app state. # livy.server.yarn.poll-interval = 1s http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/conf/spark-blacklist.conf.template ---------------------------------------------------------------------- diff --git a/conf/spark-blacklist.conf.template b/conf/spark-blacklist.conf.template index f0919b0..b9f0ec2 100644 --- a/conf/spark-blacklist.conf.template +++ b/conf/spark-blacklist.conf.template @@ -16,4 +16,4 @@ spark.yarn.jars spark.yarn.archive # Don't allow users to override the RSC timeout. -livy.rsc.server.idle_timeout +livy.rsc.server.idle-timeout http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala ---------------------------------------------------------------------- diff --git a/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala b/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala index d520269..6aa02f1 100644 --- a/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala +++ b/integration-test/src/main/scala/com/cloudera/livy/test/framework/MiniCluster.scala @@ -176,7 +176,7 @@ object MiniLivyMain extends MiniClusterBase { // server. Do it atomically since it's used by MiniCluster to detect when the Livy server // is up and ready. eventually(timeout(30 seconds), interval(1 second)) { - val serverUrlConf = Map("livy.server.serverUrl" -> server.serverUrl()) + val serverUrlConf = Map("livy.server.server-url" -> server.serverUrl()) saveProperties(serverUrlConf, new File(configPath + "/serverUrl.conf")) } } @@ -297,7 +297,7 @@ class MiniCluster(config: Map[String, String]) extends Cluster with MiniClusterU val localLivy = start(MiniLivyMain.getClass, confFile, extraJavaArgs = jacocoArgs) val props = loadProperties(confFile) - livyUrl = props("livy.server.serverUrl") + livyUrl = props("livy.server.server-url") // Wait until Livy server responds. val httpClient = new AsyncHttpClient() http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala ---------------------------------------------------------------------- diff --git a/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala b/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala index fd082eb..533741d 100644 --- a/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala +++ b/repl/src/main/scala/com/cloudera/livy/repl/SparkContextInitializer.scala @@ -46,7 +46,7 @@ trait SparkContextInitializer extends Logging { val loader = Option(Thread.currentThread().getContextClassLoader) .getOrElse(getClass.getClassLoader) if (loader.getResource("hive-site.xml") == null) { - warn("livy.repl.enableHiveContext is true but no hive-site.xml found on classpath.") + warn("livy.repl.enable-hive-context is true but no hive-site.xml found on classpath.") } sqlContext = Class.forName("org.apache.spark.sql.hive.HiveContext") @@ -87,7 +87,7 @@ trait SparkContextInitializer extends Logging { val loader = Option(Thread.currentThread().getContextClassLoader) .getOrElse(getClass.getClassLoader) if (loader.getResource("hive-site.xml") == null) { - warn("livy.repl.enableHiveContext is true but no hive-site.xml found on classpath.") + warn("livy.repl.enable-hive-context is true but no hive-site.xml found on classpath.") } builder.getClass.getMethod("enableHiveSupport").invoke(builder) http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java ---------------------------------------------------------------------- diff --git a/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java b/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java index 0d7b1c1..d1b8b39 100644 --- a/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java +++ b/rsc/src/main/java/com/cloudera/livy/rsc/RSCConf.java @@ -21,15 +21,13 @@ import java.io.IOException; import java.net.Inet4Address; import java.net.InetAddress; import java.net.NetworkInterface; +import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.Map; import java.util.Properties; import javax.security.sasl.Sasl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.cloudera.livy.client.common.ClientConf; public class RSCConf extends ClientConf<RSCConf> { @@ -37,14 +35,14 @@ public class RSCConf extends ClientConf<RSCConf> { public static final String SPARK_CONF_PREFIX = "spark."; public static final String LIVY_SPARK_PREFIX = SPARK_CONF_PREFIX + "__livy__."; - private static final Logger LOG = LoggerFactory.getLogger(RSCConf.class); + private static final String RSC_CONF_PREFIX = "livy.rsc."; public static enum Entry implements ConfEntry { CLIENT_ID("client.auth.id", null), CLIENT_SECRET("client.auth.secret", null), - CLIENT_IN_PROCESS("client.do_not_use.run_driver_in_process", false), - CLIENT_SHUTDOWN_TIMEOUT("client.shutdown_timeout", "10s"), - DRIVER_CLASS("driver_class", null), + CLIENT_IN_PROCESS("client.do-not-use.run-driver-in-process", false), + CLIENT_SHUTDOWN_TIMEOUT("client.shutdown-timeout", "10s"), + DRIVER_CLASS("driver-class", null), SESSION_KIND("session.kind", null), LIVY_JARS("jars", null), @@ -56,9 +54,9 @@ public class RSCConf extends ClientConf<RSCConf> { LAUNCHER_PORT("launcher.port", -1), // How long will the RSC wait for a connection for a Livy server before shutting itself down. - SERVER_IDLE_TIMEOUT("server.idle_timeout", "10m"), + SERVER_IDLE_TIMEOUT("server.idle-timeout", "10m"), - PROXY_USER("proxy_user", null), + PROXY_USER("proxy-user", null), RPC_SERVER_ADDRESS("rpc.server.address", null), RPC_CLIENT_HANDSHAKE_TIMEOUT("server.connect.timeout", "90s"), @@ -71,19 +69,19 @@ public class RSCConf extends ClientConf<RSCConf> { SASL_MECHANISMS("rpc.sasl.mechanisms", "DIGEST-MD5"), SASL_QOP("rpc.sasl.qop", null), - TEST_STUCK_END_SESSION("test.do_not_use.stuck_end_session", false), - TEST_STUCK_START_DRIVER("test.do_not_use.stuck_start_driver", false), + TEST_STUCK_END_SESSION("test.do-not-use.stuck-end-session", false), + TEST_STUCK_START_DRIVER("test.do-not-use.stuck-start-driver", false), - JOB_CANCEL_TRIGGER_INTERVAL("job_cancel.trigger_interval", "100ms"), - JOB_CANCEL_TIMEOUT("job_cancel.timeout", "30s"), + JOB_CANCEL_TRIGGER_INTERVAL("job-cancel.trigger-interval", "100ms"), + JOB_CANCEL_TIMEOUT("job-cancel.timeout", "30s"), - RETAINED_STATEMENT_NUMBER("retained_statements", 100); + RETAINED_STATEMENT_NUMBER("retained-statements", 100); private final String key; private final Object dflt; private Entry(String key, Object dflt) { - this.key = "livy.rsc." + key; + this.key = RSC_CONF_PREFIX + key; this.dflt = dflt; } @@ -146,4 +144,67 @@ public class RSCConf extends ClientConf<RSCConf> { return address.getCanonicalHostName(); } + private static final Map<String, DeprecatedConf> configsWithAlternatives + = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>() {{ + put(RSCConf.Entry.CLIENT_IN_PROCESS.key, DepConf.CLIENT_IN_PROCESS); + put(RSCConf.Entry.CLIENT_SHUTDOWN_TIMEOUT.key, DepConf.CLIENT_SHUTDOWN_TIMEOUT); + put(RSCConf.Entry.DRIVER_CLASS.key, DepConf.DRIVER_CLASS); + put(RSCConf.Entry.SERVER_IDLE_TIMEOUT.key, DepConf.SERVER_IDLE_TIMEOUT); + put(RSCConf.Entry.PROXY_USER.key, DepConf.PROXY_USER); + put(RSCConf.Entry.TEST_STUCK_END_SESSION.key, DepConf.TEST_STUCK_END_SESSION); + put(RSCConf.Entry.TEST_STUCK_START_DRIVER.key, DepConf.TEST_STUCK_START_DRIVER); + put(RSCConf.Entry.JOB_CANCEL_TRIGGER_INTERVAL.key, DepConf.JOB_CANCEL_TRIGGER_INTERVAL); + put(RSCConf.Entry.JOB_CANCEL_TIMEOUT.key, DepConf.JOB_CANCEL_TIMEOUT); + put(RSCConf.Entry.RETAINED_STATEMENT_NUMBER.key, DepConf.RETAINED_STATEMENT_NUMBER); + }}); + + // Maps deprecated key to DeprecatedConf with the same key. + // There are no deprecated configs without alternatives currently. + private static final Map<String, DeprecatedConf> deprecatedConfigs + = Collections.unmodifiableMap(new HashMap<String, DeprecatedConf>()); + + protected Map<String, DeprecatedConf> getConfigsWithAlternatives() { + return configsWithAlternatives; + } + + protected Map<String, DeprecatedConf> getDeprecatedConfigs() { + return deprecatedConfigs; + } + + static enum DepConf implements DeprecatedConf { + CLIENT_IN_PROCESS("client.do_not_use.run_driver_in_process", "0.4"), + CLIENT_SHUTDOWN_TIMEOUT("client.shutdown_timeout", "0.4"), + DRIVER_CLASS("driver_class", "0.4"), + SERVER_IDLE_TIMEOUT("server.idle_timeout", "0.4"), + PROXY_USER("proxy_user", "0.4"), + TEST_STUCK_END_SESSION("test.do_not_use.stuck_end_session", "0.4"), + TEST_STUCK_START_DRIVER("test.do_not_use.stuck_start_driver", "0.4"), + JOB_CANCEL_TRIGGER_INTERVAL("job_cancel.trigger_interval", "0.4"), + JOB_CANCEL_TIMEOUT("job_cancel.timeout", "0.4"), + RETAINED_STATEMENT_NUMBER("retained_statements", "0.4"); + + private final String key; + private final String version; + private final String deprecationMessage; + + private DepConf(String key, String version) { + this(key, version, ""); + } + + private DepConf(String key, String version, String deprecationMessage) { + this.key = RSC_CONF_PREFIX + key; + this.version = version; + this.deprecationMessage = deprecationMessage; + } + + @Override + public String key() { return key; } + + @Override + public String version() { return version; } + + @Override + public String deprecationMessage() { return deprecationMessage; } + } + } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/server/src/main/scala/com/cloudera/livy/LivyConf.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/com/cloudera/livy/LivyConf.scala b/server/src/main/scala/com/cloudera/livy/LivyConf.scala index 1db1b8f..8fc4777 100644 --- a/server/src/main/scala/com/cloudera/livy/LivyConf.scala +++ b/server/src/main/scala/com/cloudera/livy/LivyConf.scala @@ -20,11 +20,15 @@ package com.cloudera.livy import java.io.File import java.lang.{Boolean => JBoolean, Long => JLong} +import java.util.{Map => JMap} + +import scala.collection.JavaConverters._ import org.apache.hadoop.conf.Configuration import com.cloudera.livy.client.common.ClientConf import com.cloudera.livy.client.common.ClientConf.ConfEntry +import com.cloudera.livy.client.common.ClientConf.DeprecatedConf object LivyConf { @@ -40,31 +44,31 @@ object LivyConf { val SPARK_HOME = Entry("livy.server.spark-home", null) val LIVY_SPARK_MASTER = Entry("livy.spark.master", "local") - val LIVY_SPARK_DEPLOY_MODE = Entry("livy.spark.deployMode", null) + val LIVY_SPARK_DEPLOY_MODE = Entry("livy.spark.deploy-mode", null) // Two configurations to specify Spark and related Scala version. These are internal // configurations will be set by LivyServer and used in session creation. It is not required to // set usually unless running with unofficial Spark + Scala versions // (like Spark 2.0 + Scala 2.10, Spark 1.6 + Scala 2.11) - val LIVY_SPARK_SCALA_VERSION = Entry("livy.spark.scalaVersion", null) + val LIVY_SPARK_SCALA_VERSION = Entry("livy.spark.scala-version", null) val LIVY_SPARK_VERSION = Entry("livy.spark.version", null) val SESSION_STAGING_DIR = Entry("livy.session.staging-dir", null) val FILE_UPLOAD_MAX_SIZE = Entry("livy.file.upload.max.size", 100L * 1024 * 1024) val LOCAL_FS_WHITELIST = Entry("livy.file.local-dir-whitelist", null) - val ENABLE_HIVE_CONTEXT = Entry("livy.repl.enableHiveContext", false) + val ENABLE_HIVE_CONTEXT = Entry("livy.repl.enable-hive-context", false) val ENVIRONMENT = Entry("livy.environment", "production") val SERVER_HOST = Entry("livy.server.host", "0.0.0.0") val SERVER_PORT = Entry("livy.server.port", 8998) - val CSRF_PROTECTION = LivyConf.Entry("livy.server.csrf_protection.enabled", false) + val CSRF_PROTECTION = LivyConf.Entry("livy.server.csrf-protection.enabled", false) val IMPERSONATION_ENABLED = Entry("livy.impersonation.enabled", false) val SUPERUSERS = Entry("livy.superusers", null) - val ACCESS_CONTROL_ENABLED = Entry("livy.server.access_control.enabled", false) - val ACCESS_CONTROL_USERS = Entry("livy.server.access_control.users", null) + val ACCESS_CONTROL_ENABLED = Entry("livy.server.access-control.enabled", false) + val ACCESS_CONTROL_USERS = Entry("livy.server.access-control.users", null) val SSL_KEYSTORE = Entry("livy.keystore", null) val SSL_KEYSTORE_PASSWORD = Entry("livy.keystore.password", null) @@ -73,7 +77,7 @@ object LivyConf { val AUTH_TYPE = Entry("livy.server.auth.type", null) val AUTH_KERBEROS_PRINCIPAL = Entry("livy.server.auth.kerberos.principal", null) val AUTH_KERBEROS_KEYTAB = Entry("livy.server.auth.kerberos.keytab", null) - val AUTH_KERBEROS_NAME_RULES = Entry("livy.server.auth.kerberos.name_rules", "DEFAULT") + val AUTH_KERBEROS_NAME_RULES = Entry("livy.server.auth.kerberos.name-rules", "DEFAULT") val HEARTBEAT_WATCHDOG_INTERVAL = Entry("livy.server.heartbeat-watchdog.interval", "1m") @@ -82,9 +86,9 @@ object LivyConf { val LAUNCH_KERBEROS_KEYTAB = LivyConf.Entry("livy.server.launch.kerberos.keytab", null) val LAUNCH_KERBEROS_REFRESH_INTERVAL = - LivyConf.Entry("livy.server.launch.kerberos.refresh_interval", "1h") + LivyConf.Entry("livy.server.launch.kerberos.refresh-interval", "1h") val KINIT_FAIL_THRESHOLD = - LivyConf.Entry("livy.server.launch.kerberos.kinit_fail_threshold", 5) + LivyConf.Entry("livy.server.launch.kerberos.kinit-fail-threshold", 5) /** * Recovery mode of Livy. Possible values: @@ -124,9 +128,9 @@ object LivyConf { val RSC_JARS = Entry("livy.rsc.jars", null) // How long to check livy session leakage - val YARN_APP_LEAKAGE_CHECK_TIMEOUT = Entry("livy.server.yarn.app-leakage.check_timeout", "600s") + val YARN_APP_LEAKAGE_CHECK_TIMEOUT = Entry("livy.server.yarn.app-leakage.check-timeout", "600s") // how often to check livy session leakage - val YARN_APP_LEAKAGE_CHECK_INTERVAL = Entry("livy.server.yarn.app-leakage.check_interval", "60s") + val YARN_APP_LEAKAGE_CHECK_INTERVAL = Entry("livy.server.yarn.app-leakage.check-interval", "60s") // Whether session timeout should be checked, by default it will be checked, which means inactive // session will be stopped after "livy.server.session.timeout" @@ -166,6 +170,37 @@ object LivyConf { "spark.yarn.jars" ) + case class DepConf( + override val key: String, + override val version: String, + override val deprecationMessage: String = "") + extends DeprecatedConf + + private val configsWithAlternatives: Map[String, DeprecatedConf] = Map[String, DepConf]( + LIVY_SPARK_DEPLOY_MODE.key -> DepConf("livy.spark.deployMode", "0.4"), + LIVY_SPARK_SCALA_VERSION.key -> DepConf("livy.spark.scalaVersion", "0.4"), + ENABLE_HIVE_CONTEXT.key -> DepConf("livy.repl.enableHiveContext", "0.4"), + CSRF_PROTECTION.key -> DepConf("livy.server.csrf_protection.enabled", "0.4"), + ACCESS_CONTROL_ENABLED.key -> DepConf("livy.server.access_control.enabled", "0.4"), + ACCESS_CONTROL_USERS.key -> DepConf("livy.server.access_control.users", "0.4"), + AUTH_KERBEROS_NAME_RULES.key -> DepConf("livy.server.auth.kerberos.name_rules", "0.4"), + LAUNCH_KERBEROS_REFRESH_INTERVAL.key -> + DepConf("livy.server.launch.kerberos.refresh_interval", "0.4"), + KINIT_FAIL_THRESHOLD.key -> DepConf("livy.server.launch.kerberos.kinit_fail_threshold", "0.4"), + YARN_APP_LEAKAGE_CHECK_TIMEOUT.key -> + DepConf("livy.server.yarn.app-leakage.check_timeout", "0.4"), + YARN_APP_LEAKAGE_CHECK_INTERVAL.key -> + DepConf("livy.server.yarn.app-leakage.check_interval", "0.4") + ) + + private val deprecatedConfigs: Map[String, DeprecatedConf] = { + val configs: Seq[DepConf] = Seq( + // There are no deprecated configs without alternatives currently. + ) + + Map(configs.map { cfg => (cfg.key -> cfg) }: _*) + } + } /** @@ -250,4 +285,12 @@ class LivyConf(loadDefaults: Boolean) extends ClientConf[LivyConf](null) { Option(get(entry)).map(_.split("[, ]+").toSeq).getOrElse(Nil) } + override def getConfigsWithAlternatives: JMap[String, DeprecatedConf] = { + configsWithAlternatives.asJava + } + + override def getDeprecatedConfigs: JMap[String, DeprecatedConf] = { + deprecatedConfigs.asJava + } + } http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala b/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala index 0499d48..4bd5635 100644 --- a/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala +++ b/server/src/main/scala/com/cloudera/livy/server/LivyServer.scala @@ -232,7 +232,7 @@ class LivyServer extends Logging { }) _serverUrl = Some(s"${server.protocol}://${server.host}:${server.port}") - sys.props("livy.server.serverUrl") = _serverUrl.get + sys.props("livy.server.server-url") = _serverUrl.get } def runKinit(keytab: String, principal: String): Boolean = { http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/221aa9cf/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala index 605edc6..b53dea7 100644 --- a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala +++ b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala @@ -100,7 +100,7 @@ object InteractiveSession extends Logging { info(s"Creating LivyClient for sessionId: $id") val builder = new LivyClientBuilder() .setAll(builderProperties.asJava) - .setConf("livy.client.sessionId", id.toString) + .setConf("livy.client.session-id", id.toString) .setConf(RSCConf.Entry.DRIVER_CLASS.key(), "com.cloudera.livy.repl.ReplDriver") .setConf(RSCConf.Entry.PROXY_USER.key(), proxyUser.orNull) .setURI(new URI("rsc:/"))
