This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new c864c4d [Submarine-339] Separate SubmarineConfVars from
SubmarineConfiguration
c864c4d is described below
commit c864c4dd208c3f4b37afe1962a9883026326d3ea
Author: kevin85421 <[email protected]>
AuthorDate: Tue Jan 14 10:17:31 2020 +0800
[Submarine-339] Separate SubmarineConfVars from SubmarineConfiguration
### What is this PR for?
SubmarineConfiguration becomes larger and larger. Hence, to facilitate the
management of configuration items, I separate the class SubmarineConfVars from
the class SubmarineConfiguration.
### What type of PR is it?
[Refactoring]
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-339
### How should this be tested?
It will be tested by Travis.
### Screenshots (if appropriate)
https://travis-ci.org/apache/submarine/builds/636675316?utm_source=github_status&utm_medium=notification
### Questions:
* Does the licenses files need to be updated? No
* Are there breaking changes for older versions? No
* Does this needs documentation? No
Author: kevin85421 <[email protected]>
Closes #151 from kevin85421/SUBMARINE-339 and squashes the following
commits:
885d409 [kevin85421] [SUBMARINE-339] Separate SubmarineConfVars from
SubmarineConfiguration
e37765d [kevin85421] [SUBMARINE-339] Separate SubmarineConfVars from
SubmarineConfiguration
7306645 [kevin85421] [SUBMARINE-339] Separate SubmarineConfVars from
SubmarineConfiguration
---
.../java/org/apache/submarine/client/cli/Cli.java | 4 +-
.../submarine/client/cli/remote/RpcContext.java | 5 +-
.../client/cli/remote/RpcRuntimeFactory.java | 4 +-
.../submarine/commons/runtime/RuntimeFactory.java | 3 +-
.../submarine/commons/utils/SubmarineConfVars.java | 223 +++++++++++++++
.../commons/utils/SubmarineConfiguration.java | 305 ++++-----------------
.../apache/submarine/server/SubmarineServer.java | 13 +-
.../server/AbstractSubmarineServerTest.java | 6 +-
.../submarine/server/rpc/SubmarineRpcServer.java | 3 +-
.../apache/submarine/server/rpc/MockRpcServer.java | 3 +-
.../submarine/server/rpc/SubmarineRpcClient.java | 5 +-
.../server/submitter/k8s/K8sJobSubmitter.java | 3 +-
.../yarnservice/FileSystemOperations.java | 3 +-
.../YarnServiceRunJobCliLocalizationTest.java | 3 +-
.../yarnservice/FileSystemOperationsTest.java | 9 +-
15 files changed, 314 insertions(+), 278 deletions(-)
diff --git
a/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java
b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java
index 296d6cc..4cfa663 100644
--- a/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java
+++ b/submarine-client/src/main/java/org/apache/submarine/client/cli/Cli.java
@@ -25,7 +25,7 @@ import org.apache.submarine.client.cli.runjob.RunJobCli;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.submarine.commons.runtime.ClientContext;
import org.apache.submarine.commons.runtime.RuntimeFactory;
-import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,7 +50,7 @@ public class Cli {
ClientContext clientContext = new ClientContext();
RuntimeFactory runtimeFactory;
if (clientContext.getSubmarineConfig().getBoolean(
- SubmarineConfiguration.ConfVars.
+ SubmarineConfVars.ConfVars.
SUBMARINE_SERVER_REMOTE_EXECUTION_ENABLED)) {
runtimeFactory = new RpcRuntimeFactory(clientContext);
} else {
diff --git
a/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcContext.java
b/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcContext.java
index 891a8f5..7259798 100644
---
a/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcContext.java
+++
b/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcContext.java
@@ -20,6 +20,7 @@
package org.apache.submarine.client.cli.remote;
import org.apache.submarine.commons.runtime.ClientContext;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
import java.util.HashMap;
@@ -41,9 +42,9 @@ public class RpcContext {
private void addSubmarineJobConfiguration(RpcContext rpcContext,
SubmarineConfiguration submarineConfig) {
rpcContext.getSubmarineJobConfigMap().put(
- SubmarineConfiguration.ConfVars.SUBMARINE_RUNTIME_CLASS.getVarName(),
+ SubmarineConfVars.ConfVars.SUBMARINE_RUNTIME_CLASS.getVarName(),
submarineConfig.getString(
- SubmarineConfiguration.ConfVars.SUBMARINE_RUNTIME_CLASS)
+ SubmarineConfVars.ConfVars.SUBMARINE_RUNTIME_CLASS)
);
}
diff --git
a/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcRuntimeFactory.java
b/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcRuntimeFactory.java
index 135570c..7596d8a 100644
---
a/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcRuntimeFactory.java
+++
b/submarine-client/src/main/java/org/apache/submarine/client/cli/remote/RpcRuntimeFactory.java
@@ -25,7 +25,7 @@ import org.apache.submarine.commons.runtime.JobSubmitter;
import org.apache.submarine.commons.runtime.RuntimeFactory;
import org.apache.submarine.commons.runtime.fs.MemorySubmarineStorage;
import org.apache.submarine.commons.runtime.fs.SubmarineStorage;
-import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
/**
* Implementation of RuntimeFactory with rpc server
@@ -37,7 +37,7 @@ public class RpcRuntimeFactory extends RuntimeFactory {
super(clientContext);
String remoteHost = clientContext.getSubmarineConfig().getServerAddress();
int port = clientContext.getSubmarineConfig().getInt(
-
SubmarineConfiguration.ConfVars.SUBMARINE_SERVER_REMOTE_EXECUTION_PORT);
+ SubmarineConfVars.ConfVars.SUBMARINE_SERVER_REMOTE_EXECUTION_PORT);
submitter = new JobSubmitterRpcImpl(remoteHost, port, clientContext);
}
diff --git
a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java
b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java
index b838a87..aad555e 100644
---
a/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java
+++
b/submarine-commons/commons-runtime/src/main/java/org/apache/submarine/commons/runtime/RuntimeFactory.java
@@ -21,6 +21,7 @@ package org.apache.submarine.commons.runtime;
import com.google.common.annotations.VisibleForTesting;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import
org.apache.submarine.commons.runtime.exception.SubmarineRuntimeException;
import org.apache.submarine.commons.runtime.fs.SubmarineStorage;
@@ -41,7 +42,7 @@ public abstract class RuntimeFactory {
SubmarineConfiguration submarineConfiguration =
clientContext.getSubmarineConfig();
String runtimeClass = submarineConfiguration.getString(
- SubmarineConfiguration.ConfVars.SUBMARINE_RUNTIME_CLASS);
+ SubmarineConfVars.ConfVars.SUBMARINE_RUNTIME_CLASS);
try {
Class<?> runtimeClazz = Class.forName(runtimeClass);
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
new file mode 100644
index 0000000..7e6e586
--- /dev/null
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfVars.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.submarine.commons.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SubmarineConfVars {
+ private static final Logger LOG =
LoggerFactory.getLogger(SubmarineConfVars.class);
+ public enum ConfVars {
+ SUBMARINE_CONF_DIR("submarine.conf.dir", "conf"),
+ SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB(
+ "submarine.localization.max-allowed-file-size-mb", 2048L),
+ SUBMARINE_SERVER_ADDR("submarine.server.addr", "0.0.0.0"),
+ SUBMARINE_SERVER_PORT("submarine.server.port", 8080),
+ SUBMARINE_SERVER_SSL("submarine.server.ssl", false),
+ SUBMARINE_SERVER_SSL_PORT("submarine.server.ssl.port", 8443),
+
SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX("submarine.server.jetty.thread.pool.max",
400),
+
SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN("submarine.server.jetty.thread.pool.min",
8),
+
SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT("submarine.server.jetty.thread.pool.timeout",
30),
+
SUBMARINE_SERVER_JETTY_REQUEST_HEADER_SIZE("submarine.server.jetty.request.header.size",
8192),
+ SUBMARINE_SERVER_SSL_CLIENT_AUTH("submarine.server.ssl.client.auth",
false),
+ SUBMARINE_SERVER_SSL_KEYSTORE_PATH("submarine.server.ssl.keystore.path",
"keystore"),
+ SUBMARINE_SERVER_SSL_KEYSTORE_TYPE("submarine.server.ssl.keystore.type",
"JKS"),
+
SUBMARINE_SERVER_SSL_KEYSTORE_PASSWORD("submarine.server.ssl.keystore.password",
""),
+
SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD("submarine.server.ssl.key.manager.password",
null),
+
SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH("submarine.server.ssl.truststore.path",
null),
+
SUBMARINE_SERVER_SSL_TRUSTSTORE_TYPE("submarine.server.ssl.truststore.type",
null),
+
SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD("submarine.server.ssl.truststore.password",
null),
+ SUBMARINE_CLUSTER_ADDR("submarine.cluster.addr", ""),
+ SUBMARINE_SERVER_REMOTE_EXECUTION_ENABLED(
+ "submarine.server.remote.execution.enabled", false),
+ SUBMARINE_SERVER_REMOTE_EXECUTION_PORT(
+ "submarine.server.remote.execution.port", 8980),
+ CLUSTER_HEARTBEAT_INTERVAL("cluster.heartbeat.interval", 3000),
+ CLUSTER_HEARTBEAT_TIMEOUT("cluster.heartbeat.timeout", 9000),
+
+ JDBC_DRIVERCLASSNAME("jdbc.driverClassName", "com.mysql.jdbc.Driver"),
+ JDBC_URL("jdbc.url", "jdbc:mysql://127.0.0.1:3306/submarine" +
+
"?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" +
+
"failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"),
+ JDBC_USERNAME("jdbc.username", "submarine"),
+ JDBC_PASSWORD("jdbc.password", "password"),
+ METASTORE_JDBC_URL("metastore.jdbc.url",
"jdbc:mysql://127.0.0.1:3306/metastore" +
+
"?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" +
+
"failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"),
+ METASTORE_JDBC_USERNAME("metastore.jdbc.username", "metastore"),
+ METASTORE_JDBC_PASSWORD("metastore.jdbc.password", "password"),
+
+ WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE(
+ "workbench.websocket.max.text.message.size", "1024000"),
+ WORKBENCH_WEB_WAR("workbench.web.war",
"submarine-workbench/workbench-web/dist"),
+ SUBMARINE_RUNTIME_CLASS("submarine.runtime.class",
+ "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"),
+ SUBMARINE_SUBMITTERS("submarine.submitters", ""),
+ SUBMARINE_SUBMITTERS_CLASS("submarine.submitters.%s.class", ""),
+ SUBMARINE_SUBMITTERS_CLASSPATH("submarine.submitters.%s.classpath", ""),
+ SUBMARINE_K8S_KUBE_CONFIG("submarine.k8s.kube.config", "");
+
+ private String varName;
+ @SuppressWarnings("rawtypes")
+ private Class varClass;
+ private String stringValue;
+ private VarType type;
+ private int intValue;
+ private float floatValue;
+ private boolean booleanValue;
+ private long longValue;
+
+
+ ConfVars(String varName, String varValue) {
+ this.varName = varName;
+ this.varClass = String.class;
+ this.stringValue = varValue;
+ this.intValue = -1;
+ this.floatValue = -1;
+ this.longValue = -1;
+ this.booleanValue = false;
+ this.type = VarType.STRING;
+ }
+
+ ConfVars(String varName, int intValue) {
+ this.varName = varName;
+ this.varClass = Integer.class;
+ this.stringValue = null;
+ this.intValue = intValue;
+ this.floatValue = -1;
+ this.longValue = -1;
+ this.booleanValue = false;
+ this.type = VarType.INT;
+ }
+
+ ConfVars(String varName, long longValue) {
+ this.varName = varName;
+ this.varClass = Integer.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.floatValue = -1;
+ this.longValue = longValue;
+ this.booleanValue = false;
+ this.type = VarType.LONG;
+ }
+
+ ConfVars(String varName, float floatValue) {
+ this.varName = varName;
+ this.varClass = Float.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.longValue = -1;
+ this.floatValue = floatValue;
+ this.booleanValue = false;
+ this.type = VarType.FLOAT;
+ }
+
+ ConfVars(String varName, boolean booleanValue) {
+ this.varName = varName;
+ this.varClass = Boolean.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.longValue = -1;
+ this.floatValue = -1;
+ this.booleanValue = booleanValue;
+ this.type = VarType.BOOLEAN;
+ }
+
+ public String getVarName() {
+ return varName;
+ }
+
+ @SuppressWarnings("rawtypes")
+ public Class getVarClass() {
+ return varClass;
+ }
+
+ public int getIntValue() {
+ return intValue;
+ }
+
+ public long getLongValue() {
+ return longValue;
+ }
+
+ public float getFloatValue() {
+ return floatValue;
+ }
+
+ public String getStringValue() {
+ return stringValue;
+ }
+
+ public boolean getBooleanValue() {
+ return booleanValue;
+ }
+
+ public VarType getType() {
+ return type;
+ }
+
+ enum VarType {
+ STRING {
+ @Override
+ void checkType(String value) throws Exception {
+ }
+ },
+ INT {
+ @Override
+ void checkType(String value) throws Exception {
+ Integer.valueOf(value);
+ }
+ },
+ LONG {
+ @Override
+ void checkType(String value) throws Exception {
+ Long.valueOf(value);
+ }
+ },
+ FLOAT {
+ @Override
+ void checkType(String value) throws Exception {
+ Float.valueOf(value);
+ }
+ },
+ BOOLEAN {
+ @Override
+ void checkType(String value) throws Exception {
+ Boolean.valueOf(value);
+ }
+ };
+
+ boolean isType(String value) {
+ try {
+ checkType(value);
+ } catch (Exception e) {
+ LOG.error("Exception in SubmarineConfiguration while isType", e);
+ return false;
+ }
+ return true;
+ }
+
+ String typeString() {
+ return name().toUpperCase();
+ }
+
+ abstract void checkType(String value) throws Exception;
+ }
+ }
+}
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
index e5af925..6ed4fb1 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
@@ -67,17 +67,17 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
private SubmarineConfiguration() {
- ConfVars[] vars = ConfVars.values();
- for (ConfVars v : vars) {
- if (v.getType() == ConfVars.VarType.BOOLEAN) {
+ SubmarineConfVars.ConfVars[] vars = SubmarineConfVars.ConfVars.values();
+ for (SubmarineConfVars.ConfVars v : vars) {
+ if (v.getType() == SubmarineConfVars.ConfVars.VarType.BOOLEAN) {
this.setProperty(v.getVarName(), v.getBooleanValue());
- } else if (v.getType() == ConfVars.VarType.LONG) {
+ } else if (v.getType() == SubmarineConfVars.ConfVars.VarType.LONG) {
this.setProperty(v.getVarName(), v.getLongValue());
- } else if (v.getType() == ConfVars.VarType.INT) {
+ } else if (v.getType() == SubmarineConfVars.ConfVars.VarType.INT) {
this.setProperty(v.getVarName(), v.getIntValue());
- } else if (v.getType() == ConfVars.VarType.FLOAT) {
+ } else if (v.getType() == SubmarineConfVars.ConfVars.VarType.FLOAT) {
this.setProperty(v.getVarName(), v.getFloatValue());
- } else if (v.getType() == ConfVars.VarType.STRING) {
+ } else if (v.getType() == SubmarineConfVars.ConfVars.VarType.STRING) {
this.setProperty(v.getVarName(), v.getStringValue());
} else {
throw new RuntimeException("Unsupported VarType");
@@ -133,41 +133,41 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
public String getServerAddress() {
- return getString(ConfVars.SUBMARINE_SERVER_ADDR);
+ return getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_ADDR);
}
public boolean useSsl() {
- return getBoolean(ConfVars.SUBMARINE_SERVER_SSL);
+ return getBoolean(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL);
}
public int getServerPort() {
- return getInt(ConfVars.SUBMARINE_SERVER_PORT);
+ return getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_PORT);
}
@VisibleForTesting
public void setServerPort(int port) {
- properties.put(ConfVars.SUBMARINE_SERVER_PORT.getVarName(),
String.valueOf(port));
+
properties.put(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_PORT.getVarName(),
String.valueOf(port));
}
public int getServerSslPort() {
- return getInt(ConfVars.SUBMARINE_SERVER_SSL_PORT);
+ return getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_PORT);
}
public String getKeyStorePath() {
- String path = getString(ConfVars.SUBMARINE_SERVER_SSL_KEYSTORE_PATH);
+ String path =
getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_KEYSTORE_PATH);
return path;
}
public String getKeyStoreType() {
- return getString(ConfVars.SUBMARINE_SERVER_SSL_KEYSTORE_TYPE);
+ return
getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_KEYSTORE_TYPE);
}
public String getKeyStorePassword() {
- return getString(ConfVars.SUBMARINE_SERVER_SSL_KEYSTORE_PASSWORD);
+ return
getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_KEYSTORE_PASSWORD);
}
public String getKeyManagerPassword() {
- String password =
getString(ConfVars.SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD);
+ String password =
getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD);
if (password == null) {
return getKeyStorePassword();
} else {
@@ -176,11 +176,11 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
public boolean useClientAuth() {
- return getBoolean(ConfVars.SUBMARINE_SERVER_SSL_CLIENT_AUTH);
+ return
getBoolean(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_CLIENT_AUTH);
}
public String getTrustStorePath() {
- String path = getString(ConfVars.SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH);
+ String path =
getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH);
if (path == null) {
path = getKeyStorePath();
}
@@ -188,7 +188,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
public String getTrustStoreType() {
- String type = getString(ConfVars.SUBMARINE_SERVER_SSL_TRUSTSTORE_TYPE);
+ String type =
getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_TRUSTSTORE_TYPE);
if (type == null) {
return getKeyStoreType();
} else {
@@ -197,7 +197,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
public String getTrustStorePassword() {
- String password =
getString(ConfVars.SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD);
+ String password =
getString(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD);
if (password == null) {
return getKeyStorePassword();
} else {
@@ -206,10 +206,10 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
public Integer getJettyRequestHeaderSize() {
- return getInt(ConfVars.SUBMARINE_SERVER_JETTY_REQUEST_HEADER_SIZE);
+ return
getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_REQUEST_HEADER_SIZE);
}
- public String getRelativeDir(ConfVars c) {
+ public String getRelativeDir(SubmarineConfVars.ConfVars c) {
return getRelativeDir(getString(c));
}
@@ -226,74 +226,74 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
public String getJdbcDriverClassName() {
- return getString(ConfVars.JDBC_DRIVERCLASSNAME);
+ return getString(SubmarineConfVars.ConfVars.JDBC_DRIVERCLASSNAME);
}
public String getJdbcUrl() {
- return getString(ConfVars.JDBC_URL);
+ return getString(SubmarineConfVars.ConfVars.JDBC_URL);
}
public String getMetastoreJdbcUrl() {
- return getString(ConfVars.METASTORE_JDBC_URL);
+ return getString(SubmarineConfVars.ConfVars.METASTORE_JDBC_URL);
}
@VisibleForTesting
public void setMetastoreJdbcUrl(String testMetastoreJdbcUrl) {
- properties.put(ConfVars.METASTORE_JDBC_URL.getVarName(),
testMetastoreJdbcUrl);
+ properties.put(SubmarineConfVars.ConfVars.METASTORE_JDBC_URL.getVarName(),
testMetastoreJdbcUrl);
}
@VisibleForTesting
public void setJdbcUrl(String testJdbcUrl) {
- properties.put(ConfVars.JDBC_URL.getVarName(), testJdbcUrl);
+ properties.put(SubmarineConfVars.ConfVars.JDBC_URL.getVarName(),
testJdbcUrl);
}
public String getJdbcUserName() {
- return getString(ConfVars.JDBC_USERNAME);
+ return getString(SubmarineConfVars.ConfVars.JDBC_USERNAME);
}
public String getMetastoreJdbcUserName() {
- return getString(ConfVars.METASTORE_JDBC_USERNAME);
+ return getString(SubmarineConfVars.ConfVars.METASTORE_JDBC_USERNAME);
}
@VisibleForTesting
public void setJdbcUserName(String userName) {
- properties.put(ConfVars.JDBC_USERNAME.getVarName(), userName);
+ properties.put(SubmarineConfVars.ConfVars.JDBC_USERNAME.getVarName(),
userName);
}
@VisibleForTesting
public void setMetastoreJdbcUserName(String metastoreUserName) {
- properties.put(ConfVars.METASTORE_JDBC_USERNAME.getVarName(),
metastoreUserName);
+
properties.put(SubmarineConfVars.ConfVars.METASTORE_JDBC_USERNAME.getVarName(),
metastoreUserName);
}
public String getJdbcPassword() {
- return getString(ConfVars.JDBC_PASSWORD);
+ return getString(SubmarineConfVars.ConfVars.JDBC_PASSWORD);
}
public String getMetastoreJdbcPassword() {
- return getString(ConfVars.METASTORE_JDBC_PASSWORD);
+ return getString(SubmarineConfVars.ConfVars.METASTORE_JDBC_PASSWORD);
}
@VisibleForTesting
public void setJdbcPassword(String password) {
- properties.put(ConfVars.JDBC_PASSWORD.getVarName(), password);
+ properties.put(SubmarineConfVars.ConfVars.JDBC_PASSWORD.getVarName(),
password);
}
@VisibleForTesting
public void setMetastoreJdbcPassword(String metastorePassword) {
- properties.put(ConfVars.METASTORE_JDBC_PASSWORD.getVarName(),
metastorePassword);
+
properties.put(SubmarineConfVars.ConfVars.METASTORE_JDBC_PASSWORD.getVarName(),
metastorePassword);
}
public String getClusterAddress() {
- return getString(ConfVars.SUBMARINE_CLUSTER_ADDR);
+ return getString(SubmarineConfVars.ConfVars.SUBMARINE_CLUSTER_ADDR);
}
public void setClusterAddress(String clusterAddr) {
- properties.put(ConfVars.SUBMARINE_CLUSTER_ADDR.getVarName(), clusterAddr);
+
properties.put(SubmarineConfVars.ConfVars.SUBMARINE_CLUSTER_ADDR.getVarName(),
clusterAddr);
}
public boolean isClusterMode() {
- String clusterAddr = getString(ConfVars.SUBMARINE_CLUSTER_ADDR);
+ String clusterAddr =
getString(SubmarineConfVars.ConfVars.SUBMARINE_CLUSTER_ADDR);
if (StringUtils.isEmpty(clusterAddr)) {
return false;
}
@@ -302,15 +302,15 @@ public class SubmarineConfiguration extends
XMLConfiguration {
}
public int getClusterHeartbeatInterval() {
- return getInt(ConfVars.CLUSTER_HEARTBEAT_INTERVAL);
+ return getInt(SubmarineConfVars.ConfVars.CLUSTER_HEARTBEAT_INTERVAL);
}
public int getClusterHeartbeatTimeout() {
- return getInt(ConfVars.CLUSTER_HEARTBEAT_TIMEOUT);
+ return getInt(SubmarineConfVars.ConfVars.CLUSTER_HEARTBEAT_TIMEOUT);
}
public String getWebsocketMaxTextMessageSize() {
- return getString(ConfVars.WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
+ return
getString(SubmarineConfVars.ConfVars.WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
}
/**
@@ -320,7 +320,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
public List<String> listSubmitter() {
List<String> values = new ArrayList<>();
- String submitters = getString(ConfVars.SUBMARINE_SUBMITTERS);
+ String submitters =
getString(SubmarineConfVars.ConfVars.SUBMARINE_SUBMITTERS);
if (submitters != null) {
final String delim = ",";
StringTokenizer tokenizer = new StringTokenizer(submitters, delim);
@@ -338,7 +338,8 @@ public class SubmarineConfiguration extends
XMLConfiguration {
* @return class name
*/
public String getSubmitterClass(String name) {
- return
getStringValue(String.format(ConfVars.SUBMARINE_SUBMITTERS_CLASS.getVarName(),
name), "");
+ return
getStringValue(String.format(SubmarineConfVars.ConfVars.SUBMARINE_SUBMITTERS_CLASS.getVarName(),
+ name), "");
}
/**
@@ -347,8 +348,8 @@ public class SubmarineConfiguration extends
XMLConfiguration {
* @return classpath
*/
public String getSubmitterClassPath(String name) {
- return
getStringValue(String.format(ConfVars.SUBMARINE_SUBMITTERS_CLASSPATH.getVarName(),
- name), "");
+ return
getStringValue(String.format(SubmarineConfVars.ConfVars.SUBMARINE_SUBMITTERS_CLASSPATH.
+ getVarName(), name), "");
}
private String getStringValue(String name, String d) {
@@ -391,7 +392,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return d;
}
- public String getString(ConfVars c) {
+ public String getString(SubmarineConfVars.ConfVars c) {
return getString(c.name(), c.getVarName(), c.getStringValue());
}
@@ -406,15 +407,15 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return getStringValue(propertyName, defaultValue);
}
- public void setString(ConfVars c, String value) {
+ public void setString(SubmarineConfVars.ConfVars c, String value) {
properties.put(c.getVarName(), value);
}
- public int getInt(ConfVars c) {
+ public int getInt(SubmarineConfVars.ConfVars c) {
return getInt(c.name(), c.getVarName(), c.getIntValue());
}
- public void setInt(ConfVars c, int value) {
+ public void setInt(SubmarineConfVars.ConfVars c, int value) {
properties.put(c.getVarName(), String.valueOf(value));
}
@@ -429,11 +430,11 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return getIntValue(propertyName, defaultValue);
}
- public long getLong(ConfVars c) {
+ public long getLong(SubmarineConfVars.ConfVars c) {
return getLong(c.name(), c.getVarName(), c.getLongValue());
}
- public void setLong(ConfVars c, long value) {
+ public void setLong(SubmarineConfVars.ConfVars c, long value) {
properties.put(c.getVarName(), String.valueOf(value));
}
@@ -448,7 +449,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return getLongValue(propertyName, defaultValue);
}
- public float getFloat(ConfVars c) {
+ public float getFloat(SubmarineConfVars.ConfVars c) {
return getFloat(c.name(), c.getVarName(), c.getFloatValue());
}
@@ -462,7 +463,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
return getFloatValue(propertyName, defaultValue);
}
- public boolean getBoolean(ConfVars c) {
+ public boolean getBoolean(SubmarineConfVars.ConfVars c) {
return getBoolean(c.name(), c.getVarName(), c.getBooleanValue());
}
@@ -480,202 +481,4 @@ public class SubmarineConfiguration extends
XMLConfiguration {
public void updateConfiguration(String name, String value) {
properties.put(name, value);
}
-
- public enum ConfVars {
- SUBMARINE_CONF_DIR("submarine.conf.dir", "conf"),
- SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB(
- "submarine.localization.max-allowed-file-size-mb", 2048L),
- SUBMARINE_SERVER_ADDR("submarine.server.addr", "0.0.0.0"),
- SUBMARINE_SERVER_PORT("submarine.server.port", 8080),
- SUBMARINE_SERVER_SSL("submarine.server.ssl", false),
- SUBMARINE_SERVER_SSL_PORT("submarine.server.ssl.port", 8443),
-
SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX("submarine.server.jetty.thread.pool.max",
400),
-
SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN("submarine.server.jetty.thread.pool.min",
8),
-
SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT("submarine.server.jetty.thread.pool.timeout",
30),
-
SUBMARINE_SERVER_JETTY_REQUEST_HEADER_SIZE("submarine.server.jetty.request.header.size",
8192),
- SUBMARINE_SERVER_SSL_CLIENT_AUTH("submarine.server.ssl.client.auth",
false),
- SUBMARINE_SERVER_SSL_KEYSTORE_PATH("submarine.server.ssl.keystore.path",
"keystore"),
- SUBMARINE_SERVER_SSL_KEYSTORE_TYPE("submarine.server.ssl.keystore.type",
"JKS"),
-
SUBMARINE_SERVER_SSL_KEYSTORE_PASSWORD("submarine.server.ssl.keystore.password",
""),
-
SUBMARINE_SERVER_SSL_KEY_MANAGER_PASSWORD("submarine.server.ssl.key.manager.password",
null),
-
SUBMARINE_SERVER_SSL_TRUSTSTORE_PATH("submarine.server.ssl.truststore.path",
null),
-
SUBMARINE_SERVER_SSL_TRUSTSTORE_TYPE("submarine.server.ssl.truststore.type",
null),
-
SUBMARINE_SERVER_SSL_TRUSTSTORE_PASSWORD("submarine.server.ssl.truststore.password",
null),
- SUBMARINE_CLUSTER_ADDR("submarine.cluster.addr", ""),
- SUBMARINE_SERVER_REMOTE_EXECUTION_ENABLED(
- "submarine.server.remote.execution.enabled", false),
- SUBMARINE_SERVER_REMOTE_EXECUTION_PORT(
- "submarine.server.remote.execution.port", 8980),
- CLUSTER_HEARTBEAT_INTERVAL("cluster.heartbeat.interval", 3000),
- CLUSTER_HEARTBEAT_TIMEOUT("cluster.heartbeat.timeout", 9000),
-
- JDBC_DRIVERCLASSNAME("jdbc.driverClassName", "com.mysql.jdbc.Driver"),
- JDBC_URL("jdbc.url", "jdbc:mysql://127.0.0.1:3306/submarine" +
-
"?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" +
-
"failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"),
- JDBC_USERNAME("jdbc.username", "submarine"),
- JDBC_PASSWORD("jdbc.password", "password"),
- METASTORE_JDBC_URL("metastore.jdbc.url",
"jdbc:mysql://127.0.0.1:3306/metastore" +
-
"?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" +
-
"failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"),
- METASTORE_JDBC_USERNAME("metastore.jdbc.username", "metastore"),
- METASTORE_JDBC_PASSWORD("metastore.jdbc.password", "password"),
-
- WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE(
- "workbench.websocket.max.text.message.size", "1024000"),
- WORKBENCH_WEB_WAR("workbench.web.war",
"submarine-workbench/workbench-web/dist"),
- SUBMARINE_RUNTIME_CLASS("submarine.runtime.class",
- "org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory"),
- SUBMARINE_SUBMITTERS("submarine.submitters", ""),
- SUBMARINE_SUBMITTERS_CLASS("submarine.submitters.%s.class", ""),
- SUBMARINE_SUBMITTERS_CLASSPATH("submarine.submitters.%s.classpath", ""),
- SUBMARINE_K8S_KUBE_CONFIG("submarine.k8s.kube.config", "");
-
- private String varName;
- @SuppressWarnings("rawtypes")
- private Class varClass;
- private String stringValue;
- private VarType type;
- private int intValue;
- private float floatValue;
- private boolean booleanValue;
- private long longValue;
-
-
- ConfVars(String varName, String varValue) {
- this.varName = varName;
- this.varClass = String.class;
- this.stringValue = varValue;
- this.intValue = -1;
- this.floatValue = -1;
- this.longValue = -1;
- this.booleanValue = false;
- this.type = VarType.STRING;
- }
-
- ConfVars(String varName, int intValue) {
- this.varName = varName;
- this.varClass = Integer.class;
- this.stringValue = null;
- this.intValue = intValue;
- this.floatValue = -1;
- this.longValue = -1;
- this.booleanValue = false;
- this.type = VarType.INT;
- }
-
- ConfVars(String varName, long longValue) {
- this.varName = varName;
- this.varClass = Integer.class;
- this.stringValue = null;
- this.intValue = -1;
- this.floatValue = -1;
- this.longValue = longValue;
- this.booleanValue = false;
- this.type = VarType.LONG;
- }
-
- ConfVars(String varName, float floatValue) {
- this.varName = varName;
- this.varClass = Float.class;
- this.stringValue = null;
- this.intValue = -1;
- this.longValue = -1;
- this.floatValue = floatValue;
- this.booleanValue = false;
- this.type = VarType.FLOAT;
- }
-
- ConfVars(String varName, boolean booleanValue) {
- this.varName = varName;
- this.varClass = Boolean.class;
- this.stringValue = null;
- this.intValue = -1;
- this.longValue = -1;
- this.floatValue = -1;
- this.booleanValue = booleanValue;
- this.type = VarType.BOOLEAN;
- }
-
- public String getVarName() {
- return varName;
- }
-
- @SuppressWarnings("rawtypes")
- public Class getVarClass() {
- return varClass;
- }
-
- public int getIntValue() {
- return intValue;
- }
-
- public long getLongValue() {
- return longValue;
- }
-
- public float getFloatValue() {
- return floatValue;
- }
-
- public String getStringValue() {
- return stringValue;
- }
-
- public boolean getBooleanValue() {
- return booleanValue;
- }
-
- public VarType getType() {
- return type;
- }
-
- enum VarType {
- STRING {
- @Override
- void checkType(String value) throws Exception {
- }
- },
- INT {
- @Override
- void checkType(String value) throws Exception {
- Integer.valueOf(value);
- }
- },
- LONG {
- @Override
- void checkType(String value) throws Exception {
- Long.valueOf(value);
- }
- },
- FLOAT {
- @Override
- void checkType(String value) throws Exception {
- Float.valueOf(value);
- }
- },
- BOOLEAN {
- @Override
- void checkType(String value) throws Exception {
- Boolean.valueOf(value);
- }
- };
-
- boolean isType(String value) {
- try {
- checkType(value);
- } catch (Exception e) {
- LOG.error("Exception in SubmarineConfiguration while isType", e);
- return false;
- }
- return true;
- }
-
- String typeString() {
- return name().toUpperCase();
- }
-
- abstract void checkType(String value) throws Exception;
- }
- }
}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java
index 86f174b..8ef560a 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/SubmarineServer.java
@@ -48,7 +48,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
-import org.apache.submarine.commons.utils.SubmarineConfiguration.ConfVars;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -166,8 +166,9 @@ public class SubmarineServer extends ResourceConfig {
SubmarineConfiguration conf) {
WebAppContext webApp = new WebAppContext();
webApp.setContextPath("/");
- File warPath = new File(conf.getString(ConfVars.WORKBENCH_WEB_WAR));
- LOG.info("workbench web war file path is {}.",
conf.getString(ConfVars.WORKBENCH_WEB_WAR));
+ File warPath = new
File(conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
+ LOG.info("workbench web war file path is {}.",
+ conf.getString(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR));
if (warPath.isDirectory()) {
// Development mode, read from FS
// webApp.setDescriptor(warPath+"/WEB-INF/web.xml");
@@ -189,9 +190,9 @@ public class SubmarineServer extends ResourceConfig {
private static Server setupJettyServer(SubmarineConfiguration conf) {
ThreadPool threadPool =
- new
QueuedThreadPool(conf.getInt(ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX),
- conf.getInt(ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN),
- conf.getInt(ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT));
+ new
QueuedThreadPool(conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MAX),
+
conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_MIN),
+
conf.getInt(SubmarineConfVars.ConfVars.SUBMARINE_SERVER_JETTY_THREAD_POOL_TIMEOUT));
final Server server = new Server(threadPool);
ServerConnector connector;
diff --git
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
index 88e797b..4b414ad 100644
---
a/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
+++
b/submarine-server/server-core/src/test/java/org/apache/submarine/server/AbstractSubmarineServerTest.java
@@ -30,7 +30,7 @@ import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.server.utils.TestUtils;
import org.hamcrest.Description;
import org.hamcrest.TypeSafeMatcher;
@@ -98,9 +98,9 @@ public abstract class AbstractSubmarineServerTest {
confDir = new File(submarineServerHome, "conf_" + testClassName);
confDir.mkdirs();
-
System.setProperty(SubmarineConfiguration.ConfVars.WORKBENCH_WEB_WAR.getVarName(),
+
System.setProperty(SubmarineConfVars.ConfVars.WORKBENCH_WEB_WAR.getVarName(),
new File("../workbench-web/dist").getAbsolutePath());
-
System.setProperty(SubmarineConfiguration.ConfVars.SUBMARINE_CONF_DIR.getVarName(),
+
System.setProperty(SubmarineConfVars.ConfVars.SUBMARINE_CONF_DIR.getVarName(),
confDir.getAbsolutePath());
// some test profile does not build workbench-web.
diff --git
a/submarine-server/server-rpc/src/main/java/org/apache/submarine/server/rpc/SubmarineRpcServer.java
b/submarine-server/server-rpc/src/main/java/org/apache/submarine/server/rpc/SubmarineRpcServer.java
index 671cd7d..dc3e936 100644
---
a/submarine-server/server-rpc/src/main/java/org/apache/submarine/server/rpc/SubmarineRpcServer.java
+++
b/submarine-server/server-rpc/src/main/java/org/apache/submarine/server/rpc/SubmarineRpcServer.java
@@ -36,6 +36,7 @@ import
org.apache.submarine.commons.runtime.exception.SubmarineException;
import org.apache.submarine.commons.runtime.RuntimeFactory;
import org.apache.submarine.commons.runtime.param.Parameter;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -136,7 +137,7 @@ public class SubmarineRpcServer {
SubmarineConfiguration submarineConfiguration =
SubmarineConfiguration.getInstance();
int rpcServerPort = submarineConfiguration.getInt(
-
SubmarineConfiguration.ConfVars.SUBMARINE_SERVER_REMOTE_EXECUTION_PORT);
+ SubmarineConfVars.ConfVars.SUBMARINE_SERVER_REMOTE_EXECUTION_PORT);
SubmarineRpcServer server = new SubmarineRpcServer(rpcServerPort);
server.start();
return server;
diff --git
a/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/MockRpcServer.java
b/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/MockRpcServer.java
index c85aefd..13858c6 100644
---
a/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/MockRpcServer.java
+++
b/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/MockRpcServer.java
@@ -23,6 +23,7 @@ import org.apache.submarine.client.cli.CliUtils;
import org.apache.submarine.commons.runtime.ClientContext;
import org.apache.submarine.commons.runtime.param.Parameter;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import java.io.IOException;
@@ -54,7 +55,7 @@ public class MockRpcServer extends SubmarineRpcServer {
SubmarineConfiguration submarineConfiguration =
SubmarineConfiguration.getInstance();
int rpcServerPort = submarineConfiguration.getInt(
-
SubmarineConfiguration.ConfVars.SUBMARINE_SERVER_REMOTE_EXECUTION_PORT);
+ SubmarineConfVars.ConfVars.SUBMARINE_SERVER_REMOTE_EXECUTION_PORT);
SubmarineRpcServer server = new MockRpcServer(rpcServerPort);
server.start();
return server;
diff --git
a/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/SubmarineRpcClient.java
b/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/SubmarineRpcClient.java
index f15eadb..7be095c 100644
---
a/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/SubmarineRpcClient.java
+++
b/submarine-server/server-rpc/src/test/java/org/apache/submarine/server/rpc/SubmarineRpcClient.java
@@ -26,6 +26,7 @@ import
org.apache.submarine.commons.rpc.SubmarineServerProtocolGrpc;
import
org.apache.submarine.commons.rpc.SubmarineServerProtocolGrpc.SubmarineServerProtocolBlockingStub;
import
org.apache.submarine.commons.rpc.SubmarineServerProtocolGrpc.SubmarineServerProtocolStub;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,9 +47,9 @@ public class SubmarineRpcClient extends RpcServerTestUtils {
public SubmarineRpcClient(SubmarineConfiguration config) {
this(ManagedChannelBuilder.forAddress(
config.getString(
- SubmarineConfiguration.ConfVars.SUBMARINE_SERVER_ADDR),
+ SubmarineConfVars.ConfVars.SUBMARINE_SERVER_ADDR),
config.getInt(
- SubmarineConfiguration.ConfVars.
+ SubmarineConfVars.ConfVars.
SUBMARINE_SERVER_REMOTE_EXECUTION_PORT))
.usePlaintext());
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
index afc187a..680c633 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sJobSubmitter.java
@@ -34,6 +34,7 @@ import io.kubernetes.client.models.V1DeleteOptionsBuilder;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.server.api.JobSubmitter;
import org.apache.submarine.server.api.exception.UnsupportedJobTypeException;
import org.apache.submarine.server.api.job.Job;
@@ -73,7 +74,7 @@ public class K8sJobSubmitter implements JobSubmitter {
supportedCRDMap.put("TFJob", "tfjobs");
if (confPath == null || confPath.trim().isEmpty()) {
- confPath =
conf.getString(SubmarineConfiguration.ConfVars.SUBMARINE_K8S_KUBE_CONFIG);
+ confPath =
conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_K8S_KUBE_CONFIG);
}
loadClientConfiguration(confPath);
}
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
index 229b9e2..197f0cc 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
import org.apache.submarine.commons.runtime.ClientContext;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.runtime.conf.SubmarineLogs;
import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
import org.apache.submarine.server.submitter.yarnservice.utils.ZipUtilities;
@@ -262,7 +263,7 @@ public class FileSystemOperations {
private long getMaxRemoteFileSizeMB() {
return submarineConfig.getLong(
- SubmarineConfiguration.ConfVars.
+ SubmarineConfVars.ConfVars.
SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB);
}
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
index c8a07cb..7eae1c5 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.submarine.client.cli.runjob.RunJobCli;
import org.apache.submarine.commons.runtime.MockClientContext;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.apache.submarine.commons.runtime.conf.SubmarineLogs;
import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
@@ -425,7 +426,7 @@ public class YarnServiceRunJobCliLocalizationTest {
// Max 10MB, mock remote will always return file size 100MB.
submarineConf.setLong(
- SubmarineConfiguration.ConfVars.
+ SubmarineConfVars.ConfVars.
SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 10L);
mockClientContext.setSubmarineConfig(submarineConf);
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperationsTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperationsTest.java
index 4f36212..19655a6 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperationsTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperationsTest.java
@@ -26,6 +26,7 @@ import org.apache.submarine.commons.runtime.ClientContext;
import org.apache.submarine.commons.runtime.MockClientContext;
import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.apache.submarine.commons.utils.SubmarineConfVars;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
@@ -216,8 +217,8 @@ public class FileSystemOperationsTest {
SubmarineConfiguration config =
SubmarineConfiguration.newInstance();
- config.setLong(SubmarineConfiguration.ConfVars
- .SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 21L);
+ config.setLong(SubmarineConfVars
+ .ConfVars.SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 21L);
when(clientContext.getSubmarineConfig()).thenReturn(config);
fileSystemOperations = new FileSystemOperations(clientContext);
@@ -237,8 +238,8 @@ public class FileSystemOperationsTest {
SubmarineConfiguration config =
SubmarineConfiguration.newInstance();
- config.setLong(SubmarineConfiguration.ConfVars
- .SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 19L);
+ config.setLong(SubmarineConfVars
+ .ConfVars.SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 19L);
when(clientContext.getSubmarineConfig()).thenReturn(config);
fileSystemOperations = new FileSystemOperations(clientContext);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
