This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 3d815f4 SUBMARINE-332. Fix submarine server database connection
failure
3d815f4 is described below
commit 3d815f46962405ad888eb7c8bf61a1c380d99bce
Author: Xun Liu <[email protected]>
AuthorDate: Sat Jan 4 18:45:08 2020 +0800
SUBMARINE-332. Fix submarine server database connection failure
### What is this PR for?
The submarine server cannot access the database due to the following issues
1. Mysql configuration file is missing parameters in `submairne-site.xml`.
2. In the pom file submarine project, without introducing mysql connector
jar package
### What type of PR is it?
[Bug Fix | Improvement ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/SUBMARINE-332
### How should this be tested?
* [CI Pass](https://travis-ci.org/liuxunorg/submarine/builds/632581397)
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Xun Liu <[email protected]>
Closes #142 from liuxunorg/SUBMARINE-332 and squashes the following commits:
708fe66 [Xun Liu] use SUBMARINE_HOME.
c10652d [Xun Liu] SUBMARINE-332. Fix submarine server database connection
failure
---
bin/common.sh | 12 ++++++-
bin/submarine-daemon.sh | 4 +--
conf/submarine-site.xml | 30 ++++++++++++++++
conf/submarine-site.xml.template | 30 ++++++++++++++++
dev-support/mini-submarine/README.md | 8 +++--
pom.xml | 3 +-
.../commons/utils/SubmarineConfiguration.java | 3 +-
submarine-dist/src/assembly/distribution.xml | 2 ++
submarine-server/server-core/pom.xml | 15 +++++++-
.../src/main/resources}/submarine-site.xml | 42 ++++++++++++----------
10 files changed, 122 insertions(+), 27 deletions(-)
diff --git a/bin/common.sh b/bin/common.sh
index 70bfed6..0bccf5d 100755
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -73,8 +73,18 @@ function download_mysql_jdbc_jar(){
fi
MYSQL_JAR_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_VERSION}/mysql-connector-java-${MYSQL_VERSION}.jar";
fi
+
echo "Downloading mysql jdbc jar from ${MYSQL_JAR_URL}."
- wget ${MYSQL_JAR_URL} -P "${BIN}/../lib"
+ if type wget >/dev/null 2>&1; then
+ wget ${MYSQL_JAR_URL} -P "${SUBMARINE_HOME}/lib" --no-check-certificate
+ elif type curl >/dev/null 2>&1; then
+ curl -o "${SUBMARINE_HOME}/lib/mysql-connector-java-${MYSQL_VERSION}.jar"
${MYSQL_JAR_URL}
+ else
+ echo 'We need a tool to transfer data from or to a server. Such as
wget/curl.'
+ echo 'Bye, bye!'
+ exit -1
+ fi
+
echo "Mysql jdbc jar is downloaded and put in the path of submarine/lib."
}
diff --git a/bin/submarine-daemon.sh b/bin/submarine-daemon.sh
index 4614d7f..01b332c 100755
--- a/bin/submarine-daemon.sh
+++ b/bin/submarine-daemon.sh
@@ -38,8 +38,8 @@ SUBMARINE_SERVER_LOGFILE="${SUBMARINE_LOG_DIR}/submarine.log"
SUBMARINE_SERVER_MAIN=org.apache.submarine.server.SubmarineServer
JAVA_OPTS+="${SUBMARINE_SERVER_JAVA_OPTS} ${SUBMARINE_SERVER_MEM}
-Dsubmarine.log.file=${SUBMARINE_SERVER_LOGFILE}"
-add_each_jar_in_dir "${BIN}/../lib"
-add_each_jar_in_dir "${BIN}/../lib/submitter"
+add_jar_in_dir "${BIN}/../lib"
+add_jar_in_dir "${BIN}/../lib/submitter"
function initialize_default_directories() {
if [[ ! -d "${SUBMARINE_LOG_DIR}" ]]; then
diff --git a/conf/submarine-site.xml b/conf/submarine-site.xml
index da74af0..c3a9919 100755
--- a/conf/submarine-site.xml
+++ b/conf/submarine-site.xml
@@ -108,6 +108,36 @@
</property>
<property>
+ <name>jdbc.driverClassName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ </property>
+ <property>
+ <name>jdbc.url</name>
+
<value>jdbc:mysql://127.0.0.1:3306/submarine?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false</value>
+ </property>
+ <property>
+ <name>jdbc.username</name>
+ <value>submarine</value>
+ </property>
+ <property>
+ <name>jdbc.password</name>
+ <value>password</value>
+ </property>
+
+ <property>
+ <name>metastore.jdbc.url</name>
+
<value>jdbc:mysql://127.0.0.1:3306/metastore?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false</value>
+ </property>
+ <property>
+ <name>metastore.jdbc.username</name>
+ <value>metastore</value>
+ </property>
+ <property>
+ <name>metastore.jdbc.password</name>
+ <value>password</value>
+ </property>
+
+ <property>
<name>submarine.runtime.class</name>
<value>org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory</value>
<description>RuntimeFactory for Submarine jobs</description>
diff --git a/conf/submarine-site.xml.template b/conf/submarine-site.xml.template
index 86dc941..0583752 100755
--- a/conf/submarine-site.xml.template
+++ b/conf/submarine-site.xml.template
@@ -108,6 +108,36 @@
</property>
<property>
+ <name>jdbc.driverClassName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ </property>
+ <property>
+ <name>jdbc.url</name>
+
<value>jdbc:mysql://127.0.0.1:3306/submarine?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false</value>
+ </property>
+ <property>
+ <name>jdbc.username</name>
+ <value>submarine</value>
+ </property>
+ <property>
+ <name>jdbc.password</name>
+ <value>password</value>
+ </property>
+
+ <property>
+ <name>metastore.jdbc.url</name>
+
<value>jdbc:mysql://127.0.0.1:3306/metastore?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false</value>
+ </property>
+ <property>
+ <name>metastore.jdbc.username</name>
+ <value>metastore</value>
+ </property>
+ <property>
+ <name>metastore.jdbc.password</name>
+ <value>password</value>
+ </property>
+
+ <property>
<name>submarine.runtime.class</name>
<value>org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory</value>
<description>RuntimeFactory for Submarine jobs</description>
diff --git a/dev-support/mini-submarine/README.md
b/dev-support/mini-submarine/README.md
index 74afd0d..41abc3e 100644
--- a/dev-support/mini-submarine/README.md
+++ b/dev-support/mini-submarine/README.md
@@ -90,7 +90,7 @@ hdfs dfs -ls /user
## Run workbench server
1. Setup mysql mariadb server
-> Because mysql and mariadb use the GPL license, you need to manually execute
the script to install it.
+> Because mysql and mariadb use the GPL license, So there is no binary file
containing mysql in the image, you need to manually execute the script to
install it.
```
/tmp/hadoop-config/setup-mysql.sh
@@ -104,8 +104,10 @@ su yarn
/opt/submarine-current/bin/submarine-daemon.sh start getMysqlJar
```
-3. login submarine workbench
-You can use the following command to get the login URL of the submarine
workbench
+3. Login submarine workbench
+
+Execute the following command in your host machine, Get the access URL of the
submarine workbench running in docker
+
```shell
echo "http://localhost:$(docker inspect --format='{{(index (index
.NetworkSettings.Ports "8080/tcp") 0).HostPort}}' mini-submarine)"
```
diff --git a/pom.xml b/pom.xml
index 48170f9..e33a6c7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,7 +58,8 @@
<plugin.enforcer.version>1.3.1</plugin.enforcer.version>
<plugin.checkstyle.version>2.17</plugin.checkstyle.version>
<plugin.war.version>2.3</plugin.war.version>
- <plugin.maven.antrun>1.8</plugin.maven.antrun>
+ <plugin.maven.antrun.version>1.8</plugin.maven.antrun.version>
+ <plugin.maven.jar.version>3.0.2</plugin.maven.jar.version>
<!-- submarine shaded dependency prefix-->
<shaded.dependency.prefix>org.apache.submarine.shaded</shaded.dependency.prefix>
diff --git
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
index 4e63a09..e5af925 100644
---
a/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
+++
b/submarine-commons/commons-utils/src/main/java/org/apache/submarine/commons/utils/SubmarineConfiguration.java
@@ -508,11 +508,11 @@ public class SubmarineConfiguration extends
XMLConfiguration {
"submarine.server.remote.execution.port", 8980),
CLUSTER_HEARTBEAT_INTERVAL("cluster.heartbeat.interval", 3000),
CLUSTER_HEARTBEAT_TIMEOUT("cluster.heartbeat.timeout", 9000),
+
JDBC_DRIVERCLASSNAME("jdbc.driverClassName", "com.mysql.jdbc.Driver"),
JDBC_URL("jdbc.url", "jdbc:mysql://127.0.0.1:3306/submarine" +
"?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&" +
"failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"),
-
JDBC_USERNAME("jdbc.username", "submarine"),
JDBC_PASSWORD("jdbc.password", "password"),
METASTORE_JDBC_URL("metastore.jdbc.url",
"jdbc:mysql://127.0.0.1:3306/metastore" +
@@ -520,6 +520,7 @@ public class SubmarineConfiguration extends
XMLConfiguration {
"failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false"),
METASTORE_JDBC_USERNAME("metastore.jdbc.username", "metastore"),
METASTORE_JDBC_PASSWORD("metastore.jdbc.password", "password"),
+
WORKBENCH_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE(
"workbench.websocket.max.text.message.size", "1024000"),
WORKBENCH_WEB_WAR("workbench.web.war",
"submarine-workbench/workbench-web/dist"),
diff --git a/submarine-dist/src/assembly/distribution.xml
b/submarine-dist/src/assembly/distribution.xml
index 703ed5a..bbb0c2e 100644
--- a/submarine-dist/src/assembly/distribution.xml
+++ b/submarine-dist/src/assembly/distribution.xml
@@ -154,6 +154,8 @@
<exclude>commons-rpc-${project.version}.jar</exclude>
<exclude>grpc-*.jar</exclude>
<exclude>protobuf-java*.jar</exclude>
+ <!-- mysql-connector-java uses the GPL license. So we need exclude
mysql-connector-java jar -->
+ <exclude>mysql-connector-java-*.jar</exclude>
</excludes>
</fileSet>
<fileSet>
diff --git a/submarine-server/server-core/pom.xml
b/submarine-server/server-core/pom.xml
index 86339a2..17d4116 100644
--- a/submarine-server/server-core/pom.xml
+++ b/submarine-server/server-core/pom.xml
@@ -310,7 +310,7 @@
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql-connector-java.version}</version>
- <scope>provided</scope>
+ <!-- mysql-connector-java uses the GPL license. When we release the
version in Submarine-dist, we exclude mysql-connector-java jar -->
</dependency>
<dependency>
@@ -437,6 +437,19 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${plugin.maven.jar.version}</version>
+ <configuration>
+ <excludes>
+ <!-- Because SubmarineConfiguration class will load the
submarine-site.xml file from the jar package,
+ which will cause the conf/submarine-site.xml to fail to load
properly, so needs to be excluded from the package. -->
+ <exclude>submarine-site.xml</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
diff --git a/conf/submarine-site.xml
b/submarine-server/server-core/src/main/resources/submarine-site.xml
similarity index 80%
copy from conf/submarine-site.xml
copy to submarine-server/server-core/src/main/resources/submarine-site.xml
index da74af0..a9fdc34 100755
--- a/conf/submarine-site.xml
+++ b/submarine-server/server-core/src/main/resources/submarine-site.xml
@@ -103,38 +103,44 @@
<property>
<name>workbench.web.war</name>
- <value>../workbench-web-ng.war</value>
+
<value>submarine-workbench/workbench-web-ng/target/workbench-web-ng.war</value>
<description>Submarine workbench web war file path.</description>
</property>
<property>
- <name>submarine.runtime.class</name>
-
<value>org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory</value>
- <description>RuntimeFactory for Submarine jobs</description>
+ <name>jdbc.driverClassName</name>
+ <value>com.mysql.jdbc.Driver</value>
+ </property>
+ <property>
+ <name>jdbc.url</name>
+
<value>jdbc:mysql://127.0.0.1:3306/submarine?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false</value>
+ </property>
+ <property>
+ <name>jdbc.username</name>
+ <value>submarine</value>
+ </property>
+ <property>
+ <name>jdbc.password</name>
+ <value>password</value>
</property>
- <!-- Submarine Submitters Configuration -->
<property>
- <name>submarine.submitters</name>
- <value>k8s</value>
- <description>Submitter list for the server</description>
+ <name>metastore.jdbc.url</name>
+
<value>jdbc:mysql://127.0.0.1:3306/metastore?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&zeroDateTimeBehavior=convertToNull&useSSL=false</value>
</property>
<property>
- <name>submarine.submitters.k8s.class</name>
- <value>org.apache.submarine.server.submitter.k8s.K8sJobSubmitter</value>
- <description>The entry class for the specified submitter</description>
+ <name>metastore.jdbc.username</name>
+ <value>metastore</value>
</property>
<property>
- <name>submarine.submitters.k8s.classpath</name>
- <value>../lib/submitter/k8s/</value>
- <description>The libs for Kubernetes submitter</description>
+ <name>metastore.jdbc.password</name>
+ <value>password</value>
</property>
- <!-- K8s Configuration -->
<property>
- <name>submarine.k8s.kube.config</name>
- <value>../conf/k8s/config</value>
- <description>Kube config for kubernetes, you should get the config from
cluster</description>
+ <name>submarine.runtime.class</name>
+
<value>org.apache.submarine.server.submitter.yarn.YarnRuntimeFactory</value>
+ <description>RuntimeFactory for Submarine jobs</description>
</property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
