This is an automated email from the ASF dual-hosted git repository.
zhouquan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 82fae82 SUBMARINE-371. Remove tony submodule.
82fae82 is described below
commit 82fae8227fb18e4be5137360d16d06f3a61bda8a
Author: Quan zhou <[email protected]>
AuthorDate: Mon Feb 3 16:33:10 2020 +0800
SUBMARINE-371. Remove tony submodule.
### What is this PR for?
To work with tony community more closely, we need to remove tony submodule
and use tony dependency directly. We should not maintain tony code by
ourselves. If a patch to tony is needed, we should submit a PR to tony
repository
### What type of PR is it?
Improvement
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-371
### How should this be tested?
https://travis-ci.org/yuanzac/hadoop-submarine/builds/646704729?utm_source=github_status&utm_medium=notification
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? Yes
Author: Quan zhou <[email protected]>
Closes #173 from yuanzac/topic/SUBMARINE-371 and squashes the following
commits:
e045f11 [Quan zhou] SUBMARINE-371. Remove tony submodule.
---
.gitmodules | 4 ----
.travis.yml | 3 +--
dev-support/docker-images/submarine/build.sh | 6 -----
dev-support/mini-submarine/build_mini-submarine.sh | 7 ------
docs/development/BuildFromCode.md | 21 ++---------------
pom.xml | 3 +--
.../server-submitter/submitter-yarn/pom.xml | 26 ++++++++++++++++++++++
.../submarine/server/submitter/yarn/YarnUtils.java | 5 ++---
.../src/test/java/YarnUtilsTest.java | 6 +++--
submodules/tony | 1 -
10 files changed, 36 insertions(+), 46 deletions(-)
diff --git a/.gitmodules b/.gitmodules
index adf143c..e69de29 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +0,0 @@
-[submodule "submodules/tony"]
- path = submodules/tony
- url = https://github.com/hadoopsubmarine/linkedin-tony.git
- branch = submarine
diff --git a/.travis.yml b/.travis.yml
index 58cf9dc..b0b82b0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -46,7 +46,6 @@ env:
-
EXCLUDE_SUBMARINE="!submarine-all,!submarine-client,!submarine-commons,!submarine-commons/commons-runtime,!submarine-dist,!submarine-server/server-submitter/submitter-yarn,!submarine-server/server-core,!submarine-server/server-rpc"
-
EXCLUDE_WORKBENCH="!submarine-workbench,!submarine-workbench/workbench-web,!submarine-workbench/workbench-web-ng"
-
EXCLUDE_INTERPRETER="!submarine-workbench/interpreter,!submarine-workbench/interpreter/interpreter-engine,!submarine-workbench/interpreter/python-interpreter,!submarine-workbench/interpreter/spark-interpreter"
- -
EXCLUDE_SUBMODULE_TONY="!submodules/tony,!submodules/tony/tony-mini,!submodules/tony/tony-core,!submodules/tony/tony-proxy,!submodules/tony/tony-portal,!submodules/tony/tony-azkaban,!submodules/tony/tony-cli"
- EXCLUDE_K8S="!submarine-server/server-submitter/submitter-k8s"
- EXCLUDE_COMMON_RPC="!submarine-commons/commons-rpc"
- MOZ_HEADLESS=1
@@ -176,7 +175,7 @@ matrix:
language: java
jdk: "openjdk8"
dist: xenial
- env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install
-DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl
${EXCLUDE_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH},${EXCLUDE_SUBMODULE_TONY}"
TEST_MODULES="-pl $(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')"
TEST_PROJECTS=""
+ env: PYTHON="3" PROFILE="-Phadoop-2.9" BUILD_FLAG="clean package install
-DskipTests" TEST_FLAG="test -DskipRat -am" MODULES="-pl
${EXCLUDE_K8S},${EXCLUDE_SUBMARINE},${EXCLUDE_WORKBENCH}" TEST_MODULES="-pl
$(echo ${EXCLUDE_INTERPRETER} | sed 's/!//g')" TEST_PROJECTS=""
- name: Test submarine-sdk
language: python
diff --git a/dev-support/docker-images/submarine/build.sh
b/dev-support/docker-images/submarine/build.sh
index 831b3e2..691af90 100755
--- a/dev-support/docker-images/submarine/build.sh
+++ b/dev-support/docker-images/submarine/build.sh
@@ -33,12 +33,6 @@ fi
submarine_dist_exists=$(find -L "${SUBMARINE_HOME}/submarine-dist/target"
-name "submarine-dist-${SUBMARINE_VERSION}*.tar.gz")
# Build source code if the package doesn't exist.
if [[ -z "${submarine_dist_exists}" ]]; then
- # update tony code
- if is_empty_dir "${SUBMARINE_HOME}/submodules/tony" ]; then
- git submodule update --init --recursive
- else
- git submodule update --recursive
- fi
cd "${SUBMARINE_HOME}"
mvn clean package -DskipTests
fi
diff --git a/dev-support/mini-submarine/build_mini-submarine.sh
b/dev-support/mini-submarine/build_mini-submarine.sh
index da0a246..7cc797e 100755
--- a/dev-support/mini-submarine/build_mini-submarine.sh
+++ b/dev-support/mini-submarine/build_mini-submarine.sh
@@ -74,13 +74,6 @@ fi
# Build source code if the package doesn't exist.
if [[ -z "${submarine_dist_exists}" ]]; then
- # update tony code
- if is_empty_dir "${SUBMARINE_PROJECT_PATH}/submodules/tony" ]; then
- git submodule update --init --recursive
- else
- git submodule update --recursive
- fi
-
cd "${SUBMARINE_PROJECT_PATH}"
mvn clean package -DskipTests
fi
diff --git a/docs/development/BuildFromCode.md
b/docs/development/BuildFromCode.md
index 0a7f7c0..309b9f0 100644
--- a/docs/development/BuildFromCode.md
+++ b/docs/development/BuildFromCode.md
@@ -22,23 +22,6 @@
## Quick Start
-### Init submodule
-LinkedIn TensorFlow on YARN (TonY) project is included in the Submarine as a
-submodule. If you just clone Submarine project, please initialize the TonY
-project using the following command.
-
-```
-git submodule update --init --recursive
-```
-
-### Update submodule
-After the initialization, when you want to update tony project, please execute
a
-command like this.
-
-```
-git submodule update --recursive
-```
-
### Building the code
+ Checking releases for licenses
@@ -85,5 +68,5 @@ mvn clean install package -DskipTests -Psrc
## TonY code modification
-If it is needed to make modifications to TonY project, please make sure
-both submarine and tony project are committed.
+If it is needed to make modifications to TonY project, please make a PR
+to [Tony repository](https://github.com/linkedin/TonY).
diff --git a/pom.xml b/pom.xml
index eec7832..7b4ee4e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,7 +65,7 @@
<shaded.dependency.prefix>org.apache.submarine.shaded</shaded.dependency.prefix>
<!--library versions-->
- <tony.version>0.3.21</tony.version>
+ <tony.version>0.3.23</tony.version>
<jersey.version>2.27</jersey.version>
<jersey.servlet.version>1.13</jersey.servlet.version>
<jetty.version>9.4.18.v20190429</jetty.version>
@@ -140,7 +140,6 @@
<module>submarine-client</module>
<!-- There is a problem with the submarine-cloud pom, the compilation is
very slow, and it is temporarily closed
<module>submarine-cloud</module-->
- <module>submodules/tony</module>
<module>submarine-server</module>
<module>submarine-all</module>
<module>submarine-workbench</module>
diff --git a/submarine-server/server-submitter/submitter-yarn/pom.xml
b/submarine-server/server-submitter/submitter-yarn/pom.xml
index fb831dd..296f9fe 100644
--- a/submarine-server/server-submitter/submitter-yarn/pom.xml
+++ b/submarine-server/server-submitter/submitter-yarn/pom.xml
@@ -281,6 +281,32 @@
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>hadoop-3.1</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop-3.2</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<build>
<finalName>submarine-${artifactId}-${project.version}</finalName>
<plugins>
diff --git
a/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java
b/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java
index 41e058b..07a26ed 100644
---
a/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java
+++
b/submarine-server/server-submitter/submitter-yarn/src/main/java/org/apache/submarine/server/submitter/yarn/YarnUtils.java
@@ -39,8 +39,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import static
org.apache.submarine.commons.utils.SubmarineConfiguration.SUBMARINE_RUNTIME_APP_TYPE;
-
/**
* Utilities for YARN Runtime.
*/
@@ -132,7 +130,8 @@ public final class YarnUtils {
envs.stream().map(env -> env.replaceAll("DOCKER_", ""))
.toArray(String[]::new));
}
- tonyConf.setStrings(TonyConfigurationKeys.APPLICATION_TYPE,
SUBMARINE_RUNTIME_APP_TYPE);
+ // Update after SUBMARINE-104 is merged into tony.
+ // tonyConf.setStrings(TonyConfigurationKeys.APPLICATION_TYPE,
SUBMARINE_RUNTIME_APP_TYPE);
// Set up running command
if (parameters.getOptionValue(CliConstants.WORKER_LAUNCH_CMD) != null) {
tonyConf.set(
diff --git
a/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
b/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
index 5366e34..e919e41 100644
---
a/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
+++
b/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
@@ -116,7 +116,8 @@ public class YarnUtilsTest {
Constants.VCORES)));
Assert.assertEquals(tensorFlowParams.getPSLaunchCmd(),
tonyConf.get(TonyConfigurationKeys.getExecuteCommandKey("ps")));
- Assert.assertEquals("SUBMARINE",
tonyConf.get(TonyConfigurationKeys.APPLICATION_TYPE));
+ // Update after SUBMARINE-104 is merged into tony.
+ // Assert.assertEquals("SUBMARINE",
tonyConf.get(TonyConfigurationKeys.APPLICATION_TYPE));
}
@Test
@@ -155,6 +156,7 @@ public class YarnUtilsTest {
.getResourceKey(Constants.WORKER_JOB_NAME, Constants.MEMORY)));
Assert.assertEquals("4", tonyConf.get(TonyConfigurationKeys
.getResourceKey(Constants.WORKER_JOB_NAME, Constants.VCORES)));
- Assert.assertEquals("SUBMARINE",
tonyConf.get(TonyConfigurationKeys.APPLICATION_TYPE));
+ // Update after SUBMARINE-104 is merged into tony.
+ // Assert.assertEquals("SUBMARINE",
tonyConf.get(TonyConfigurationKeys.APPLICATION_TYPE));
}
}
diff --git a/submodules/tony b/submodules/tony
deleted file mode 160000
index 1e0cfbe..0000000
--- a/submodules/tony
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 1e0cfbea6d115a3f1757ca40200df7747790d762
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
