This is an automated email from the ASF dual-hosted git repository.
jin pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
The following commit(s) were added to refs/heads/master by this push:
new adf6801e [improve][build] Improve package & cli & build image (#200)
adf6801e is described below
commit adf6801e8788dfbf4ac1e3ef12d2b91a7011f6c5
Author: Cong Zhao <[email protected]>
AuthorDate: Fri Nov 25 23:30:39 2022 +0800
[improve][build] Improve package & cli & build image (#200)
* improve package
* Refactor computer: abstract computer-api from computer-core (#196)
* address comment
* improve pom and options
* fix package
* improve build image
* improve start-computer.sh
* improve log file
* improve start-computer.sh
* improve build image
* trigger maven cache invalidation
* Update ci.yml
Co-authored-by: imbajin <[email protected]>
---
.github/workflows/publish_images.yml | 11 +--
computer-algorithm/Dockerfile | 23 -----
computer-algorithm/pom.xml | 1 +
computer-core/pom.xml | 6 --
.../computer/core/util/ComputerContextUtil.java | 3 +-
.../hugegraph/computer/core/util/ShutdownHook.java | 3 -
.../computer/core/worker/WorkerService.java | 4 +-
computer-dist/Dockerfile | 7 +-
computer-dist/pom.xml | 5 ++
.../src/assembly/static/bin/start-computer.sh | 97 ++++++++++++----------
.../src/assembly/static/conf/computer.properties | 16 ++++
.../src/assembly/static/conf}/log4j2.xml | 2 +-
computer-dist/src/assembly/travis/build-images.sh | 13 +--
.../hugegraph/computer/dist/HugeGraphComputer.java | 19 +++--
.../src/main/resources/computer.properties | 0
computer-k8s-operator/Dockerfile | 2 +-
.../operator/controller/ComputerJobDeployer.java | 2 +-
.../baidu/hugegraph/computer/k8s/Constants.java | 2 +-
.../computer/k8s/config/KubeDriverOptions.java | 2 +-
computer-k8s/src/main/resources/docker_push.sh | 2 +-
computer-test/conf/images/docker_push_test.sh | 2 +-
.../computer/dist/HugeGraphComputerTest.java | 6 +-
.../src/main/resources/computer-worker1.properties | 15 ----
...puter-master.properties => computer.properties} | 2 +-
.../src/main/resources/log4j2.xml | 0
pom.xml | 6 +-
26 files changed, 120 insertions(+), 131 deletions(-)
diff --git a/.github/workflows/publish_images.yml
b/.github/workflows/publish_images.yml
index 608c9a74..ba4c02b4 100644
--- a/.github/workflows/publish_images.yml
+++ b/.github/workflows/publish_images.yml
@@ -19,13 +19,11 @@ jobs:
map: |
{
"push": {
- "FRAMEWORK_IMAGE_URL":
"hugegraph/hugegraph-computer-framework:latest",
- "ALGORITHM_IMAGE_URL":
"hugegraph/hugegraph-builtin-algorithms:latest",
+ "COMPUTER_IMAGE_URL": "hugegraph/hugegraph-computer:latest",
"OPERATOR_IMAGE_URL":
"hugegraph/hugegraph-computer-operator:latest"
},
"release": {
- "FRAMEWORK_IMAGE_URL":
"hugegraph/hugegraph-computer-framework:${{ github.ref_name }}",
- "ALGORITHM_IMAGE_URL":
"hugegraph/hugegraph-builtin-algorithms:${{ github.ref_name }}",
+ "COMPUTER_IMAGE_URL": "hugegraph/hugegraph-computer:${{
github.ref_name }}",
"OPERATOR_IMAGE_URL":
"hugegraph/hugegraph-computer-operator:${{ github.ref_name }}"
}
}
@@ -58,13 +56,12 @@ jobs:
- name: Build images
env:
TRAVIS_DIR: computer-dist/src/assembly/travis
- run: $TRAVIS_DIR/build-images.sh ${{ env.FRAMEWORK_IMAGE_URL }} ${{
env.ALGORITHM_IMAGE_URL }} ${{ env.OPERATOR_IMAGE_URL }}
+ run: $TRAVIS_DIR/build-images.sh ${{ env.COMPUTER_IMAGE_URL }} ${{
env.OPERATOR_IMAGE_URL }}
- name: Docker images
run: docker images
- name: Publish
run: |
- docker push ${{ env.FRAMEWORK_IMAGE_URL }}
- docker push ${{ env.ALGORITHM_IMAGE_URL }}
+ docker push ${{ env.COMPUTER_IMAGE_URL }}
docker push ${{ env.OPERATOR_IMAGE_URL }}
diff --git a/computer-algorithm/Dockerfile b/computer-algorithm/Dockerfile
deleted file mode 100644
index 26c04aef..00000000
--- a/computer-algorithm/Dockerfile
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM hugegraph/hugegraph-computer-framework:latest
-
-LABEL maintainer="HugeGraph Docker Maintainers <[email protected]>"
-
-ARG jarFilePath="/opt/jars/hugegraph-builtin-algorithms.jar"
-COPY target/computer-algorithm-*.jar ${jarFilePath}
-ENV JAR_FILE_PATH=${jarFilePath}
diff --git a/computer-algorithm/pom.xml b/computer-algorithm/pom.xml
index 4c641775..b35c7eed 100644
--- a/computer-algorithm/pom.xml
+++ b/computer-algorithm/pom.xml
@@ -32,6 +32,7 @@
<groupId>com.baidu.hugegraph</groupId>
<artifactId>computer-api</artifactId>
<version>${version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.mvel</groupId>
diff --git a/computer-core/pom.xml b/computer-core/pom.xml
index 30f4ae56..cb19ed06 100644
--- a/computer-core/pom.xml
+++ b/computer-core/pom.xml
@@ -67,12 +67,6 @@
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-loader</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>spark-core_2.12</artifactId>
- <groupId>org.apache.spark</groupId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
diff --git
a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java
b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java
index f55af98b..ca16ea7a 100644
---
a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java
+++
b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java
@@ -45,8 +45,7 @@ public class ComputerContextUtil {
algorithmParams = (AlgorithmParams) Class.forName(
algorithmParamsName).newInstance();
} catch (Exception e) {
- throw new ComputerException("Can't create algorithmParams, " +
- "algorithmParamsName = {}",
+ throw new ComputerException("Can't create algorithmParams,
algorithmParamsName = %s",
algorithmParamsName);
}
algorithmParams.setAlgorithmParameters(params);
diff --git
a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java
b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java
index a31f5355..fa11175d 100644
---
a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java
+++
b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java
@@ -21,7 +21,6 @@ package com.baidu.hugegraph.computer.core.util;
import java.io.Closeable;
import org.apache.hugegraph.util.Log;
-import org.apache.logging.log4j.LogManager;
import org.slf4j.Logger;
public class ShutdownHook {
@@ -41,8 +40,6 @@ public class ShutdownHook {
} catch (Throwable e) {
LOG.warn("Failed to execute shutdown hook: {}",
e.getMessage(), e);
- } finally {
- LogManager.shutdown();
}
});
Runtime.getRuntime().addShutdownHook(this.threadShutdownHook);
diff --git
a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
index 515b34df..aba86e31 100644
---
a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
+++
b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
@@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.baidu.hugegraph.computer.core.aggregator.Aggregator;
@@ -56,8 +58,6 @@ import
com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager;
import com.baidu.hugegraph.computer.core.sort.sorting.SortManager;
import com.baidu.hugegraph.computer.core.store.FileManager;
import com.baidu.hugegraph.computer.core.util.ShutdownHook;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
public class WorkerService implements Closeable {
diff --git a/computer-dist/Dockerfile b/computer-dist/Dockerfile
index 6ab4d462..d7649a64 100644
--- a/computer-dist/Dockerfile
+++ b/computer-dist/Dockerfile
@@ -15,9 +15,12 @@
# limitations under the License.
#
FROM openjdk:11-jre
-LABEL maintainer="HugeGraph Docker Maintainers <[email protected]>"
+LABEL maintainer="HugeGraph Docker Maintainers <[email protected]>"
# use ParallelGC which is more friendly to olap system
ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC
-XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
-COPY . /etc/local/hugegraph-computer
+COPY ./bin /etc/local/hugegraph-computer/bin
+COPY ./lib /etc/local/hugegraph-computer/lib
+COPY ./algorithm /etc/local/hugegraph-computer/algorithm
+COPY ./conf /etc/local/hugegraph-computer/conf
WORKDIR /etc/local/hugegraph-computer
RUN apt-get update && apt-get -y install gettext-base && apt-get -y install
wget
diff --git a/computer-dist/pom.xml b/computer-dist/pom.xml
index 61d7ebf4..2e483526 100644
--- a/computer-dist/pom.xml
+++ b/computer-dist/pom.xml
@@ -116,6 +116,11 @@
<configuration>
<tasks>
<echo file="${top.level.dir}/dist.sh">
+ mkdir -p
${top.level.dir}/${final.name}/algorithm/
+ cp
${top.level.dir}/computer-algorithm/target/computer-algorithm-${project.version}.jar
${top.level.dir}/${final.name}/algorithm/builtin-algorithm.jar
+ mkdir -p
${top.level.dir}/${final.name}/k8s-operator/
+ cp
${top.level.dir}/computer-k8s-operator/target/hugegraph-computer-operator-${project.version}.jar
${top.level.dir}/${final.name}/k8s-operator/hugegraph-computer-operator.jar
+
tar -zcvf \
${top.level.dir}/${final.name}.tar.gz \
${final.name} || exit 1
diff --git a/computer-dist/src/assembly/static/bin/start-computer.sh
b/computer-dist/src/assembly/static/bin/start-computer.sh
index dcbff4ad..3f119d5c 100644
--- a/computer-dist/src/assembly/static/bin/start-computer.sh
+++ b/computer-dist/src/assembly/static/bin/start-computer.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,12 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-#!/usr/bin/env bash
-set -e
+
BIN_DIR=$(cd "$(dirname "$0")" && pwd -P)
BASE_DIR=$(cd "${BIN_DIR}/.." && pwd -P)
LIB_DIR=${BASE_DIR}/lib
CONF_DIR="${BASE_DIR}/conf"
+DEFAULT_ALGORITHM_DIR="${BASE_DIR}/algorithm"
COMPUTER_CONF_PATH="${COMPUTER_CONF_PATH}"
LOG4J_CONF_PATH="${LOG4J_CONF_PATH}"
@@ -34,10 +35,12 @@ ROLE_WORKER="worker"
usage() {
echo "Usage:"
- echo " start-computer.sh <-c|--conf conf_file_path>"
- echo " <-a|--algorithm algorithm_jar_path>"
+ echo " start-computer.sh"
+ echo " [-c|--conf conf_file_path>"
+ echo " [-a|--algorithm algorithm_jar_path>"
echo " [-l|--log4 log4_conf_path]"
echo " <-d|--drive drive_type(local|k8s|yarn)>"
+ echo " <-r|--role role(master|worker)>"
}
if [ $# -lt 4 ]; then
@@ -124,23 +127,13 @@ parse_opts() {
parse_opts $*
-echo "************************************"
-echo "COMPUTER_CONF_PATH=${COMPUTER_CONF_PATH}"
-echo "LOG4J_CONF_PATH=${LOG4J_CONF_PATH}"
-echo "JAR_FILE_PATH=${JAR_FILE_PATH}"
-echo "DRIVE=${DRIVE}"
-echo "************************************"
-
-if [ "${JAR_FILE_PATH}" = "" ]; then
- echo "graph algorithm jar file missed";
- usage;
- exit 1;
-fi
-
if [ "${COMPUTER_CONF_PATH}" = "" ]; then
- echo "conf file missed";
- usage;
- exit 1;
+ if [ "$DRIVE" = "$K8S_DRIVE" ]; then
+ echo "conf file missed";
+ usage;
+ exit 1;
+ fi
+ COMPUTER_CONF_PATH=${CONF_DIR}/computer.properties
fi
if [ "${DRIVE}" = "" ]; then
@@ -155,21 +148,37 @@ if [ "${ROLE}" = "" ]; then
exit 1;
fi
+if [ "${LOG4J_CONF_PATH}" = "" ];then
+ LOG4J_CONF_PATH=${CONF_DIR}/log4j2.xml
+fi
+
+echo "************************************"
+echo "COMPUTER_CONF_PATH=${COMPUTER_CONF_PATH}"
+echo "LOG4J_CONF_PATH=${LOG4J_CONF_PATH}"
+echo "ALGORITHM_JAR_FILE_PATH=${JAR_FILE_PATH}"
+echo "DRIVE=${DRIVE}"
+echo "ROLE=${ROLE}"
+echo "************************************"
+
CP=$(find "${LIB_DIR}" -name "*.jar" | tr "\n" ":")
-CP="$JAR_FILE_PATH":${CP}
+CP=${CP}:"${DEFAULT_ALGORITHM_DIR}/*"
+
+if [ "${JAR_FILE_PATH}" != "" ]; then
+ CP=${CP}:${JAR_FILE_PATH}
+fi
# Download remote job JAR file.
-if [[ "${JOB_JAR_URI}" == http://* || "${JOB_JAR_URI}" == https://* ]]; then
+if [[ "${REMOTE_JAR_URI}" == http://* || "${REMOTE_JAR_URI}" == https://* ]];
then
mkdir -p "${BASE_DIR}/job"
- echo "Downloading job JAR ${JOB_JAR_URI} to ${BASE_DIR}/job/"
- wget -nv -P "${BASE_DIR}/job/" "${JOB_JAR_URI}"
+ echo "Downloading job JAR ${REMOTE_JAR_URI} to ${BASE_DIR}/job/"
+ wget -nv -P "${BASE_DIR}/job/" "${REMOTE_JAR_URI}"
JOB_JAR=$(find "${BASE_DIR}/job" -name "*.jar" | tr "\n" ":")
if [[ "$JOB_JAR" != "" ]]; then
CP="${JOB_JAR}"$CP
fi
-elif [[ "${JOB_JAR_URI}" != "" ]]; then
- echo "Unsupported protocol for ${JOB_JAR_URI}"
+elif [[ "${REMOTE_JAR_URI}" != "" ]]; then
+ echo "Unsupported protocol for ${REMOTE_JAR_URI}"
exit 1
fi
@@ -195,26 +204,30 @@ if [ ! -a "${CONF_DIR}" ];then
mkdir -p "${CONF_DIR}"
fi
-COPY_CONF_DIR="${CONF_DIR}/copy"
-if [ ! -a "${COPY_CONF_DIR}" ]; then
- mkdir -p "${COPY_CONF_DIR}"
- chmod 777 "${COPY_CONF_DIR}"
+if [ "$DRIVE" = "$K8S_DRIVE" ]; then
+ COPY_CONF_DIR="${CONF_DIR}/copy"
+ if [ ! -a "${COPY_CONF_DIR}" ]; then
+ mkdir -p "${COPY_CONF_DIR}"
+ chmod 777 "${COPY_CONF_DIR}"
+ fi
+
+ NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename
"${COMPUTER_CONF_PATH}")"
+ envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' <
"${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}"
+ chmod 777 "${NEW_COMPUTER_CONF_PATH}"
+ COMPUTER_CONF_PATH=${NEW_COMPUTER_CONF_PATH}
fi
-NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename "${COMPUTER_CONF_PATH}")"
-envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' <
"${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}"
-chmod 777 "${NEW_COMPUTER_CONF_PATH}"
+LOG4j_CONF=-Dlog4j.configurationFile="${LOG4J_CONF_PATH}"
-if [ "${LOG4J_CONF_PATH}" != "" ];then
- LOG4j_CONF=-Dlog4j.configurationFile="${LOG4J_CONF_PATH}"
+if [ "${ROLE}" = "${ROLE_MASTER}" ]; then
+ LOG_NAME=-Dlog.name=hugegraph-computer-master
+fi
+
+if [ "${ROLE}" = "${ROLE_WORKER}" ]; then
+ LOG_NAME=-Dlog.name=hugegraph-computer-worker
fi
MAIN_CLASS=com.baidu.hugegraph.computer.dist.HugeGraphComputer
-if [ "${LOG4j_CONF}" != "" ]; then
- exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JAVA_OPTS}
${JVM_OPTIONS} \
- -cp "${CP}" ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
-else
- exec ${JAVA} -Dname="hugegraph-computer" ${JAVA_OPTS} ${JVM_OPTIONS} -cp
"${CP}" \
- ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
-fi
+exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${LOG_NAME}
${JAVA_OPTS} ${JVM_OPTIONS} \
+ -cp "${CP}" ${MAIN_CLASS} "${COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
diff --git a/computer-dist/src/assembly/static/conf/computer.properties
b/computer-dist/src/assembly/static/conf/computer.properties
new file mode 100644
index 00000000..1e760932
--- /dev/null
+++ b/computer-dist/src/assembly/static/conf/computer.properties
@@ -0,0 +1,16 @@
+job.id=local_001
+job.workers_count=1
+
+rpc.server_host=localhost
+rpc.server_port=8190
+
+bsp.etcd_endpoints=http://127.0.0.1:2379
+
+hugegraph.url=http://127.0.0.1:8080
+hugegraph.name=hugegraph
+
+algorithm.params_class=com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams
+
+bsp.register_timeout=100000
+bsp.log_interval=30000
+bsp.max_super_step=2
diff --git a/computer-core/src/main/resources/log4j2.xml
b/computer-dist/src/assembly/static/conf/log4j2.xml
similarity index 97%
copy from computer-core/src/main/resources/log4j2.xml
copy to computer-dist/src/assembly/static/conf/log4j2.xml
index 91f2ca8a..250d7f6a 100644
--- a/computer-core/src/main/resources/log4j2.xml
+++ b/computer-dist/src/assembly/static/conf/log4j2.xml
@@ -19,7 +19,7 @@
<configuration status="error" monitorInterval="60">
<properties>
<property name="LOG_PATH">logs</property>
- <property name="FILE_NAME">hugegraph-computer</property>
+ <property
name="FILE_NAME">${sys:log.name:-hugegraph-computer}</property>
</properties>
<appenders>
diff --git a/computer-dist/src/assembly/travis/build-images.sh
b/computer-dist/src/assembly/travis/build-images.sh
index 1dededfe..1cc0d602 100755
--- a/computer-dist/src/assembly/travis/build-images.sh
+++ b/computer-dist/src/assembly/travis/build-images.sh
@@ -20,7 +20,6 @@ set -ev
BASE_DIR=$(cd "$(dirname "$0")" && pwd -P)
PROJECT_PATH="$(cd "${BASE_DIR}/../../../.." && pwd -P)"
PROJECT_POM_PATH="${PROJECT_PATH}/pom.xml"
-JAR_FILE_PATH=/opt/jars/hugegraph-builtin-algorithms.jar
mvn -f "${PROJECT_POM_PATH}" clean package -DskipTests
@@ -29,16 +28,6 @@ CONTEXT_PATH=$(mvn -f "${PROJECT_POM_PATH}" -q -N \
-Dexec.executable='echo' -Dexec.args='${final.name}')
CONTEXT_PATH="${PROJECT_PATH}/${CONTEXT_PATH}"
-PROJECT_VERSION=$(mvn -f "${PROJECT_POM_PATH}" -q -N \
- org.codehaus.mojo:exec-maven-plugin:1.3.1:exec \
- -Dexec.executable='echo' -Dexec.args='${project.version}')
-
docker build -t $1 $CONTEXT_PATH -f $PROJECT_PATH/computer-dist/Dockerfile
-echo "FROM $1
-LABEL maintainer='HugeGraph Docker Maintainers <[email protected]>'
-COPY target/computer-algorithm-*.jar $JAR_FILE_PATH
-ENV JAR_FILE_PATH=$JAR_FILE_PATH" | \
-docker build -t $2 -f - $PROJECT_PATH/computer-algorithm
-
-docker build -t $3 -f $PROJECT_PATH/computer-k8s-operator/Dockerfile
$PROJECT_PATH/computer-k8s-operator
+docker build -t $2 -f $PROJECT_PATH/computer-k8s-operator/Dockerfile
$PROJECT_PATH/computer-k8s-operator
diff --git
a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
index 9b2a60f2..9b53608e 100644
---
a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
+++
b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
@@ -25,6 +25,11 @@ import java.util.Properties;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hugegraph.config.OptionSpace;
+import org.apache.hugegraph.config.RpcOptions;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
+import org.apache.logging.log4j.LogManager;
import org.slf4j.Logger;
import com.baidu.hugegraph.computer.core.common.ComputerContext;
@@ -34,9 +39,6 @@ import com.baidu.hugegraph.computer.core.master.MasterService;
import com.baidu.hugegraph.computer.core.network.message.MessageType;
import com.baidu.hugegraph.computer.core.util.ComputerContextUtil;
import com.baidu.hugegraph.computer.core.worker.WorkerService;
-import org.apache.hugegraph.config.OptionSpace;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
public class HugeGraphComputer {
@@ -56,6 +58,8 @@ public class HugeGraphComputer {
public static void main(String[] args) throws IOException,
ClassNotFoundException {
+ Runtime.getRuntime().addShutdownHook(new Thread(LogManager::shutdown));
+
E.checkArgument(ArrayUtils.getLength(args) == 3,
"Argument count must be three, " +
"the first is conf path;" +
@@ -69,7 +73,7 @@ public class HugeGraphComputer {
setUncaughtExceptionHandler();
loadClass();
registerOptions();
- ComputerContext context = parseContext(args[0]);
+ ComputerContext context = parseContext(args[0], role);
switch (role) {
case ROLE_MASTER:
executeMasterService(context);
@@ -124,12 +128,17 @@ public class HugeGraphComputer {
}
}
- private static ComputerContext parseContext(String conf)
+ private static ComputerContext parseContext(String conf, String role)
throws IOException {
Properties properties = new Properties();
BufferedReader bufferedReader = new BufferedReader(
new FileReader(conf));
properties.load(bufferedReader);
+ properties.remove(RpcOptions.RPC_REMOTE_URL.name());
+ if (ROLE_WORKER.equals(role)) {
+ properties.remove(RpcOptions.RPC_SERVER_HOST.name());
+ properties.remove(RpcOptions.RPC_SERVER_PORT.name());
+ }
ComputerContextUtil.initContext(properties);
return ComputerContext.instance();
}
diff --git a/computer-dist/src/main/resources/computer.properties
b/computer-dist/src/main/resources/computer.properties
deleted file mode 100644
index e69de29b..00000000
diff --git a/computer-k8s-operator/Dockerfile b/computer-k8s-operator/Dockerfile
index 37172ce2..93b0fad6 100644
--- a/computer-k8s-operator/Dockerfile
+++ b/computer-k8s-operator/Dockerfile
@@ -15,7 +15,7 @@
# limitations under the License.
#
FROM openjdk:11-jre
-LABEL maintainer="HugeGraph Docker Maintainers <[email protected]>"
+LABEL maintainer="HugeGraph Docker Maintainers <[email protected]>"
WORKDIR /opt/app
COPY target/hugegraph-computer-operator-*.jar hugegraph-computer-operator.jar
ENTRYPOINT ["java", "-jar", "hugegraph-computer-operator.jar"]
diff --git
a/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
b/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
index c465660f..2251ceb7 100644
---
a/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
+++
b/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java
@@ -473,7 +473,7 @@ public class ComputerJobDeployer {
String remoteJarUri = spec.getRemoteJarUri();
if (StringUtils.isNotBlank(remoteJarUri)) {
EnvVar jobJarURI = new EnvVarBuilder()
- .withName(Constants.ENV_JOB_JAR_URI)
+ .withName(Constants.ENV_REMOTE_JAR_URI)
.withValue(remoteJarUri)
.build();
envVars.add(jobJarURI);
diff --git
a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java
b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java
index 279ed8c0..7467b020 100644
--- a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java
+++ b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java
@@ -40,7 +40,7 @@ public class Constants {
public static final String ENV_COMPUTER_CONF_PATH = "COMPUTER_CONF_PATH";
public static final String ENV_LOG4J_CONF_PATH = "LOG4J_CONF_PATH";
public static final String ENV_JAR_FILE_PATH = "JAR_FILE_PATH";
- public static final String ENV_JOB_JAR_URI = "JOB_JAR_URI";
+ public static final String ENV_REMOTE_JAR_URI = "REMOTE_JAR_URI";
public static final String ENV_JVM_OPTIONS = "JVM_OPTIONS";
public static final String ENV_CPU_LIMIT = "CPU_LIMIT";
public static final String ENV_MEMORY_LIMIT = "MEMORY_LIMIT";
diff --git
a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java
b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java
index a5c2d588..6bfd7c6e 100644
---
a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java
+++
b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java
@@ -67,7 +67,7 @@ public class KubeDriverOptions extends OptionHolder {
"k8s.framework_image_url",
"The image url of computer framework.",
disallowEmpty(),
- "hugegraph/hugegraph-computer-framework:latest"
+ "hugegraph/hugegraph-computer:latest"
);
public static final ConfigOption<String> BUILD_IMAGE_BASH_PATH =
diff --git a/computer-k8s/src/main/resources/docker_push.sh
b/computer-k8s/src/main/resources/docker_push.sh
index 48163a18..dbe3e6eb 100755
--- a/computer-k8s/src/main/resources/docker_push.sh
+++ b/computer-k8s/src/main/resources/docker_push.sh
@@ -29,7 +29,7 @@ function build_image() {
JAR_FILE=""
IMG_URL=""
FRAMEWORK_IMG_URL=""
- MAINTAINER="HugeGraph Docker Maintainers <[email protected]>"
+ MAINTAINER="HugeGraph Docker Maintainers <[email protected]>"
while getopts "r:u:p:s:j:i:f:" arg; do
case ${arg} in
diff --git a/computer-test/conf/images/docker_push_test.sh
b/computer-test/conf/images/docker_push_test.sh
index 46078ea1..5c45e801 100644
--- a/computer-test/conf/images/docker_push_test.sh
+++ b/computer-test/conf/images/docker_push_test.sh
@@ -29,7 +29,7 @@ function build_image() {
JAR_FILE=""
IMG_URL=""
FRAMEWORK_IMG_URL=""
- MAINTAINER="HugeGraph Docker Maintainers <[email protected]>"
+ MAINTAINER="HugeGraph Docker Maintainers <[email protected]>"
while getopts "r:u:p:s:j:i:f:" arg; do
case ${arg} in
diff --git
a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
index a54d54f9..52e31fba 100644
---
a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
+++
b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
@@ -41,9 +41,9 @@ public class HugeGraphComputerTest extends UnitTestBase {
CountDownLatch countDownLatch = new CountDownLatch(2);
Throwable[] exceptions = new Throwable[2];
String masterConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-master.properties").getPath();
+ "/computer.properties").getPath();
String work1ConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-worker1.properties").getPath();
+ "/computer.properties").getPath();
pool.submit(() -> {
try {
Thread.sleep(2000L);
@@ -79,7 +79,7 @@ public class HugeGraphComputerTest extends UnitTestBase {
@Test
public void testServiceWithError() {
String work1ConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-worker1.properties").getPath();
+ "/computer.properties").getPath();
Assert.assertThrows(IllegalArgumentException.class,
() -> {
String[] args1 = {work1ConfPath, "worker111",
diff --git a/computer-test/src/main/resources/computer-worker1.properties
b/computer-test/src/main/resources/computer-worker1.properties
deleted file mode 100644
index 8852515b..00000000
--- a/computer-test/src/main/resources/computer-worker1.properties
+++ /dev/null
@@ -1,15 +0,0 @@
-job.id=local_002
-job.workers_count=1
-
-transport.server_port=8086
-
-bsp.register_timeout=100000
-bsp.log_interval=30000
-bsp.max_super_step=2
-
-worker.computation_class=com.baidu.hugegraph.computer.core.worker.MockComputation
-
-algorithm.result_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue
-algorithm.message_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue
-algorithm.params_class=com.baidu.hugegraph.computer.core.worker.MockComputationParams
-
diff --git a/computer-test/src/main/resources/computer-master.properties
b/computer-test/src/main/resources/computer.properties
similarity index 95%
rename from computer-test/src/main/resources/computer-master.properties
rename to computer-test/src/main/resources/computer.properties
index 37968263..dd52c3b6 100644
--- a/computer-test/src/main/resources/computer-master.properties
+++ b/computer-test/src/main/resources/computer.properties
@@ -1,5 +1,5 @@
rpc.server_host=localhost
-rpc.server_port=8090
+rpc.server_port=8190
job.id=local_002
job.workers_count=1
diff --git a/computer-core/src/main/resources/log4j2.xml
b/computer-test/src/main/resources/log4j2.xml
similarity index 100%
rename from computer-core/src/main/resources/log4j2.xml
rename to computer-test/src/main/resources/log4j2.xml
diff --git a/pom.xml b/pom.xml
index 7707774e..ed586f0e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<name>hugegraph-computer</name>
<url>https://github.com/apache/hugegraph-computer</url>
<description>
- hugegraph-computer is a fast-speed, highly-scalable, fault-tolerance
graph processing system developed by baidu.
+ hugegraph-computer is a fast-speed, highly-scalable, fault-tolerance
graph processing system developed by apache.
</description>
<inceptionYear>2020</inceptionYear>
@@ -140,6 +140,10 @@
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
