This is an automated email from the ASF dual-hosted git repository. vgalaxies pushed a commit to branch intro-pd-fin in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git
commit 5ec18f81ec652c28b9720a917fb26c4d6a0f9092 Author: VGalaxies <[email protected]> AuthorDate: Thu Apr 25 10:54:40 2024 +0800 git add hugegraph-pd/hg-pd-dist/ --- hugegraph-pd/hg-pd-dist/pom.xml | 151 +++++++++ .../src/assembly/descriptor/server-assembly.xml | 57 ++++ .../src/assembly/static/bin/start-hugegraph-pd.sh | 169 ++++++++++ .../src/assembly/static/bin/stop-hugegraph-pd.sh | 48 +++ .../hg-pd-dist/src/assembly/static/bin/util.sh | 372 +++++++++++++++++++++ .../src/assembly/static/conf/application.yml | 78 +++++ .../assembly/static/conf/application.yml.template | 71 ++++ .../src/assembly/static/conf/hugegraph.license | Bin 0 -> 856 bytes .../hg-pd-dist/src/assembly/static/conf/log4j2.xml | 135 ++++++++ .../src/assembly/static/conf/verify-license.json | 6 + 10 files changed, 1087 insertions(+) diff --git a/hugegraph-pd/hg-pd-dist/pom.xml b/hugegraph-pd/hg-pd-dist/pom.xml new file mode 100644 index 000000000..719fc11c4 --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/pom.xml @@ -0,0 +1,151 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns="http://maven.apache.org/POM/4.0.0"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>hugegraph-pd</artifactId> + <groupId>org.apache.hugegraph</groupId> + <version>${revision}</version> + <relativePath>../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>hg-pd-dist</artifactId> + + <properties> + <dist.dir>${project.parent.basedir}/dist</dist.dir> + <shell-executable>bash</shell-executable> + <assembly.dir>${project.basedir}/src/assembly</assembly.dir> + <assembly.descriptor.dir>${assembly.dir}/descriptor</assembly.descriptor.dir> + <assembly.static.dir>${assembly.dir}/static</assembly.static.dir> + <final.name>hugegraph-pd-${project.parent.version}</final.name> + <executable.jar.name>hg-pd-service</executable.jar.name> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.hugegraph</groupId> + <artifactId>hg-pd-service</artifactId> + <version>${revision}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.4</version> + <executions> + <execution> + <id>assembly-hugegraph-pd</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <attach>false</attach> + <appendAssemblyId>false</appendAssemblyId> + <outputDirectory>${dist.dir}</outputDirectory> + <descriptors> + <descriptor> + ${assembly.descriptor.dir}/server-assembly.xml + </descriptor> + </descriptors> + <finalName>${final.name}</finalName> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <configuration> + <filesets> + <fileset> + <directory>${dist.dir}</directory> + </fileset> + </filesets> + </configuration> + <executions> + <execution> + <phase>initialize</phase> + <goals> + <goal>clean</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.3</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <tasks> + <taskdef resource="net/sf/antcontrib/antcontrib.properties" + classpathref="maven.plugin.classpath"/> + <if> + <contains string="${os.name}" substring="Linux"/> + <then> + <echo file="${dist.dir}/dist.sh"> + tar zcvf \ + ${dist.dir}/${final.name}.tar.gz \ + ${final.name} || exit 1 + rm -f ${dist.dir}/dist.sh + echo + echo "HugeGraph dist tar.gz available at: + ${dist.dir}/${final.name}.tar.gz" + echo + </echo> + <exec executable="${shell-executable}" + dir="${dist.dir}" failonerror="true"> + <arg line="./dist.sh"/> + </exec> + </then> + <else> + <!-- WARN: keep msg in one line --> + <echo message="We only tar package in Linux env, you could check pom.xml in pd-dist to modify it"/> + </else> + </if> + </tasks> + </configuration> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>ant-contrib</groupId> + <artifactId>ant-contrib</artifactId> + <version>1.0b3</version> + <exclusions> + <exclusion> + <groupId>ant</groupId> + <artifactId>ant</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + </plugin> + + </plugins> + </build> + +</project> diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml new file mode 100644 index 000000000..c43a86835 --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/descriptor/server-assembly.xml @@ -0,0 +1,57 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<assembly> + <id>distribution</id> + <includeBaseDirectory>false</includeBaseDirectory> + + <formats> + <format>dir</format> + </formats> + + <fileSets> + <fileSet> + <directory>${assembly.static.dir}/bin</directory> + <outputDirectory>bin</outputDirectory> + <includes> + <include>*</include> + </includes> + <fileMode>755</fileMode> + </fileSet> + <fileSet> + <directory>${assembly.static.dir}/conf</directory> + <outputDirectory>conf</outputDirectory> + <includes> + <include>*</include> + </includes> + </fileSet> + </fileSets> + + <dependencySets> + <!-- code jars --> + <dependencySet> + <outputDirectory>/lib</outputDirectory> + <unpack>false</unpack> + <scope>runtime</scope> + <useProjectArtifact>false</useProjectArtifact> + <includes> + <include>org.apache.hugegraph:${executable.jar.name}:jar:*</include> + </includes> + </dependencySet> + </dependencySets> + +</assembly> diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh new file mode 100644 index 000000000..d4264e841 --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/start-hugegraph-pd.sh @@ -0,0 +1,169 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if [ -z "$GC_OPTION" ];then + GC_OPTION="" +fi +if [ -z "$USER_OPTION" ];then + USER_OPTION="" +fi +if [ -z "$OPEN_TELEMETRY" ];then + OPEN_TELEMETRY="false" +fi + +while getopts "g:j:y:" arg; do + case ${arg} in + g) GC_OPTION="$OPTARG" ;; + j) USER_OPTION="$OPTARG" ;; + # Telemetry is used to collect metrics, traces and logs + y) OPEN_TELEMETRY="$OPTARG" ;; + ?) echo "USAGE: $0 [-g g1] [-j xxx] [-y true|false]" && exit 1 ;; + esac +done + +function abs_path() { + SOURCE="${BASH_SOURCE[0]}" + while [ -h "$SOURCE" ]; do + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + echo "$( cd -P "$( dirname "$SOURCE" )" && pwd )" +} + +BIN=$(abs_path) +TOP="$(cd "$BIN"/../ && pwd)" +CONF="$TOP/conf" +LIB="$TOP/lib" +PLUGINS="$TOP/plugins" +LOGS="$TOP/logs" +OUTPUT=${LOGS}/hugegraph-pd-stdout.log +GITHUB="https://github.com"; +PID_FILE="$BIN/pid" + +. "$BIN"/util.sh + +ensure_path_writable "$LOGS" +ensure_path_writable "$PLUGINS" + +# The maximum and minium heap memory that service can use +MAX_MEM=$((32 * 1024)) +MIN_MEM=$((1 * 512)) +EXPECT_JDK_VERSION=11 + +# Change to $BIN's parent +cd "${TOP}" || exit + +# Find Java +if [ "$JAVA_HOME" = "" ]; then + JAVA="java" +else + JAVA="$JAVA_HOME/bin/java" +fi + +# check jdk version +JAVA_VERSION=$($JAVA -version 2>&1 | awk 'NR==1{gsub(/"/,""); print $3}' | awk -F'_' '{print $1}') +if [[ $? -ne 0 || $JAVA_VERSION < $EXPECT_JDK_VERSION ]]; then + echo "Please make sure that the JDK is installed and the version >= $EXPECT_JDK_VERSION" >> ${OUTPUT} + exit 1 +fi + +# Set Java options +if [ "$JAVA_OPTIONS" = "" ]; then + XMX=$(calc_xmx $MIN_MEM $MAX_MEM) + if [ $? -ne 0 ]; then + echo "Failed to start HugeGraphPDServer, requires at least ${MIN_MEM}m free memory" \ + >> ${OUTPUT} + exit 1 + fi + JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}" + + # Rolling out detailed GC logs + #JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseGCLogFileRotation -XX:GCLogFileSize=10M -XX:NumberOfGCLogFiles=3 \ + # -Xloggc:./logs/gc.log -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +fi + +# Using G1GC as the default garbage collector (Recommended for large memory machines) +case "$GC_OPTION" in + g1) + echo "Using G1GC as the default garbage collector" + JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseG1GC -XX:+ParallelRefProcEnabled \ + -XX:InitiatingHeapOccupancyPercent=50 -XX:G1RSetUpdatingPauseTimePercent=5" + ;; + "") ;; + *) + echo "Unrecognized gc option: '$GC_OPTION', only support 'g1' now" >> ${OUTPUT} + exit 1 +esac + +if [ "${OPEN_TELEMETRY}" == "true" ]; then + OT_JAR="opentelemetry-javaagent.jar" + OT_JAR_PATH="${PLUGINS}/${OT_JAR}" + + if [[ ! -e "${OT_JAR_PATH}" ]]; then + echo "## Downloading ${OT_JAR}..." + download "${PLUGINS}" \ + "${GITHUB}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.1.0/${OT_JAR}" + + if [[ ! -e "${OT_JAR_PATH}" ]]; then + echo "## Error: Failed to download ${OT_JAR}." >>${OUTPUT} + exit 1 + fi + fi + + # Note: remember update it if we change the jar + expected_md5="e3bcbbe8ed9b6d840fa4c333b36f369f" + actual_md5=$(md5sum "${OT_JAR_PATH}" | awk '{print $1}') + + if [[ "${expected_md5}" != "${actual_md5}" ]]; then + echo "## Error: MD5 checksum verification failed for ${OT_JAR_PATH}." >>${OUTPUT} + echo "## Tips: Remove the file and try again." >>${OUTPUT} + exit 1 + fi + + # Note: check carefully if multi "javeagent" params are set + export JAVA_TOOL_OPTIONS="-javaagent:${PLUGINS}/${OT_JAR}" + export OTEL_TRACES_EXPORTER=otlp + export OTEL_METRICS_EXPORTER=none + export OTEL_LOGS_EXPORTER=none + export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc + # 127.0.0.1:4317 is the port of otel-collector running in Docker located in + # 'hugegraph-server/hugegraph-dist/docker/example/docker-compose-trace.yaml'. + # Make sure the otel-collector is running before starting HugeGraphPD. + export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4317 + export OTEL_RESOURCE_ATTRIBUTES=service.name=pd +fi + +#if [ "${JMX_EXPORT_PORT}" != "" ] && [ ${JMX_EXPORT_PORT} -ne 0 ] ; then +# JAVA_OPTIONS="${JAVA_OPTIONS} -javaagent:${LIB}/jmx_prometheus_javaagent-0.16.1.jar=${JMX_EXPORT_PORT}:${CONF}/jmx_exporter.yml" +#fi +if [ $(ps -ef|grep -v grep| grep java|grep -cE ${CONF}) -ne 0 ]; then + echo "HugeGraphPDServer is already running..." + exit 0 +fi +echo "Starting HugeGraphPDServer..." + +# Turn on security check +exec ${JAVA} -Dname="HugeGraphPD" ${JAVA_OPTIONS} -jar \ + -Dspring.config.location=${CONF}/application.yml ${LIB}/hg-pd-service-*.jar >> ${OUTPUT} 2>&1 & + +PID="$!" +# Write pid to file +echo "$PID" > "$PID_FILE" +echo "[+pid] $PID" diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/stop-hugegraph-pd.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/stop-hugegraph-pd.sh new file mode 100644 index 000000000..2df4942d3 --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/stop-hugegraph-pd.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +abs_path() { + SOURCE="${BASH_SOURCE[0]}" + while [ -h "$SOURCE" ]; do + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + done + echo "$( cd -P "$( dirname "$SOURCE" )" && pwd )" +} + +BIN=$(abs_path) +TOP="$(cd $BIN/../ && pwd)" + +. "$BIN"/util.sh + +PID_FILE=$BIN/pid +SERVER_SHUTDOWN_TIMEOUT_S=30 + +if [ ! -f ${PID_FILE} ]; then + echo "The pid file $PID_FILE doesn't exist" + exit 0 +fi + +PID=`cat $PID_FILE` +kill_process_and_wait "HugeGraphPDServer" "$PID" "$SERVER_SHUTDOWN_TIMEOUT_S" + +if [ $? -eq 0 ]; then + rm "$PID_FILE" +fi diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh new file mode 100644 index 000000000..807da92fa --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/bin/util.sh @@ -0,0 +1,372 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function command_available() { + local cmd=$1 + if [ $(command -v $cmd >/dev/null 2>&1) ]; then + return 1 + else + return 0 + fi +} + +# read a property from .properties file +function read_property() { + # file path + file_name=$1 + # replace "." to "\." + property_name=`echo $2 | sed 's/\./\\\./g'` + cat $file_name | sed -n -e "s/^[ ]*//g;/^#/d;s/^$property_name=//p" | tail -1 +} + +function write_property() { + local file=$1 + local key=$2 + local value=$3 + + local os=`uname` + case $os in + # Note: in mac os should use sed -i '' "xxx" to replace string, + # otherwise prompt 'command c expects \ followed by text'. + # See http://www.cnblogs.com/greedy-day/p/5952899.html + Darwin) sed -i '' "s!$key=.*!$key=$value!g" "$file" ;; + *) sed -i "s!$key=.*!$key=$value!g" "$file" ;; + esac +} + +function parse_yaml() { + local file=$1 + local version=$2 + local module=$3 + + cat $file | tr -d '\n {}'| awk -F',+|:' '''{ + pre=""; + for(i=1; i<=NF; ) { + if(match($i, /version/)) { + pre=$i; + i+=1 + } else { + result[pre"-"$i] = $(i+1); + i+=2 + } + } + } END {for(e in result) {print e": "result[e]}}''' \ + | grep "$version-$module" | awk -F':' '{print $2}' | tr -d ' ' && echo +} + +function process_num() { + num=`ps -ef | grep $1 | grep -v grep | wc -l` + return $num +} + +function process_id() { + pid=`ps -ef | grep $1 | grep -v grep | awk '{print $2}'` + return $pid +} + +# check the port of rest server is occupied +function check_port() { + local port=`echo $1 | awk -F':' '{print $3}'` + if ! command_available "lsof"; then + echo "Required lsof but it is unavailable" + exit 1 + fi + lsof -i :$port >/dev/null + if [ $? -eq 0 ]; then + echo "The port $port has already been used" + exit 1 + fi +} + +function crontab_append() { + local job="$1" + crontab -l | grep -F "$job" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + return 1 + fi + (crontab -l ; echo "$job") | crontab - +} + +function crontab_remove() { + local job="$1" + # check exist before remove + crontab -l | grep -F "$job" >/dev/null 2>&1 + if [ $? -eq 1 ]; then + return 0 + fi + + crontab -l | grep -Fv "$job" | crontab - + + # Check exist after remove + crontab -l | grep -F "$job" >/dev/null 2>&1 + if [ $? -eq 0 ]; then + return 1 + else + return 0 + fi +} + +# wait_for_startup friendly_name host port timeout_s +function wait_for_startup() { + local pid="$1" + local server_name="$2" + local server_url="$3" + local timeout_s="$4" + + local now_s=`date '+%s'` + local stop_s=$(( $now_s + $timeout_s )) + + local status + + echo -n "Connecting to $server_name ($server_url)" + while [ $now_s -le $stop_s ]; do + echo -n . + process_status "$server_name" "$pid" >/dev/null + if [ $? -eq 1 ]; then + echo "Starting $server_name failed" + return 1 + fi + + status=`curl -o /dev/null -s -k -w %{http_code} $server_url` + if [[ $status -eq 200 || $status -eq 401 ]]; then + echo "OK" + echo "Started [pid $pid]" + return 0 + fi + sleep 2 + now_s=`date '+%s'` + done + + echo "The operation timed out when attempting to connect to $server_url" >&2 + return 1 +} + +function free_memory() { + local free="" + local os=`uname` + if [ "$os" == "Linux" ]; then + local mem_free=`cat /proc/meminfo | grep -w "MemFree" | awk '{print $2}'` + local mem_buffer=`cat /proc/meminfo | grep -w "Buffers" | awk '{print $2}'` + local mem_cached=`cat /proc/meminfo | grep -w "Cached" | awk '{print $2}'` + if [[ "$mem_free" == "" || "$mem_buffer" == "" || "$mem_cached" == "" ]]; then + echo "Failed to get free memory" + exit 1 + fi + free=`expr $mem_free + $mem_buffer + $mem_cached` + free=`expr $free / 1024` + elif [ "$os" == "Darwin" ]; then + local pages_free=`vm_stat | awk '/Pages free/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` + local pages_inactive=`vm_stat | awk '/Pages inactive/{print $0}' | awk -F'[:.]+' '{print $2}' | tr -d " "` + local pages_available=`expr $pages_free + $pages_inactive` + free=`expr $pages_available \* 4096 / 1024 / 1024` + else + echo "Unsupported operating system $os" + exit 1 + fi + echo $free +} + +function calc_xmx() { + local min_mem=$1 + local max_mem=$2 + # Get machine available memory + local free=`free_memory` + local half_free=$[free/2] + + local xmx=$min_mem + if [[ "$free" -lt "$min_mem" ]]; then + exit 1 + elif [[ "$half_free" -ge "$max_mem" ]]; then + xmx=$max_mem + elif [[ "$half_free" -lt "$min_mem" ]]; then + xmx=$min_mem + else + xmx=$half_free + fi + echo $xmx +} + +function remove_with_prompt() { + local path=$1 + local tips="" + + if [ -d "$path" ]; then + tips="Remove directory '$path' and all sub files [y/n]?" + elif [ -f "$path" ]; then + tips="Remove file '$path' [y/n]?" + else + return 0 + fi + + read -p "$tips " yn + case $yn in + [Yy]* ) rm -rf "$path";; + * ) ;; + esac +} + +function ensure_path_writable() { + local path=$1 + # Ensure input path exist + if [ ! -d "${path}" ]; then + mkdir -p ${path} + fi + # Check for write permission + if [ ! -w "${path}" ]; then + echo "No write permission on directory ${path}" + exit 1 + fi +} + +function get_ip() { + local os=`uname` + local loopback="127.0.0.1" + local ip="" + case $os in + Linux) + if command_available "ifconfig"; then + ip=`ifconfig | grep 'inet addr:' | grep -v "$loopback" | cut -d: -f2 | awk '{ print $1}'` + elif command_available "ip"; then + ip=`ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | awk -F"/" '{print $1}'` + else + ip=$loopback + fi + ;; + FreeBSD|OpenBSD|Darwin) + if command_available "ifconfig"; then + ip=`ifconfig | grep -E 'inet.[0-9]' | grep -v "$loopback" | awk '{ print $2}'` + else + ip=$loopback + fi + ;; + SunOS) + if command_available "ifconfig"; then + ip=`ifconfig -a | grep inet | grep -v "$loopback" | awk '{ print $2} '` + else + ip=$loopback + fi + ;; + *) ip=$loopback;; + esac + echo $ip +} + +function download() { + local path=$1 + local link_url=$2 + + if command_available "wget"; then + wget --help | grep -q '\--show-progress' && progress_opt="-q --show-progress" || progress_opt="" + wget ${link_url} -P ${path} $progress_opt + elif command_available "curl"; then + curl ${link_url} -o ${path}/${link_url} + else + echo "Required wget or curl but they are unavailable" + exit 1 + fi +} + +function ensure_package_exist() { + local path=$1 + local dir=$2 + local tar=$3 + local link=$4 + + if [ ! -d ${path}/${dir} ]; then + if [ ! -f ${path}/${tar} ]; then + echo "Downloading the compressed package '${tar}'" + download ${path} ${link} + if [ $? -ne 0 ]; then + echo "Failed to download, please ensure the network is available and link is valid" + exit 1 + fi + echo "[OK] Finished download" + fi + echo "Unzip the compressed package '$tar'" + tar -zxvf ${path}/${tar} -C ${path} >/dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Failed to unzip, please check the compressed package" + exit 1 + fi + echo "[OK] Finished unzip" + fi +} + +########################################################################### + +function wait_for_shutdown() { + local process_name="$1" + local pid="$2" + local timeout_s="$3" + + local now_s=`date '+%s'` + local stop_s=$(( $now_s + $timeout_s )) + + echo -n "Killing $process_name(pid $pid)" >&2 + while [ $now_s -le $stop_s ]; do + echo -n . + process_status "$process_name" "$pid" >/dev/null + if [ $? -eq 1 ]; then + echo "OK" + return 0 + fi + sleep 2 + now_s=`date '+%s'` + done + echo "$process_name shutdown timeout(exceeded $timeout_s seconds)" >&2 + return 1 +} + +function process_status() { + local process_name="$1" + local pid="$2" + + ps -p "$pid" + if [ $? -eq 0 ]; then + echo "$process_name is running with pid $pid" + return 0 + else + echo "The process $process_name does not exist" + return 1 + fi +} + +function kill_process() { + local process_name="$1" + local pid="$2" + + if [ -z "$pid" ]; then + echo "The process $pid does not exist" + return 0 + fi + + case "`uname`" in + CYGWIN*) taskkill /F /PID "$pid" ;; + *) kill "$pid" ;; + esac +} + +function kill_process_and_wait() { + local process_name="$1" + local pid="$2" + local timeout_s="$3" + + kill_process "$process_name" "$pid" + wait_for_shutdown "$process_name" "$pid" "$timeout_s" +} diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml new file mode 100644 index 000000000..4bc160263 --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: hugegraph-pd + +management: + metrics: + export: + prometheus: + enabled: true + endpoints: + web: + exposure: + include: "*" + +logging: + config: 'file:./conf/log4j2.xml' +license: + verify-path: ./conf/verify-license.json + license-path: ./conf/hugegraph.license +grpc: + port: 8686 + # grpc 的服务地址，部署时需要改为本地实际 IPv4 地址 + host: 127.0.0.1 + +server: + # rest 服务端口号 + port: 8620 + +pd: + # 存储路径 + data-path: ./pd_data + # 自动扩容的检查周期，定时检查每个 store 的分区数量，自动进行分区数量平衡 + patrol-interval: 1800 + # 初始 store 列表，在列表内的 store 自动激活 + initial-store-count: 1 + # grpc IP:grpc port + initial-store-list: 127.0.0.1:8500 + +raft: + # 本机 raft 服务地址 + address: 127.0.0.1:8610 + # pd 集群服务地址 + peers-list: 127.0.0.1:8610 + +store: + # store 下线时间。超过该时间，认为 store 永久不可用，分配副本到其他机器，单位秒 + max-down-time: 172800 + # 是否开启 store 监控数据存储 + monitor_data_enabled: true + # 监控数据的间隔，minute (默认), hour, second + # default: 1 min * 1 day = 1440 + monitor_data_interval: 1 minute + # 监控数据的保留时间 1 天; day, month, year + monitor_data_retention: 1 day + initial-store-count: 1 + +partition: + # 默认每个分区副本数 + default-shard-count: 1 + # 默认每机器最大副本数，初始分区数 = store-max-shard-count * store-number / default-shard-count + store-max-shard-count: 12 diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml.template b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml.template new file mode 100644 index 000000000..43f52df60 --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/application.yml.template @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: hugegraph-pd +logging: + config: file:./conf/log4j2.xml + +management: + metrics: + export: + prometheus: + enabled: true + endpoints: + web: + exposure: + include: "*" + +grpc: + port: $GRPC_PORT$ + # grpc的服务地址, + #注意：部署时需要改为本地实际IPv4地址。 + host: $GRPC_HOST$ + netty-server: + max-inbound-message-size: 100MB + +server: + port : $SERVER_PORT$ + +pd: + # 集群ID，区分不同的PD集群 + + patrol-interval: 2147483647 + data-path: $PD_DATA_PATH$ + +raft: + address: $RAFT_ADDRESS$ + # raft集群 + peers-list: $RAFT_PEERS_LIST$ + # 快照生成时间间隔，单位秒 + snapshotInterval: 300 + metrics: true +store: + # store心跳超时时间，超过该时间，认为store临时不可用，转移Leader到其他副本,单位秒 + keepAlive-timeout: 60 + # store下线时间。超过该时间，认为store永久不可用，分配副本到其他机器，单位秒 + max-down-time: 1800 +partition: + # 默认分区总数 + default-total-count: 30 + # 默认每个分区副本数 + default-shard-count: 3 + +discovery: + #客户端注册后，无心跳最长次数，超过后，之前的注册信息会被删除 + heartbeat-try-count: 3 diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/hugegraph.license b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/hugegraph.license new file mode 100644 index 000000000..bca71af9a Binary files /dev/null and b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/hugegraph.license differ diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/log4j2.xml b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/log4j2.xml new file mode 100644 index 000000000..4e2eee329 --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/log4j2.xml @@ -0,0 +1,135 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!-- Config will be auto loaded every 60s --> +<configuration status="error" monitorInterval="60"> + <properties> + <property name="LOG_PATH">logs</property> + <property name="FILE_NAME">hugegraph-pd</property> + </properties> + + <appenders> + <Console name="console" target="SYSTEM_OUT"> + <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/> + </Console> + + <!-- Normal server log config --> + <RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log" + filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log" + bufferedIO="true" bufferSize="524288" immediateFlush="false"> + <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/> + <!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true"> + <KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/> + </JsonLayout--> + <!-- Trigger after exceeding 1day or 50MB --> + <Policies> + <SizeBasedTriggeringPolicy size="128MB"/> + <TimeBasedTriggeringPolicy interval="1" modulate="true"/> + </Policies> + <!-- Keep 5 files per day & auto delete after over 2GB or 100 files --> + <DefaultRolloverStrategy max="16"> + <Delete basePath="${LOG_PATH}" maxDepth="2"> + <IfFileName glob="*/*.log"/> + <!-- Limit log amount & size --> + <IfAny> + <IfAccumulatedFileSize exceeds="2GB"/> + <IfAccumulatedFileCount exceeds="100"/> + </IfAny> + </Delete> + </DefaultRolloverStrategy> + </RollingRandomAccessFile> + + <!-- jraft server log config --> + <RollingRandomAccessFile name="raft_file" fileName="${LOG_PATH}/${FILE_NAME}_raft.log" + filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log" + bufferedIO="true" bufferSize="524288" immediateFlush="false"> + <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/> + <!--JsonLayout compact="true" eventEol="true" complete="true" locationInfo="true"> + <KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/> + </JsonLayout--> + <!-- Trigger after exceeding 1day or 50MB --> + <Policies> + <SizeBasedTriggeringPolicy size="128MB"/> + <TimeBasedTriggeringPolicy interval="1" modulate="true"/> + </Policies> + <!-- Keep 5 files per day & auto Delete after over 2GB or 100 files --> + <DefaultRolloverStrategy max="16"> + <Delete basePath="${LOG_PATH}" maxDepth="2"> + <IfFileName glob="*/*.log"/> + <!-- Limit log amount & size --> + <IfAny> + <IfAccumulatedFileSize exceeds="2GB"/> + <IfAccumulatedFileCount exceeds="100"/> + </IfAny> + </Delete> + </DefaultRolloverStrategy> + </RollingRandomAccessFile> + + <!-- Separate & compress audit log, buffer size is 512KB --> + <RollingRandomAccessFile name="audit" fileName="${LOG_PATH}/audit-${FILE_NAME}.log" + filePattern="${LOG_PATH}/$${date:yyyy-MM}/audit-${FILE_NAME}-%d{yyyy-MM-dd-HH}-%i.gz" + bufferedIO="true" bufferSize="524288" immediateFlush="false"> + <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/> + <!-- Use simple format for audit log to speed up --> + <!-- PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} - %m%n"/ --> + <JsonLayout compact="true" eventEol="true" locationInfo="true"> + <KeyValuePair key="timestamp" value="$${date:yyyy-MM-dd HH:mm:ss.SSS}"/> + </JsonLayout> + <!-- Trigger after exceeding 1hour or 500MB --> + <Policies> + <SizeBasedTriggeringPolicy size="512MB"/> + <TimeBasedTriggeringPolicy interval="1" modulate="true"/> + </Policies> + <!-- Keep 2 files per hour & auto delete [after 60 days] or [over 5GB or 500 files] --> + <DefaultRolloverStrategy max="16"> + <Delete basePath="${LOG_PATH}" maxDepth="2"> + <IfFileName glob="*/*.gz"/> + <IfLastModified age="60d"/> + <IfAny> + <IfAccumulatedFileSize exceeds="5GB"/> + <IfAccumulatedFileCount exceeds="500"/> + </IfAny> + </Delete> + </DefaultRolloverStrategy> + </RollingRandomAccessFile> + </appenders> + + <loggers> + <root level="INFO"> + <appender-ref ref="file"/> + </root> + <logger name="com.alipay.sofa" level="INFO" additivity="false"> + <appender-ref ref="raft_file"/> + </logger> + <logger name="io.netty" level="INFO" additivity="false"> + <appender-ref ref="file"/> + </logger> + <logger name="org.apache.commons" level="INFO" additivity="false"> + <appender-ref ref="file"/> + </logger> + <!-- Use mixed async way to output logs --> + <logger name="org.apache.hugegraph" level="INFO" additivity="false"> + <appender-ref ref="file"/> + <appender-ref ref="console"/> + </logger> + </loggers> +</configuration> diff --git a/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/verify-license.json b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/verify-license.json new file mode 100644 index 000000000..868ccbebb --- /dev/null +++ b/hugegraph-pd/hg-pd-dist/src/assembly/static/conf/verify-license.json @@ -0,0 +1,6 @@ +{ + "subject": "hugegraph-license", + "public_alias": "publiccert", + "store_ticket": "803b6cc3-d144-47e8-948f-ec8b39c8881e", + "publickey_path": "/public-certs.store" +}
