This is an automated email from the ASF dual-hosted git repository.
lvshaokang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new dae35df3a [Improve] start-script support docker start (#2694)
dae35df3a is described below
commit dae35df3a4379850df7fba657bbc064e0b149f04
Author: benjobs <[email protected]>
AuthorDate: Tue Apr 25 11:50:05 2023 +0800
[Improve] start-script support docker start (#2694)
* [Improve] start-script suppord docker start
* Dockerfile bug fixed.
* application.yml/ streampark.sh updated
---
.github/workflows/docker-push.yml | 5 -
deploy/docker/.env | 2 +-
deploy/docker/Dockerfile | 3 -
.../conf/streampark-console-config/application.yml | 26 ++++-
.../src/main/assembly/bin/streampark.sh | 117 +++++++++++++++++----
.../src/main/resources/application.yml | 2 +-
6 files changed, 122 insertions(+), 33 deletions(-)
diff --git a/.github/workflows/docker-push.yml
b/.github/workflows/docker-push.yml
index f21b858a0..1c261be80 100644
--- a/.github/workflows/docker-push.yml
+++ b/.github/workflows/docker-push.yml
@@ -56,11 +56,6 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- - name: init
- run: |
- sed -i -e 's/eval $NOHUP/eval/'
streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
- sed -i -e 's/>> "$APP_OUT" 2>&1 "&"//'
streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
-
- name: Build project release package
run: |
./mvnw clean install -DskipTests -Pscala-2.12,shaded,dist,webapp
diff --git a/deploy/docker/.env b/deploy/docker/.env
index 956e576af..fe9dec2d7 100644
--- a/deploy/docker/.env
+++ b/deploy/docker/.env
@@ -28,6 +28,6 @@ SPRING_PROFILES_ACTIVE=h2 #mysql, pgsql
FLINK=flink1.14.5
FLINK_IMAGE=flink:1.14.5-scala_2.12
-RUN_COMMAND='/bin/sh -c "wget -P lib
https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.31/mysql-connector-j-8.0.31.jar
&& bash bin/startup.sh"'
+RUN_COMMAND='/bin/sh -c "wget -P lib
https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.31/mysql-connector-j-8.0.31.jar
&& bash bin/streampark.sh startDocker "'
JOB_MANAGER_RPC_ADDRESS=flink-jobmanager
diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile
index 7c8767493..6e58347c4 100644
--- a/deploy/docker/Dockerfile
+++ b/deploy/docker/Dockerfile
@@ -28,9 +28,6 @@ COPY --from=deps-stage /streampark /streampark
ENV NODE_VERSION=16.1.0
ENV NPM_VERSION=7.11.2
-RUN sed -i -e 's/eval $NOHUP/eval/' bin/streampark.sh \
- && sed -i -e 's/>> "$APP_OUT" 2>&1 "&"//' bin/streampark.sh
-
RUN apk add openjdk8 \
&& apk add maven \
&& apk add wget \
diff --git
a/deploy/helm/streampark/conf/streampark-console-config/application.yml
b/deploy/helm/streampark/conf/streampark-console-config/application.yml
index 422fd0ae3..309e7e6af 100755
--- a/deploy/helm/streampark/conf/streampark-console-config/application.yml
+++ b/deploy/helm/streampark/conf/streampark-console-config/application.yml
@@ -28,9 +28,6 @@ logging:
level:
root: info
-swagger:
- enable: true
-
knife4j:
enable: true
basic:
@@ -39,6 +36,13 @@ knife4j:
username: admin
password: streampark
+springdoc:
+ api-docs:
+ enabled: true
+ swagger-ui:
+ path: /swagger-ui.html
+ packages-to-scan: org.apache.streampark.console
+
spring:
profiles.active: h2 #[h2,pgsql,mysql]
application.name: StreamPark
@@ -54,10 +58,14 @@ spring:
jackson:
date-format: yyyy-MM-dd HH:mm:ss
time-zone: GMT+8
+ deserialization:
+ fail-on-unknown-properties: false
main:
allow-circular-references: true
banner-mode: off
-
+ mvc:
+ converters:
+ preferred-json-mapper: jackson
management:
endpoints:
@@ -70,13 +78,18 @@ management:
show-details: always
probes:
enabled: true
+ health:
+ ldap:
+ enabled: false
streampark:
proxy:
# knox process address
https://cdpsit02.example.cn:8443/gateway/cdp-proxy/yarn
yarn-url:
+ # lark alert proxy,default https://open.feishu.cn
+ lark-url:
yarn:
- # default sample, or kerberos
+ # default sample, or kerberos
http-auth: sample
# HADOOP_USER_NAME
@@ -93,6 +106,7 @@ streampark:
max-connections: 10000
connection-timeout-sec: 10000
response-timeout-sec: 12000
+ docker-host: ""
# flink-k8s tracking configuration
flink-k8s:
@@ -119,6 +133,8 @@ streampark:
anonUrl: >
ldap:
+ # Is ldap enabled? If so, please modify the urls
+ enable: false
## AD server IP, default port 389
urls: ldap://99.99.99.99:389
## Login Account
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
index 23dbfb1ac..f1c552709 100755
---
a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
+++
b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
@@ -241,6 +241,24 @@ if [ "$USE_NOHUP" = "true" ]; then
NOHUP="nohup"
fi
+
+PARAM_CLI="org.apache.streampark.flink.core.conf.ParameterCli"
+
+APP_MAIN="org.apache.streampark.console.StreamParkConsoleBootstrap"
+
+DEFAULT_OPTS="""
+ -ea
+ -server
+ -Xms1024m
+ -Xmx1024m
+ -Xmn256m
+ -XX:NewSize=100m
+ -XX:+UseConcMarkSweepGC
+ -XX:CMSInitiatingOccupancyFraction=70
+ -XX:ThreadStackSize=512
+ -Xloggc:${APP_HOME}/logs/gc.log
+ """
+
# ----- Execute The Requested Command -----------------------------------------
print_logo() {
@@ -370,35 +388,26 @@ start() {
APP_CLASSPATH+=":${HADOOP_HOME}/etc/hadoop"
fi
- PARAM_CLI="org.apache.streampark.flink.core.conf.ParameterCli"
# shellcheck disable=SC2034
# shellcheck disable=SC2006
vmOption=`$_RUNJAVA -cp "$APP_CLASSPATH" $PARAM_CLI --vmopt`
JAVA_OPTS="""
$vmOption
- -ea
- -server
- -Xms1024m
- -Xmx1024m
- -Xmn256m
- -XX:NewSize=100m
- -XX:+UseConcMarkSweepGC
- -XX:CMSInitiatingOccupancyFraction=70
- -XX:ThreadStackSize=512
- -Xloggc:${APP_HOME}/logs/gc.log
+ $DEFAULT_OPTS
$DEBUG_OPTS
"""
- eval $NOHUP "\"$_RUNJAVA\"" $JAVA_OPTS \
- -classpath "\"$APP_CLASSPATH\"" \
- -Dapp.home="\"${APP_HOME}\"" \
- -Dlogging.config="\"${APP_CONF}\"/logback-spring.xml" \
- -Dspring.config.location="\"${PROPER}\"" \
- -Djava.io.tmpdir="\"$APP_TMPDIR\"" \
- org.apache.streampark.console.StreamParkConsoleBootstrap >> "$APP_OUT"
2>&1 "&"
+ eval $NOHUP $_RUNJAVA $JAVA_OPTS \
+ -classpath "$APP_CLASSPATH" \
+ -Dapp.home="${APP_HOME}" \
+ -Dlogging.config="${APP_CONF}/logback-spring.xml" \
+ -Dspring.config.location="${PROPER}" \
+ -Djava.io.tmpdir="$APP_TMPDIR" \
+ $APP_MAIN >> "$APP_OUT" 2>&1 "&"
local PID=$!
+ local IS_NUMBER="^[0-9]+$"
# Add to pid file if successful start
if [[ ${PID} =~ ${IS_NUMBER} ]] && kill -0 $PID > /dev/null 2>&1 ; then
@@ -410,6 +419,75 @@ start() {
fi
}
+# shellcheck disable=SC2120
+startDocker() {
+ # Bugzilla 37848: only output this if we have a TTY
+ if [[ ${have_tty} -eq 1 ]]; then
+ echo_w "Using APP_BASE: $APP_BASE"
+ echo_w "Using APP_HOME: $APP_HOME"
+ if [[ "$1" = "debug" ]] ; then
+ echo_w "Using JAVA_HOME: $JAVA_HOME"
+ else
+ echo_w "Using JRE_HOME: $JRE_HOME"
+ fi
+ echo_w "Using APP_PID: $APP_PID"
+ fi
+
+ PROPER="${APP_CONF}/application.yml"
+ if [[ ! -f "$PROPER" ]] ; then
+ echo_r "ERROR: config file application.yml invalid or not found! ";
+ exit 1;
+ else
+ echo_g "Usage: config file: $PROPER ";
+ fi
+
+ if [ "${HADOOP_HOME}"x == ""x ]; then
+ echo_y "WARN: HADOOP_HOME is undefined on your system env,please check it."
+ else
+ echo_w "Using HADOOP_HOME: ${HADOOP_HOME}"
+ fi
+
+ # classpath options:
+ # 1): java env (lib and jre/lib)
+ # 2): StreamPark
+ # 3): hadoop conf
+ # shellcheck disable=SC2091
+ APP_CLASSPATH=".:${JAVA_HOME}/lib:${JAVA_HOME}/jre/lib"
+ # shellcheck disable=SC2206
+ # shellcheck disable=SC2010
+ JARS=$(ls "$APP_LIB"/*.jar | grep -v
"$APP_LIB/streampark-flink-shims_.*.jar$")
+ # shellcheck disable=SC2128
+ for jar in $JARS;do
+ APP_CLASSPATH=$APP_CLASSPATH:$jar
+ done
+
+ if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
+ echo_w "Using HADOOP_CONF_DIR: ${HADOOP_CONF_DIR}"
+ APP_CLASSPATH+=":${HADOOP_CONF_DIR}"
+ else
+ APP_CLASSPATH+=":${HADOOP_HOME}/etc/hadoop"
+ fi
+
+ # shellcheck disable=SC2034
+ # shellcheck disable=SC2006
+ vmOption=`$_RUNJAVA -cp "$APP_CLASSPATH" $PARAM_CLI --vmopt`
+
+ JAVA_OPTS="""
+ $vmOption
+ $DEFAULT_OPTS
+ $DEBUG_OPTS
+ """
+
+ $_RUNJAVA $JAVA_OPTS \
+ -classpath "$APP_CLASSPATH" \
+ -Dapp.home="${APP_HOME}" \
+ -Dlogging.config="${APP_CONF}/logback-spring.xml" \
+ -Dspring.config.location="${PROPER}" \
+ -Djava.io.tmpdir="$APP_TMPDIR" \
+ $APP_MAIN
+
+}
+
debug() {
if [ ! -n "$DEBUG_PORT" ]; then
echo_r "If start with debug mode,Please fill in the debug port like: bash
streampark.sh debug 10002 "
@@ -578,6 +656,9 @@ main() {
"start")
start
;;
+ "startDocker")
+ startDocker
+ ;;
"stop")
stop
;;
diff --git
a/streampark-console/streampark-console-service/src/main/resources/application.yml
b/streampark-console/streampark-console-service/src/main/resources/application.yml
index 813c7d4a2..5b832f547 100644
---
a/streampark-console/streampark-console-service/src/main/resources/application.yml
+++
b/streampark-console/streampark-console-service/src/main/resources/application.yml
@@ -96,7 +96,7 @@ streampark:
hadoop-user-name: hdfs
# local workspace, used to store source code and build dir etc.
workspace:
- local:
+ local: /opt/streampark_workspace
remote: hdfs://hdfscluster/streampark # support hdfs:///streampark/ 、
/streampark 、hdfs://host:ip/streampark/
# remote docker register namespace for streampark
