This is an automated email from the ASF dual-hosted git repository.
jinrongtong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
The following commit(s) were added to refs/heads/master by this push:
new 01d41f76 [ISSUES #215] Support docker (#329)
01d41f76 is described below
commit 01d41f76c838f049cac9fb14293cc8429eb23f53
Author: zhoubo <[email protected]>
AuthorDate: Mon Apr 3 15:52:10 2023 +0800
[ISSUES #215] Support docker (#329)
* connect support docker
* connect support docker bug fix
* connect support docker bug fix
* connect support docker
* remove check_java_opts
* connect support docker
* connect support docker
* connect support docker
* connect support docker
* connect support docker
* connect support docker
* connect support docker
* connect support docker
* connect support docker
* add connect support docker doc
---------
Co-authored-by: Fan Lin <[email protected]>
---
distribution/bin/runconnect.sh | 6 ---
distribution/conf/connect-distributed.conf | 8 +--
docker/connect/Dockerfile | 56 +++++++++++++++++++++
docker/connect/README.md | 29 +++++++++++
docker/connect/bin/connect-distributed.sh | 51 +++++++++++++++++++
docker/connect/bin/connect-standalone.sh | 51 +++++++++++++++++++
docker/connect/bin/connectAdmin | 22 ++++++++
docker/connect/bin/connectshutdown.sh | 27 ++++++++++
{distribution => docker/connect}/bin/runconnect.sh | 23 +--------
docker/connect/build_image.sh | 58 ++++++++++++++++++++++
.../src/main/resources/package.xml | 1 +
.../src/main/resources/connect-distributed.conf | 10 ++--
.../src/main/resources/connect-standalone.conf | 10 ++--
.../src/main/resources/package.xml | 1 +
14 files changed, 312 insertions(+), 41 deletions(-)
diff --git a/distribution/bin/runconnect.sh b/distribution/bin/runconnect.sh
index 0c9c5d0d..308182ed 100644
--- a/distribution/bin/runconnect.sh
+++ b/distribution/bin/runconnect.sh
@@ -100,12 +100,6 @@ else
fi
JAVA_OPT="${JAVA_OPT} -DisSyncFlush=false"
-#===========================================================================================
-# Log Configuration for auth module.
-#===========================================================================================
-JAVA_OPT="${JAVA_OPT} -Dons.api.loggingCharset=UTF-8"
-JAVA_OPT="${JAVA_OPT} -Dons.api.loggingLevel=INFO"
-
#===========================================================================================
# SSL Configuration
#===========================================================================================
diff --git a/distribution/conf/connect-distributed.conf
b/distribution/conf/connect-distributed.conf
index 85f1984e..030a7a78 100644
--- a/distribution/conf/connect-distributed.conf
+++ b/distribution/conf/connect-distributed.conf
@@ -14,13 +14,13 @@
# limitations under the License.
workerId=DEFAULT_WORKER_1
-storePathRootDir=/tmp/storeRoot
+storePathRootDir=/home/connect/mq-connect/storeRoot
## Http port for user to access REST API
httpPort=8082
# Rocketmq namesrvAddr
-namesrvAddr=localhost:9876
+# namesrvAddr=localhost:9876
# RocketMQ acl
aclEnable=false
@@ -28,10 +28,10 @@ accessKey=rocketmq
secretKey=12345678
autoCreateGroupEnable=false
-clusterName="DefaultCluster"
+clusterName="cluster1"
# Source or sink connector jar file dir,The default value is
rocketmq-connect-sample
-pluginPaths=rocketmq-connect-sample/target/rocketmq-connect-sample-0.0.1-SNAPSHOT.jar
+pluginPaths=/home/connect/mq-connect/plugins,rocketmq-connect-sample/target/rocketmq-connect-sample-0.0.1-SNAPSHOT.jar
diff --git a/docker/connect/Dockerfile b/docker/connect/Dockerfile
new file mode 100644
index 00000000..3addda32
--- /dev/null
+++ b/docker/connect/Dockerfile
@@ -0,0 +1,56 @@
+FROM centos:7
+
+RUN yum install -y java-1.8.0-openjdk-devel.x86_64 unzip gettext nmap-ncat
openssl, which gnupg, telnet \
+ && yum clean all -y
+
+ARG user=connect
+ARG group=connect
+ARG uid=3000
+ARG gid=3000
+
+# RocketMQ is run with user `rocketmq`, uid = 3000
+# If you bind mount a volume from the host or a data container,
+# ensure you use the same uid
+RUN groupadd -g ${gid} ${group} \
+ && useradd -u ${uid} -g ${gid} -m -s /bin/bash ${user}
+
+ARG version
+
+# RocketMQ Connect Version
+ENV CONNECT_VERSION ${version}
+
+# Rocketmq home
+ENV CONNECT_HOME /home/connect/mq-connect
+
+WORKDIR /home/connect
+
+RUN chown -R ${uid}:${gid} /home/connect
+
+
+# 设置环境变量 USERNAME 默认值为 admin，后面可以通过docker run -e
USERNAME="XXXXXX"修改，这个环境变量在容器里也可以$USERNAME获取
+ENV NAMESRV_ADDR=localhost:9876
+# expose namesrv port
+EXPOSE 8082
+
+USER ${user}
+#WORKDIR /home/connect
+
+RUN mkdir -p /home/connect/mq-connect/bin /home/connect/mq-connect/plugins
/home/connect/mq-connect/target /home/connect/mq-connect/cli && \
+ chown -R connect:connect /home/connect/ && \
+ echo "export JAVA_HOME=/lib/jvm/java-1.8.0-openjdk" >> ~/.bash_profile && \
+ echo "export PATH=$PATH:/lib/jvm/java-1.8.0-openjdk/bin" >>
~/.bash_profile && \
+ echo "export CLASSPATH=.:/lib/jvm/java-1.8.0-openjdk/lib" >>
~/.bash_profile && \
+ source ~/.bash_profile
+
+COPY ./plugins/*.jar /home/connect/mq-connect/plugins/
+ADD runtime.tar.gz /home/connect/mq-connect/
+ADD connect-cli.tar.gz /home/connect/mq-connect/cli/
+
+COPY bin/connect-distributed.sh \
+ bin/connect-standalone.sh \
+ bin/connectshutdown.sh \
+ bin/runconnect.sh \
+ bin/connectAdmin \
+ /home/connect/mq-connect/bin/
+
+ENTRYPOINT ["/usr/bin/sh",
"/home/connect/mq-connect/bin/connect-distributed.sh", "-c",
"/home/connect/mq-connect/conf/connect-distributed.conf"]
diff --git a/docker/connect/README.md b/docker/connect/README.md
new file mode 100644
index 00000000..e802358b
--- /dev/null
+++ b/docker/connect/README.md
@@ -0,0 +1,29 @@
+#connect docker start
+##准备
+
+###环境
+1. Linux/Unix/Mac
+2. 64bit JDK 1.8+;
+3. Maven 3.2.x或以上版本;
+4. 安装docker
+
+###启动RocketMQ
+
+RocketMQ镜像下载
+
+https://hub.docker.com/r/apache/rocketmq/tags
+
+RocketMQ镜像启动
+
+RocketMQ镜像 quick start
+
+###构建Connector镜像
+```
+sh build_image.sh
+```
+
+##启动Connector
+
+```
+docker run --name rmqconnect --link rmqnamesrv:namesrv -e
"NAMESRV_ADDR=namesrv:9876" -p 8082:8082 apache/rocketmqconnect:0.0.1-SNAPSHOT
sh /home/connect/mq-connect/bin/connect-distributed.sh -c
/home/connect/mq-connect/conf/connect-distributed.conf
+```
\ No newline at end of file
diff --git a/docker/connect/bin/connect-distributed.sh
b/docker/connect/bin/connect-distributed.sh
new file mode 100644
index 00000000..08ea6103
--- /dev/null
+++ b/docker/connect/bin/connect-distributed.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+#
+#
+
+if [ -z "$CONNECT_HOME" ] ; then
+ ## resolve links - $0 may be a link to maven's home
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+
+ saveddir=`pwd`
+
+ CONNECT_HOME=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ CONNECT_HOME=`cd "$CONNECT_HOME" && pwd`
+
+ cd "$saveddir"
+fi
+
+export CONNECT_HOME
+
+exec sh ${CONNECT_HOME}/bin/runconnect.sh
org.apache.rocketmq.connect.runtime.DistributedConnectStartup $@
diff --git a/docker/connect/bin/connect-standalone.sh
b/docker/connect/bin/connect-standalone.sh
new file mode 100644
index 00000000..b7947bde
--- /dev/null
+++ b/docker/connect/bin/connect-standalone.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+#
+#
+
+if [ -z "$CONNECT_HOME" ] ; then
+ ## resolve links - $0 may be a link to maven's home
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+
+ saveddir=`pwd`
+
+ CONNECT_HOME=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ CONNECT_HOME=`cd "$CONNECT_HOME" && pwd`
+
+ cd "$saveddir"
+fi
+
+export CONNECT_HOME
+
+exec sh ${CONNECT_HOME}/bin/runconnect.sh
org.apache.rocketmq.connect.runtime.StandaloneConnectStartup $@
diff --git a/docker/connect/bin/connectAdmin b/docker/connect/bin/connectAdmin
new file mode 100644
index 00000000..8a382b8a
--- /dev/null
+++ b/docker/connect/bin/connectAdmin
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+
+#cd target/distribution/ && java -cp .:./conf/:./lib/*
org.apache.rocketmq.connect.cli.ConnectAdminStartup -s conf/connect.conf $@
+cd /home/admin/mq-connect/cli && java -cp .:./conf/:./lib/*
org.apache.rocketmq.connect.cli.ConnectAdminStartup -s conf/connect.conf $@
diff --git a/docker/connect/bin/connectshutdown.sh
b/docker/connect/bin/connectshutdown.sh
new file mode 100644
index 00000000..5dfddb7c
--- /dev/null
+++ b/docker/connect/bin/connectshutdown.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+pid=$(ps ax | grep -i 'ConnectStartup' | grep java | grep -v grep | awk
'{print $1}')
+if [ -z "$pid" ]; then
+ echo "No connect running."
+ exit -1
+fi
+
+echo "The connect(${pid}) is running..."
+
+kill ${pid}
+
+echo "Send shutdown request to connect(${pid}) OK"
diff --git a/distribution/bin/runconnect.sh b/docker/connect/bin/runconnect.sh
similarity index 86%
copy from distribution/bin/runconnect.sh
copy to docker/connect/bin/runconnect.sh
index 0c9c5d0d..76b617a6 100644
--- a/distribution/bin/runconnect.sh
+++ b/docker/connect/bin/runconnect.sh
@@ -45,20 +45,6 @@ check_java_version ()
echo $flag
}
-find_java_home()
-{
- case "`uname`" in
- Darwin)
- JAVA_HOME=$(/usr/libexec/java_home)
- ;;
- *)
- JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))
- ;;
- esac
-}
-
-find_java_home
-
[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java
[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/usr/java
[ ! -e "$JAVA_HOME/bin/java" ] && error_exit "Please set the JAVA_HOME
variable in your environment, We need java(x64)!"
@@ -85,7 +71,7 @@ if [[ $(check_java_version "$JAVA" "1.8") == "false" ]]; then
JAVA_OPT="${JAVA_OPT} -XX:PermSize=128m -XX:MaxPermSize=320m"
fi
if [[ $(check_java_version "$JAVA" "9") == "false" ]]; then
- JAVA_OPT="${JAVA_OPT} -Xloggc:/dev/shm/mq_gc_%p.log -XX:+PrintGCDetails
-XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime
-XX:+PrintAdaptiveSizePolicy"
+ JAVA_OPT="${JAVA_OPT} -Xloggc:/dev/shm/mq_gc_%p.log -XX:+PrintGCDetails
-XX:+PrintGCDateStamps -XX:+PrintAdaptiveSizePolicy"
JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5
-XX:GCLogFileSize=30m"
JAVA_OPT="${JAVA_OPT}
-Djava.ext.dirs=${BASE_DIR}/lib:${JAVA_HOME}/jre/lib/ext"
JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}"
@@ -100,12 +86,6 @@ else
fi
JAVA_OPT="${JAVA_OPT} -DisSyncFlush=false"
-#===========================================================================================
-# Log Configuration for auth module.
-#===========================================================================================
-JAVA_OPT="${JAVA_OPT} -Dons.api.loggingCharset=UTF-8"
-JAVA_OPT="${JAVA_OPT} -Dons.api.loggingLevel=INFO"
-
#===========================================================================================
# SSL Configuration
#===========================================================================================
@@ -113,6 +93,7 @@ JAVA_OPT="${JAVA_OPT} -Dtls.server.mode=disabled"
JAVA_OPT="${JAVA_OPT} -Dtls.private.key.encrypted=false"
JAVA_OPT="${JAVA_OPT} -Djdk.tls.rejectClientInitiatedRenegotiation=true"
+
if [[ $(check_java_version "$JAVA" "1.7") == "false" ]]; then
error_exit "Java version is too low, we need java(x64) 1.7+!"
fi
diff --git a/docker/connect/build_image.sh b/docker/connect/build_image.sh
new file mode 100644
index 00000000..b36e3d8f
--- /dev/null
+++ b/docker/connect/build_image.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+cd ../..
+cd `dirname $0`
+
+WORK_DIR=`pwd`
+
+mvn clean package -DskipTests -Dmaven.test.skip=true -U
+
+if test -f ${WORK_DIR}/docker/connect/distribution.tar.gz; then
+ rm -f ${WORK_DIR}/docker/connect/distribution.tar.gz
+fi
+
+if test -f ${WORK_DIR}/docker/connect/distribution.tar.gz; then
+ rm -f ${WORK_DIR}/docker/connect/distribution.tar.gz
+fi
+
+if test -f
${WORK_DIR}/docker/connect/rocketmq-connect-sample-0.0.1-SNAPSHOT.jar; then
+ rm -f ${WORK_DIR}/docker/connect/rocketmq-connect-sample-0.0.1-SNAPSHOT.jar
+fi
+
+cd ${WORK_DIR}/rocketmq-connect-runtime/target/
+
+mv distribution.tar.gz runtime.tar.gz
+cp runtime.tar.gz ${WORK_DIR}/docker/connect/
+
+cd ${WORK_DIR}/rocketmq-connect-cli/target
+mv distribution.tar.gz connect-cli.tar.gz
+cp connect-cli.tar.gz ${WORK_DIR}/docker/connect/
+
+mkdir -p ${WORK_DIR}/docker/connect/plugins/
+
+cd ${WORK_DIR}/rocketmq-connect-sample/target
+
+cp rocketmq-connect-sample-0.0.1-SNAPSHOT.jar
${WORK_DIR}/docker/connect/plugins/
+
+CONNECT_VERSION=0.0.1-SNAPSHOT
+
+cd ${WORK_DIR}/docker/connect
+docker build --no-cache -f Dockerfile -t
apache/rocketmqconnect:${CONNECT_VERSION} .
+
diff --git a/rocketmq-connect-cli/src/main/resources/package.xml
b/rocketmq-connect-cli/src/main/resources/package.xml
index 91fce854..449f5e05 100644
--- a/rocketmq-connect-cli/src/main/resources/package.xml
+++ b/rocketmq-connect-cli/src/main/resources/package.xml
@@ -22,6 +22,7 @@
<id>package</id>
<formats>
<format>dir</format>
+ <format>tar.gz</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
diff --git
a/rocketmq-connect-runtime/src/main/resources/connect-distributed.conf
b/rocketmq-connect-runtime/src/main/resources/connect-distributed.conf
index 5b259c00..e86b42af 100644
--- a/rocketmq-connect-runtime/src/main/resources/connect-distributed.conf
+++ b/rocketmq-connect-runtime/src/main/resources/connect-distributed.conf
@@ -14,13 +14,13 @@
# limitations under the License.
workerId=DEFAULT_WORKER_1
-storePathRootDir=/xxx/storeRoot
+storePathRootDir=/home/connect/mq-connect/storeRoot
## Http port for user to access REST API
httpPort=8082
-# Rocketmq namesrvAddr
-namesrvAddr=localhost:9876
+# Rocketmq namesrvAddr use NAMESRV_ADDR=localhost:9876 by default
+#namesrvAddr=localhost:9876
# RocketMQ acl
aclEnable=false
@@ -28,10 +28,10 @@ accessKey=rocketmq
secretKey=12345678
autoCreateGroupEnable=false
-clusterName="DefaultCluster"
+clusterName="cluster1"
# Source or sink connector jar file dir,The default value is
rocketmq-connect-sample
-pluginPaths=/xxx/connector-plugins
+pluginPaths=/home/connect/mq-connect/plugins
# metrics config path
metricsConfigPath=/xxx/metrics.conf
diff --git
a/rocketmq-connect-runtime/src/main/resources/connect-standalone.conf
b/rocketmq-connect-runtime/src/main/resources/connect-standalone.conf
index 8879ed12..e2df207c 100644
--- a/rocketmq-connect-runtime/src/main/resources/connect-standalone.conf
+++ b/rocketmq-connect-runtime/src/main/resources/connect-standalone.conf
@@ -14,13 +14,13 @@
# limitations under the License.
workerId=standalone-worker
-storePathRootDir=/tmp/storeRoot
+storePathRootDir=/home/connect/mq-connect/storeRoot
## Http port for user to access REST API
httpPort=8082
-# Rocketmq namesrvAddr
-namesrvAddr=localhost:9876
+# Rocketmq namesrvAddr use NAMESRV_ADDR=localhost:9876 by default
+#namesrvAddr=localhost:9876
# RocketMQ acl
aclEnable=false
@@ -28,10 +28,10 @@ accessKey=rocketmq
secretKey=12345678
autoCreateGroupEnable=false
-clusterName="DefaultCluster"
+clusterName="cluster1"
# Source or sink connector jar file dir,The default value is
rocketmq-connect-sample
-pluginPaths=/xxx/connector-plugins
+pluginPaths=/home/connect/mq-connect/plugins
# metrics config path
metricsConfigPath=/xxx/metrics.conf
diff --git a/rocketmq-connect-runtime/src/main/resources/package.xml
b/rocketmq-connect-runtime/src/main/resources/package.xml
index 0993bbde..fee0ad82 100644
--- a/rocketmq-connect-runtime/src/main/resources/package.xml
+++ b/rocketmq-connect-runtime/src/main/resources/package.xml
@@ -22,6 +22,7 @@
<id>package</id>
<formats>
<format>dir</format>
+ <format>tar.gz</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
