This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch docker/engine in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
commit 45410b266dfe90b861a4293420a5cc597b35c334 Author: kezhenxu94 <[email protected]> AuthorDate: Sat Apr 8 17:38:38 2023 +0800 Add Docker image for engine --- .github/workflows/codeql.yaml | 12 ++----- .github/workflows/docker.yml | 10 ++---- .github/workflows/publish-docker.yaml | 18 ++++------- .github/workflows/schedule_backend.yml | 2 +- .gitignore | 5 +-- Makefile | 59 ++++++++++++++++++++++++++++++++++ bin/install-plugin.sh | 14 ++++---- config/log4j2.properties | 2 +- docker/seatunnel-engine/Dockerfile | 30 +++++++++++++++++ 9 files changed, 113 insertions(+), 39 deletions(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 86a0dc9dc..28fefc0af 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -17,7 +17,7 @@ name: "CodeQL" on: schedule: - - cron: '0 0 12 * * ?' + - cron: '0 0 12 * *' jobs: analyze: @@ -38,17 +38,11 @@ jobs: with: submodules: true - name: Set up JDK 1.8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: java-version: 8 distribution: 'adopt' - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + cache: maven - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b1354cb48..3e4865437 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,17 +29,11 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up JDK 1.8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: java-version: 8 distribution: 'adopt' - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- + cache: maven - name: Build and Package run: | ./mvnw -B package \ diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index e3b53331a..ecc43480b 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -20,7 +20,7 @@ on: push: branches: - dev - - docker + - docker/* paths-ignore: - 'docs/**' - '**/*.md' @@ -39,18 +39,12 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v2 - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - name: Set up JDK 1.8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: java-version: 8 distribution: 'adopt' + cache: 'maven' - name: Log in to the Container registry uses: docker/[email protected] with: @@ -65,12 +59,14 @@ jobs: env: MAVEN_OPTS: -Xmx2G -Xms2G run: | - ./mvnw -B clean deploy \ + ./mvnw -B clean install deploy \ -Dmaven.test.skip \ -Dmaven.javadoc.skip \ -Dlicense.skipAddThirdParty=true \ -Dmaven.deploy.skip \ -Ddocker.tag=${{ github.sha }} \ -Ddocker.hub=${{ env.HUB }} \ - -Pdocker \ + -Pdocker,release \ --no-snapshot-updates + - name: Build and push engine images + run: make clean push.docker diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml index d78d2dc57..bb9bf6e82 100644 --- a/.github/workflows/schedule_backend.yml +++ b/.github/workflows/schedule_backend.yml @@ -18,7 +18,7 @@ name: Schedule Backend on: schedule: - - cron: '0 0 03 * * ?' + - cron: '0 0 03 * *' concurrency: group: backend-${{ github.event.pull_request.number || github.ref }} diff --git a/.gitignore b/.gitignore index 25977068e..6507089ce 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ logs.zip # Intellij Idea files .idea/ *.iml -.idea/* .DS_Store @@ -48,4 +47,6 @@ test.conf spark-warehouse *.flattened-pom.xml -seatunnel-examples \ No newline at end of file +seatunnel-examples + +dist diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..1d3a468c6 --- /dev/null +++ b/Makefile @@ -0,0 +1,59 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SHELL := /bin/bash -o pipefail + +ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) + +HUB ?= apache +IMAGE ?= seatunnel-engine +TAG ?= $(shell git rev-parse HEAD) + +CONTEXT ?= $(ROOT)/dist/docker-build +DIST_TAR ?= $(ROOT)/seatunnel-dist/target/apache-seatunnel-*-bin.tar.gz + +BUILD_ARGS ?= + +docker: PLATFORMS = +docker: LOAD_OR_PUSH = --load +push.docker: PLATFORMS = --platform linux/amd64,linux/arm64 +push.docker: LOAD_OR_PUSH = --push + +$(DIST_TAR): + ./mvnw -B clean install -Dmaven.test.skip -Prelease + +$(CONTEXT)/$(IMAGE): $(ROOT)/docker/seatunnel-engine/* $(DIST_TAR) + mkdir -p $(CONTEXT)/$(IMAGE) + cp -r $< $(CONTEXT)/$(IMAGE) + tar -zxf $(DIST_TAR) --strip-components=1 -C $@ + $@/bin/install-plugin.sh $(shell ./mvnw help:evaluate -q -DforceStdout -D"expression=project.version") + +.PHONY: clean +clean: + rm -rf $(CONTEXT) + docker buildx rm seatunnel_$(IMAGE) > /dev/null 2>&1 || true + +.PHONY: docker push.docker +docker push.docker: $(CONTEXT)/$(IMAGE) + docker buildx create --driver docker-container --name seatunnel_$(IMAGE) > /dev/null 2>&1 || true + docker buildx build $(PLATFORMS) $(LOAD_OR_PUSH) \ + --no-cache $(BUILD_ARGS) \ + --builder seatunnel_$(IMAGE) \ + -t $(HUB)/$(IMAGE):$(TAG) \ + -t $(HUB)/$(IMAGE):latest \ + $(CONTEXT)/$(IMAGE) + docker buildx rm seatunnel_$(IMAGE) || true diff --git a/bin/install-plugin.sh b/bin/install-plugin.sh index 35a4dbec2..d2a11db9f 100644 --- a/bin/install-plugin.sh +++ b/bin/install-plugin.sh @@ -16,7 +16,7 @@ # limitations under the License. # -#This script is used to download the connector plug-ins required during the running process. +#This script is used to download the connector plug-ins required during the running process. #All are downloaded by default. You can also choose what you need. #You only need to configure the plug-in name in config/plugin_config. @@ -32,7 +32,7 @@ fi echo "Install hadoop shade jar, usage version is ${version}" -${SEATUNNEL_HOME}/mvnw dependency:get -DgroupId=org.apache.seatunnel -Dclassifier=optional -DartifactId=seatunnel-hadoop3-3.1.4-uber -Dversion=${version} -Ddest=${SEATUNNEL_HOME}/lib +${SEATUNNEL_HOME}/mvnw dependency:copy -Dartifact=org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:${version}:jar:optional -DoutputDirectory=${SEATUNNEL_HOME}/lib echo "Install SeaTunnel connectors plugins, usage version is ${version}" @@ -40,21 +40,21 @@ echo "Install SeaTunnel connectors plugins, usage version is ${version}" if [ ! -d ${SEATUNNEL_HOME}/connectors ]; then mkdir ${SEATUNNEL_HOME}/connectors - echo "create connectors directory" + echo "Create connectors directory" fi # create the seatunnel connectors directory (for v2) if [ ! -d ${SEATUNNEL_HOME}/connectors/seatunnel ]; then mkdir ${SEATUNNEL_HOME}/connectors/seatunnel - echo "create seatunnel connectors directory" + echo "Create seatunnel connectors directory" fi while read line; do if [ ${line:0:1} != "-" ] && [ ${line:0:1} != "#" ] then - echo "install connector : " $line - ${SEATUNNEL_HOME}/mvnw dependency:get -DgroupId=org.apache.seatunnel -DartifactId=${line} -Dversion=${version} -Ddest=${SEATUNNEL_HOME}/connectors/seatunnel + echo "Install connector : " $line + ${SEATUNNEL_HOME}/mvnw dependency:copy -Dartifact=org.apache.seatunnel:${line}:${version}:jar -DoutputDirectory=${SEATUNNEL_HOME}/connectors/seatunnel fi -done < ${SEATUNNEL_HOME}/config/plugin_config \ No newline at end of file +done < ${SEATUNNEL_HOME}/config/plugin_config diff --git a/config/log4j2.properties b/config/log4j2.properties index fb1a07c6d..749d099ab 100644 --- a/config/log4j2.properties +++ b/config/log4j2.properties @@ -81,4 +81,4 @@ appender.file.strategy.action.condition.nested_condition.type = IfAny appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl} appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount -appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count} \ No newline at end of file +appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count} diff --git a/docker/seatunnel-engine/Dockerfile b/docker/seatunnel-engine/Dockerfile new file mode 100644 index 000000000..bf8351aa7 --- /dev/null +++ b/docker/seatunnel-engine/Dockerfile @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE='eclipse-temurin:8-jre' + +FROM $BASE_IMAGE + +ENV SEATUNNEL_HOME /opt/seatunnel + +WORKDIR $SEATUNNEL_HOME + +COPY . $SEATUNNEL_HOME + +EXPOSE 5801 + +CMD [ "/bin/bash", "-c", "$SEATUNNEL_HOME/bin/seatunnel-cluster.sh" ]
