This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 9e61969 [GOBBLIN-903] Initialize docker file for gobblin-standalone
and fix docker compose
9e61969 is described below
commit 9e619697467841af213faa5c3a1fe4b8f0f58c9a
Author: William Lo <[email protected]>
AuthorDate: Thu Oct 10 20:56:19 2019 -0700
[GOBBLIN-903] Initialize docker file for gobblin-standalone and fix docker
compose
Closes #2756 from Will-Lo/dockerize-gobblin-
standalone
---
.dockerignore | 2 +
conf/gobblin-as-service/application.conf | 2 +-
.../gobblin-service/alpine-gaas-latest/Dockerfile | 27 +++---
.../alpine-gaas-latest/docker-compose.yml | 11 ++-
.../alpine-gaas-latest/entrypoint.sh | 108 +--------------------
.../alpine-gobblin-latest/Dockerfile | 23 +++++
.../alpine-gobblin-latest/entrypoint.sh | 21 ++++
.../user-guide/Building-Gobblin-as-a-Service.md | 16 ++-
8 files changed, 88 insertions(+), 122 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 7157a84..e130ea6 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -8,8 +8,10 @@
.project
*.md
+build/
gobblin-docker/
gobblin-docs/
+gobblin-dist/
# Intellij related
**/.idea
diff --git a/conf/gobblin-as-service/application.conf
b/conf/gobblin-as-service/application.conf
index 7b244a1..1bf7fea 100644
--- a/conf/gobblin-as-service/application.conf
+++ b/conf/gobblin-as-service/application.conf
@@ -16,7 +16,7 @@
#
# Gobblin-As-Service configuration properties
-gobblin.service.work.dir=/tools/gobblin-dist/gobblin-as-a-service-work-dir
+gobblin.service.work.dir=/tmp/gobblin-as-service
fs.uri="file:///"
# Topology Catalog and Store
diff --git a/gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
b/gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
index be81ebb..97c464a 100644
--- a/gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
+++ b/gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
@@ -1,18 +1,23 @@
-ARG GOBBLIN_ROOT=${GOBBLIN_WORK_DIR}
+ARG GOBBLIN_ROOT=${GOBBLIN_REPO_ROOT}
FROM openjdk:8-alpine as build
WORKDIR /home/gobblin
+ARG GOBBLIN_VERSION=latest
COPY ${GOBBLIN_ROOT} .
RUN apk add --no-cache bash && \
- ./gradlew :gobblin-distribution:buildDistributionTar -Pversion=latest
+ ./gradlew :gobblin-distribution:buildDistributionTar -Pversion=latest && \
+ tar -xvf apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz && \
+ rm apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz
FROM openjdk:8-jre-alpine
-ARG GOBBLIN_VERSION=latest
-WORKDIR /home/gobblin
-COPY
${GOBBLIN_ROOT}/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
./gobblin-dist/bin/entrypoint.sh
-COPY --from=build
/home/gobblin/apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz .
-RUN tar -xvf apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz && \
- mkdir /tmp/templateCatalog/ && \
- chmod +x ./gobblin-dist/bin/gobblin-service.sh && \
- rm apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz
-ENTRYPOINT ["./gobblin-dist/bin/entrypoint.sh", "start"]
\ No newline at end of file
+WORKDIR /home/gobblin/
+COPY
${GOBBLIN_ROOT}/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
./bin/entrypoint.sh
+COPY --from=build /home/gobblin/gobblin-dist .
+
+RUN apk add --no-cache bash && \
+ mkdir /tmp/templateCatalog && \
+ mkdir /tmp/gobblin-as-service && \
+ mkdir /tmp/gobblin-as-service/logs && \
+ chmod +x ./bin/entrypoint.sh
+
+CMD ["./bin/entrypoint.sh"]
\ No newline at end of file
diff --git
a/gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml
b/gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml
index 82607f7..72fe9f8 100644
--- a/gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml
+++ b/gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml
@@ -4,5 +4,14 @@ services:
build:
context: ../../../
dockerfile: gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
+ volumes:
+ - "${LOCAL_DATAPACK_DIR}:/tmp/templateCatalog"
+ - "${LOCAL_JOB_DIR}:/tmp/gobblin-as-service/jobs"
ports:
- - 6956:6956
\ No newline at end of file
+ - 6956:6956
+ gobblin-standalone:
+ build:
+ context: ../../../
+ dockerfile:
gobblin-docker/gobblin-standalone/alpine-gobblin-latest/Dockerfile
+ volumes:
+ - "${LOCAL_JOB_DIR}:/tmp/gobblin-standalone/jobs"
diff --git a/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
b/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
index 75a5426..68b870e 100755
--- a/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
+++ b/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,108 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+GOBBLIN_HOME="$(cd `dirname $0`/..; pwd)"
-# Print an error message and exit
-
-function start() {
- for jarFile in `ls ${FWDIR_LIB}/*`
- do
- GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
- done
-
- export HADOOP_USER_CLASSPATH_FIRST=true
-
-
CLASSPATH=${FWDIR_CONF}:${GOBBLIN_JARS}:${SERVICE_CONF_DIR}:${HADOOP_HOME}/lib
- if [ -n "$EXTRA_JARS" ]; then
- CLASSPATH=$CLASSPATH:"$EXTRA_JARS"
- fi
-
- LOG_ARGS="1>${FWDIR_LOGS}/GobblinService.stdout
2>${FWDIR_LOGS}/GobblinService.stderr"
-
-# LOG4J_ARGS="-Dlog4j.configuration=conf/log4j.xml"
-
DEBUGGER_ARGS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006"
-
- LOG4J_PATH=file://${FWDIR_CONF}/log4j-service.properties
- $JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH
$JVM_FLAGS $LOG4J_ARGS $DEBUGGER_ARGS
org.apache.gobblin.service.modules.core.GobblinServiceManager --service_name
$SERVICE_NAME $LOG_ARGS
-}
-
-function stop() {
- if [ -f "$PID" ]; then
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo 'Stopping the Gobblin service'
- kill $PID_VALUE
- else
- echo "Process $PID_VALUE is not running"
- fi
- else
- echo "No pid file found"
- fi
-}
-
-FWDIR="$(cd `dirname $0`/..; pwd)"
-FWDIR_LIB=${FWDIR}/lib
-FWDIR_CONF=${FWDIR}/conf/service
-FWDIR_BIN=${FWDIR}/bin
-FWDIR_LOGS=${FWDIR}/logs
-SERVICE_NAME="gobblin_service"
-
-. ${FWDIR_BIN}/gobblin-env.sh
-
-for i in "$@"
-do
- case "$1" in
- start|stop)
- ACTION="$1"
- ;;
- --jvmflags)
- JVM_FLAGS="$2"
- shift
- ;;
- --jars)
- EXTRA_JARS="$2"
- shift
- ;;
- --service)
- SERVICE_NAME="$2"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
-if [ -n "$JVM_FLAGS" ]; then
- JVM_FLAGS="-Xmx1g -Xms512m"
-fi
-
-PID="$FWDIR/.gobblin-service-app-pid"
-
-if [ -f "$PID" ]; then
- PID_VALUE=`cat $PID` > /dev/null 2>&1
-else
- PID_VALUE=""
-fi
-
-case "$ACTION" in
- "start")
- start
- ;;
- "stop")
- stop
- ;;
- *)
- print_usage
- exit 1
- ;;
-esac
+./bin/gobblin.sh service gobblin-as-service start
alpine-gaas-latest_gobblin-standalone
+busybox tail -F $GOBBLIN_HOME/logs/gobblin-as-service.out
diff --git a/gobblin-docker/gobblin-standalone/alpine-gobblin-latest/Dockerfile
b/gobblin-docker/gobblin-standalone/alpine-gobblin-latest/Dockerfile
new file mode 100644
index 0000000..c054593
--- /dev/null
+++ b/gobblin-docker/gobblin-standalone/alpine-gobblin-latest/Dockerfile
@@ -0,0 +1,23 @@
+ARG GOBBLIN_ROOT=${GOBBLIN_REPO_ROOT}
+
+FROM openjdk:8-alpine as build
+WORKDIR /home/gobblin
+ARG GOBBLIN_VERSION=latest
+COPY ${GOBBLIN_ROOT} .
+RUN apk add --no-cache bash && \
+ ./gradlew :gobblin-distribution:buildDistributionTar -Pversion=latest && \
+ tar -xvf apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz && \
+ rm apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz
+
+FROM openjdk:8-jre-alpine
+WORKDIR /home/gobblin/
+COPY
${GOBBLIN_ROOT}/gobblin-docker/gobblin-standalone/alpine-gobblin-latest/entrypoint.sh
./bin/entrypoint.sh
+COPY --from=build /home/gobblin/gobblin-dist .
+RUN apk add --no-cache bash && \
+ mkdir /tmp/gobblin-standalone && \
+ mkdir /tmp/gobblin-standalone/jobs && \
+ chmod +x ./bin/entrypoint.sh
+ENV GOBBLIN_WORK_DIR=/tmp/gobblin-standalone/jobs
+ENV GOBBLIN_JOB_CONFIG_DIR=/tmp/gobblin-standalone/jobs
+
+CMD ["./bin/entrypoint.sh"]
\ No newline at end of file
diff --git
a/gobblin-docker/gobblin-standalone/alpine-gobblin-latest/entrypoint.sh
b/gobblin-docker/gobblin-standalone/alpine-gobblin-latest/entrypoint.sh
new file mode 100644
index 0000000..a122e9d
--- /dev/null
+++ b/gobblin-docker/gobblin-standalone/alpine-gobblin-latest/entrypoint.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+GOBBLIN_HOME="$(cd `dirname $0`/..; pwd)"
+
+./bin/gobblin.sh service standalone start
+busybox tail -F ${GOBBLIN_HOME}/logs/standalone.out
\ No newline at end of file
diff --git a/gobblin-docs/user-guide/Building-Gobblin-as-a-Service.md
b/gobblin-docs/user-guide/Building-Gobblin-as-a-Service.md
index 857680d..8cb6fcf 100644
--- a/gobblin-docs/user-guide/Building-Gobblin-as-a-Service.md
+++ b/gobblin-docs/user-guide/Building-Gobblin-as-a-Service.md
@@ -19,10 +19,16 @@ Currently the setup only runs a portion of the service, but
work will be done to
The service can now be accessed on `localhost:6956`
# Running Gobblin as a Service with Docker
-There is also a Dockerfile to create new images of Gobblin based on the source
code that can be easily run.
+There are also Dockerfiles to create new images of Gobblin based on the source
code that can be easily run independently.
-1. `export GOBBLIN_WORK_DIR=<root_directory_of_gobblin>`
-2. `docker compose -f
gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml build`
-3. `docker compose -f
gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml up`
+The Docker compose is set up to easily create a working end-to-end workflow of
Gobblin as a Service, which communicates Gobblin Standalone through a local
volume filesystem.
+
+To run the full docker compose:
+
+1. `export GOBBLIN_ROOT_DIR=<root_directory_of_gobblin>`
+2. `export LOCAL_DATAPACK_DIR=<local_directory_of_templateUris>`
+3. `export LOCAL_JOB_DIR=<local_directory_to_read_and_write_jobs>`
+4. `docker compose -f
gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml build`
+5. `docker compose -f
gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml up`
-The docker container exposes the endpoints which can be accessed on
`localhost:6956`
\ No newline at end of file
+The docker container exposes the endpoints from Gobblin as a Service which can
be accessed on `localhost:6956`
\ No newline at end of file
