This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new f4c7513 [GOBBLIN-1357] Updates the cluster to use up-to-date images
and non-deprecated ingress Fixes bug where standalone could not pick up jobs
due to env variable not being set
f4c7513 is described below
commit f4c75139bf38b73b3fc052cc5c2010cdca9bc4c0
Author: William Lo <[email protected]>
AuthorDate: Fri Jan 22 15:31:36 2021 -0800
[GOBBLIN-1357] Updates the cluster to use up-to-date images and
non-deprecated ingress
Fixes bug where standalone could not pick up jobs due to env variable not
being set
Closes #3196 from Will-Lo/update-k8s-gaas
---
conf/gobblin-as-service/application.conf | 2 +-
.../gobblin-service-standalone/docker-compose.yml} | 41 ++++++++++------------
.../gobblin-recipes/kafka-hdfs/docker-compose.yml | 4 +--
.../gobblin/alpine-gobblin-latest/Dockerfile | 5 +--
.../gobblin/alpine-gobblin-latest/entrypoint.sh | 13 ++++---
gobblin-docs/user-guide/Docker-Integration.md | 4 +--
.../gobblin-service/base-cluster/deployment.yaml | 17 ++++++---
.../gobblin-service/base-cluster/ingress.yaml | 17 ++++++---
.../gobblin-service/mysql-cluster/deployment.yaml | 17 +++++----
.../mysql-cluster/gaas-application.conf | 2 +-
.../mysql-cluster/mysql-deployment.yaml | 2 +-
11 files changed, 69 insertions(+), 55 deletions(-)
diff --git a/conf/gobblin-as-service/application.conf
b/conf/gobblin-as-service/application.conf
index 5145325..452e52b 100644
--- a/conf/gobblin-as-service/application.conf
+++ b/conf/gobblin-as-service/application.conf
@@ -16,7 +16,7 @@
#
# Gobblin-As-Service configuration properties
-gobblin.service.work.dir=/tmp/gobblin-as-service
+gobblin.service.work.dir=/etc/gobblin-as-service
fs.uri="file:///"
# Topology Catalog and Store
diff --git a/gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile
b/gobblin-docker/gobblin-recipes/gobblin-service-standalone/docker-compose.yml
similarity index 50%
copy from gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile
copy to
gobblin-docker/gobblin-recipes/gobblin-service-standalone/docker-compose.yml
index 071129f..99a1cf0 100644
--- a/gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile
+++
b/gobblin-docker/gobblin-recipes/gobblin-service-standalone/docker-compose.yml
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -14,26 +15,20 @@
# limitations under the License.
#
-FROM gradle:5.6.4-jdk8 as build
-WORKDIR /home/gobblin
-ARG GOBBLIN_FLAVOR=standard
-
-# copy the entire project folder
-COPY . .
-RUN gradle build -x findbugsMain -x test -x rat -x checkstyleMain -x javadoc
-x checkstyleTest -PgobblinFlavor=${GOBBLIN_FLAVOR} --no-daemon && \
- tar -xvf ./build/gobblin-distribution/distributions/*.tar.gz && \
- rm *.tar.gz
-
-FROM openjdk:8-jre-alpine
-WORKDIR /home/gobblin/
-COPY ./gobblin-docker/gobblin/alpine-gobblin-latest/entrypoint.sh
./bin/entrypoint.sh
-COPY --from=build /home/gobblin/gobblin-dist .
-RUN apk add --no-cache bash && \
- mkdir /tmp/gobblin && \
- mkdir /tmp/gobblin/jobs && \
- chmod +x ./bin/entrypoint.sh
-
-ENV GOBBLIN_WORK_DIR=/tmp/gobblin/
-ENV GOBBLIN_JOB_CONFIG_DIR=/tmp/gobblin/jobs
-
-ENTRYPOINT ["./bin/entrypoint.sh"]
\ No newline at end of file
+version: '3'
+services:
+ gobblin-service:
+ image: apache/gobblin:latest
+ command: --mode "gobblin-as-service"
+ volumes:
+ - "${LOCAL_DATAPACK_DIR}:/etc/templateCatalog"
+ - "${LOCAL_JOB_DIR}:/etc/gobblin-as-service/jobs"
+ ports:
+ - 6956:6956
+ gobblin-standalone:
+ image: apache/gobblin:latest
+ command: --mode "standalone"
+ volumes:
+ - "${LOCAL_JOB_DIR}:/etc/gobblin-standalone/jobs"
+ environment:
+ - GOBBLIN_JOB_CONFIG_DIR=/etc/gobblin-standalone/jobs
diff --git a/gobblin-docker/gobblin-recipes/kafka-hdfs/docker-compose.yml
b/gobblin-docker/gobblin-recipes/kafka-hdfs/docker-compose.yml
index 2a4a332..7c82439 100644
--- a/gobblin-docker/gobblin-recipes/kafka-hdfs/docker-compose.yml
+++ b/gobblin-docker/gobblin-recipes/kafka-hdfs/docker-compose.yml
@@ -18,10 +18,10 @@
version: '3'
services:
gobblin-standalone:
- image: apache/gobblin:master
+ image: apache/gobblin:latest
command: --mode "standalone"
volumes:
- - "${LOCAL_JOB_DIR}:/tmp/gobblin-standalone/jobs"
+ - "${LOCAL_JOB_DIR}:/etc/gobblin-standalone/jobs"
zookeeper:
image: wurstmeister/zookeeper
ports:
diff --git a/gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile
b/gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile
index 071129f..db5ec05 100644
--- a/gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile
+++ b/gobblin-docker/gobblin/alpine-gobblin-latest/Dockerfile
@@ -30,10 +30,11 @@ COPY
./gobblin-docker/gobblin/alpine-gobblin-latest/entrypoint.sh ./bin/entrypoi
COPY --from=build /home/gobblin/gobblin-dist .
RUN apk add --no-cache bash && \
mkdir /tmp/gobblin && \
- mkdir /tmp/gobblin/jobs && \
+ mkdir /etc/gobblin && \
+ mkdir /etc/gobblin/jobs && \
chmod +x ./bin/entrypoint.sh
ENV GOBBLIN_WORK_DIR=/tmp/gobblin/
-ENV GOBBLIN_JOB_CONFIG_DIR=/tmp/gobblin/jobs
+ENV GOBBLIN_JOB_CONFIG_DIR=/etc/gobblin/jobs
ENTRYPOINT ["./bin/entrypoint.sh"]
\ No newline at end of file
diff --git a/gobblin-docker/gobblin/alpine-gobblin-latest/entrypoint.sh
b/gobblin-docker/gobblin/alpine-gobblin-latest/entrypoint.sh
index ff717b8..bd7b8b1 100755
--- a/gobblin-docker/gobblin/alpine-gobblin-latest/entrypoint.sh
+++ b/gobblin-docker/gobblin/alpine-gobblin-latest/entrypoint.sh
@@ -30,18 +30,17 @@ GOBBLIN_HOME="$(cd `dirname $0`/..; pwd)"
EXECUTION_MODE=''
CONF_PATH=''
-ARGS="$@"
+JVM_OPTS=''
+ARGS=("$@")
shopt -s nocasematch
-for i in $ARGS
+for i in "${!ARGS[@]}"
do
- case "$1" in
+ case "${ARGS[$i]}" in
--mode )
- EXECUTION_MODE="$2"
- shift
+ EXECUTION_MODE="${ARGS[$i+1]}"
;;
esac
- shift
done
if [[ -z "$EXECUTION_MODE" ]]; then
@@ -50,4 +49,4 @@ if [[ -z "$EXECUTION_MODE" ]]; then
exit 1
fi
-./bin/gobblin.sh service $EXECUTION_MODE start --log-to-stdout $ARGS
+./bin/gobblin.sh service $EXECUTION_MODE start --log-to-stdout "$@"
diff --git a/gobblin-docs/user-guide/Docker-Integration.md
b/gobblin-docs/user-guide/Docker-Integration.md
index c38020d..cdd4cfa 100644
--- a/gobblin-docs/user-guide/Docker-Integration.md
+++ b/gobblin-docs/user-guide/Docker-Integration.md
@@ -44,7 +44,7 @@ Run these commands to start the docker image:
`docker pull apache/gobblin:latest`
-`docker run -v $LOCAL_JOB_DIR:/tmp/gobblin-standalone/jobs
apache/gobblin:latest --mode standalone`
+`docker run -v $LOCAL_JOB_DIR:/etc/gobblin-standalone/jobs
apache/gobblin:latest --mode standalone`
After the container spins up, put the
[wikipedia.pull](https://github.com/apache/incubator-gobblin/blob/master/gobblin-example/src/main/resources/wikipedia.pull)
in ${LOCAL_JOB_DIR}. You will see the Gobblin daemon pick up the job, and the
result output is in ${LOCAL_JOB_DIR}/job-output/.
@@ -98,7 +98,7 @@ Similar to standalone working directory settings:
Run these commands to start the docker image:
-`docker run -p 6956:6956 -v $GAAS_JOB_DIR:/tmp/gobblin-as-service/jobs -v
$LOCAL_DATAPACK_DIR:/tmp/templateCatalog apache/gobblin --mode
gobblin-as-service`
+`docker run -p 6956:6956 -v $GAAS_JOB_DIR:/etc/gobblin-as-service/jobs -v
$LOCAL_DATAPACK_DIR:/etc/templateCatalog apache/gobblin --mode
gobblin-as-service`
The GaaS will be started, and the service can now be accessed on
localhost:6956.
diff --git a/gobblin-kubernetes/gobblin-service/base-cluster/deployment.yaml
b/gobblin-kubernetes/gobblin-service/base-cluster/deployment.yaml
index 57bec9d..a788896 100644
--- a/gobblin-kubernetes/gobblin-service/base-cluster/deployment.yaml
+++ b/gobblin-kubernetes/gobblin-service/base-cluster/deployment.yaml
@@ -27,12 +27,14 @@ spec:
name: flowconfig-templates
containers:
- name: gobblin-service
- image: will97/gobblin-as-a-service:latest
+ image: apache/gobblin:latest
+ args: ["--mode", "gobblin-as-service", "--jvmopts",
"-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap"]
volumeMounts:
- name: shared-jobs
- mountPath: /tmp/gobblin-as-service/jobs
+ mountPath: /etc/gobblin-as-service/jobs
- name: flowconfig-templates
- mountPath: /tmp/templateCatalog
+ mountPath: /etc/templateCatalog
+ imagePullPolicy: Never # TODO: Remove this once docker images are
deployed to Apache DockerHub post-graduation
---
apiVersion: apps/v1
kind: Deployment
@@ -57,7 +59,12 @@ spec:
claimName: shared-jobs-claim
containers:
- name: gobblin-standalone
- image: will97/gobblin-standalone:latest
+ image: apache/gobblin:latest
+ args: ["--mode", "standalone", "--jvmopts",
"-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap"]
volumeMounts:
- name: shared-jobs
- mountPath: /tmp/gobblin-standalone/jobs
+ mountPath: /etc/gobblin-standalone/jobs
+ env:
+ - name: GOBBLIN_JOB_CONFIG_DIR
+ value: /etc/gobblin-standalone/jobs
+ imagePullPolicy: Never # TODO: Remove this once docker images are
deployed to Apache DockerHub post-graduation
diff --git a/gobblin-kubernetes/gobblin-service/base-cluster/ingress.yaml
b/gobblin-kubernetes/gobblin-service/base-cluster/ingress.yaml
index c50c50b..a862824 100644
--- a/gobblin-kubernetes/gobblin-service/base-cluster/ingress.yaml
+++ b/gobblin-kubernetes/gobblin-service/base-cluster/ingress.yaml
@@ -1,8 +1,17 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gaas-ingress
+ annotations:
+ nginx.ingress.kubernetes.io/rewrite-target: /
spec:
- backend:
- serviceName: gaas-svc
- servicePort: 6956
+ rules:
+ - http:
+ paths:
+ - path: /
+ pathType: Prefix
+ backend:
+ service:
+ name: gaas-svc
+ port:
+ number: 6956
diff --git a/gobblin-kubernetes/gobblin-service/mysql-cluster/deployment.yaml
b/gobblin-kubernetes/gobblin-service/mysql-cluster/deployment.yaml
index c71a4ad..9a9e703 100644
--- a/gobblin-kubernetes/gobblin-service/mysql-cluster/deployment.yaml
+++ b/gobblin-kubernetes/gobblin-service/mysql-cluster/deployment.yaml
@@ -30,9 +30,8 @@ spec:
name: gaas-config
containers:
- name: gobblin-service
- image: will97/gobblin-as-a-service:latest
- command: ["./bin/entrypoint.sh"]
- args: ["--jvmopts", "-DmysqlCredentials.user=$(MYSQL_USERNAME)
-DmysqlCredentials.password=$(MYSQL_PASSWORD)"]
+ image: apache/gobblin:latest
+ args: ["--mode", "gobblin-as-service", "--jvmopts",
"-DmysqlCredentials.user=$(MYSQL_USERNAME)
-DmysqlCredentials.password=$(MYSQL_PASSWORD) -XX:+UnlockExperimentalVMOptions
-XX:+UseCGroupMemoryLimitForHeap"]
env:
- name: MYSQL_USERNAME
valueFrom:
@@ -46,9 +45,9 @@ spec:
key: password
volumeMounts:
- name: shared-jobs
- mountPath: /tmp/gobblin-as-service/jobs
+ mountPath: /etc/gobblin-as-service/jobs
- name: flowconfig-templates
- mountPath: /tmp/templateCatalog
+ mountPath: /etc/templateCatalog
- name: gaas-config
mountPath: /home/gobblin/conf/gobblin-as-service/application.conf
subPath: gaas-application.conf
@@ -86,10 +85,14 @@ spec:
name: standalone-config
containers:
- name: gobblin-standalone
- image: will97/gobblin-standalone:latest
+ image: apache/gobblin:latest
+ args: ["--mode", "standalone"]
volumeMounts:
- name: shared-jobs
- mountPath: /tmp/gobblin-standalone/jobs
+ mountPath: /etc/gobblin-standalone/jobs
- name: standalone-config
mountPath: /home/gobblin/conf/standalone/application.conf
subPath: standalone-application.conf
+ env:
+ - name: GOBBLIN_JOB_CONFIG_DIR
+ value: /etc/gobblin-standalone/jobs
diff --git
a/gobblin-kubernetes/gobblin-service/mysql-cluster/gaas-application.conf
b/gobblin-kubernetes/gobblin-service/mysql-cluster/gaas-application.conf
index 7c89cbe..bed21c4 100644
--- a/gobblin-kubernetes/gobblin-service/mysql-cluster/gaas-application.conf
+++ b/gobblin-kubernetes/gobblin-service/mysql-cluster/gaas-application.conf
@@ -18,7 +18,7 @@
# Sample configuration properties for the Gobblin Service
# Topology Catalog and Store
-gobblin.service.work.dir=/tmp/gobblin-as-service
+gobblin.service.work.dir=/etc/gobblin-as-service
# TopologySpec Factory
topologySpec.store.dir=${gobblin.service.work.dir}/topologySpecStore
diff --git
a/gobblin-kubernetes/gobblin-service/mysql-cluster/mysql-deployment.yaml
b/gobblin-kubernetes/gobblin-service/mysql-cluster/mysql-deployment.yaml
index ff11411..745666f 100644
--- a/gobblin-kubernetes/gobblin-service/mysql-cluster/mysql-deployment.yaml
+++ b/gobblin-kubernetes/gobblin-service/mysql-cluster/mysql-deployment.yaml
@@ -30,7 +30,7 @@ spec:
persistentVolumeClaim:
claimName: mysql-pv-claim
containers:
- - image: mysql:5.6.45
+ - image: mysql:8.0
name: mysql
env:
- name: MYSQL_RANDOM_ROOT_PASSWORD
