This is an automated email from the ASF dual-hosted git repository. himanshug pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push: new 38b6047 Update docker build (#8244) 38b6047 is described below commit 38b6047aa96e2ea3a6651aed138ee46566d596f6 Author: ilhanadiyaman <ilhanadiyam...@gmail.com> AuthorDate: Tue Aug 6 17:32:59 2019 +0200 Update docker build (#8244) --- distribution/docker/Dockerfile | 6 ++++- distribution/docker/README.md | 6 ++--- distribution/docker/druid.sh | 55 +++++++++++++++++++++++++----------------- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index fa80ba2..9db493c 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -19,9 +19,13 @@ FROM maven:3-jdk-8 as builder +RUN apt-get update && apt-get install --no-install-recommends -y python3-pip python3-setuptools python3-wheel\ + && rm -rf /var/lib/apt/lists/* +RUN pip3 install --no-cache-dir pyyaml + COPY . /src WORKDIR /src -RUN mvn install -ff -DskipTests -Dforbiddenapis.skip=true -Pdist -Pbundle-contrib-exts +RUN mvn dependency:go-offline install -ff -q -B -DskipTests -Dforbiddenapis.skip=true -Pdist -Pbundle-contrib-exts RUN \ VER=$(mvn -B org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate -Dexpression=project.version -q -DforceStdout=true -f pom.xml 2>/dev/null) \ diff --git a/distribution/docker/README.md b/distribution/docker/README.md index b029522..64f51d6 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -23,9 +23,9 @@ From the root of the repo, run `docker build -t druid:tag -f distribution/docker ## Run -Edit `environment` to suite. Run 'docker-compose -f distribution/docker/docker-compose.yml up` +Edit `environment` to suite. Run `docker-compose -f distribution/docker/docker-compose.yml up` -## mysql database connector +## MySQL Database Connector This image contains solely the postgres metadata database connector. If you need the mysql metadata storage connector, consider adding these lines before the `addgroup` @@ -37,6 +37,6 @@ RUN wget -O /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5. && ln -s /opt/druid/extensions/mysql-metadata-storage/mysql-connector-java-5.1.38.jar /opt/druid/lib ``` -Alternatively, cd src/distribution/docker; docker build -t druid:mysql --build-arg DRUID_RELEASE=upstream -f Dockerfile.mysql . +Alternatively, `cd src/distribution/docker; docker build -t druid:mysql --build-arg DRUID_RELEASE=upstream -f Dockerfile.mysql .` where `upstream` is the version to use as the base (e.g. druid:0.14.0 from Dockerhub) diff --git a/distribution/docker/druid.sh b/distribution/docker/druid.sh index b142ec6..c60b62a 100755 --- a/distribution/docker/druid.sh +++ b/distribution/docker/druid.sh @@ -45,45 +45,56 @@ echo "$(date -Is) startup service $SERVICE" # We put all the config in /tmp/conf to allow for a # read-only root filesystem -cp -r /opt/druid/conf /tmp/conf +mkdir -p /tmp/conf/ +cp -r /opt/druid/conf/druid /tmp/conf/druid + +getConfPath() { + cluster_conf_base=/tmp/conf/druid/cluster + case "$1" in + _common) echo $cluster_conf_base/_common ;; + historical) echo $cluster_conf_base/data/historical ;; + middleManager) echo $cluster_conf_base/data/middleManager ;; + coordinator | overlord) echo $cluster_conf_base/master/coordinator-overlord ;; + broker) echo $cluster_conf_base/query/broker ;; + router) echo $cluster_conf_base/query/router ;; + esac +} +COMMON_CONF_DIR=$(getConfPath _common) +SERVICE_CONF_DIR=$(getConfPath ${SERVICE}) # Delete the old key (if existing) and append new key=value setKey() { service="$1" key="$2" value="$3" - case "$service" in - _common) - fname=common.runtime.properties ;; - *) - fname=runtime.properties ;; - esac + service_conf=$(getConfPath $service)/runtime.properties # Delete from all - sed -ri "/$key=/d" /tmp/conf/druid/_common/common.runtime.properties - [ -f /tmp/conf/druid/$service/$fname ] && sed -ri "/$key=/d" /tmp/conf/druid/$service/$fname - [ -f /tmp/conf/druid/$service/$fname ] && echo "$key=$value" >> /tmp/conf/druid/$service/$fname - [ -f /tmp/conf/druid/$service/$fname ] || echo "$key=$value" >> /tmp/conf/druid/_common/$fname + sed -ri "/$key=/d" $COMMON_CONF_DIR/common.runtime.properties + [ -f $service_conf ] && sed -ri "/$key=/d" $service_conf + [ -f $service_conf ] && echo "$key=$value" >>$service_conf + [ -f $service_conf ] || echo "$key=$value" >>$COMMON_CONF_DIR/common.runtime.properties + + echo "Setting $key=$value in $service_conf" } setJavaKey() { service="$1" key=$2 value=$3 - file=/tmp/conf/druid/$service/jvm.config + file=$(getConfPath $service)/jvm.config sed -ri "/$key/d" $file echo $value >> $file } ## Setup host names -if [ -n "${ZOOKEEPER}" ] +if [ -n "${ZOOKEEPER}" ]; then setKey _common druid.zk.service.host "${ZOOKEEPER}" fi setKey $SERVICE druid.host $(ip r get 1 | awk '{print $7;exit}') - -env |grep ^druid_ | while read evar +env | grep ^druid_ | while read evar; do # Can't use IFS='=' to parse since var might have = in it (e.g. password) val=$(echo "$evar" | sed -e 's?[^=]*=??') @@ -95,14 +106,14 @@ env |grep ^s3service | while read evar do val=$(echo "$evar" | sed -e 's?[^=]*=??') var=$(echo "$evar" | sed -e 's?^\([^=]*\)=.*?\1?g' -e 's?_?.?' -e 's?_?-?g') - echo "$var=$val" >> /tmp/conf/druid/_common/jets3t.properties + echo "$var=$val" >>$COMMON_CONF_DIR/jets3t.properties done # This is to allow configuration via a Kubernetes configMap without # e.g. using subPath (you can also mount the configMap on /tmp/conf/druid) if [ -n "$DRUID_CONFIG_COMMON" ] then - cp -f "$DRUID_CONFIG_COMMON" /tmp/conf/druid/_common/common.runtime.properties + cp -f "$DRUID_CONFIG_COMMON" $COMMON_CONF_DIR/common.runtime.properties fi SCONFIG=$(printf "%s_%s" DRUID_CONFIG ${SERVICE}) @@ -110,7 +121,7 @@ SCONFIG=$(eval echo \$$(echo $SCONFIG)) if [ -n "${SCONFIG}" ] then - cp -f "${SCONFIG}" /tmp/conf/druid/${SERVICE}/runtime.properties + cp -f "${SCONFIG}" $SERVICE_CONF_DIR/runtime.properties fi # Now do the java options @@ -121,17 +132,17 @@ if [ -n "$DRUID_MAXNEWSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxNewSize -XX:Max if [ -n "$DRUID_NEWSIZE" ]; then setJavaKey ${SERVICE} -XX:NewSize -XX:MaxNewSize=${DRUID_NEWSIZE}; fi if [ -n "$DRUID_MAXDIRECTMEMORYSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxDirectMemorySize -XX:MaxDirectMemorySize=${DRUID_MAXDIRECTMEMORYSIZE}; fi -JAVA_OPTS="$JAVA_OPTS $(cat /tmp/conf/druid/${SERVICE}/jvm.config | xargs)" +JAVA_OPTS="$JAVA_OPTS $(cat $SERVICE_CONF_DIR/jvm.config | xargs)" if [ -n "$DRUID_LOG_LEVEL" ] then - sed -ri 's/"info"/"'$DRUID_LOG_LEVEL'"/g' /tmp/conf/druid/_common/log4j2.xml + sed -ri 's/"info"/"'$DRUID_LOG_LEVEL'"/g' $COMMON_CONF_DIR/log4j2.xml fi if [ -n "$DRUID_LOG4J" ] then - echo "$DRUID_LOG4J" > /tmp/conf/druid/_common/log4j2.xml + echo "$DRUID_LOG4J" >$COMMON_CONF_DIR/log4j2.xml fi mkdir -p var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/hadoop-tmp var/druid/segment-cache -exec java ${JAVA_OPTS} -cp /tmp/conf/druid/_common:/tmp/conf/druid/${SERVICE}:lib/*: org.apache.druid.cli.Main server $@ +exec java ${JAVA_OPTS} -cp $COMMON_CONF_DIR:$SERVICE_CONF_DIR:lib/*: org.apache.druid.cli.Main server $@ \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org