This is an automated email from the ASF dual-hosted git repository. kdoran pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push: new 6259269 NIFI-8779 correctly pass MIRROR_BASE_URL from DockerBuild to Dockerfile; address shellcheck issues with Docker-related scripts 6259269 is described below commit 6259269f73a486b7d834d701e44362880b8a6a85 Author: Chris Sampson <chris.sampso...@gmail.com> AuthorDate: Tue Jul 13 09:29:23 2021 +0100 NIFI-8779 correctly pass MIRROR_BASE_URL from DockerBuild to Dockerfile; address shellcheck issues with Docker-related scripts Allow override of DISTRO_PATH for NiFi dockerhub build to enable image creation from the Apache Dev server during Release Candidate voting Rationalise nifi-docker builds and integration-tests (remove duplicate script directories) This closes #5213. Signed-off-by: Kevin Doran <kdo...@apache.org> --- nifi-docker/dockerhub/.dockerignore | 6 +- nifi-docker/dockerhub/DockerBuild.sh | 32 +++-- nifi-docker/dockerhub/DockerRun.sh | 14 ++- nifi-docker/dockerhub/Dockerfile | 5 +- nifi-docker/dockerhub/pom.xml | 17 ++- .../{dockerhub => dockermaven}/.dockerignore | 14 ++- nifi-docker/dockermaven/Dockerfile | 3 +- nifi-docker/dockermaven/integration-test.sh | 31 +++-- nifi-docker/dockermaven/pom.xml | 27 ++++- nifi-docker/dockermaven/sh/common.sh | 36 ------ nifi-docker/dockermaven/sh/secure.sh | 81 ------------- nifi-docker/dockermaven/sh/start.sh | 134 --------------------- nifi-docker/dockermaven/sh/toolkit.sh | 32 ----- .../sh/update_cluster_state_management.sh | 31 ----- .../dockermaven/sh/update_login_providers.sh | 47 -------- nifi-docker/pom.xml | 4 +- 16 files changed, 102 insertions(+), 412 deletions(-) diff --git a/nifi-docker/dockerhub/.dockerignore b/nifi-docker/dockerhub/.dockerignore index 30a2650..0058a31 100644 --- a/nifi-docker/dockerhub/.dockerignore +++ b/nifi-docker/dockerhub/.dockerignore @@ -16,4 +16,8 @@ # Place files you want to exclude from the docker build here similar to .gitignore https://docs.docker.com/engine/reference/builder/#dockerignore-file DockerBuild.sh DockerRun.sh -DockerImage.txt \ No newline at end of file +DockerImage.txt + +target/ +*.iml +pom.xml diff --git a/nifi-docker/dockerhub/DockerBuild.sh b/nifi-docker/dockerhub/DockerBuild.sh index 2285aac..09806b0 100755 --- a/nifi-docker/dockerhub/DockerBuild.sh +++ b/nifi-docker/dockerhub/DockerBuild.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -13,24 +15,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -#!/bin/bash - -DOCKER_UID=1000 -if [ -n "$1" ]; then - DOCKER_UID="$1" -fi +set -e +set -o pipefail -DOCKER_GID=1000 -if [ -n "$2" ]; then - DOCKER_GID="$2" -fi +DOCKER_UID="${1:-1000}" +DOCKER_GID="${2:-1000}" +MIRROR="${3:-https://archive.apache.org/dist}" +BASE="${4:-${MIRROR}}" +DISTRO_PATH="${5:-}" -MIRROR=https://archive.apache.org/dist -if [ -n "$3" ]; then - MIRROR="$3" +DOCKER_IMAGE="$(grep -Ev '(^#|^\s*$|^\s*\t*#)' DockerImage.txt)" +NIFI_IMAGE_VERSION="$(echo "${DOCKER_IMAGE}" | cut -d : -f 2)" +if [ -z "${DISTRO_PATH}" ]; then + DISTRO_PATH="${NIFI_VERSION}" fi -DOCKER_IMAGE="$(egrep -v '(^#|^\s*$|^\s*\t*#)' DockerImage.txt)" -NIFI_IMAGE_VERSION="$(echo $DOCKER_IMAGE | cut -d : -f 2)" -echo "Building NiFi Image: '$DOCKER_IMAGE' Version: $NIFI_IMAGE_VERSION Mirror: $MIRROR" -docker build --build-arg UID="$DOCKER_UID" --build-arg GID="$DOCKER_GID" --build-arg NIFI_VERSION="$NIFI_IMAGE_VERSION" --build-arg MIRROR="$MIRROR" -t $DOCKER_IMAGE . +echo "Building NiFi Image: '${DOCKER_IMAGE}' Version: '${NIFI_IMAGE_VERSION}' Mirror: '${MIRROR}' Base: '${BASE} Path: '${DISTRO_PATH}' User/Group: '${DOCKER_UID}/${DOCKER_GID}'" +docker build --build-arg UID="${DOCKER_UID}" --build-arg GID="${DOCKER_GID}" --build-arg NIFI_VERSION="${NIFI_IMAGE_VERSION}" --build-arg MIRROR_BASE_URL="${MIRROR}" --build-arg BASE_URL="${BASE}" --build-arg DISTRO_PATH="${DISTRO_PATH}" -t "${DOCKER_IMAGE}" . diff --git a/nifi-docker/dockerhub/DockerRun.sh b/nifi-docker/dockerhub/DockerRun.sh index d25551d..97d0092 100755 --- a/nifi-docker/dockerhub/DockerRun.sh +++ b/nifi-docker/dockerhub/DockerRun.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -13,7 +15,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -#!/bin/bash -DOCKER_IMAGE="$(egrep -v '(^#|^\s*$|^\s*\t*#)' DockerImage.txt)" -echo "Running Docker Image: $DOCKER_IMAGE" -docker run -it -d -p 8080:8080 -p 8181:8181 $DOCKER_IMAGE \ No newline at end of file +set -e +set -o pipefail + +DOCKER_IMAGE="$(grep -Ev '(^#|^\s*$|^\s*\t*#)' DockerImage.txt)" +NIFI_IMAGE_VERSION="$(echo "${DOCKER_IMAGE}" | cut -d : -f 2)" + +echo "Running Docker Image: ${DOCKER_IMAGE}" +docker run -d --name "nifi-${NIFI_IMAGE_VERSION}" -p 8080:8080 -p 8443:8443 -p 10000:10000 -p 8000:8000 -p 8181:8181 "${DOCKER_IMAGE}" diff --git a/nifi-docker/dockerhub/Dockerfile b/nifi-docker/dockerhub/Dockerfile index d6e5d7b..f3de2f4 100644 --- a/nifi-docker/dockerhub/Dockerfile +++ b/nifi-docker/dockerhub/Dockerfile @@ -27,8 +27,9 @@ ARG GID=1000 ARG NIFI_VERSION=1.15.0 ARG BASE_URL=https://archive.apache.org/dist ARG MIRROR_BASE_URL=${MIRROR_BASE_URL:-${BASE_URL}} -ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSION}-bin.zip} -ARG NIFI_TOOLKIT_BINARY_PATH=${NIFI_TOOLKIT_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-toolkit-${NIFI_VERSION}-bin.zip} +ARG DISTRO_PATH=${DISTRO_PATH:-${NIFI_VERSION}} +ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${DISTRO_PATH}/nifi-${NIFI_VERSION}-bin.zip} +ARG NIFI_TOOLKIT_BINARY_PATH=${NIFI_TOOLKIT_BINARY_PATH:-/nifi/${DISTRO_PATH}/nifi-toolkit-${NIFI_VERSION}-bin.zip} ENV NIFI_BASE_DIR=/opt/nifi ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current diff --git a/nifi-docker/dockerhub/pom.xml b/nifi-docker/dockerhub/pom.xml index adcec5f..b445572 100644 --- a/nifi-docker/dockerhub/pom.xml +++ b/nifi-docker/dockerhub/pom.xml @@ -24,17 +24,26 @@ <docker.image.name>openjdk</docker.image.name> <docker.image.tag>8-jre</docker.image.tag> <docker.maintainer><![CDATA[Apache NiFi <d...@nifi.apache.org>]]></docker.maintainer> + <!-- files need to exist at https://archive.apache.org/dist/nifi/${docker.nifi.version}/nifi-[toolkit-]${docker.nifi.version}-bin.zip + so this version needs to be *before* the current ${project.version} --> + <docker.nifi.version>1.14.0</docker.nifi.version> </properties> <profiles> <profile> <id>docker</id> + <dependencies> + <dependency> + <groupId>javax.activation</groupId> + <artifactId>activation</artifactId> + <version>1.1.1</version> + </dependency> + </dependencies> <build> <plugins> <plugin> <groupId>com.spotify</groupId> <artifactId>dockerfile-maven-plugin</artifactId> - <version>1.3.5</version> <executions> <execution> <id>default</id> @@ -48,7 +57,7 @@ <MAINTAINER>${docker.maintainer}</MAINTAINER> <UID>1000</UID> <GID>1000</GID> - <NIFI_VERSION>1.7.0</NIFI_VERSION> + <NIFI_VERSION>${docker.nifi.version}</NIFI_VERSION> </buildArgs> <repository>apache/nifi</repository> <!-- Right now we can only test against the latest released NiFi version to check our Dockerfile --> @@ -57,7 +66,6 @@ </execution> </executions> </plugin> - <!-- Disabled integration test pending release of Single User Authentication <plugin> <artifactId>exec-maven-plugin</artifactId> <groupId>org.codehaus.mojo</groupId> @@ -71,14 +79,13 @@ <configuration> <arguments> <argument>${project.version}-dockerhub</argument> - <argument>1.14.0</argument> + <argument>${docker.nifi.version}</argument> </arguments> <executable>${project.basedir}/../dockermaven/integration-test.sh</executable> </configuration> </execution> </executions> </plugin> - --> </plugins> </build> </profile> diff --git a/nifi-docker/dockerhub/.dockerignore b/nifi-docker/dockermaven/.dockerignore similarity index 86% copy from nifi-docker/dockerhub/.dockerignore copy to nifi-docker/dockermaven/.dockerignore index 30a2650..cac492b 100644 --- a/nifi-docker/dockerhub/.dockerignore +++ b/nifi-docker/dockermaven/.dockerignore @@ -14,6 +14,14 @@ # limitations under the License. # Place files you want to exclude from the docker build here similar to .gitignore https://docs.docker.com/engine/reference/builder/#dockerignore-file -DockerBuild.sh -DockerRun.sh -DockerImage.txt \ No newline at end of file +integration-test.sh +*.iml +pom.xml + +# ignore target/ +target/ + +# except nifi assembly files and scripts +!target/*.zip +!target/sh/ +!target/sh/*.sh diff --git a/nifi-docker/dockermaven/Dockerfile b/nifi-docker/dockermaven/Dockerfile index 40fbf92..2bd31d0 100644 --- a/nifi-docker/dockermaven/Dockerfile +++ b/nifi-docker/dockermaven/Dockerfile @@ -25,6 +25,7 @@ LABEL maintainer="${MAINTAINER}" ARG NIFI_VERSION ARG NIFI_BINARY ARG NIFI_TOOLKIT_BINARY +ARG NIFI_SCRIPTS ENV NIFI_BASE_DIR /opt/nifi ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current @@ -32,7 +33,7 @@ ENV NIFI_TOOLKIT_HOME ${NIFI_BASE_DIR}/nifi-toolkit-current ENV NIFI_PID_DIR=${NIFI_HOME}/run ENV NIFI_LOG_DIR=${NIFI_HOME}/logs -ADD sh/ ${NIFI_BASE_DIR}/scripts/ +ADD ${NIFI_SCRIPTS} ${NIFI_BASE_DIR}/scripts/ RUN chmod -R +x ${NIFI_BASE_DIR}/scripts/*.sh COPY $NIFI_BINARY $NIFI_BASE_DIR diff --git a/nifi-docker/dockermaven/integration-test.sh b/nifi-docker/dockermaven/integration-test.sh index e7d3dc4..726d89c 100755 --- a/nifi-docker/dockermaven/integration-test.sh +++ b/nifi-docker/dockermaven/integration-test.sh @@ -20,34 +20,39 @@ set -exuo pipefail TAG=$1 VERSION=$2 -trap "{ docker ps -qaf Name=nifi-${TAG}-integration-test | xargs docker rm -f; }" EXIT + +trap '{ docker ps -qaf Name="nifi-${TAG}-integration-test" | xargs --no-run-if-empty docker rm -f; }' EXIT + +echo "Deleting any existing nifi-${TAG}-integration-test containers" +docker ps -qaf Name="nifi-${TAG}-integration-test" | xargs --no-run-if-empty docker rm -f; echo "Checking that all files are owned by NiFi" -test -z $(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "find /opt/nifi ! -user nifi") +test -z "$(docker run --rm --entrypoint /bin/bash "apache/nifi:${TAG}" -c "find /opt/nifi ! -user nifi")" echo "Checking environment variables" -test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_HOME')" -test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "readlink \${NIFI_BASE_DIR}/nifi-${VERSION}")" -test "/opt/nifi/nifi-toolkit-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "readlink \${NIFI_BASE_DIR}/nifi-toolkit-${VERSION}")" +test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash "apache/nifi:${TAG}" -c 'echo -n $NIFI_HOME')" +test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash "apache/nifi:${TAG}" -c "readlink \${NIFI_BASE_DIR}/nifi-${VERSION}")" +test "/opt/nifi/nifi-toolkit-current" = "$(docker run --rm --entrypoint /bin/bash "apache/nifi:${TAG}" -c "readlink \${NIFI_BASE_DIR}/nifi-toolkit-${VERSION}")" -test "/opt/nifi/nifi-current/logs" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_LOG_DIR')" -test "/opt/nifi/nifi-current/run" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_PID_DIR')" -test "/opt/nifi" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_BASE_DIR')" +test "/opt/nifi/nifi-current/logs" = "$(docker run --rm --entrypoint /bin/bash "apache/nifi:${TAG}" -c 'echo -n $NIFI_LOG_DIR')" +test "/opt/nifi/nifi-current/run" = "$(docker run --rm --entrypoint /bin/bash "apache/nifi:${TAG}" -c 'echo -n $NIFI_PID_DIR')" +test "/opt/nifi" = "$(docker run --rm --entrypoint /bin/bash "apache/nifi:${TAG}" -c 'echo -n $NIFI_BASE_DIR')" echo "Starting NiFi container..." -docker run -d --name nifi-${TAG}-integration-test apache/nifi:${TAG} +docker run -d --name "nifi-${TAG}-integration-test" "apache/nifi:${TAG}" -IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nifi-${TAG}-integration-test) +IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "nifi-${TAG}-integration-test") for i in $(seq 1 10) :; do - if docker exec nifi-${TAG}-integration-test bash -c "ss -ntl | grep 8443"; then + echo "Iteration: ${i}" + if docker exec "nifi-${TAG}-integration-test" bash -c " echo Running < /dev/tcp/${IP}/8443"; then break fi sleep 10 done echo "Checking NiFi REST API Access" -test "200" = $(docker exec nifi-${TAG}-integration-test bash -c "curl -s -o /dev/null -w %{http_code} -k https://$IP:8443/nifi-api/access") +test "200" = "$(docker exec "nifi-${TAG}-integration-test" bash -c "curl -s -o /dev/null -w %{http_code} -k https://${IP}:8443/nifi-api/access")" echo "Stopping NiFi container" -time docker stop nifi-${TAG}-integration-test \ No newline at end of file +time docker stop "nifi-${TAG}-integration-test" diff --git a/nifi-docker/dockermaven/pom.xml b/nifi-docker/dockermaven/pom.xml index 994c908..590596a 100644 --- a/nifi-docker/dockermaven/pom.xml +++ b/nifi-docker/dockermaven/pom.xml @@ -29,12 +29,18 @@ <profiles> <profile> <id>docker</id> + <dependencies> + <dependency> + <groupId>javax.activation</groupId> + <artifactId>activation</artifactId> + <version>1.1.1</version> + </dependency> + </dependencies> <build> <plugins> <plugin> <groupId>com.spotify</groupId> <artifactId>dockerfile-maven-plugin</artifactId> - <version>1.3.5</version> <executions> <execution> <id>default</id> @@ -51,6 +57,7 @@ <NIFI_VERSION>${project.version}</NIFI_VERSION> <NIFI_BINARY>target/nifi-${nifi.version}-bin.zip</NIFI_BINARY> <NIFI_TOOLKIT_BINARY>target/nifi-toolkit-${nifi.version}-bin.zip</NIFI_TOOLKIT_BINARY> + <NIFI_SCRIPTS>target/sh</NIFI_SCRIPTS> </buildArgs> <repository>apache/nifi</repository> <tag>${project.version}-dockermaven</tag> @@ -61,9 +68,25 @@ <!-- Copy generated artifact to nifi-docker --> <plugin> <artifactId>maven-antrun-plugin</artifactId> - <version>1.8</version> + <version>3.0.0</version> <executions> <execution> + <id>copy-sh-for-docker</id> + <phase>process-sources</phase> + <configuration> + <target name="copy docker scripts to nifi-docker for image build"> + <copy todir="${project.basedir}/target/sh" overwrite="true" flatten="true"> + <fileset dir="${project.basedir}/../dockerhub/sh" includes="*.sh"> + <include name="*.sh" /> + </fileset> + </copy> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + <execution> <id>copy-for-docker</id> <phase>process-sources</phase> <configuration> diff --git a/nifi-docker/dockermaven/sh/common.sh b/nifi-docker/dockermaven/sh/common.sh deleted file mode 100755 index f3ea435..0000000 --- a/nifi-docker/dockermaven/sh/common.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -e -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# 1 - value to search for -# 2 - value to replace -# 3 - file to perform replacement inline -prop_replace () { - target_file=${3:-${nifi_props_file}} - echo 'replacing target file ' ${target_file} - sed -i -e "s|^$1=.*$|$1=$2|" ${target_file} -} - -uncomment() { - target_file=${2} - echo "Uncommenting ${target_file}" - sed -i -e "s|^\#$1|$1|" ${target_file} -} - -# NIFI_HOME is defined by an ENV command in the backing Dockerfile -export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf -export nifi_props_file=${NIFI_HOME}/conf/nifi.properties -export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties -export hostname=$(hostname) diff --git a/nifi-docker/dockermaven/sh/secure.sh b/nifi-docker/dockermaven/sh/secure.sh deleted file mode 100755 index 4fff214..0000000 --- a/nifi-docker/dockermaven/sh/secure.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -scripts_dir='/opt/nifi/scripts' - -[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" - -# Perform idempotent changes of configuration to support secure environments -echo 'Configuring environment with SSL settings' - -: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."} -if [ ! -f "${KEYSTORE_PATH}" ]; then - echo "Keystore file specified (${KEYSTORE_PATH}) does not exist." - exit 1 -fi -: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."} -: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."} - -: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."} -if [ ! -f "${TRUSTSTORE_PATH}" ]; then - echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist." - exit 1 -fi -: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."} -: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."} - -prop_replace 'nifi.security.keystore' "${KEYSTORE_PATH}" -prop_replace 'nifi.security.keystoreType' "${KEYSTORE_TYPE}" -prop_replace 'nifi.security.keystorePasswd' "${KEYSTORE_PASSWORD}" -prop_replace 'nifi.security.keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" -prop_replace 'nifi.security.truststore' "${TRUSTSTORE_PATH}" -prop_replace 'nifi.security.truststoreType' "${TRUSTSTORE_TYPE}" -prop_replace 'nifi.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}" - -prop_replace 'keystore' "${KEYSTORE_PATH}" ${nifi_toolkit_props_file} -prop_replace 'keystoreType' "${KEYSTORE_TYPE}" ${nifi_toolkit_props_file} -prop_replace 'keystorePasswd' "${KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file} -prop_replace 'keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file} -prop_replace 'truststore' "${TRUSTSTORE_PATH}" ${nifi_toolkit_props_file} -prop_replace 'truststoreType' "${TRUSTSTORE_TYPE}" ${nifi_toolkit_props_file} -prop_replace 'truststorePasswd' "${TRUSTSTORE_PASSWORD}" ${nifi_toolkit_props_file} - -# Disable HTTP and enable HTTPS -prop_replace 'nifi.web.http.port' '' -prop_replace 'nifi.web.http.host' '' -prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}" -prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}" -prop_replace 'nifi.remote.input.secure' 'true' -# Enable the property only for cluster install -prop_replace 'nifi.cluster.protocol.is.secure' "${NIFI_CLUSTER_IS_NODE:-false}" - -# Setup nifi-toolkit -prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}:${NIFI_WEB_HTTPS_PORT:-8443}" ${nifi_toolkit_props_file} - -# Configure Authorizer and Login Identity Provider -prop_replace 'nifi.security.user.authorizer' "${NIFI_SECURITY_USER_AUTHORIZER:-managed-authorizer}" -prop_replace 'nifi.security.user.login.identity.provider' "${NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER}" - -# Establish initial user and an associated admin identity -sed -i -e 's|<property name="Initial User Identity 1"></property>|<property name="Initial User Identity 1">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml -sed -i -e 's|<property name="Initial Admin Identity"></property>|<property name="Initial Admin Identity">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml - -if [ -n "${NODE_IDENTITY}" ]; then - sed -i -e 's|<property name="Node Identity 1"></property>|<property name="Node Identity 1">'"${NODE_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml -fi - -prop_replace 'proxiedEntity' "${INITIAL_ADMIN_IDENTITY}" ${nifi_toolkit_props_file} diff --git a/nifi-docker/dockermaven/sh/start.sh b/nifi-docker/dockermaven/sh/start.sh deleted file mode 100755 index 617a39d..0000000 --- a/nifi-docker/dockermaven/sh/start.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -scripts_dir='/opt/nifi/scripts' - -[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" - -# Override JVM memory settings -if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then - prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file} -fi - -if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then - prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file} -fi - -if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then - uncomment "java.arg.debug" ${nifi_bootstrap_file} -fi - -# Establish baseline properties -prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}" -prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}" -prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}" -prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}" -prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" -prop_replace 'nifi.remote.input.secure' 'true' -prop_replace 'nifi.cluster.protocol.is.secure' 'true' - -# Set nifi-toolkit properties files and baseUrl -"${scripts_dir}/toolkit.sh" -prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}:${NIFI_WEB_HTTPS_PORT:-8443}" ${nifi_toolkit_props_file} - -prop_replace 'keystore' "${NIFI_HOME}/conf/keystore.p12" ${nifi_toolkit_props_file} -prop_replace 'keystoreType' "PKCS12" ${nifi_toolkit_props_file} -prop_replace 'truststore' "${NIFI_HOME}/conf/truststore.p12" ${nifi_toolkit_props_file} -prop_replace 'truststoreType' "PKCS12" ${nifi_toolkit_props_file} - -if [ -n "${NIFI_WEB_HTTP_PORT}" ]; then - prop_replace 'nifi.web.https.port' '' - prop_replace 'nifi.web.https.host' '' - prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT}" - prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}" - prop_replace 'nifi.remote.input.secure' 'false' - prop_replace 'nifi.cluster.protocol.is.secure' 'false' - prop_replace 'nifi.security.keystore' '' - prop_replace 'nifi.security.keystoreType' '' - prop_replace 'nifi.security.truststore' '' - prop_replace 'nifi.security.truststoreType' '' - prop_replace 'nifi.security.user.login.identity.provider' '' - prop_replace 'keystore' '' ${nifi_toolkit_props_file} - prop_replace 'keystoreType' '' ${nifi_toolkit_props_file} - prop_replace 'truststore' '' ${nifi_toolkit_props_file} - prop_replace 'truststoreType' '' ${nifi_toolkit_props_file} - prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT}" ${nifi_toolkit_props_file} - - if [ -n "${NIFI_WEB_PROXY_HOST}" ]; then - echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Unsetting nifi.web.proxy.host.' - prop_replace 'nifi.web.proxy.host' '' - fi -else - if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then - echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping or connecting through a proxy.' - fi -fi - -prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" -prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}" -prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}" -prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" -prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" -prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}" -prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}" -prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}" -prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}" -prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}" - -# Set analytics properties -prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" -prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" -prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" -prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}" -prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}" -prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}" - -if [ -n "${NIFI_SENSITIVE_PROPS_KEY}" ]; then - prop_replace 'nifi.sensitive.props.key' "${NIFI_SENSITIVE_PROPS_KEY}" -fi - -if [ -n "${SINGLE_USER_CREDENTIALS_USERNAME}" ] && [ -n "${SINGLE_USER_CREDENTIALS_PASSWORD}" ]; then - ${NIFI_HOME}/bin/nifi.sh set-single-user-credentials "${SINGLE_USER_CREDENTIALS_USERNAME}" "${SINGLE_USER_CREDENTIALS_PASSWORD}" -fi - -. "${scripts_dir}/update_cluster_state_management.sh" - -# Check if we are secured or unsecured -case ${AUTH} in - tls) - echo 'Enabling Two-Way SSL user authentication' - . "${scripts_dir}/secure.sh" - ;; - ldap) - echo 'Enabling LDAP user authentication' - # Reference ldap-provider in properties - export NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER="ldap-provider" - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_login_providers.sh" - ;; -esac - -# Continuously provide logs so that 'docker logs' can produce them -tail -F "${NIFI_HOME}/logs/nifi-app.log" & -"${NIFI_HOME}/bin/nifi.sh" run & -nifi_pid="$!" - -trap "echo Received trapped signal, beginning shutdown...;" KILL TERM HUP INT EXIT; - -echo NiFi running with PID ${nifi_pid}. -wait ${nifi_pid} diff --git a/nifi-docker/dockermaven/sh/toolkit.sh b/nifi-docker/dockermaven/sh/toolkit.sh deleted file mode 100755 index 5262c91..0000000 --- a/nifi-docker/dockermaven/sh/toolkit.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cat <<EOT > ${nifi_toolkit_props_file} -baseUrl= -keystore= -keystoreType= -keystorePasswd= -keyPasswd= -truststore= -truststoreType= -truststorePasswd= -proxiedEntity= -EOT - -cat <<EOT > ${HOME}/.nifi-cli.config -nifi.props=${nifi_toolkit_props_file} -EOT \ No newline at end of file diff --git a/nifi-docker/dockermaven/sh/update_cluster_state_management.sh b/nifi-docker/dockermaven/sh/update_cluster_state_management.sh deleted file mode 100755 index 718e52d..0000000 --- a/nifi-docker/dockermaven/sh/update_cluster_state_management.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -state_providers_file=${NIFI_HOME}/conf/state-management.xml -property_xpath='/stateManagement/cluster-provider/property' - -edit_property() { - property_name=$1 - property_value=$2 - - if [ -n "${property_value}" ]; then - xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}" - fi -} - -edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}" -edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}" \ No newline at end of file diff --git a/nifi-docker/dockermaven/sh/update_login_providers.sh b/nifi-docker/dockermaven/sh/update_login_providers.sh deleted file mode 100755 index e124960..0000000 --- a/nifi-docker/dockermaven/sh/update_login_providers.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -login_providers_file=${NIFI_HOME}/conf/login-identity-providers.xml -property_xpath='//loginIdentityProviders/provider/property' - -# Update a given property in the login-identity-providers file if a value is specified -edit_property() { - property_name=$1 - property_value=$2 - - if [ -n "${property_value}" ]; then - xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}" - fi -} - -# Remove comments to enable the ldap-provider -sed -i '/To enable the ldap-provider remove/d' "${login_providers_file}" - -edit_property 'Authentication Strategy' "${LDAP_AUTHENTICATION_STRATEGY}" -edit_property 'Manager DN' "${LDAP_MANAGER_DN}" -edit_property 'Manager Password' "${LDAP_MANAGER_PASSWORD}" -edit_property 'TLS - Keystore' "${LDAP_TLS_KEYSTORE}" -edit_property 'TLS - Keystore Password' "${LDAP_TLS_KEYSTORE_PASSWORD}" -edit_property 'TLS - Keystore Type' "${LDAP_TLS_KEYSTORE_TYPE}" -edit_property 'TLS - Truststore' "${LDAP_TLS_TRUSTSTORE}" -edit_property 'TLS - Truststore Password' "${LDAP_TLS_TRUSTSTORE_PASSWORD}" -edit_property 'TLS - Truststore Type' "${LDAP_TLS_TRUSTSTORE_TYPE}" -edit_property 'TLS - Protocol' "${LDAP_TLS_PROTOCOL}" -edit_property 'Url' "${LDAP_URL}" -edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}" -edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}" -edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}" \ No newline at end of file diff --git a/nifi-docker/pom.xml b/nifi-docker/pom.xml index 43770bd..5a81791 100644 --- a/nifi-docker/pom.xml +++ b/nifi-docker/pom.xml @@ -18,9 +18,7 @@ language governing permissions and limitations under the License. --> <version>1.16.0-SNAPSHOT</version> </parent> - <groupId>org.apache.nifi</groupId> <artifactId>nifi-docker</artifactId> - <version>1.16.0-SNAPSHOT</version> <packaging>pom</packaging> <properties> @@ -38,7 +36,7 @@ language governing permissions and limitations under the License. --> <plugin> <groupId>com.spotify</groupId> <artifactId>dockerfile-maven-plugin</artifactId> - <version>1.3.5</version> + <version>1.4.13</version> <executions> <execution> <id>default</id>