This is an automated email from the ASF dual-hosted git repository.
aboda pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 48acd26 MINIFICPP-1633 Add ccache to docker CI builds
48acd26 is described below
commit 48acd26e3c07ea1bb05f91fecad601ec67bbae59
Author: Gabor Gyimesi <[email protected]>
AuthorDate: Mon Jul 26 10:17:55 2021 +0200
MINIFICPP-1633 Add ccache to docker CI builds
Signed-off-by: Arpad Boda <[email protected]>
This closes #1164
---
.github/workflows/ci.yml | 15 ++++++++++++---
.gitignore | 1 +
CMakeLists.txt | 1 +
cmake/DockerConfig.cmake | 1 +
docker/DockerBuild.sh | 29 +++++++++++++++++++++++------
docker/Dockerfile | 11 +++++++----
6 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 29d024b..79dae16 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -258,15 +258,24 @@ jobs:
docker_integration_tests:
name: "Docker integration tests"
runs-on: ubuntu-20.04
- timeout-minutes: 90
+ timeout-minutes: 120
steps:
- id: checkout
uses: actions/checkout@v2
+ - id: cache
+ uses: actions/cache@v2
+ with:
+ path: ~/.ccache
+ key: docker-ccache-${{github.ref}}-${{github.sha}}
+ restore-keys: |
+ docker-ccache-${{github.ref}}-
+ docker-ccache-refs/heads/main
- id: build
run: |
- ./bootstrap.sh -e -t
+ if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
+ mkdir build
cd build
- cmake -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DENABLE_JNI=OFF
-DDISABLE_JEMALLOC=ON -DDISABLE_SCRIPTING=ON -DENABLE_AWS=ON
-DENABLE_LIBRDKAFKA=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON ..
+ cmake -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DENABLE_JNI=OFF
-DDISABLE_JEMALLOC=ON -DDISABLE_SCRIPTING=ON -DENABLE_AWS=ON
-DENABLE_LIBRDKAFKA=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON -DDOCKER_BUILD_ONLY=ON
-DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache ..
make docker
- id: install_deps
run: |
diff --git a/.gitignore b/.gitignore
index bcc7486..9dd9de9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -77,6 +77,7 @@ __pycache__/
.vs/slnx.sqlite
/.ccls-cache
/.vscode
+.ccache
# generated files from WEL test
extensions/windows-event-log/tests/custom-provider/unit-test-provider.cs
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fd1de67..355bb6d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -28,6 +28,7 @@ set(BUILD_NUMBER "" CACHE STRING "Build number")
# Base Alpine image to be used in the Docker build instead of the default
Alpine image
set(DOCKER_BASE_IMAGE "" CACHE STRING "Docker build Alpine base image")
+set(DOCKER_CCACHE_DUMP_LOCATION "" CACHE STRING "Directory to dump ccache to
after docker build for later reuse")
include(CMakeDependentOption)
include(CheckIncludeFile)
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index c6059eb..d841c49 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -56,6 +56,7 @@ add_custom_target(
-c DISABLE_PYTHON_SCRIPTING=${DISABLE_PYTHON_SCRIPTING}
-c DISABLE_CONTROLLER=${DISABLE_CONTROLLER}
-c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE}
+ -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION}
-c BUILD_NUMBER=${BUILD_NUMBER}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/)
diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh
index 397fa8c..d6e7734 100755
--- a/docker/DockerBuild.sh
+++ b/docker/DockerBuild.sh
@@ -28,6 +28,7 @@ IMAGE_TAG=
DUMP_LOCATION=
DISTRO_NAME=
BUILD_NUMBER=
+DOCKER_CCACHE_DUMP_LOCATION=
function usage {
echo "Usage: ./DockerBuild.sh -v <MINIFI_VERSION> [additional options]"
@@ -58,10 +59,10 @@ while [[ $# -gt 0 ]]; do
shift
;;
-v|--minifi-version)
- MINIFI_VERSION="$2"
- shift
- shift
- ;;
+ MINIFI_VERSION="$2"
+ shift
+ shift
+ ;;
-t|--tag)
IMAGE_TAG="$2"
shift
@@ -84,6 +85,8 @@ while [[ $# -gt 0 ]]; do
BUILD_NUMBER="${ARR[1]}"
elif [ "${ARR[0]}" == "DOCKER_BASE_IMAGE" ]; then
BUILD_ARGS="${BUILD_ARGS} --build-arg BASE_ALPINE_IMAGE=${ARR[1]}"
+ elif [ "${ARR[0]}" == "DOCKER_CCACHE_DUMP_LOCATION" ]; then
+ DOCKER_CCACHE_DUMP_LOCATION="${ARR[1]}"
else
BUILD_ARGS="${BUILD_ARGS} --build-arg ${ARR[0]}=${ARR[1]}"
fi
@@ -133,14 +136,28 @@ if [ -n "${BUILD_NUMBER}" ]; then
TARGZ_TAG="${TARGZ_TAG}-${BUILD_NUMBER}"
fi
-DOCKER_COMMAND="docker build "
+DOCKER_BUILD_START="docker build "
BUILD_ARGS="--build-arg UID=${UID_ARG} \
--build-arg GID=${GID_ARG} \
--build-arg MINIFI_VERSION=${MINIFI_VERSION} \
--build-arg DUMP_LOCATION=${DUMP_LOCATION} \
--build-arg DISTRO_NAME=${DISTRO_NAME} ${BUILD_ARGS}"
-DOCKER_COMMAND="${DOCKER_COMMAND} ${BUILD_ARGS} \
+if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then
+ DOCKER_COMMAND="${DOCKER_BUILD_START} ${BUILD_ARGS} \
+ -f ${DOCKERFILE} \
+ --target build \
+ -t \
+ minifi_build .."
+ echo "Build image Docker Command: 'DOCKER_BUILDKIT=1 ${DOCKER_COMMAND}'"
+ DOCKER_BUILDKIT=1 ${DOCKER_COMMAND}
+ container_id=$(docker run --rm -d minifi_build sh -c "while true; do sleep
1; done")
+ mkdir -p "${DOCKER_CCACHE_DUMP_LOCATION}"
+ docker cp "${container_id}:/home/minificpp/.ccache/."
"${DOCKER_CCACHE_DUMP_LOCATION}"
+ docker rm -f "${container_id}"
+fi
+
+DOCKER_COMMAND="${DOCKER_BUILD_START} ${BUILD_ARGS} \
-f ${DOCKERFILE} \
-t \
apacheminificpp:${TAG} .."
diff --git a/docker/Dockerfile b/docker/Dockerfile
index bbe9930..da1fbc2 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -87,7 +87,8 @@ RUN apk --no-cache add gcc \
gpsd-dev \
python3-dev \
boost-dev \
- doxygen
+ doxygen \
+ ccache
ENV USER minificpp
ENV MINIFI_BASE_DIR /opt/minifi
@@ -100,9 +101,11 @@ ENV MINIFI_VERSION ${MINIFI_VERSION}
RUN addgroup -g ${GID} ${USER} && adduser -u ${UID} -D -G ${USER} -g ""
${USER} && \
install -d -o ${USER} -g ${USER} ${MINIFI_BASE_DIR}
COPY --chown=${USER}:${USER} . ${MINIFI_BASE_DIR}
+RUN if [ -d "${MINIFI_BASE_DIR}/.ccache" ]; then mv ${MINIFI_BASE_DIR}/.ccache
/home/${USER}/.ccache; fi
USER ${USER}
+ENV PATH /usr/lib/ccache/bin:${PATH}
RUN mkdir ${MINIFI_BASE_DIR}/build
WORKDIR ${MINIFI_BASE_DIR}/build
RUN cmake -DSTATIC_BUILD= -DSKIP_TESTS=true -DENABLE_ALL="${ENABLE_ALL}"
-DENABLE_PYTHON="${ENABLE_PYTHON}" -DENABLE_OPS="${ENABLE_OPS}" \
@@ -115,9 +118,9 @@ RUN cmake -DSTATIC_BUILD= -DSKIP_TESTS=true
-DENABLE_ALL="${ENABLE_ALL}" -DENABL
-DDISABLE_EXPRESSION_LANGUAGE="${DISABLE_EXPRESSION_LANGUAGE}"
-DDISABLE_ROCKSDB="${DISABLE_ROCKSDB}" \
-DDISABLE_LIBARCHIVE="${DISABLE_LIBARCHIVE}"
-DDISABLE_LZMA="${DISABLE_LZMA}" -DDISABLE_BZIP2="${DISABLE_BZIP2}" \
-DDISABLE_SCRIPTING="${DISABLE_SCRIPTING}"
-DDISABLE_PYTHON_SCRIPTING="${DISABLE_PYTHON_SCRIPTING}"
-DDISABLE_CONTROLLER="${DISABLE_CONTROLLER}" \
- -DENABLE_ENCRYPT_CONFIG="${ENABLE_ENCRYPT_CONFIG}"
-DAWS_ENABLE_UNITY_BUILD=OFF -DEXCLUDE_BOOST=ON
-DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \
- && make -j "$(nproc)" package \
- && tar -xzvf
"${MINIFI_BASE_DIR}/build/nifi-minifi-cpp-${MINIFI_VERSION}-bin.tar.gz" -C
"${MINIFI_BASE_DIR}"
+ -DENABLE_ENCRYPT_CONFIG="${ENABLE_ENCRYPT_CONFIG}"
-DAWS_ENABLE_UNITY_BUILD=OFF -DEXCLUDE_BOOST=ON
-DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \
+ make -j "$(nproc)" package && \
+ tar -xzvf
"${MINIFI_BASE_DIR}/build/nifi-minifi-cpp-${MINIFI_VERSION}-bin.tar.gz" -C
"${MINIFI_BASE_DIR}"
# Release image
