This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/master by this push:
new 786d288 YETUS-791. Add support for --cache-from to speed up building
786d288 is described below
commit 786d2886f0b52d3a6fc01b4fa30dcc58baecf43d
Author: Allen Wittenauer <[email protected]>
AuthorDate: Sat Feb 2 09:56:55 2019 -0800
YETUS-791. Add support for --cache-from to speed up building
Signed-off-by: Allen Wittenauer <[email protected]>
---
Jenkinsfile | 2 ++
.../source/documentation/in-progress/precommit-docker.md | 4 ++++
precommit/src/main/shell/core.d/docker.sh | 15 +++++++++++++++
start-build-env.sh | 7 +++++--
4 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index e067994..ff9a624 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -172,8 +172,10 @@ pipeline {
# run in docker mode and specifically point to our
# Dockerfile since we don't want to use the auto-pulled
version.
if [[ "${USE_DOCKER_FLAG}" == true ]]; then
+ docker pull ubuntu:xenial
YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${YETUS_DOCKERFILE}")
+ YETUS_ARGS+=("--docker-cache-from=apache/yetus-base:master")
else
# need to figure this out programmatically; hard-coded for
now
export JAVA_HOME=/home/jenkins/tools/java/latest1.8
diff --git a/asf-site-src/source/documentation/in-progress/precommit-docker.md
b/asf-site-src/source/documentation/in-progress/precommit-docker.md
index f65d1bf..c051a1c 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-docker.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-docker.md
@@ -57,6 +57,10 @@ In order to use both 'YETUS CUT HERE' and a Dockerfile that
uses COPY and ADD di
Instead of processing a Dockerfile, test-patch can pull a tag from a
repository using the `--docker-tag` parameter. Note that the repository must
already be logged into and configured prior to executing test-patch.
+## Using a cache
+
+With the `--docker-cache-from` parameter, other images may be utilized to
provide a cache when building a Dockerfile. This comma delimited list will
automatically be pulled (errors are ignored) and given to the docker command
line to use.
+
## Platforms
When either building or pull an image, test-patch supports the
`--docker-platform` flag to pass in the Docker `--platform` flag. This allows
you full control over what kind of image the software either creates or fetches.
diff --git a/precommit/src/main/shell/core.d/docker.sh
b/precommit/src/main/shell/core.d/docker.sh
index d298d19..12621d3 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -62,6 +62,7 @@ function docker_usage
yetus_add_option "--dockercmd=<file>" "Command to use as docker executable
(default: '${DOCKERCMD}')"
if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
yetus_add_option "--docker-bash-debug=<bool>" "Enable bash -x mode running
in a container (default: ${YETUS_DOCKER_BASH_DEBUG})"
+ yetus_add_option "--docker-cache-from=<image>" "Comma delimited images to
use as a cache when building"
yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the
base (default: '${DOCKERFILE_DEFAULT}')"
yetus_add_option "--dockerind=<bool>" "Enable Docker-in-Docker by mounting
the Docker socket in the container (default: '${DOCKER_IN_DOCKER}')"
yetus_add_option "--docker-platform=<plat>" "Use a platform string for
building and pulling (default: ${DOCKER_PLATFORM})"
@@ -93,6 +94,9 @@ function docker_parse_args
YETUS_DOCKER_BASH_DEBUG=${i#*=}
add_docker_env YETUS_DOCKER_BASH_DEBUG
;;
+ --docker-cache-from=*)
+ DOCKER_CACHE_FROM=${i#*=}
+ ;;
--dockercmd=*)
#shellcheck disable=SC2034
DOCKERCMD=${i#*=}
@@ -561,6 +565,8 @@ function docker_run_image
declare lines
declare dockerversion
declare -a dockplat
+ declare -a cachefrom
+ declare -a images
big_console_header "Docker Image Creation"
start_clock
@@ -633,8 +639,17 @@ function docker_run_image
cp -p "${buildfile}" "${PATCH_DIR}/Dockerfile"
fi
+ if [[ -n "${DOCKER_CACHE_FROM}" ]]; then
+ yetus_comma_to_array images "${DOCKER_CACHE_FROM}"
+ for i in "${images[@]}"; do
+ docker pull "${i}" || true
+ done
+
cachefrom=("--cache-from=yetus/${PROJECT_NAME}:${gitfilerev},${DOCKER_CACHE_FROM}")
+ fi
+
if ! dockercmd build \
"${dockplat[@]}" \
+ "${cachefrom[@]}" \
--label org.apache.yetus=\"\" \
--label org.apache.yetus.testpatch.project="${PROJECT_NAME}" \
--tag "${baseimagename}" \
diff --git a/start-build-env.sh b/start-build-env.sh
index d2d024a..dc7160d 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -26,11 +26,14 @@ if [[ "${BRANCH}" =~ HEAD ]]; then
BRANCH=$(git branch | grep '\*' | awk '{print $NF}' | sed -e s,rel/,,g -e
s,\),,g )
fi
-echo "Attempting a pull of apache/yetus-base:${BRANCH} and
apache/yetus-base:master to save time"
+echo "Attempting a few pulls of apache/yetus and apache/yetus-base to save
time"
echo "Errors here will be ignored!"
docker pull "apache/yetus-base:${BRANCH}" || docker pull
"apache/yetus-base:master" || true
+docker pull "apache/yetus:${BRANCH}" || docker pull "apache/yetus:master" ||
true
-docker build -t "apache/yetus-build:${BRANCH}" .
+docker build \
+
--cache-from="apache/yetus-base:${BRANCH},apache/yetus-base:master,apache/yetus:${BRANCH},apache/yetus:master"
\
+ -t "apache/yetus-build:${BRANCH}" .
USER_NAME=${SUDO_USER:=$USER}
USER_ID=$(id -u "${USER_NAME}")
