jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/366726 )
Change subject: Dockerfiles use build container pattern
......................................................................
Dockerfiles use build container pattern
This change modifies the dockerfiles/build.sh script to always look for
a `Dockerfile.build` file from which a container will be created. The
container created from `Dockerfile.build` will have everything from
`/tmp/cache` copied into the dockerfile directory where the CI image is
created.
This pattern allows `Dockerfile.build` files that cache a repo and all
testing dependencies within the container itself.
Bug: T166888
Change-Id: I6b72d194e9eb353bff64c27a7308d3a6d5552bd6
---
M .gitignore
M dockerfiles/build.sh
M dockerfiles/contint-operations-puppet/Dockerfile
A dockerfiles/contint-operations-puppet/Dockerfile.build
M dockerfiles/contint-operations-puppet/run.sh
D dockerfiles/share/wikimedia.list
D dockerfiles/share/wikimedia.pref
M jjb/operations-puppet.yaml
8 files changed, 112 insertions(+), 60 deletions(-)
Approvals:
Thcipriani: Looks good to me, approved
jenkins-bot: Verified
diff --git a/.gitignore b/.gitignore
index e14c72b..5e5c309 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,5 @@
/Gemfile.lock
-dockerfiles/share/wikimedia-archive-keyring.gpg
+/dockerfiles/*/cache
+/dockerfiles/*/.cache-buster
diff --git a/dockerfiles/build.sh b/dockerfiles/build.sh
index 2518136..dbf6caf 100755
--- a/dockerfiles/build.sh
+++ b/dockerfiles/build.sh
@@ -3,15 +3,54 @@
set -eu
-BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+info() {
+ printf "[$(tput setaf 3)INFO$(tput sgr 0)] %b\n" "$@"
+}
-wget -O "$BASE_DIR/share/wikimedia-archive-keyring.gpg" \
- http://apt.wikimedia.org/autoinstall/keyring/wikimedia-archive-keyring.gpg
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DOCKER_HUB_USER=wmfreleng
for dockerbuild in "$BASE_DIR"/contint-*/Dockerfile; do
CONTAINER_DIR="${dockerbuild%/*}"
CONTAINER_NAME="${CONTAINER_DIR##*/}"
- docker build -t contint/"${CONTAINER_NAME:8}" -f "${dockerbuild}"
"$BASE_DIR"
-done
-rm -rf "$BASE_DIR/share/wikimedia-archive-keyring.gpg" \
+ IMG_TAG="${DOCKER_HUB_USER}/${CONTAINER_NAME:8}"
+
+ BUILD_IMG_TAG="${IMG_TAG}-img"
+ BUILD_CONTAINER_TAG="${CONTAINER_NAME:8}-build"
+
+ pushd "$CONTAINER_DIR" &>/dev/null
+ info "BUILDING $CONTAINER_NAME"
+
+ # Run a standard build step if there is one
+ if [[ -f "Dockerfile.build" ]]; then
+ info "...Creating build image ${BUILD_IMG_TAG}"
+ touch .cache-buster
+ docker build \
+ -t "${BUILD_IMG_TAG}" \
+ -f "Dockerfile.build" .
+
+ info "...Creating build container ${BUILD_CONTAINER_TAG} from
${BUILD_IMG_TAG}"
+
+ # Remove any container that may exist with that container name first
+ docker rm $BUILD_CONTAINER_TAG || /bin/true
+
+ docker create --name "${BUILD_CONTAINER_TAG}" "${BUILD_IMG_TAG}"
+
+ # Make sure to purge old cache before making new one
+ if [[ -d "cache" ]]; then
+ info "...Purging old 'cache'"
+ rm -rf "cache"
+ fi
+
+ info "...Copying ${BUILD_CONTAINER_TAG}:/tmp/cache artifact to 'cache'"
+ docker cp "${BUILD_CONTAINER_TAG}:/tmp/cache" cache
+ fi
+
+ info "...Creating final image ${IMG_TAG}"
+ docker build \
+ -t "${IMG_TAG}" \
+ -f "Dockerfile" .
+
+ popd &>/dev/null
+done
diff --git a/dockerfiles/contint-operations-puppet/Dockerfile
b/dockerfiles/contint-operations-puppet/Dockerfile
index 47d77ff..3f40389 100644
--- a/dockerfiles/contint-operations-puppet/Dockerfile
+++ b/dockerfiles/contint-operations-puppet/Dockerfile
@@ -1,24 +1,20 @@
-FROM debian:jessie-slim
-COPY share /opt
+FROM docker-registry.wikimedia.org/wikimedia-jessie:latest
-RUN apt-key add /opt/wikimedia-archive-keyring.gpg && \
- cp /opt/wikimedia.list /etc/apt/sources.list.d/wikimedia.list && \
- cp /opt/wikimedia.pref /etc/apt/preferences.d/wikimedia.pref
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
RUN apt-get update && \
apt-get install --yes --no-install-recommends \
build-essential \
bundler \
git \
- libmysqlclient-dev \
locales \
python-dev \
python-pip \
rubygems-integration \
rake \
ruby \
- ruby-dev \
- && \
+ ruby-dev && \
+ apt-get clean && rm -rf /var/lib/apt/lists/* && \
pip install pip==8.1.2 && \
pip install tox==1.9.2 setuptools && \
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
@@ -29,9 +25,12 @@
--home-dir /var/lib/jenkins \
--create-home jenkins
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
+COPY cache /var/lib/jenkins/.cache
+
+RUN chown -R 2947:500 /var/lib/jenkins/.cache
+
USER jenkins
WORKDIR /var/lib/jenkins
ENTRYPOINT /bin/bash /run.sh
-COPY contint-operations-puppet/run.sh /run.sh
+COPY run.sh /run.sh
diff --git a/dockerfiles/contint-operations-puppet/Dockerfile.build
b/dockerfiles/contint-operations-puppet/Dockerfile.build
new file mode 100644
index 0000000..0db4637
--- /dev/null
+++ b/dockerfiles/contint-operations-puppet/Dockerfile.build
@@ -0,0 +1,30 @@
+FROM docker-registry.wikimedia.org/wikimedia-jessie:latest
+
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
+
+RUN apt-get update && \
+ apt-get install --yes --no-install-recommends \
+ build-essential \
+ bundler \
+ git \
+ libmysqlclient-dev \
+ locales \
+ python-dev \
+ python-pip \
+ rubygems-integration \
+ rake \
+ ruby \
+ ruby-dev \
+ && \
+ pip install pip==8.1.2 && \
+ pip install tox==1.9.2 setuptools
+
+COPY .cache-buster /.cache-buster
+
+# Make all kinds of artifacts
+RUN mkdir -p /tmp/cache && \
+ git clone https://gerrit.wikimedia.org/r/operations/puppet
/tmp/cache/puppet && \
+ cd /tmp/cache/puppet && \
+ bundle install --clean --path="/tmp/cache/bundle" && \
+ tox -v || /bin/true && \
+ mv .tox /tmp/cache/tox
diff --git a/dockerfiles/contint-operations-puppet/run.sh
b/dockerfiles/contint-operations-puppet/run.sh
index a1f16d8..bff2ea4 100644
--- a/dockerfiles/contint-operations-puppet/run.sh
+++ b/dockerfiles/contint-operations-puppet/run.sh
@@ -2,34 +2,43 @@
set -euxo pipefail
-TEMP_DIR=$(mktemp -d)
+# Has to be in the same directory as in Dockerfile.build because of tox caching
+PUPPET_DIR="/tmp/cache/puppet"
+
LOG_DIR="$HOME/log"
CACHE_DIR="$HOME/.cache"
capture_logs() {
# Save logs
- mv "${TEMP_DIR}"/puppet/.tox/*/log/*.log "${LOG_DIR}/" || /bin/true
- mv "${TEMP_DIR}"/puppet/.tox/log/* "${LOG_DIR}/" || /bin/true
+ mv "${PUPPET_DIR}"/.tox/*/log/*.log "${LOG_DIR}/" || /bin/true
+ mv "${PUPPET_DIR}"/.tox/log/* "${LOG_DIR}/" || /bin/true
}
trap capture_logs EXIT
-git clone \
- --quiet \
- --reference "/srv/git/${ZUUL_PROJECT}.git" \
- "$ZUUL_URL/$ZUUL_PROJECT" \
- "$TEMP_DIR/puppet"
+mkdir -p /tmp/cache
-cd "$TEMP_DIR/puppet"
+mv "${CACHE_DIR}/puppet" "$PUPPET_DIR"
+cd "$PUPPET_DIR"
-git fetch --quiet origin $ZUUL_REF
+# Prepare patch set from zuul merger
+git remote add zuul "${ZUUL_URL}/${ZUUL_PROJECT}"
+git pull --quiet zuul production
+git fetch --quiet zuul "$ZUUL_REF"
git checkout --quiet FETCH_HEAD
git submodule --quiet update --init --recursive
-[ -d /cache/.cache ] && {
- mkdir -p "$CACHE_DIR"
- cp -R /cache/.cache/* "${CACHE_DIR}/"
-}
+# Tox setup
+mv "${CACHE_DIR}/tox" '.tox'
+
+# Bundle setup
+mkdir -p .bundle
+cat <<BUNDLE > .bundle/config
+---
+BUNDLE_PATH: "/var/lib/jenkins/.cache/bundle"
+BUNDLE_CLEAN: true
+BUNDLE_DISABLE_SHARED_GEMS: '1'
+BUNDLE
# Run tox tests
{
@@ -41,8 +50,9 @@
# Run rake tests
{
- bundle install --clean
+ set -o pipefail
bundle exec rake test | tee "${LOG_DIR}/rake.log"
+ set +o pipefail
} &
PID_TWO=$!
diff --git a/dockerfiles/share/wikimedia.list b/dockerfiles/share/wikimedia.list
deleted file mode 100644
index ae55272..0000000
--- a/dockerfiles/share/wikimedia.list
+++ /dev/null
@@ -1,2 +0,0 @@
-deb http://apt.wikimedia.org/wikimedia jessie-wikimedia main experimental
backports
-deb-src http://apt.wikimedia.org/wikimedia jessie-wikimedia main backports
diff --git a/dockerfiles/share/wikimedia.pref b/dockerfiles/share/wikimedia.pref
deleted file mode 100644
index d338d23..0000000
--- a/dockerfiles/share/wikimedia.pref
+++ /dev/null
@@ -1,3 +0,0 @@
-Package: *
-Pin: release o=Wikimedia
-Pin-Priority: 1001
diff --git a/jjb/operations-puppet.yaml b/jjb/operations-puppet.yaml
index d07722b..b395401 100644
--- a/jjb/operations-puppet.yaml
+++ b/jjb/operations-puppet.yaml
@@ -122,29 +122,9 @@
set -x
rm -rf log
- rm -rf cache
rm -rf .env
- # XXX: modify castor job before moving to this fully
- CASTOR_URL=castor02.integration.eqiad.wmflabs
- NAMESPACE=operations-puppet/production/operations-puppet-tests-jessie
- CACHE="$(pwd)/cache"
- BUNDLE_PATH=/cache/workspace/vendor/bundle
-
- rsync \
- --archive \
- --compress \
- --contimeout 3 \
- rsync://${CASTOR_URL}:/caches/$NAMESPACE/ $CACHE \
- || :
-
- # In case rsync fails
- mkdir -p "$(pwd)${BUNDLE_PATH}"
-
- git -C /srv/git/operations/puppet.git remote update -p
-
cat <<ZUUL > .env
- BUNDLE_PATH=$BUNDLE_PATH
ZUUL_URL=$ZUUL_URL
ZUUL_PROJECT=$ZUUL_PROJECT
ZUUL_COMMIT=$ZUUL_COMMIT
@@ -157,10 +137,8 @@
docker run \
--rm --tty \
--env-file .env \
- --volume /srv/git:/srv/git:ro \
--volume "$(pwd)"/log:/var/lib/jenkins/log \
- --volume "$CACHE":/cache \
-
ci-staging-docker-registry01.wmflabs.org/contint/operations-puppet:v0.0.4
+ wmfreleng/operations-puppet:0.0.1
wrappers:
- timeout:
timeout: 1 # minute
--
To view, visit https://gerrit.wikimedia.org/r/366726
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6b72d194e9eb353bff64c27a7308d3a6d5552bd6
Gerrit-PatchSet: 2
Gerrit-Project: integration/config
Gerrit-Branch: master
Gerrit-Owner: Thcipriani <[email protected]>
Gerrit-Reviewer: Hashar <[email protected]>
Gerrit-Reviewer: Paladox <[email protected]>
Gerrit-Reviewer: Thcipriani <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
