This is an automated email from the ASF dual-hosted git repository. marcoabreu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new f107397 [MXNET-454] Move distributed Docker cache from S3 to Docker Hub (#11018) f107397 is described below commit f107397b753ff08bb19e7572bc1a9ebedd832f88 Author: Marco de Abreu <marcoab...@users.noreply.github.com> AuthorDate: Sun Jun 3 15:45:30 2018 +0200 [MXNET-454] Move distributed Docker cache from S3 to Docker Hub (#11018) * Add test cases for Docker cache and move from S3 to Docker registry * Remove unnecessary args * Fix lint * Address review --- Jenkinsfile | 2 +- ci/Jenkinsfile_docker_cache | 8 +- ci/build.py | 67 ++++++----- ci/docker_cache.py | 264 ++++++++++++++++---------------------------- ci/test_docker_cache.py | 252 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 391 insertions(+), 202 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c3fe535..288f9a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,7 +93,7 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum } def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') { - def command = "ci/build.py --download-docker-cache --docker-cache-bucket ${env.DOCKER_CACHE_BUCKET} %USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" + def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%" command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '') command = command.replaceAll('%PLATFORM%', platform) command = command.replaceAll('%FUNCTION_NAME%', function_name) diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache index 8a0428b..eba3a49 100644 --- a/ci/Jenkinsfile_docker_cache +++ b/ci/Jenkinsfile_docker_cache @@ -49,11 +49,11 @@ def init_git() { try { stage("Docker cache build & publish") { - node('mxnetlinux-cpu') { + node('restricted-mxnetlinux-cpu') { ws('workspace/docker_cache') { timeout(time: total_timeout, unit: 'MINUTES') { init_git() - sh "ci/docker_cache.py --docker-cache-bucket ${env.DOCKER_CACHE_BUCKET}" + sh "ci/docker_cache.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}" } } } @@ -62,13 +62,13 @@ try { // set build status to success at the end currentBuild.result = "SUCCESS" } catch (caughtError) { - node("mxnetlinux-cpu") { + node("restricted-mxnetlinux-cpu") { sh "echo caught ${caughtError}" err = caughtError currentBuild.result = "FAILURE" } } finally { - node("mxnetlinux-cpu") { + node("restricted-mxnetlinux-cpu") { // Only send email if master failed if (currentBuild.result == "FAILURE" && env.BRANCH_NAME == "master") { emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' diff --git a/ci/build.py b/ci/build.py index deae1d7..e52fa79 100755 --- a/ci/build.py +++ b/ci/build.py @@ -48,8 +48,8 @@ def get_platforms(path: Optional[str]="docker"): return platforms -def get_docker_tag(platform: str) -> str: - return "mxnet/build.{0}".format(platform) +def get_docker_tag(platform: str, registry: str) -> str: + return "{0}/build.{1}".format(registry, platform) def get_dockerfile(platform: str, path="docker") -> str: @@ -60,19 +60,19 @@ def get_docker_binary(use_nvidia_docker: bool) -> str: return "nvidia-docker" if use_nvidia_docker else "docker" -def build_docker(platform: str, docker_binary: str) -> None: +def build_docker(platform: str, docker_binary: str, registry: str) -> None: """ Build a container for the given platform :param platform: Platform :param docker_binary: docker binary to use (docker/nvidia-docker) + :param registry: Dockerhub registry name :return: Id of the top level image """ - tag = get_docker_tag(platform) + tag = get_docker_tag(platform=platform, registry=registry) logging.info("Building container tagged '%s' with %s", tag, docker_binary) cmd = [docker_binary, "build", "-f", get_dockerfile(platform), - "--rm=false", # Keep intermediary layers to prime the build cache "--build-arg", "USER_ID={}".format(os.getuid()), "--cache-from", tag, "-t", tag, @@ -118,11 +118,12 @@ def buildir() -> str: def container_run(platform: str, docker_binary: str, + docker_registry: str, shared_memory_size: str, command: List[str], dry_run: bool = False, into_container: bool = False) -> str: - tag = get_docker_tag(platform) + tag = get_docker_tag(platform=platform, registry=docker_registry) mx_root = get_mxnet_root() local_build_folder = buildir() # We need to create it first, otherwise it will be created by the docker daemon with root only permissions @@ -159,6 +160,19 @@ def container_run(platform: str, def list_platforms() -> str: print("\nSupported platforms:\n{}".format('\n'.join(get_platforms()))) + +def load_docker_cache(tag, docker_registry) -> None: + if docker_registry: + try: + import docker_cache + logging.info('Docker cache download is enabled') + docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag) + except Exception: + logging.exception('Unable to retrieve Docker cache. Continue without...') + else: + logging.info('Distributed docker cache disabled') + + def main() -> int: # We need to be in the same directory than the script so the commands in the dockerfiles work as # expected. But the script can be invoked from a different path @@ -207,12 +221,9 @@ def main() -> int: help="go in a shell inside the container", action='store_true') - parser.add_argument("--download-docker-cache", - help="Download the docker cache from our central repository instead of rebuilding locally", - action='store_true') - - parser.add_argument("--docker-cache-bucket", - help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache", + parser.add_argument("--docker-registry", + help="Dockerhub registry name to retrieve cache from", + default='mxnetci', type=str) parser.add_argument("command", @@ -220,6 +231,7 @@ def main() -> int: nargs='*', action='append', type=str) args = parser.parse_args() + docker_registry = args.docker_registry command = list(chain(*args.command)) docker_binary = get_docker_binary(args.nvidiadocker) shared_memory_size = args.shared_memory_size @@ -229,44 +241,43 @@ def main() -> int: list_platforms() elif args.platform: platform = args.platform - tag = get_docker_tag(platform) - if args.download_docker_cache: - import docker_cache - logging.info('Docker cache download is enabled') - docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag) - build_docker(platform, docker_binary) + tag = get_docker_tag(platform=platform, registry=docker_registry) + load_docker_cache(tag=tag, docker_registry=args.docker_registry) + build_docker(platform, docker_binary, registry=docker_registry) if args.build_only: logging.warning("Container was just built. Exiting due to build-only.") return 0 if command: - container_run(platform, docker_binary, shared_memory_size, command) + container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size, + command=command, docker_registry=docker_registry) elif args.print_docker_run: - print(container_run(platform, docker_binary, shared_memory_size, [], True)) + print(container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size, + command=[], dry_run=True, docker_registry=docker_registry)) elif args.into_container: - container_run(platform, docker_binary, shared_memory_size, [], False, True) + container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size, + command=[], dry_run=False, into_container=True, docker_registry=docker_registry) else: cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)] logging.info("No command specified, trying default build: %s", ' '.join(cmd)) - container_run(platform, docker_binary, shared_memory_size, cmd) + container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size, + command=cmd, docker_registry=docker_registry) elif args.all: platforms = get_platforms() logging.info("Building for all architectures: {}".format(platforms)) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: - if args.download_docker_cache: - import docker_cache - tag = get_docker_tag(platform) - logging.info('Docker cache download is enabled') - docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag) + tag = get_docker_tag(platform=platform, registry=docker_registry) + load_docker_cache(tag=tag, docker_registry=args.docker_registry) build_docker(platform, docker_binary) if args.build_only: continue build_platform = "build_{}".format(platform) cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform] shutil.rmtree(buildir(), ignore_errors=True) - container_run(platform, docker_binary, shared_memory_size, cmd) + container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size, + command=cmd, docker_registry=docker_registry) plat_buildir = os.path.join(get_mxnet_root(), build_platform) shutil.move(buildir(), plat_buildir) logging.info("Built files left in: %s", plat_buildir) diff --git a/ci/docker_cache.py b/ci/docker_cache.py index 7fdfbcf..16abb9e 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -28,237 +28,164 @@ import os import logging import argparse import sys -import boto3 -import tempfile -import pprint -import threading -import build as build_util -import botocore import subprocess -from botocore.handlers import disable_signing -from subprocess import call, check_call, CalledProcessError +import json +import build as build_util from joblib import Parallel, delayed -S3_METADATA_IMAGE_ID_KEY = 'docker-image-id' -LOG_PROGRESS_PERCENTAGE_THRESHOLD = 10 - -cached_aws_session = None - - -class ProgressPercentage(object): - def __init__(self, object_name, size): - self._object_name = object_name - self._size = size - self._seen_so_far = 0 - self._last_percentage = 0 - self._lock = threading.Lock() - def __call__(self, bytes_amount) -> None: - # To simplify we'll assume this is hooked up - # to a single filename. - with self._lock: - self._seen_so_far += bytes_amount - percentage = int((self._seen_so_far / self._size) * 100) - if (percentage - self._last_percentage) >= LOG_PROGRESS_PERCENTAGE_THRESHOLD: - self._last_percentage = percentage - logging.info('{}% of {}'.format(percentage, self._object_name)) - -def build_save_containers(platforms, bucket) -> int: +def build_save_containers(platforms, registry, load_cache) -> int: """ Entry point to build and upload all built dockerimages in parallel :param platforms: List of platforms - :param bucket: S3 bucket name + :param registry: Docker registry name + :param load_cache: Load cache before building :return: 1 if error occurred, 0 otherwise """ if len(platforms) == 0: return 0 platform_results = Parallel(n_jobs=len(platforms), backend="multiprocessing")( - delayed(_build_save_container)(platform, bucket) + delayed(_build_save_container)(platform, registry, load_cache) for platform in platforms) is_error = False for platform_result in platform_results: if platform_result is not None: - logging.error('Failed to generate {}'.format(platform_result)) + logging.error('Failed to generate %s', platform_result) is_error = True return 1 if is_error else 0 -def _build_save_container(platform, bucket) -> str: +def _build_save_container(platform, registry, load_cache) -> str: """ Build image for passed platform and upload the cache to the specified S3 bucket :param platform: Platform - :param bucket: Target s3 bucket + :param registry: Docker registry name + :param load_cache: Load cache before building :return: Platform if failed, None otherwise """ - docker_tag = build_util.get_docker_tag(platform) + docker_tag = build_util.get_docker_tag(platform=platform, registry=registry) # Preload cache - # TODO: Allow to disable this in order to allow clean rebuilds - load_docker_cache(bucket_name=bucket, docker_tag=docker_tag) + if load_cache: + load_docker_cache(registry=registry, docker_tag=docker_tag) # Start building - logging.debug('Building {} as {}'.format(platform, docker_tag)) + logging.debug('Building %s as %s', platform, docker_tag) try: - image_id = build_util.build_docker(docker_binary='docker', platform=platform) - logging.info('Built {} as {}'.format(docker_tag, image_id)) + image_id = build_util.build_docker(docker_binary='docker', platform=platform, registry=registry) + logging.info('Built %s as %s', docker_tag, image_id) - # Compile and upload tarfile - _compile_upload_cache_file(bucket_name=bucket, docker_tag=docker_tag, image_id=image_id) + # Push cache to registry + _upload_image(registry=registry, docker_tag=docker_tag, image_id=image_id) return None except Exception: - logging.exception('Unexpected exception during build of {}'.format(docker_tag)) + logging.exception('Unexpected exception during build of %s', docker_tag) return platform # Error handling is done by returning the errorous platform name. This is necessary due to # Parallel being unable to handle exceptions -def _compile_upload_cache_file(bucket_name, docker_tag, image_id) -> None: +def _upload_image(registry, docker_tag, image_id) -> None: """ Upload the passed image by id, tag it with docker tag and upload to S3 bucket - :param bucket_name: S3 bucket name + :param registry: Docker registry name :param docker_tag: Docker tag :param image_id: Image id :return: None """ - session = _get_aws_session() - s3_object = session.resource('s3').Object(bucket_name, docker_tag) - - remote_image_id = _get_remote_image_id(s3_object) - if remote_image_id == image_id: - logging.info('{} ({}) for {} has not been updated - skipping'.format(docker_tag, image_id, docker_tag)) - return - else: - logging.debug('Cached image {} differs from local {} for {}'.format(remote_image_id, image_id, docker_tag)) + _login_dockerhub() + # We don't have to retag the image since it is already in the right format + logging.info('Uploading %s (%s) to %s', docker_tag, image_id, registry) + push_cmd = ['docker', 'push', docker_tag] + subprocess.check_call(push_cmd) - # Compile layers into tarfile - with tempfile.TemporaryDirectory() as temp_dir: - tar_file_path = _format_docker_cache_filepath(output_dir=temp_dir, docker_tag=docker_tag) - logging.debug('Writing layers of {} to {}'.format(docker_tag, tar_file_path)) - history_cmd = ['docker', 'history', '-q', docker_tag] - image_ids_b = subprocess.check_output(history_cmd) - image_ids_str = image_ids_b.decode('utf-8').strip() - layer_ids = [id.strip() for id in image_ids_str.split('\n') if id != '<missing>'] - - # docker_tag is important to preserve the image name. Otherwise, the --cache-from feature will not be able to - # reference the loaded cache later on. The other layer ids are added to ensure all intermediary layers - # are preserved to allow resuming the cache at any point - cmd = ['docker', 'save', '-o', tar_file_path, docker_tag] - cmd.extend(layer_ids) - try: - check_call(cmd) - except CalledProcessError as e: - logging.error('Error during save of {} at {}. Command: {}'. - format(docker_tag, tar_file_path, pprint.pprint(cmd))) - return - - # Upload file - logging.info('Uploading {} to S3'.format(docker_tag)) - with open(tar_file_path, 'rb') as data: - s3_object.upload_fileobj( - Fileobj=data, - Callback=ProgressPercentage(object_name=docker_tag, size=os.path.getsize(tar_file_path)), - ExtraArgs={"Metadata": {S3_METADATA_IMAGE_ID_KEY: image_id}}) - logging.info('Uploaded {} to S3'.format(docker_tag)) - - -def _get_remote_image_id(s3_object) -> str: +def _login_dockerhub(): """ - Get the image id of the docker cache which is represented by the S3 object - :param s3_object: S3 object - :return: Image id as string or None if object does not exist + Login to the Docker Hub account + :return: None """ - try: - if S3_METADATA_IMAGE_ID_KEY in s3_object.metadata: - cached_image_id = s3_object.metadata[S3_METADATA_IMAGE_ID_KEY] - return cached_image_id - else: - logging.debug('No cached image available for {}'.format(s3_object.key)) - except botocore.exceptions.ClientError as e: - if e.response['Error']['Code'] == "404": - logging.debug('{} does not exist in S3 yet'.format(s3_object.key)) - else: - raise + dockerhub_credentials = _get_dockerhub_credentials() + login_cmd = ['docker', 'login', '--username', dockerhub_credentials['username'], '--password', + dockerhub_credentials['password']] + subprocess.check_call(login_cmd) - return None - -def load_docker_cache(bucket_name, docker_tag) -> None: +def load_docker_cache(registry, docker_tag) -> None: """ - Load the precompiled docker cache from the passed S3 bucket - :param bucket_name: S3 bucket name + Load the precompiled docker cache from the registry + :param registry: Docker registry name :param docker_tag: Docker tag to load :return: None """ - # Allow anonymous access - s3_resource = boto3.resource('s3') - s3_resource.meta.client.meta.events.register('choose-signer.s3.*', disable_signing) - s3_object = s3_resource.Object(bucket_name, docker_tag) - - # Check if cache is still valid and exists - remote_image_id = _get_remote_image_id(s3_object) - if remote_image_id: - if _docker_layer_exists(remote_image_id): - logging.info('Local docker cache already present for {}'.format(docker_tag)) - return - else: - logging.info('Local docker cache not present for {}'.format(docker_tag)) - - # Download using public S3 endpoint (without requiring credentials) - with tempfile.TemporaryDirectory() as temp_dir: - tar_file_path = os.path.join(temp_dir, 'layers.tar') - s3_object.download_file( - Filename=tar_file_path, - Callback=ProgressPercentage(object_name=docker_tag, size=s3_object.content_length)) - - # Load layers - cmd = ['docker', 'load', '-i', tar_file_path] - try: - check_call(cmd) - logging.info('Docker cache for {} loaded successfully'.format(docker_tag)) - except CalledProcessError as e: - logging.error('Error during load of docker cache for {} at {}'.format(docker_tag, tar_file_path)) - logging.exception(e) - return - else: - logging.info('No cached remote image of {} present'.format(docker_tag)) - - -def _docker_layer_exists(layer_id) -> bool: - """ - Check if the docker cache contains the layer with the passed id - :param layer_id: layer id - :return: True if exists, False otherwise - """ - cmd = ['docker', 'images', '-q'] - image_ids_b = subprocess.check_output(cmd) - image_ids_str = image_ids_b.decode('utf-8').strip() - return layer_id in [id.strip() for id in image_ids_str.split('\n')] + # We don't have to retag the image since it's already in the right format + logging.info('Loading Docker cache for %s from %s', docker_tag, registry) + pull_cmd = ['docker', 'pull', docker_tag] + subprocess.call(pull_cmd) # Don't throw an error if the image does not exist -def _get_aws_session() -> boto3.Session: # pragma: no cover +def delete_local_docker_cache(docker_tag): """ - Get the boto3 AWS session - :return: Session object + Delete the local docker cache for the entire docker image chain + :param docker_tag: Docker tag + :return: None """ - global cached_aws_session - if cached_aws_session: - return cached_aws_session - - session = boto3.Session() # Uses IAM user credentials - cached_aws_session = session - return session + history_cmd = ['docker', 'history', '-q', docker_tag] + try: + image_ids_b = subprocess.check_output(history_cmd) + image_ids_str = image_ids_b.decode('utf-8').strip() + layer_ids = [id.strip() for id in image_ids_str.split('\n') if id != '<missing>'] -def _format_docker_cache_filepath(output_dir, docker_tag) -> str: - return os.path.join(output_dir, docker_tag.replace('/', '_') + '.tar') + delete_cmd = ['docker', 'image', 'rm', '--force'] + delete_cmd.extend(layer_ids) + subprocess.check_call(delete_cmd) + except subprocess.CalledProcessError as error: + # Could be caused by the image not being present + logging.debug('Error during local cache deletion %s', error) + + +def _get_dockerhub_credentials(): # pragma: no cover + import boto3 + import botocore + secret_name = os.environ['DOCKERHUB_SECRET_NAME'] + endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL'] + region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION'] + + session = boto3.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name, + endpoint_url=endpoint_url + ) + try: + get_secret_value_response = client.get_secret_value( + SecretId=secret_name + ) + except botocore.exceptions.ClientError as client_error: + if client_error.response['Error']['Code'] == 'ResourceNotFoundException': + logging.exception("The requested secret %s was not found", secret_name) + elif client_error.response['Error']['Code'] == 'InvalidRequestException': + logging.exception("The request was invalid due to:") + elif client_error.response['Error']['Code'] == 'InvalidParameterException': + logging.exception("The request had invalid params:") + else: + raise + else: + secret = get_secret_value_response['SecretString'] + secret_dict = json.loads(secret) + return secret_dict def main() -> int: + """ + Utility to create and publish the Docker cache to Docker Hub + :return: + """ # We need to be in the same directory than the script so the commands in the dockerfiles work as # expected. But the script can be invoked from a different path base = os.path.split(os.path.realpath(__file__))[0] @@ -275,17 +202,16 @@ def main() -> int: logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name())) - parser = argparse.ArgumentParser(description="Utility for preserving and loading Docker cache",epilog="") - parser.add_argument("--docker-cache-bucket", - help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache", + parser = argparse.ArgumentParser(description="Utility for preserving and loading Docker cache", epilog="") + parser.add_argument("--docker-registry", + help="Docker hub registry name", type=str, required=True) args = parser.parse_args() platforms = build_util.get_platforms() - _get_aws_session() # Init AWS credentials - return build_save_containers(platforms=platforms, bucket=args.docker_cache_bucket) + return build_save_containers(platforms=platforms, registry=args.docker_registry, load_cache=True) if __name__ == '__main__': diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py new file mode 100644 index 0000000..fa8833f --- /dev/null +++ b/ci/test_docker_cache.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Distributed Docker cache tests +""" + +import unittest.mock +import tempfile +import os +import logging +import subprocess +import sys + +sys.path.append(os.path.dirname(__file__)) +import docker_cache +import build as build_util + +DOCKERFILE_DIR = 'docker' +DOCKER_REGISTRY_NAME = 'test_registry' +DOCKER_REGISTRY_PORT = 5000 +DOCKER_REGISTRY_PATH = 'localhost:{}'.format(DOCKER_REGISTRY_PORT) + +class RedirectSubprocessOutput(object): + """ + Redirect the output of all subprocess.call calls to a readable buffer instead of writing it to stdout/stderr. + The output can then be retrieved with get_output. + """ + def __enter__(self): + self.buf_output = tempfile.TemporaryFile() + + def trampoline(*popenargs, **kwargs): + self.call(*popenargs, **kwargs) + + self.old_method = subprocess.call + subprocess.call = trampoline + return self + + def __exit__(self, *args): + logging.info('Releasing docker output buffer:\n%s', self.get_output()) + subprocess.call = self.old_method + self.buf_output.close() + + def call(self, *popenargs, **kwargs): + """ + Replace subprocess.call + :param popenargs: + :param timeout: + :param kwargs: + :return: + """ + kwargs['stderr'] = subprocess.STDOUT + kwargs['stdout'] = self.buf_output + return self.old_method(*popenargs, **kwargs) + + def get_output(self): + self.buf_output.seek(0) + return self.buf_output.read().decode('utf-8') + + +class TestDockerCache(unittest.TestCase): + """ + Test utility class + """ + def setUp(self): + logging.getLogger().setLevel(logging.DEBUG) + + # We need to be in the same directory than the script so the commands in the dockerfiles work as + # expected. But the script can be invoked from a different path + base = os.path.split(os.path.realpath(__file__))[0] + os.chdir(base) + + # Stop in case previous execution was dirty + try: + self._stop_local_docker_registry() + except Exception: + pass + + # Start up docker registry + self._start_local_docker_registry() + + def tearDown(self): + # Stop docker registry + self._stop_local_docker_registry() + + @classmethod + def _start_local_docker_registry(cls): + # https://docs.docker.com/registry/deploying/#run-a-local-registrys + start_cmd = [ + 'docker', 'run', '-d', '-p', '{}:{}'.format(DOCKER_REGISTRY_PORT, DOCKER_REGISTRY_PORT), + '--name', DOCKER_REGISTRY_NAME, 'registry:2' + ] + subprocess.check_call(start_cmd) + + @classmethod + def _stop_local_docker_registry(cls): + # https://docs.docker.com/registry/deploying/#run-a-local-registry + stop_cmd = ['docker', 'container', 'stop', DOCKER_REGISTRY_NAME] + subprocess.check_call(stop_cmd) + + clean_cmd = ['docker', 'container', 'rm', '-v', DOCKER_REGISTRY_NAME] + subprocess.check_call(clean_cmd) + + def test_full_cache(self): + """ + Test whether it's possible to restore cache entirely + :return: + """ + dockerfile_content = """ + FROM busybox + RUN touch ~/file1 + RUN touch ~/file2 + RUN touch ~/file3 + RUN touch ~/file4 + """ + platform = 'test_full_cache' + docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + try: + with open(dockerfile_path, 'w') as dockerfile_handle: + dockerfile_handle.write(dockerfile_content) + + # Warm up + docker_cache.delete_local_docker_cache(docker_tag=docker_tag) + + def warm_up_lambda_func(): + build_util.build_docker(docker_binary='docker', platform=platform, registry=DOCKER_REGISTRY_PATH) + _assert_docker_build(lambda_func=warm_up_lambda_func, expected_cache_hit_count=0, + expected_cache_miss_count=4) + + # Assert local cache is properly primed + def primed_cache_lambda_func(): + build_util.build_docker(docker_binary='docker', platform=platform, registry=DOCKER_REGISTRY_PATH) + _assert_docker_build(lambda_func=primed_cache_lambda_func, expected_cache_hit_count=4, + expected_cache_miss_count=0) + + # Upload and clean local cache + docker_cache.build_save_containers(platforms=[platform], registry=DOCKER_REGISTRY_PATH, load_cache=False) + docker_cache.delete_local_docker_cache(docker_tag=docker_tag) + + # Build with clean local cache and cache loading enabled + def clean_cache_lambda_func(): + docker_cache.build_save_containers( + platforms=[platform], registry=DOCKER_REGISTRY_PATH, load_cache=True) + _assert_docker_build(lambda_func=clean_cache_lambda_func, expected_cache_hit_count=4, + expected_cache_miss_count=0) + finally: + # Delete dockerfile + os.remove(dockerfile_path) + docker_cache.delete_local_docker_cache(docker_tag=docker_tag) + + + + def test_partial_cache(self): + """ + Test whether it's possible to restore cache and then pit it up partially by using a Dockerfile which shares + some parts + :return: + """ + # These two dockerfiles diverge at the fourth RUN statement. Their common parts (1-3) should be re-used + dockerfile_content_1 = """ + FROM busybox + RUN touch ~/file1 + RUN touch ~/file2 + RUN touch ~/file3 + RUN touch ~/file4 + """ + dockerfile_content_2 = """ + FROM busybox + RUN touch ~/file1 + RUN touch ~/file2 + RUN touch ~/file3 + RUN touch ~/file5 + RUN touch ~/file4 + RUN touch ~/file6 + """ + platform = 'test_partial_cache' + docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + try: + # Write initial Dockerfile + with open(dockerfile_path, 'w') as dockerfile_handle: + dockerfile_handle.write(dockerfile_content_1) + + # Warm up + docker_cache.delete_local_docker_cache(docker_tag=docker_tag) + + def warm_up_lambda_func(): + build_util.build_docker(docker_binary='docker', platform=platform, registry=DOCKER_REGISTRY_PATH) + _assert_docker_build(lambda_func=warm_up_lambda_func, expected_cache_hit_count=0, + expected_cache_miss_count=4) + + # Assert local cache is properly primed + def primed_cache_lambda_func(): + build_util.build_docker(docker_binary='docker', platform=platform, registry=DOCKER_REGISTRY_PATH) + _assert_docker_build(lambda_func=primed_cache_lambda_func, expected_cache_hit_count=4, + expected_cache_miss_count=0) + + # Upload and clean local cache + docker_cache.build_save_containers(platforms=[platform], registry=DOCKER_REGISTRY_PATH, load_cache=False) + docker_cache.delete_local_docker_cache(docker_tag=docker_tag) + + # Replace Dockerfile with the second one, resulting in a partial cache hit + with open(dockerfile_path, 'w') as dockerfile_handle: + dockerfile_handle.write(dockerfile_content_2) + + # Test if partial cache is properly hit. It will attempt to load the cache from the first Dockerfile, + # resulting in a partial hit + def partial_cache_lambda_func(): + docker_cache.build_save_containers( + platforms=[platform], registry=DOCKER_REGISTRY_PATH, load_cache=True) + _assert_docker_build(lambda_func=partial_cache_lambda_func, expected_cache_hit_count=3, + expected_cache_miss_count=3) + + finally: + # Delete dockerfile + os.remove(dockerfile_path) + docker_cache.delete_local_docker_cache(docker_tag=docker_tag) + + +def _assert_docker_build(lambda_func, expected_cache_hit_count: int, expected_cache_miss_count: int): + with RedirectSubprocessOutput() as redirected_output: + lambda_func() + output = redirected_output.get_output() + assert output.count('Running in') == expected_cache_miss_count, \ + 'Expected {} "Running in", got {}. Log:{}'.\ + format(expected_cache_miss_count, output.count('Running in'), output) + assert output.count('Using cache') == expected_cache_hit_count, \ + 'Expected {} "Using cache", got {}. Log:{}'.\ + format(expected_cache_hit_count, output.count('Using cache'), output) + + +if __name__ == '__main__': + import nose2 + nose2.main() -- To stop receiving notification emails like this one, please contact marcoab...@apache.org.