marcoabreu closed pull request #13182: Add functionality to build.py to disable caching URL: https://github.com/apache/incubator-mxnet/pull/13182
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/ci/README.md b/ci/README.md index 6c8a23f03cf..5b6cc668978 100644 --- a/ci/README.md +++ b/ci/README.md @@ -5,6 +5,8 @@ Docker containers You need docker and nvidia docker if you have a GPU. +Also you need to run `pip3 install docker` as it uses the [docker python module](https://docker-py.readthedocs.io/en/stable/containers.html#) + If you are in ubuntu an easy way to install Docker CE is executing the following script: diff --git a/ci/build.py b/ci/build.py index 8f3fe2d1244..628f49e706c 100755 --- a/ci/build.py +++ b/ci/build.py @@ -101,6 +101,8 @@ def get_platforms(path: str = get_dockerfiles_path()) -> List[str]: def get_docker_tag(platform: str, registry: str) -> str: """:return: docker tag to be used for the container""" + if not registry: + registry = "mxnet_local" return "{0}/build.{1}".format(registry, platform) @@ -112,14 +114,14 @@ def get_docker_binary(use_nvidia_docker: bool) -> str: return "nvidia-docker" if use_nvidia_docker else "docker" -def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, use_cache: bool) -> str: +def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool) -> str: """ Build a container for the given platform :param platform: Platform :param docker_binary: docker binary to use (docker/nvidia-docker) :param registry: Dockerhub registry name :param num_retries: Number of retries to build the docker image - :param use_cache: will pass cache_from to docker to use the previously pulled tag + :param no_cache: pass no-cache to docker to rebuild the images :return: Id of the top level image """ tag = get_docker_tag(platform=platform, registry=registry) @@ -144,7 +146,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries: "-f", get_dockerfile(platform), "--build-arg", "USER_ID={}".format(os.getuid()), "--build-arg", "GROUP_ID={}".format(os.getgid())] - if use_cache: + if no_cache: + cmd.append("--no-cache") + elif registry: cmd.extend(["--cache-from", tag]) cmd.extend(["-t", tag, get_dockerfiles_path()]) @@ -422,7 +426,7 @@ def main() -> int: action='store_true') parser.add_argument("-d", "--docker-registry", - help="Dockerhub registry name to retrieve cache from. Default is 'mxnetci'", + help="Dockerhub registry name to retrieve cache from.", default='mxnetci', type=str) @@ -431,10 +435,8 @@ def main() -> int: default=1, type=int) - parser.add_argument("-c", "--no-dockerhub-cache", action="store_true", - help="Disables use of --cache-from option on docker build, allowing docker" - " to use local layers for caching. If absent, we use the cache from dockerhub" - " which is the default.") + parser.add_argument("--no-cache", action="store_true", + help="passes --no-cache to docker build") parser.add_argument("command", help="command to run in the container", @@ -447,9 +449,6 @@ def main() -> int: args = parser.parse_args() - def use_cache(): - return not args.no_dockerhub_cache or under_ci() - command = list(chain(*args.command)) docker_binary = get_docker_binary(args.nvidiadocker) @@ -472,10 +471,10 @@ def signal_handler(signum, _): elif args.platform: platform = args.platform tag = get_docker_tag(platform=platform, registry=args.docker_registry) - if use_cache(): + if args.docker_registry: load_docker_cache(tag=tag, docker_registry=args.docker_registry) build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry, - num_retries=args.docker_build_retries, use_cache=use_cache()) + num_retries=args.docker_build_retries, no_cache=args.no_cache) if args.build_only: logging.warning("Container was just built. Exiting due to build-only.") return 0 @@ -512,10 +511,9 @@ def signal_handler(signum, _): logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: tag = get_docker_tag(platform=platform, registry=args.docker_registry) - if use_cache(): - load_docker_cache(tag=tag, docker_registry=args.docker_registry) + load_docker_cache(tag=tag, docker_registry=args.docker_registry) build_docker(platform, docker_binary=docker_binary, registry=args.docker_registry, - num_retries=args.docker_build_retries, use_cache=use_cache()) + num_retries=args.docker_build_retries, no_cache=args.no_cache) if args.build_only: continue shutil.rmtree(buildir(), ignore_errors=True) diff --git a/ci/docker_cache.py b/ci/docker_cache.py index bebcb25fb8f..23c173c39e0 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -125,6 +125,10 @@ def load_docker_cache(registry, docker_tag) -> None: :return: None """ # We don't have to retag the image since it's already in the right format + if not registry: + return + assert docker_tag + logging.info('Loading Docker cache for %s from %s', docker_tag, registry) pull_cmd = ['docker', 'pull', docker_tag] subprocess.call(pull_cmd) # Don't throw an error if the image does not exist diff --git a/ci/requirements.txt b/ci/requirements.txt new file mode 100644 index 00000000000..8f21ead27f7 --- /dev/null +++ b/ci/requirements.txt @@ -0,0 +1 @@ +docker==3.5.0 ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
