This is an automated email from the ASF dual-hosted git repository. areusch pushed a commit to branch areusch/freeze-dependencies in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 3017139b284e5ef0383e42202878df037cfb3e3a Author: Andrew Reusch <[email protected]> AuthorDate: Fri May 13 14:02:06 2022 -0700 Adjust Dockerfile python installs --- docker/Dockerfile.ci_arm | 3 ++ docker/Dockerfile.ci_gpu | 6 ++-- docker/Dockerfile.ci_hexagon | 6 ++-- docker/Dockerfile.ci_i386 | 3 ++ docker/Dockerfile.ci_lint | 9 +++--- docker/Dockerfile.ci_qemu | 8 ++--- docker/Dockerfile.ci_wasm | 3 ++ docker/install/ubuntu1804_install_python_venv.sh | 31 ------------------- docker/install/ubuntu2004_install_python.sh | 38 +++++++++++++++++++----- 9 files changed, 54 insertions(+), 53 deletions(-) diff --git a/docker/Dockerfile.ci_arm b/docker/Dockerfile.ci_arm index c19f1ff5a4..42f9ebe705 100644 --- a/docker/Dockerfile.ci_arm +++ b/docker/Dockerfile.ci_arm @@ -44,6 +44,7 @@ ENV PATH /opt/sccache:$PATH COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh RUN bash /install/ubuntu_install_llvm.sh +COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh @@ -53,6 +54,8 @@ RUN pip config set global.no-cache-dir false COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh RUN bash /install/ubuntu_install_cmake_source.sh +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu index 5a7db5315f..62d6eb2dbe 100644 --- a/docker/Dockerfile.ci_gpu +++ b/docker/Dockerfile.ci_gpu @@ -33,12 +33,10 @@ RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -# Globally disable pip cache -RUN pip config set global.no-cache-dir false - COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh RUN bash /install/ubuntu_install_cmake_source.sh @@ -48,6 +46,8 @@ RUN bash /install/ubuntu1804_install_llvm.sh COPY install/ubuntu_install_opencl.sh /install/ubuntu_install_opencl.sh RUN bash /install/ubuntu_install_opencl.sh +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/Dockerfile.ci_hexagon b/docker/Dockerfile.ci_hexagon index 20b185ab64..0057002f9e 100644 --- a/docker/Dockerfile.ci_hexagon +++ b/docker/Dockerfile.ci_hexagon @@ -28,12 +28,10 @@ RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt COPY install/ubuntu2004_install_python.sh /install/ubuntu2004_install_python.sh RUN bash /install/ubuntu2004_install_python.sh -# Globally disable pip cache -RUN pip config set global.cache-dir false - # Rust env (build early; takes a while) COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh RUN bash /install/ubuntu_install_rust.sh @@ -41,6 +39,8 @@ ENV RUSTUP_HOME /opt/rust ENV CARGO_HOME /opt/rust ENV PATH $PATH:$CARGO_HOME/bin +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/Dockerfile.ci_i386 b/docker/Dockerfile.ci_i386 index bbf449d9c3..4a0b359836 100644 --- a/docker/Dockerfile.ci_i386 +++ b/docker/Dockerfile.ci_i386 @@ -31,6 +31,7 @@ RUN bash /install/ubuntu_install_googletest.sh COPY install/ubuntu_install_llvm.sh /install/ubuntu_install_llvm.sh RUN bash /install/ubuntu_install_llvm.sh +COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh @@ -40,6 +41,8 @@ RUN pip config set global.no-cache-dir false COPY install/ubuntu_install_cmake_source.sh /install/ubuntu_install_cmake_source.sh RUN bash /install/ubuntu_install_cmake_source.sh +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/Dockerfile.ci_lint b/docker/Dockerfile.ci_lint index 1d0c984c61..38e3567a78 100644 --- a/docker/Dockerfile.ci_lint +++ b/docker/Dockerfile.ci_lint @@ -24,16 +24,17 @@ RUN apt-get update --fix-missing RUN apt-get update && apt-get install -y wget git sudo make parallel +COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh -# Globally disable pip cache -RUN pip config set global.no-cache-dir false +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock +COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh +RUN bash /install/ubuntu_install_python_package.sh RUN apt-get update && apt-get install -y doxygen graphviz curl shellcheck -RUN pip3 install cpplint pylint==2.4.4 mypy==0.902 black==22.3.0 flake8==3.9.2 blocklint==0.2.3 jinja2==3.0.3 - # Rust env (build early; takes a while) COPY install/ubuntu_install_rust.sh /install/ubuntu_install_rust.sh RUN bash /install/ubuntu_install_rust.sh diff --git a/docker/Dockerfile.ci_qemu b/docker/Dockerfile.ci_qemu index 28bfd8962d..6082139abc 100644 --- a/docker/Dockerfile.ci_qemu +++ b/docker/Dockerfile.ci_qemu @@ -27,16 +27,15 @@ RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh COPY install/ubuntu1804_install_python_venv.sh /install/ubuntu1804_install_python_venv.sh RUN bash /install/ubuntu1804_install_python_venv.sh -ENV PATH=/opt/tvm-venv/bin:/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH - -# Globally disable pip cache -RUN pip config set global.no-cache-dir false +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh @@ -77,6 +76,7 @@ COPY install/ubuntu_init_zephyr_project.sh /install/ubuntu_init_zephyr_project.s COPY install/ubuntu_install_zephyr_sdk.sh /install/ubuntu_install_zephyr_sdk.sh RUN bash /install/ubuntu_install_zephyr.sh ENV ZEPHYR_BASE=/opt/zephyrproject/zephyr +ENV PATH=/opt/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin:$PATH # FreeRTOS deps COPY install/ubuntu_install_freertos.sh /install/ubuntu_install_freertos.sh diff --git a/docker/Dockerfile.ci_wasm b/docker/Dockerfile.ci_wasm index 1c7d3eb59b..5868e87fef 100644 --- a/docker/Dockerfile.ci_wasm +++ b/docker/Dockerfile.ci_wasm @@ -24,12 +24,15 @@ RUN bash /install/ubuntu_install_core.sh COPY install/ubuntu_install_googletest.sh /install/ubuntu_install_googletest.sh RUN bash /install/ubuntu_install_googletest.sh +COPY python/bootstrap-requirements.txt /install/python/bootstrap-requirements.txt COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh RUN bash /install/ubuntu1804_install_python.sh # Globally disable pip cache RUN pip config set global.no-cache-dir false +COPY python/build/pyproject.toml /install/python/pyproject.toml +COPY python/build/poetry.lock /install/python/poetry.lock COPY install/ubuntu_install_python_package.sh /install/ubuntu_install_python_package.sh RUN bash /install/ubuntu_install_python_package.sh diff --git a/docker/install/ubuntu1804_install_python_venv.sh b/docker/install/ubuntu1804_install_python_venv.sh deleted file mode 100755 index 5dc5efea76..0000000000 --- a/docker/install/ubuntu1804_install_python_venv.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -u -set -o pipefail - -# install python and pip, don't modify this, modify install_python_package.sh -apt-get update -apt-get install -y software-properties-common -apt-get install -y python3.7-dev python3-setuptools python3.7-venv - -python3 -mvenv /opt/tvm-venv - -# Pin pip and setuptools versions -/opt/tvm-venv/bin/pip3 install pip==19.3.1 setuptools==58.4.0 diff --git a/docker/install/ubuntu2004_install_python.sh b/docker/install/ubuntu2004_install_python.sh index 5b87a74061..31f803a90f 100755 --- a/docker/install/ubuntu2004_install_python.sh +++ b/docker/install/ubuntu2004_install_python.sh @@ -34,12 +34,34 @@ apt-get install -y software-properties-common apt-get install -y python3.8 python3.8-dev python3-pip update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 -# Pin pip and setuptools versions -# Hashes generated via: -# $ pip download <package>==<version> -# $ pip hash --algorithm sha256 <package>.whl -cat <<EOF > base-requirements.txt -pip==22.0.4 --hash=sha256:c6aca0f2f081363f689f041d90dab2a07a9a07fb840284db2218117a52da800b -setuptools==58.4.0 --hash=sha256:e8b1d3127a0441fb99a130bcc3c2bf256c2d3ead3aba8fd400e5cbbaf788e036 +function download_hash() { + cat >/tmp/hash-bootstrap-packages.py <<EOF +import os +import os.path +import subprocess +import pkginfo + +for f in sorted(os.scandir("."), key=lambda x: x.name): + if not f.is_file(): + continue + p = pkginfo.get_metadata(f.name) + if not p: + continue + print(f"{p.name}=={p.version} {subprocess.check_output(['pip3', 'hash', '-a', 'sha256', p.filename], encoding='utf-8').split()[1]} # {f.name}") EOF -pip3 install -r base-requirements.txt + mkdir packages && cd packages + pip3 install -U "$@" + pip3 download pip poetry setuptools + python3 /tmp/hash-bootstrap-packages.py + exit 2 # make docker build stop +} + +# Install bootstrap packages. You can update these with the following procedure: +# 1. Uncomment the line below, then attempt torebuild the base images (it will fail). +# 2. New hashes should be printed in the terminal log from each docker build. Copy these hashes into the +# the arch-appropriate file in docker/python/bootstrap-requirements/ +# download_hash pip setuptools pkginfo + +pip3 install -U pip -c /install/python/bootstrap-requirements.txt # Update pip to match version used to produce base-requirements.txt +pip3 config set global.no-cache-dir false +pip3 install -r /install/python/bootstrap-requirements.txt -c /install/python/bootstrap-requirements.txt
