This is an automated email from the ASF dual-hosted git repository. jieyu pushed a commit to branch 1.7.x in repository https://gitbox.apache.org/repos/asf/mesos.git
commit e2ca4186918c6519f560ebc340e4fd9c23b118ea Author: Jie Yu <[email protected]> AuthorDate: Sat Oct 27 11:44:12 2018 -0700 Refactored the CentOS RPM and Docker build. This patch enables the build of RPM packages using Docker so that you can build RPM from anywhere (e.g., your Macbook). This patch also fixes an user/group related issue with Docker which might cause permission denided failures during cleanup. The idea is to add the expected user to the docker container that's being used to build RPM. (cherry picked from commit efb6b884242eb0ccecf10982d4b407356924a35e) --- support/packaging/centos/build-docker-centos.sh | 34 ++++++++++++ support/packaging/centos/build-docker-image.sh | 57 -------------------- support/packaging/centos/build-docker-rpmbuild.sh | 33 ++++++++++++ support/packaging/centos/build-rpm-docker.sh | 66 +++++++++++++++++++++++ support/packaging/centos/build_rpm.sh | 34 ++++++++---- support/packaging/centos/centos6.dockerfile | 9 ++++ support/packaging/centos/centos7.dockerfile | 9 ++++ support/packaging/centos/user-init.sh | 18 +++++++ 8 files changed, 192 insertions(+), 68 deletions(-) diff --git a/support/packaging/centos/build-docker-centos.sh b/support/packaging/centos/build-docker-centos.sh new file mode 100755 index 0000000..d6d3d99 --- /dev/null +++ b/support/packaging/centos/build-docker-centos.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail -o verbose + +# This script builds a CentOS based docker image with Mesos installed +# using the current head of the source tree. + +CENTOS_DIR="$(cd "$(dirname "$0")"; pwd -P)" +SOURCE_DIR="$(cd "${CENTOS_DIR}/../../.."; pwd -P)" + +CENTOS_DISTRO=${CENTOS_DISTRO:-"7"} +DOCKER_IMAGE=${DOCKER_IMAGE:-"mesos/mesos-centos"} +DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"latest"} + +if ! [ -x "$(command -v docker)" ]; then + echo 'Error: docker is not installed.' >&2 + exit 1 +fi + +"${CENTOS_DIR}/build-rpm-docker.sh" + +# Build the image for running Mesos. +DOCKER_CONTEXT_DIR="${SOURCE_DIR}/centos${CENTOS_DISTRO}/rpmbuild/RPMS/x86_64" + +cat <<EOF > "${DOCKER_CONTEXT_DIR}/Dockerfile" +FROM centos:${CENTOS_DISTRO} +ADD mesos-?.?.?-*.rpm / +RUN yum --nogpgcheck -y localinstall /mesos-*.rpm +EOF + +docker build \ + --rm \ + -t "${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" \ + "${DOCKER_CONTEXT_DIR}" diff --git a/support/packaging/centos/build-docker-image.sh b/support/packaging/centos/build-docker-image.sh deleted file mode 100755 index cf7c298..0000000 --- a/support/packaging/centos/build-docker-image.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail -o verbose - -# This script builds a CentOS based docker image with Mesos installed -# using the current head of the source tree. - -CENTOS_DIR="$(cd "$(dirname "$0")"; pwd -P)" -SOURCE_DIR="$(cd ${CENTOS_DIR}/../../..; pwd -P)" - -DOCKER_IMAGE_PACKAGING=${DOCKER_IMAGE_PACKAGING:-"mesos/mesos-centos-packaging"} -DOCKER_IMAGE_DISTRO=${DOCKER_IMAGE_DISTRO:-"mesos/mesos-centos"} -DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"latest"} - -if ! [ -x "$(command -v docker)" ]; then - echo 'Error: docker is not installed.' >&2 - exit 1 -fi - -function cleanup { - rm -rf "${SOURCE_DIR}/centos7" -} - -cleanup - -trap cleanup EXIT - -# Build the image for building Mesos packages. -docker build \ - --rm \ - -t ${DOCKER_IMAGE_PACKAGING}:${DOCKER_IMAGE_TAG} \ - -f "${SOURCE_DIR}/support/packaging/centos/centos7.dockerfile" \ - "${SOURCE_DIR}/support/packaging/centos/" - -# Build the RPM. -USER_ID=`id -u` -GROUP_ID=`id -g` - -docker run \ - --rm \ - -v "${SOURCE_DIR}:${SOURCE_DIR}" \ - ${DOCKER_IMAGE_PACKAGING}:${DOCKER_IMAGE_TAG} \ - /bin/bash -c "${SOURCE_DIR}/support/packaging/centos/build_rpm.sh && chown -R ${USER_ID}:${GROUP_ID} ${SOURCE_DIR}/centos7" - -# Build the image for running Mesos. -DOCKER_CONTEXT_DIR="${SOURCE_DIR}/centos7/rpmbuild/RPMS/x86_64" - -cat <<EOF > "${DOCKER_CONTEXT_DIR}/Dockerfile" -FROM centos:7 -ADD mesos-?.?.?-*.rpm / -RUN yum --nogpgcheck -y localinstall /mesos-*.rpm -EOF - -docker build \ - --rm \ - -t ${DOCKER_IMAGE_DISTRO}:${DOCKER_IMAGE_TAG} \ - "${DOCKER_CONTEXT_DIR}" diff --git a/support/packaging/centos/build-docker-rpmbuild.sh b/support/packaging/centos/build-docker-rpmbuild.sh new file mode 100755 index 0000000..1fdd5c7 --- /dev/null +++ b/support/packaging/centos/build-docker-rpmbuild.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail -o verbose + +# This script builds a Docker image that will be used to build RPM +# package for Mesos. + +CENTOS_DIR="$(cd "$(dirname "$0")"; pwd -P)" + +CENTOS_DISTRO=${CENTOS_DISTRO:-"7"} +DOCKER_IMAGE=${DOCKER_IMAGE:-"mesos/mesos-centos-rpmbuild"} +DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"latest"} + +if ! [ -x "$(command -v docker)" ]; then + echo 'Error: docker is not installed.' >&2 + exit 1 +fi + +USER_NAME=$(id -u -n) +USER_ID=$(id -u) +GROUP_NAME=$(id -g -n) +GROUP_ID=$(id -g) + +# Build the image for building Mesos packages. +docker build \ + --rm \ + --build-arg "USER_NAME=${USER_NAME}" \ + --build-arg "USER_ID=${USER_ID}" \ + --build-arg "GROUP_NAME=${GROUP_NAME}" \ + --build-arg "GROUP_ID=${GROUP_ID}" \ + -t "${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" \ + -f "${CENTOS_DIR}/centos${CENTOS_DISTRO}.dockerfile" \ + "${CENTOS_DIR}" diff --git a/support/packaging/centos/build-rpm-docker.sh b/support/packaging/centos/build-rpm-docker.sh new file mode 100755 index 0000000..e7d9c3f --- /dev/null +++ b/support/packaging/centos/build-rpm-docker.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail -o verbose + +# This script builds RPM package for Mesos using Docker. + +CENTOS_DIR="$(cd "$(dirname "$0")"; pwd -P)" +SOURCE_DIR="$(cd "${CENTOS_DIR}/../../.."; pwd -P)" + +CENTOS_DISTRO=${CENTOS_DISTRO:-"7"} +DOCKER_IMAGE=${DOCKER_IMAGE:-"mesos/mesos-centos-rpmbuild"} +DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG:-"latest"} + +if ! [ -x "$(command -v docker)" ]; then + echo 'Error: docker is not installed.' >&2 + exit 1 +fi + +MESOS_SHA=${MESOS_SHA:-$(git rev-parse HEAD)} +MESOS_TAG_OR_SHA=$(git describe --exact-match "${MESOS_SHA}" 2>/dev/null || echo "${MESOS_SHA}") + +if [[ "${MESOS_TAG_OR_SHA}" != "${MESOS_SHA}" ]]; then + # HEAD is also a tag. + MESOS_TAG="${MESOS_TAG_OR_SHA}" +else + MESOS_TAG="" +fi + +echo "MESOS_SHA=${MESOS_SHA}" +echo "MESOS_TAG=${MESOS_TAG}" + +rm -rf "${SOURCE_DIR}/centos${CENTOS_DISTRO}" + +CENTOS_DISTRO="${CENTOS_DISTRO}" \ +DOCKER_IMAGE="${DOCKER_IMAGE}" \ +DOCKER_IMAGE_TAG="${DOCKER_IMAGE_TAG}" \ +"${CENTOS_DIR}/build-docker-rpmbuild.sh" + +USER_ID=$(id -u) +GROUP_ID=$(id -g) + +# NOTE: A shared volume has the same gid as its host volume on Linux, +# but the same group name on macOS. To run this script on both +# platforms, we run the build with the group name of the docker +# socket. +DOCKER_SOCKET_GID=$(echo /var/run/docker.sock | perl -lne 'use File::stat; print stat($_)->gid') + +# Attach to terminal if we have a TTY so that things like CTRL-C work. +if [ -t 1 ]; then + TTYARGS="-ti" +else + TTYARGS="" +fi + +docker run \ + $TTYARGS \ + --rm \ + --user "${USER_ID}:${GROUP_ID}" \ + --group-add 0 \ + --group-add "${DOCKER_SOCKET_GID}" \ + -v "${SOURCE_DIR}:${SOURCE_DIR}" \ + "${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" \ + /bin/bash -c "MAKE_DIST=true MESOS_TAG=${MESOS_TAG} ${CENTOS_DIR}/build_rpm.sh" + +echo "RPM has been built and can be found at:" +echo "${SOURCE_DIR}/centos${CENTOS_DISTRO}/rpmbuild/RPMS/x86_64" diff --git a/support/packaging/centos/build_rpm.sh b/support/packaging/centos/build_rpm.sh index abb6d4d..f3d83b6 100755 --- a/support/packaging/centos/build_rpm.sh +++ b/support/packaging/centos/build_rpm.sh @@ -19,11 +19,7 @@ if [ "$CENTOS_VERSION" = "6" ]; then source scl_source enable devtoolset-3 fi -if [ -z "$MESOS_TAG" ]; then - gitsha=$(git rev-parse --short HEAD) - snapshot_version=$(date -u +'%Y%m%d')git$gitsha - MESOS_RELEASE=0.1.pre.$snapshot_version - +make_dist() { pushd $MESOS_DIR ./bootstrap popd @@ -36,14 +32,30 @@ if [ -z "$MESOS_TAG" ]; then cp -f $TMP_BUILD_DIR/mesos-$MESOS_VERSION.tar.gz $HOME/rpmbuild/SOURCES/ rm -rf $TMP_BUILD_DIR +} + +if [ -z "$MESOS_TAG" ]; then + gitsha=$(git rev-parse --short HEAD) + snapshot_version=$(date -u +'%Y%m%d')git$gitsha + MESOS_RELEASE=0.1.pre.$snapshot_version + + make_dist elif [ "$MESOS_VERSION" = "$MESOS_TAG" ]; then - curl -sSL \ - https://dist.apache.org/repos/dist/release/mesos/${MESOS_VERSION}/mesos-${MESOS_VERSION}.tar.gz \ - -o $HOME/rpmbuild/SOURCES/mesos-${MESOS_VERSION}.tar.gz + if [ ! -z ${MAKE_DIST:-""} ]; then + make_dist + else + curl -sSL \ + https://dist.apache.org/repos/dist/release/mesos/${MESOS_VERSION}/mesos-${MESOS_VERSION}.tar.gz \ + -o $HOME/rpmbuild/SOURCES/mesos-${MESOS_VERSION}.tar.gz + fi else - curl -sSL \ - https://dist.apache.org/repos/dist/dev/mesos/${MESOS_TAG}/mesos-${MESOS_VERSION}.tar.gz \ - -o $HOME/rpmbuild/SOURCES/mesos-${MESOS_VERSION}.tar.gz + if [ ! -z ${MAKE_DIST:-""} ]; then + make_dist + else + curl -sSL \ + https://dist.apache.org/repos/dist/dev/mesos/${MESOS_TAG}/mesos-${MESOS_VERSION}.tar.gz \ + -o $HOME/rpmbuild/SOURCES/mesos-${MESOS_VERSION}.tar.gz + fi fi rpmbuild \ diff --git a/support/packaging/centos/centos6.dockerfile b/support/packaging/centos/centos6.dockerfile index ec13b24..457f48d 100644 --- a/support/packaging/centos/centos6.dockerfile +++ b/support/packaging/centos/centos6.dockerfile @@ -36,3 +36,12 @@ ADD mesos.spec /mesos.spec RUN yum makecache && \ yum-builddep -y /mesos.spec + +ADD user-init.sh /user-init.sh + +ARG USER_NAME=root +ARG USER_ID=0 +ARG GROUP_NAME=root +ARG GROUP_ID=0 + +RUN /user-init.sh $USER_NAME $USER_ID $GROUP_NAME $GROUP_ID diff --git a/support/packaging/centos/centos7.dockerfile b/support/packaging/centos/centos7.dockerfile index 95b4d28..b69cf3e 100644 --- a/support/packaging/centos/centos7.dockerfile +++ b/support/packaging/centos/centos7.dockerfile @@ -33,3 +33,12 @@ ADD mesos.spec /mesos.spec RUN yum makecache && \ yum-builddep -y /mesos.spec + +ADD user-init.sh /user-init.sh + +ARG USER_NAME=root +ARG USER_ID=0 +ARG GROUP_NAME=root +ARG GROUP_ID=0 + +RUN /user-init.sh $USER_NAME $USER_ID $GROUP_NAME $GROUP_ID diff --git a/support/packaging/centos/user-init.sh b/support/packaging/centos/user-init.sh new file mode 100755 index 0000000..9c679a1 --- /dev/null +++ b/support/packaging/centos/user-init.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail -o verbose + +USER_NAME=$1 +USER_ID=$2 +GROUP_NAME=$3 +GROUP_ID=$4 + +# Create the group. +if ! getent group "${GROUP_ID}"; then + groupadd -r -g "${GROUP_ID}" "${GROUP_NAME}" +fi + +# Create the user. +if ! getent passwd "${USER_ID}"; then + useradd -r -m -u "${USER_ID}" "${USER_NAME}" -g "${GROUP_ID}" +fi
