Repository: aurora-packaging Updated Branches: refs/heads/master 38744d13c -> 38716e6f7
Add a tool and environment for building Aurora packages. Bugs closed: AURORA-1410 Reviewed at https://reviews.apache.org/r/37107/ Project: http://git-wip-us.apache.org/repos/asf/aurora-packaging/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora-packaging/commit/38716e6f Tree: http://git-wip-us.apache.org/repos/asf/aurora-packaging/tree/38716e6f Diff: http://git-wip-us.apache.org/repos/asf/aurora-packaging/diff/38716e6f Branch: refs/heads/master Commit: 38716e6f700ba6d0c218913ca537543fd28128b0 Parents: 38744d1 Author: Bill Farner <[email protected]> Authored: Mon Aug 24 10:11:18 2015 -0700 Committer: Bill Farner <[email protected]> Committed: Mon Aug 24 10:11:18 2015 -0700 ---------------------------------------------------------------------- .gitignore | 1 + README.md | 33 ++++++++++++++++++++ build-artifact.sh | 45 ++++++++++++++++++++++++++ builder/deb/ubuntu-trusty/Dockerfile | 52 +++++++++++++++++++++++++++++++ builder/deb/ubuntu-trusty/build.sh | 33 ++++++++++++++++++++ builder/deb/ubuntu-trusty/pants.ini | 9 ++++++ builder/rpm/centos-7/Dockerfile | 37 ++++++++++++++++++++++ builder/rpm/centos-7/build.sh | 28 +++++++++++++++++ builder/rpm/centos-7/pants.ini | 9 ++++++ specs/debian/pants.ini | 9 ------ specs/debian/rules | 10 +++--- specs/rpm/Makefile | 33 ++++---------------- specs/rpm/aurora.spec | 45 +++++++------------------- 13 files changed, 270 insertions(+), 74 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..849ddff --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +dist/ http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000..e45f22f --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +## Packaging for Apache Aurora + +This repository maintains configuration and tooling for building binary +distributions of [Apache Aurora](https://aurora.apache.org/). + +### Building a binary + +Binaries are built within Docker containers that provide the appropriate build +environment for the target platform. You will need to have a working Docker +installation before proceeding. + +1. Fetch a source distribution, such as an + [official one](https://aurora.apache.org/downloads/). + +2. Run the builder script, providing the distribution platform and the source + distribution archive you downloaded in (1). The example below will build + Aurora 0.9.0 debs for Ubuntu Trusty. + + ``` + ./build-artifact.sh builder/deb/ubuntu-trusty \ + ~/Downloads/apache-aurora-0.9.0.tar.gz + ``` + +When this completes, debs will be placed in `dist/builder/deb/ubuntu-trusty/`. + +### Adding a new distribution platform + +There are only two requirements for a 'builder' to satisfy: + +- a `Dockerfile` to provide the repeatable build environment +- a `build.sh` script that creates artifacts + +Please see the makeup of other builders for examples. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/build-artifact.sh ---------------------------------------------------------------------- diff --git a/build-artifact.sh b/build-artifact.sh new file mode 100755 index 0000000..e6ea723 --- /dev/null +++ b/build-artifact.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eu + +print_available_builders() { + find builder -name Dockerfile | sed "s/\/Dockerfile$//" +} + +if [[ $# -ne 2 ]]; then + echo "usage: $0 BUILDER RELEASE_TAR" + echo 'Where BUILDER is a builder directory in:' + print_available_builders + exit 1 +else + BUILDER_DIR=$1 + RELEASE_TAR=$2 +fi + +IMAGE_NAME="aurora-$(basename $BUILDER_DIR)" +docker build -t "$IMAGE_NAME" "$BUILDER_DIR" + +ARTIFACT_DIR="$(pwd)/dist/$BUILDER_DIR" +mkdir -p $ARTIFACT_DIR +docker run \ + --rm \ + -v "$(pwd)/specs:/specs:ro" \ + -v "$(realpath $RELEASE_TAR):/src.tar.gz:ro" \ + -v "$ARTIFACT_DIR:/dist" \ + -t "$IMAGE_NAME" /build.sh + +echo 'Produced artifacts:' +ls $ARTIFACT_DIR http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/builder/deb/ubuntu-trusty/Dockerfile ---------------------------------------------------------------------- diff --git a/builder/deb/ubuntu-trusty/Dockerfile b/builder/deb/ubuntu-trusty/Dockerfile new file mode 100644 index 0000000..73f150b --- /dev/null +++ b/builder/deb/ubuntu-trusty/Dockerfile @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM ubuntu:trusty + +WORKDIR /aurora +ENV HOME /aurora +ENV DEBIAN_FRONTEND noninteractive +# TODO(wfarner): Provide external parameterization via env vars for versions. +ENV THRIFT_DEB thrift-compiler_0.9.1_amd64.deb + +RUN apt-get update && apt-get -y install \ + bison \ + debhelper \ + devscripts \ + dpkg-dev \ + curl \ + git \ + libapr1-dev \ + libcurl4-openssl-dev \ + libsvn-dev \ + python-all-dev \ + software-properties-common + +RUN add-apt-repository ppa:openjdk-r/ppa -y \ + && apt-get update \ + && apt-get install -y openjdk-8-jdk \ + && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java + +RUN curl -O http://people.apache.org/~jfarrell/thrift/0.9.1/contrib/deb/ubuntu/12.04/$THRIFT_DEB \ + && dpkg --install $THRIFT_DEB \ + && rm $THRIFT_DEB + +# Install gradle 2.5. +RUN git clone --depth 1 https://github.com/benley/gradle-packaging \ + && cd gradle-packaging \ + && apt-get install -y ruby ruby-dev unzip wget \ + && gem install fpm && ./gradle-mkdeb.sh 2.5 \ + && dpkg -i gradle-2.5_2.5-2_all.deb \ + && cd .. && rm -rf gradle-packaging + +ADD build.sh /build.sh +ADD pants.ini /pants.ini http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/builder/deb/ubuntu-trusty/build.sh ---------------------------------------------------------------------- diff --git a/builder/deb/ubuntu-trusty/build.sh b/builder/deb/ubuntu-trusty/build.sh new file mode 100755 index 0000000..6bee0e7 --- /dev/null +++ b/builder/deb/ubuntu-trusty/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +mkdir /scratch +cd /scratch + +tar --strip-components 1 -C . -xf /src.tar.gz + +cp -R /specs/debian . + +dch \ + -b \ + -v $(cat .auroraversion)~SNAPSHOT \ + -u low --maintmaint \ + "Apache Aurora package builder <[email protected]> $(date -R)" + +dpkg-buildpackage -uc -b -tc + +mv ../*.deb /dist http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/builder/deb/ubuntu-trusty/pants.ini ---------------------------------------------------------------------- diff --git a/builder/deb/ubuntu-trusty/pants.ini b/builder/deb/ubuntu-trusty/pants.ini new file mode 100644 index 0000000..ed1a5ce --- /dev/null +++ b/builder/deb/ubuntu-trusty/pants.ini @@ -0,0 +1,9 @@ +[DEFAULT] +pants_cachedir: %(homedir)s/.pants.d + +[python-setup] +download_cache: %(pants_cachedir)s/python/downloads +install_cache: %(pants_cachedir)s/python/eggs + +[python-repos] +repos: ['third_party/', 'https://svn.apache.org/repos/asf/incubator/aurora/3rdparty/ubuntu/trusty64/python/'] http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/builder/rpm/centos-7/Dockerfile ---------------------------------------------------------------------- diff --git a/builder/rpm/centos-7/Dockerfile b/builder/rpm/centos-7/Dockerfile new file mode 100644 index 0000000..05da048 --- /dev/null +++ b/builder/rpm/centos-7/Dockerfile @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM centos:7 + +RUN yum install -y \ + make \ + rpm-build \ + yum-utils \ + apr-devel \ + cyrus-sasl-devel \ + gcc \ + gcc-c++ \ + git \ + java-1.8.0-openjdk-devel \ + krb5-devel \ + libcurl-devel \ + patch \ + python \ + python-devel \ + subversion-devel \ + tar \ + unzip \ + wget \ + zlib-devel + +ADD build.sh /build.sh +ADD pants.ini /pants.ini http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/builder/rpm/centos-7/build.sh ---------------------------------------------------------------------- diff --git a/builder/rpm/centos-7/build.sh b/builder/rpm/centos-7/build.sh new file mode 100755 index 0000000..9e8eae9 --- /dev/null +++ b/builder/rpm/centos-7/build.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +mkdir -p /scratch/src +cd /scratch + +tar --strip-components 1 -C src -xf /src.tar.gz + +cp -R /specs/rpm . +cd rpm + +make srpm +yum-builddep -y ../../../dist/rpmbuild/SRPMS/* +make rpm http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/builder/rpm/centos-7/pants.ini ---------------------------------------------------------------------- diff --git a/builder/rpm/centos-7/pants.ini b/builder/rpm/centos-7/pants.ini new file mode 100644 index 0000000..00074cb --- /dev/null +++ b/builder/rpm/centos-7/pants.ini @@ -0,0 +1,9 @@ +[DEFAULT] +pants_cachedir: %(homedir)s/.pants.d + +[python-setup] +download_cache: %(pants_cachedir)s/python/downloads +install_cache: %(pants_cachedir)s/python/eggs + +[python-repos] +repos: ['third_party/', 'https://svn.apache.org/repos/asf/incubator/aurora/3rdparty/centos/7/python/'] http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/specs/debian/pants.ini ---------------------------------------------------------------------- diff --git a/specs/debian/pants.ini b/specs/debian/pants.ini deleted file mode 100644 index ed1a5ce..0000000 --- a/specs/debian/pants.ini +++ /dev/null @@ -1,9 +0,0 @@ -[DEFAULT] -pants_cachedir: %(homedir)s/.pants.d - -[python-setup] -download_cache: %(pants_cachedir)s/python/downloads -install_cache: %(pants_cachedir)s/python/eggs - -[python-repos] -repos: ['third_party/', 'https://svn.apache.org/repos/asf/incubator/aurora/3rdparty/ubuntu/trusty64/python/'] http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/specs/debian/rules ---------------------------------------------------------------------- diff --git a/specs/debian/rules b/specs/debian/rules index d981400..ef35a04 100755 --- a/specs/debian/rules +++ b/specs/debian/rules @@ -21,7 +21,7 @@ gradle = $(shell which gradle) pants = $(CURDIR)/pants -PANTS_CONFIG_OVERRIDE = $(CURDIR)/debian/pants.ini +PANTS_CONFIG_OVERRIDE = /pants.ini export PANTS_CONFIG_OVERRIDE override_dh_auto_build: __gradle_build __pants_build @@ -31,12 +31,12 @@ __gradle_build: __pants_build: mkdir -p third_party - $(pants) binary src/main/python/apache/aurora/executor:thermos_executor - $(pants) binary src/main/python/apache/aurora/kerberos:kaurora - $(pants) binary src/main/python/apache/aurora/kerberos:kaurora_admin + $(pants) binary src/main/python/apache/aurora/executor/bin:thermos_executor + $(pants) binary src/main/python/apache/aurora/client/cli:kaurora + $(pants) binary src/main/python/apache/aurora/admin:kaurora_admin $(pants) binary src/main/python/apache/aurora/tools:thermos $(pants) binary src/main/python/apache/aurora/tools:thermos_observer - $(pants) binary src/main/python/apache/thermos/runner:thermos_runner + $(pants) binary src/main/python/apache/thermos/bin:thermos_runner build-support/embed_runner_in_executor.py override_dh_installinit: http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/specs/rpm/Makefile ---------------------------------------------------------------------- diff --git a/specs/rpm/Makefile b/specs/rpm/Makefile index 1833a25..77605fe 100644 --- a/specs/rpm/Makefile +++ b/specs/rpm/Makefile @@ -10,20 +10,10 @@ # See the License for the specific language governing permissions and # limitations under the License. - -GIT_COMMIT ?= HEAD - -USING_GIT ?= $(shell git status >/dev/null 2>&1 && echo 'true' || echo 'false') -ifeq ($(USING_GIT),true) - VERSION_SUFFIX := .$(shell git rev-parse --short $(GIT_COMMIT)) -else - VERSION_SUFFIX := -endif - CURRENT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -TOPDIR ?= $(CURRENT_DIR)/../../.. -DIST_DIR ?= $(TOPDIR)/dist +TOPDIR ?= /scratch/src +DIST_DIR ?= /dist RPM_TOPDIR ?= $(DIST_DIR) DATETIME ?= $(shell date +%Y%m%d%H%M) @@ -39,7 +29,7 @@ PHONY: all nightly_version release_version get_source_target clean mkdir srpm ni all: release_rpm nightly_version: - $(eval AURORA_VERSION := $(AURORA_VERSION).$(DATETIME)$(VERSION_SUFFIX)) + $(eval AURORA_VERSION := $(AURORA_VERSION).$(DATETIME)) release_version: $(eval AURORA_VERSION := $(AURORA_VERSION)) @@ -53,20 +43,9 @@ mkdir: clean mkdir -p $(RPM_TOPDIR)/rpmbuild/RPMS mkdir -p $(RPM_TOPDIR)/rpmbuild/SOURCES mkdir -p $(RPM_TOPDIR)/rpmbuild/SRPMS + cp /src.tar.gz $(SOURCE_TARGET) -tar_source: mkdir - if [[ "$(USING_GIT)" == 'true' ]]; then \ - cd `git rev-parse --show-toplevel` && git archive --format=tar --prefix=aurora/ $(GIT_COMMIT) | gzip > $(SOURCE_TARGET); \ - else \ - tmp_dir=`mktemp -d '/tmp/aurorabuild.XXXXXX'`; \ - mkdir $${tmp_dir}/aurora; \ - cp -r $(TOPDIR)/* $${tmp_dir}/aurora; \ - cd $${tmp_dir}; \ - tar cvzf $(SOURCE_TARGET) aurora; \ - rm -rf $${tmp_dir}; \ - fi - -srpm: tar_source +srpm: mkdir rpmbuild $(RPM_OPTS) \ --define "_topdir $(RPM_TOPDIR)" \ --define "_builddir %{_topdir}/rpmbuild/BUILD" \ @@ -79,7 +58,7 @@ srpm: tar_source --define "MESOS_VERSION $(MESOS_VERSION)" \ --nodeps -bs aurora.spec -rpm: tar_source +rpm: mkdir rpmbuild $(RPM_OPTS) \ --define "_topdir $(RPM_TOPDIR)" \ --define "_builddir %{_topdir}/rpmbuild/BUILD" \ http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/38716e6f/specs/rpm/aurora.spec ---------------------------------------------------------------------- diff --git a/specs/rpm/aurora.spec b/specs/rpm/aurora.spec index 5ec516f..e6fca4e 100644 --- a/specs/rpm/aurora.spec +++ b/specs/rpm/aurora.spec @@ -37,10 +37,6 @@ %global JAVA_VERSION 1.8.0 %endif -%if %{?!MESOS_BASEURL:1}0 -%global MESOS_BASEURL https://archive.apache.org/dist/mesos -%endif - %if %{?!MESOS_VERSION:1}0 %global MESOS_VERSION 0.22.0 %endif @@ -141,7 +137,7 @@ state of all running tasks. %prep -%setup -n %{name} +%setup -n apache-%{name}-%{version} %build @@ -163,40 +159,27 @@ export PATH=/usr/lib/jvm/java-1.8.0/bin:${PATH} wget %{GRADLE_BASEURL}/gradle-%{GRADLE_VERSION}-bin.zip unzip gradle-%{GRADLE_VERSION}-bin.zip -# Creates Pants directory where we'll store our native Mesos Python eggs. -mkdir -p .pants.d/python/eggs/ - -# Builds mesos-native and mesos-interface eggs if not currently packaged. -wget "%{MESOS_BASEURL}/%{MESOS_VERSION}/mesos-%{MESOS_VERSION}.tar.gz" -tar xvzf mesos-%{MESOS_VERSION}.tar.gz -pushd mesos-%{MESOS_VERSION} -./configure --disable-java -make -find . -name '*.egg' -exec cp -v {} ../.pants.d/python/eggs/ \; -popd - # Builds the Aurora scheduler. ./gradle-%{GRADLE_VERSION}/bin/gradle installDist +# Configures pants to use our distributed platform-specific eggs. +# This avoids building mesos to produce them. +export PANTS_CONFIG_OVERRIDE=/pants.ini + # Builds Aurora client PEX binaries. -./pants binary src/main/python/apache/aurora/kerberos:kaurora -./pants binary src/main/python/apache/aurora/kerberos:kaurora_admin +./pants binary src/main/python/apache/aurora/client/cli:kaurora +mv dist/kaurora.pex dist/aurora.pex +./pants binary src/main/python/apache/aurora/admin:kaurora_admin +mv dist/kaurora_admin.pex dist/aurora_admin.pex # Builds Aurora Thermos and GC executor PEX binaries. -./pants binary src/main/python/apache/aurora/executor:thermos_executor +./pants binary src/main/python/apache/aurora/executor/bin:thermos_executor ./pants binary src/main/python/apache/aurora/tools:thermos -./pants binary src/main/python/apache/thermos/runner:thermos_runner +./pants binary src/main/python/apache/thermos/bin:thermos_runner ./pants binary src/main/python/apache/aurora/tools:thermos_observer # Packages the Thermos runner within the Thermos executor. -python <<EOF -import contextlib -import zipfile -with contextlib.closing(zipfile.ZipFile('dist/thermos_executor.pex', 'a')) as zf: - zf.writestr('apache/aurora/executor/resources/__init__.py', '') - zf.write('dist/thermos_runner.pex', 'apache/aurora/executor/resources/thermos_runner.pex') -EOF - +build-support/embed_runner_in_executor.py %install rm -rf $RPM_BUILD_ROOT @@ -224,10 +207,6 @@ for pex_binary in %{PEX_BINARIES}; do install -m 755 dist/${pex_binary}.pex %{buildroot}%{_bindir}/${pex_binary} done -# Strip the "k" from Kerberized client binaries. -mv %{buildroot}%{_bindir}/kaurora %{buildroot}%{_bindir}/aurora -mv %{buildroot}%{_bindir}/kaurora_admin %{buildroot}%{_bindir}/aurora_admin - # Installs all support scripting. %if 0%{?fedora} || 0%{?rhel} > 6 install -m 644 build-support/packaging/rpm/%{name}.service %{buildroot}%{_sysconfdir}/systemd/system/%{name}.service
