Repository: aurora-packaging Updated Branches: refs/heads/master 712ca6158 -> 2b19410bf
Adding support for Ubuntu Xenial packages Added builder and test environment for Xenial as well as updated instructions on how to test it. Added distribution to release-candidate script. Bugs closed: AURORA-1872 Reviewed at https://reviews.apache.org/r/52437/ Project: http://git-wip-us.apache.org/repos/asf/aurora-packaging/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora-packaging/commit/2b19410b Tree: http://git-wip-us.apache.org/repos/asf/aurora-packaging/tree/2b19410b Diff: http://git-wip-us.apache.org/repos/asf/aurora-packaging/diff/2b19410b Branch: refs/heads/master Commit: 2b19410bf014d6d756772e846a97e4da61f51a92 Parents: 712ca61 Author: Renan DelValle <[email protected]> Authored: Wed Feb 8 08:56:28 2017 +0100 Committer: Stephan Erb <[email protected]> Committed: Wed Feb 8 08:56:28 2017 +0100 ---------------------------------------------------------------------- build-support/release/release-candidate | 1 + builder/deb/debian-jessie/Dockerfile | 1 + builder/deb/debian-jessie/build.sh | 9 +++ builder/deb/ubuntu-trusty/Dockerfile | 1 + builder/deb/ubuntu-trusty/build.sh | 9 +++ builder/deb/ubuntu-xenial/Dockerfile | 55 +++++++++++++++++++ builder/deb/ubuntu-xenial/build.sh | 51 +++++++++++++++++ specs/debian/aurora-executor.thermos.default | 3 + specs/debian/aurora-executor.thermos.service | 13 +++-- specs/debian/aurora-pants.ini | 3 - specs/debian/aurora-scheduler.service | 6 +- specs/debian/aurora-scheduler.startup.sh | 40 ++++++++++++++ specs/debian/rules | 9 ++- test/deb/ubuntu-xenial/README.md | 67 +++++++++++++++++++++++ test/deb/ubuntu-xenial/Vagrantfile | 14 +++++ test/deb/ubuntu-xenial/provision.sh | 17 ++++++ 16 files changed, 284 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/build-support/release/release-candidate ---------------------------------------------------------------------- diff --git a/build-support/release/release-candidate b/build-support/release/release-candidate index b3ebe91..c5aad69 100755 --- a/build-support/release/release-candidate +++ b/build-support/release/release-candidate @@ -20,6 +20,7 @@ cd "${ROOT_DIR}" declare -A DIST_DIRS=( ["ubuntu-trusty"]="${ROOT_DIR}/artifacts/aurora-ubuntu-trusty/dist" + ["ubuntu-xenial"]="${ROOT_DIR}/artifacts/aurora-ubuntu-xenial/dist" ["debian-jessie"]="${ROOT_DIR}/artifacts/aurora-debian-jessie/dist" ["centos-7"]="${ROOT_DIR}/artifacts/aurora-centos-7/dist/rpmbuild/RPMS/x86_64" ) http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/debian-jessie/Dockerfile ---------------------------------------------------------------------- diff --git a/builder/deb/debian-jessie/Dockerfile b/builder/deb/debian-jessie/Dockerfile index 54cb2e0..077d2b6 100644 --- a/builder/deb/debian-jessie/Dockerfile +++ b/builder/deb/debian-jessie/Dockerfile @@ -28,6 +28,7 @@ RUN echo \ RUN apt-get update && apt-get -y install \ bison \ debhelper \ + dh-systemd \ devscripts \ dpkg-dev \ curl \ http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/debian-jessie/build.sh ---------------------------------------------------------------------- diff --git a/builder/deb/debian-jessie/build.sh b/builder/deb/debian-jessie/build.sh index e0aeaf5..04f4cb1 100755 --- a/builder/deb/debian-jessie/build.sh +++ b/builder/deb/debian-jessie/build.sh @@ -22,6 +22,15 @@ tar --strip-components 1 -C . -xf /src.tar.gz cp -R /specs/debian . +DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') +CODENAME=$(lsb_release -cs | tr '[:upper:]' '[:lower:]') +THIRD_PARTY_REPO="https://svn.apache.org/repos/asf/aurora/3rdparty/"; +THIRD_PARTY_REPO+="${DISTRO}/${CODENAME}64/python/" + +# Place the link to the correct python egg into aurora-pants.ini +echo "[python-repos]" >> ./debian/aurora-pants.ini +echo "repos: ['third_party/', '${THIRD_PARTY_REPO}']" >> ./debian/aurora-pants.ini + export DEBFULLNAME='Apache Aurora' export DEBEMAIL='[email protected]' http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-trusty/Dockerfile ---------------------------------------------------------------------- diff --git a/builder/deb/ubuntu-trusty/Dockerfile b/builder/deb/ubuntu-trusty/Dockerfile index 802049b..7712549 100644 --- a/builder/deb/ubuntu-trusty/Dockerfile +++ b/builder/deb/ubuntu-trusty/Dockerfile @@ -19,6 +19,7 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get -y install \ bison \ debhelper \ + dh-systemd \ devscripts \ dpkg-dev \ curl \ http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-trusty/build.sh ---------------------------------------------------------------------- diff --git a/builder/deb/ubuntu-trusty/build.sh b/builder/deb/ubuntu-trusty/build.sh index e0aeaf5..04f4cb1 100755 --- a/builder/deb/ubuntu-trusty/build.sh +++ b/builder/deb/ubuntu-trusty/build.sh @@ -22,6 +22,15 @@ tar --strip-components 1 -C . -xf /src.tar.gz cp -R /specs/debian . +DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') +CODENAME=$(lsb_release -cs | tr '[:upper:]' '[:lower:]') +THIRD_PARTY_REPO="https://svn.apache.org/repos/asf/aurora/3rdparty/"; +THIRD_PARTY_REPO+="${DISTRO}/${CODENAME}64/python/" + +# Place the link to the correct python egg into aurora-pants.ini +echo "[python-repos]" >> ./debian/aurora-pants.ini +echo "repos: ['third_party/', '${THIRD_PARTY_REPO}']" >> ./debian/aurora-pants.ini + export DEBFULLNAME='Apache Aurora' export DEBEMAIL='[email protected]' http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-xenial/Dockerfile ---------------------------------------------------------------------- diff --git a/builder/deb/ubuntu-xenial/Dockerfile b/builder/deb/ubuntu-xenial/Dockerfile new file mode 100644 index 0000000..a32f4bc --- /dev/null +++ b/builder/deb/ubuntu-xenial/Dockerfile @@ -0,0 +1,55 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM ubuntu:xenial + +WORKDIR /aurora +ENV HOME /aurora +ENV DEBIAN_FRONTEND noninteractive + +# Using gcc-4.8 to ensure ABI for thermos in containers running older distros +RUN apt-get update && apt-get -y install \ + bison \ + debhelper \ + devscripts \ + dh-systemd \ + dpkg-dev \ + curl \ + git \ + gcc-4.8 \ + libapr1-dev \ + libcurl4-openssl-dev \ + libsvn-dev \ + libffi-dev \ + python-all-dev \ + software-properties-common \ + libkrb5-dev + +RUN apt-get -y install openjdk-8-jdk \ + && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java + +RUN curl -sSL http://apache.org/dist/thrift/KEYS | gpg --import - && \ + gpg --export --armor 66B778F9 | apt-key add - && \ + echo 'deb http://www.apache.org/dist/thrift/debian 0.9.1 main' > /etc/apt/sources.list.d/thrift.list && \ + apt-get update && \ + apt-get -y install thrift-compiler=0.9.1 + +# Install gradle. +RUN git clone --depth 1 https://github.com/benley/gradle-packaging \ + && cd gradle-packaging \ + && apt-get -y install ruby ruby-dev unzip wget \ + && gem install fpm && ./gradle-mkdeb.sh 3.1 \ + && apt-get -y install gdebi-core \ + && gdebi --non-interactive gradle-3.1_3.1-2_all.deb \ + && cd .. && rm -rf gradle-packaging + +ADD build.sh /build.sh http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/builder/deb/ubuntu-xenial/build.sh ---------------------------------------------------------------------- diff --git a/builder/deb/ubuntu-xenial/build.sh b/builder/deb/ubuntu-xenial/build.sh new file mode 100755 index 0000000..90a9414 --- /dev/null +++ b/builder/deb/ubuntu-xenial/build.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +mkdir /scratch +cd /scratch + +tar --strip-components 1 -C . -xf /src.tar.gz + +cp -R /specs/debian . + +# Xenial tries to convert init and upstart scripts before using systemd units. +# Avoid conflict by not including them for now. +rm ./debian/*.upstart ./debian/*.init + +DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') +CODENAME=$(lsb_release -cs | tr '[:upper:]' '[:lower:]') +THIRD_PARTY_REPO="https://svn.apache.org/repos/asf/aurora/3rdparty/"; +THIRD_PARTY_REPO+="${DISTRO}/${CODENAME}64/python/" + +# Place the link to the correct python egg into aurora-pants.ini +echo "[python-repos]" >> ./debian/aurora-pants.ini +echo "repos: ['third_party/', '${THIRD_PARTY_REPO}']" >> ./debian/aurora-pants.ini + +export DEBFULLNAME='Apache Aurora' +export DEBEMAIL='[email protected]' + +dch \ + --newversion $AURORA_VERSION \ + --package apache-aurora \ + --urgency medium \ + "Apache Aurora package builder <[email protected]> $(date -R)" +dch --release '' + +dpkg-buildpackage -uc -b -tc + +mkdir /dist +mv ../*.deb /dist http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-executor.thermos.default ---------------------------------------------------------------------- diff --git a/specs/debian/aurora-executor.thermos.default b/specs/debian/aurora-executor.thermos.default index 82ab0c8..716f08e 100644 --- a/specs/debian/aurora-executor.thermos.default +++ b/specs/debian/aurora-executor.thermos.default @@ -16,3 +16,6 @@ # Listen port for thermos_observer OBSERVER_PORT=1338 + +MESOS_ROOT=/var/lib/mesos +EXTRA_THERMOS_OBSERVER_ARGS="" http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-executor.thermos.service ---------------------------------------------------------------------- diff --git a/specs/debian/aurora-executor.thermos.service b/specs/debian/aurora-executor.thermos.service index da35116..32fc004 100644 --- a/specs/debian/aurora-executor.thermos.service +++ b/specs/debian/aurora-executor.thermos.service @@ -11,16 +11,17 @@ # limitations under the License. [Unit] -Description=Thermos Observer +Description=Aurora Thermos Observer After=network.target Wants=network.target [Service] -Type=forking -IgnoreSIGPIPE=no -GuessMainPID=no -ExecStart=/etc/init.d/thermos start -ExecStop=/etc/init.d/thermos stop +EnvironmentFile=/etc/default/thermos +ExecStart=/usr/sbin/thermos_observer \ + --port=${OBSERVER_PORT} \ + --mesos-root=${MESOS_ROOT} \ + --log_to_disk=NONE \ + --log_to_stderr=google:INFO User=root Group=root Restart=always http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-pants.ini ---------------------------------------------------------------------- diff --git a/specs/debian/aurora-pants.ini b/specs/debian/aurora-pants.ini index e4f2c54..b2c622d 100644 --- a/specs/debian/aurora-pants.ini +++ b/specs/debian/aurora-pants.ini @@ -1,5 +1,2 @@ [GLOBAL] print_exception_stacktrace: True - -[python-repos] -repos: ['third_party/', 'https://svn.apache.org/repos/asf/aurora/3rdparty/ubuntu/trusty64/python/'] http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-scheduler.service ---------------------------------------------------------------------- diff --git a/specs/debian/aurora-scheduler.service b/specs/debian/aurora-scheduler.service index ae33d26..b81cb63 100644 --- a/specs/debian/aurora-scheduler.service +++ b/specs/debian/aurora-scheduler.service @@ -16,11 +16,7 @@ After=network.target Wants=network.target [Service] -Type=forking -IgnoreSIGPIPE=no -GuessMainPID=no -ExecStart=/etc/init.d/aurora-scheduler start -ExecStop=/etc/init.d/aurora-scheduler stop +ExecStart=/usr/bin/aurora-scheduler-startup User=aurora Group=aurora Restart=always http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/aurora-scheduler.startup.sh ---------------------------------------------------------------------- diff --git a/specs/debian/aurora-scheduler.startup.sh b/specs/debian/aurora-scheduler.startup.sh new file mode 100644 index 0000000..3f97a73 --- /dev/null +++ b/specs/debian/aurora-scheduler.startup.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Starts up an Aurora scheduler process. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source /etc/default/aurora-scheduler + +# Environment variables control the behavior of the Mesos scheduler driver (libmesos). +export GLOG_v LIBPROCESS_PORT LIBPROCESS_IP +export JAVA_OPTS="${JAVA_OPTS[*]}" + +# Preferences Java 1.8 over any other Java version. +export PATH=/usr/lib/jvm/java-1.8.0/bin:${PATH} + +exec /usr/sbin/aurora-scheduler \ + -cluster_name="$CLUSTER_NAME" \ + -http_port="$HTTP_PORT" \ + -native_log_quorum_size="$QUORUM_SIZE" \ + -zk_endpoints="$ZK_ENDPOINTS" \ + -mesos_master_address="$MESOS_MASTER" \ + -serverset_path="$ZK_SERVERSET_PATH" \ + -native_log_zk_group_path="$ZK_LOGDB_PATH" \ + -native_log_file_path="$LOGDB_FILE_PATH" \ + -backup_dir="$BACKUP_DIR" \ + -thermos_executor_path="$THERMOS_EXECUTOR_PATH" \ + -thermos_executor_resources="$THERMOS_EXECUTOR_RESOURCES" \ + -thermos_executor_flags="$THERMOS_EXECUTOR_FLAGS" \ + -allowed_container_types="$ALLOWED_CONTAINER_TYPES" \ + $EXTRA_SCHEDULER_ARGS http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/specs/debian/rules ---------------------------------------------------------------------- diff --git a/specs/debian/rules b/specs/debian/rules index 4effc3e..8e338b8 100755 --- a/specs/debian/rules +++ b/specs/debian/rules @@ -16,8 +16,10 @@ # Uncomment for debugging. #DH_VERBOSE = 1 +TMP = $(CURDIR)/debian + %: - dh $@ --exclude=aurora-scheduler.bat + dh $@ --exclude=aurora-scheduler.bat --with=systemd gradle = $(shell which gradle) pants = $(CURDIR)/pants @@ -42,3 +44,8 @@ __pants_build: override_dh_installinit: dh_installinit --name=thermos dh_installinit + +override_dh_auto_install: + dh_auto_install + # Install start up script for aurora-scheduler + install -D -m 0755 $(TMP)/aurora-scheduler.startup.sh $(TMP)/aurora-scheduler/usr/bin/aurora-scheduler-startup http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/test/deb/ubuntu-xenial/README.md ---------------------------------------------------------------------- diff --git a/test/deb/ubuntu-xenial/README.md b/test/deb/ubuntu-xenial/README.md new file mode 100644 index 0000000..a5284ec --- /dev/null +++ b/test/deb/ubuntu-xenial/README.md @@ -0,0 +1,67 @@ +# Installing Aurora + +## Install packages + +Two methods are described, one for installing locally built packages, the other +for installing released packages or release candidate packages. + +### Locally built + + # Install vagrant scp + vagrant plugin install vagrant-scp + + # Scp over the newly built packages + for deb in ../../../artifacts/aurora-ubuntu-xenial/dist/*.deb; do + vagrant scp $deb :$(basename $deb); + done + + # Install each deb + vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338 + sudo dpkg -i aurora*.deb thermos*.deb + +### Released + + vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338 + version=0.15.0 + pkg_root="https://apache.bintray.com/aurora/ubuntu-xenial/"; + for deb in \ + aurora-scheduler_${version}_amd64.deb \ + aurora-executor_${version}_amd64.deb \ + aurora-tools_${version}_amd64.deb; do + wget $pkg_root/$deb + sudo dpkg -i $deb + done + +## Initialize and start + +The replicated log has to be initialized manually: + + # Stop services first + sudo systemctl stop aurora-scheduler + sudo systemctl stop thermos + + sudo -u aurora mkdir -p /var/lib/aurora/scheduler/db + sudo -u aurora mesos-log initialize --path=/var/lib/aurora/scheduler/db + sudo systemctl start aurora-scheduler + sudo systemctl start thermos + +To make the Thermos observer work, you will have to follow the instructions of our +[Install Guide](https://github.com/apache/aurora/blob/master/docs/operations/installation.md#configuration). + +## Create a job + +``` +echo " +task = SequentialTask( + processes = [Process(name = 'hello', cmdline = 'echo hello')], + resources = Resources(cpu = 0.5, ram = 128*MB, disk = 128*MB)) +jobs = [Service( + task = task, cluster = 'example', role = 'vagrant', environment = 'prod', name = 'hello')]" > hello_world.aurora + +aurora job create example/vagrant/prod/hello hello_world.aurora +``` +## Troubleshooting + +* Mesos: `/var/log/mesos` +* Aurora scheduler: `cat /var/log/syslog | grep aurora-scheduler` and `sudo journalctl -u aurora-scheduler` +* Aurora observer: `sudo journalctl -u thermos` http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/test/deb/ubuntu-xenial/Vagrantfile ---------------------------------------------------------------------- diff --git a/test/deb/ubuntu-xenial/Vagrantfile b/test/deb/ubuntu-xenial/Vagrantfile new file mode 100644 index 0000000..3ce6209 --- /dev/null +++ b/test/deb/ubuntu-xenial/Vagrantfile @@ -0,0 +1,14 @@ +Vagrant.require_version ">= 1.5.0" + +Vagrant.configure(2) do |config| + config.vm.define "aurora_xenial" + config.vm.box = "bento/ubuntu-16.04" + + config.vm.hostname = "localhost" + + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", "4096"] + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + end + config.vm.provision "shell", path: "provision.sh" +end http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/2b19410b/test/deb/ubuntu-xenial/provision.sh ---------------------------------------------------------------------- diff --git a/test/deb/ubuntu-xenial/provision.sh b/test/deb/ubuntu-xenial/provision.sh new file mode 100644 index 0000000..eda79f6 --- /dev/null +++ b/test/deb/ubuntu-xenial/provision.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +apt-get update + +apt-get install -y openjdk-8-jre zookeeperd + +update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java + +# Mesos dependencies +apt-get -y install libcurl3 libevent-dev libsvn1 libcurl4-nss-dev + +package=mesos_1.0.0-2.0.86.ubuntu1604_amd64.deb +wget -c http://repos.mesosphere.com/ubuntu/pool/main/m/mesos/$package +dpkg -i $package + +systemctl start mesos-master +systemctl start mesos-slave
