Repository: aurora-packaging Updated Branches: refs/heads/master aa9bd2de9 -> 7cb3829ca
Update packaging to work with Aurora 0.13. In addition to the minimal necessary packaging changes, I have tried to align the tests of the different distributions. A few differences remain. Those are mainly due to different defaults in our DEBs and RPMs. I will leave this to another commit. Reviewed at https://reviews.apache.org/r/47647/ Project: http://git-wip-us.apache.org/repos/asf/aurora-packaging/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora-packaging/commit/7cb3829c Tree: http://git-wip-us.apache.org/repos/asf/aurora-packaging/tree/7cb3829c Diff: http://git-wip-us.apache.org/repos/asf/aurora-packaging/diff/7cb3829c Branch: refs/heads/master Commit: 7cb3829caabb5ceb5a146b82f829913016dd7883 Parents: aa9bd2d Author: Stephan Erb <[email protected]> Authored: Sat May 28 22:43:32 2016 +0200 Committer: Stephan Erb <[email protected]> Committed: Sat May 28 22:43:32 2016 +0200 ---------------------------------------------------------------------- build-support/virtualenv | 2 +- builder/deb/debian-jessie/build.sh | 2 - specs/debian/control | 2 +- specs/rpm/SOURCES/aurora.sysconfig | 2 +- specs/rpm/aurora.spec | 4 +- test/deb/debian-jessie/README.md | 76 ++++++++++++++++++++++---------- test/deb/debian-jessie/Vagrantfile | 4 +- test/deb/debian-jessie/provision.sh | 9 +++- test/deb/ubuntu-trusty/README.md | 44 +++++++++++++----- test/deb/ubuntu-trusty/Vagrantfile | 3 ++ test/deb/ubuntu-trusty/provision.sh | 8 +++- test/rpm/centos-7/README.md | 20 +++++---- test/rpm/centos-7/Vagrantfile | 4 +- test/rpm/centos-7/provision.sh | 6 ++- 14 files changed, 131 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/build-support/virtualenv ---------------------------------------------------------------------- diff --git a/build-support/virtualenv b/build-support/virtualenv index 334653b..3ed5b8c 100755 --- a/build-support/virtualenv +++ b/build-support/virtualenv @@ -14,7 +14,7 @@ # # Wrapper for self-bootstrapping virtualenv set -ex -VIRTUALENV_VERSION=12.1.1 +VIRTUALENV_VERSION=14.0.6 if which python2.7 >/dev/null; then PY=`which python2.7` http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/builder/deb/debian-jessie/build.sh ---------------------------------------------------------------------- diff --git a/builder/deb/debian-jessie/build.sh b/builder/deb/debian-jessie/build.sh index f122a51..e0aeaf5 100755 --- a/builder/deb/debian-jessie/build.sh +++ b/builder/deb/debian-jessie/build.sh @@ -13,8 +13,6 @@ # limitations under the License. # -#TODO (wfarner): Figure out how to symlink build files into Docker container. - set -ex mkdir /scratch http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/specs/debian/control ---------------------------------------------------------------------- diff --git a/specs/debian/control b/specs/debian/control index 23afb2b..c59b67e 100644 --- a/specs/debian/control +++ b/specs/debian/control @@ -22,7 +22,7 @@ Section: net Depends: ${shlibs:Depends}, ${misc:Depends}, adduser, openjdk-8-jre-headless | java8-runtime-headless, - mesos (>= 0.21.1) + mesos (>= 0.26.0) Recommends: aurora-tools Suggests: aurora-doc Description: Apache Aurora scheduler http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/specs/rpm/SOURCES/aurora.sysconfig ---------------------------------------------------------------------- diff --git a/specs/rpm/SOURCES/aurora.sysconfig b/specs/rpm/SOURCES/aurora.sysconfig index 166df4d..d812341 100644 --- a/specs/rpm/SOURCES/aurora.sysconfig +++ b/specs/rpm/SOURCES/aurora.sysconfig @@ -72,5 +72,5 @@ AURORA_FLAGS=( # The local path of the Thermos executor binary. -thermos_executor_path='/usr/bin/thermos_executor' # Flags to pass to the Thermos executor. - -thermos_executor_flags='--announcer-enable --announcer-ensemble 127.0.0.1:2181' + -thermos_executor_flags='--announcer-ensemble 127.0.0.1:2181' ) http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/specs/rpm/aurora.spec ---------------------------------------------------------------------- diff --git a/specs/rpm/aurora.spec b/specs/rpm/aurora.spec index d400706..a4fc46d 100644 --- a/specs/rpm/aurora.spec +++ b/specs/rpm/aurora.spec @@ -14,7 +14,7 @@ # Overridable variables; %if %{?!AURORA_VERSION:1}0 -%global AURORA_VERSION 0.9.0 +%global AURORA_VERSION 0.13.0 %endif %if %{?!AURORA_INTERNAL_VERSION:1}0 @@ -42,7 +42,7 @@ %endif %if %{?!MESOS_VERSION:1}0 -%global MESOS_VERSION 0.25.0 +%global MESOS_VERSION 0.26.0 %endif %if %{?!PEX_BINARIES:1}0 http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/deb/debian-jessie/README.md ---------------------------------------------------------------------- diff --git a/test/deb/debian-jessie/README.md b/test/deb/debian-jessie/README.md index 9be58bd..a695854 100644 --- a/test/deb/debian-jessie/README.md +++ b/test/deb/debian-jessie/README.md @@ -1,32 +1,60 @@ -# Within vagrant install vagrant scp -vagrant plugin install vagrant-scp -# Then scp over the newly built packages -vagrant scp \ - ~/aurora-packaging/artifacts/aurora-debian-jessie/dist/aurora-scheduler_0.12.1.uber.3_amd64.deb \ - aurora_jessie:aurora-scheduler_0.12.1.uber.3_amd64.deb +# Installing Aurora -# Install each deb via dpkg -i +## Install packages -# Start Mesos + ZK -sudo systemctl start mesos-master -sudo systemctl start mesos-slave -# Stop scheduler to setup log replication. -sudo systemctl stop aurora-scheduler +Two methods are described, one for installing locally built packages, the other +for installing released packages or release candidate packages. -# Setup log replication -sudo chown -R aurora:aurora /var/lib/aurora -sudo -u aurora mesos-log initialize --path=/var/lib/aurora/scheduler/db +### Locally built -# Start Aurora scheduler again -sudo systemctl start aurora-scheduler + # Install vagrant scp + vagrant plugin install vagrant-scp + + # Scp over the newly built packages + for deb in ../../../artifacts/aurora-debian-jessie/dist/*.deb; do + vagrant scp $deb :$(basename $deb) + done + + # Install each rpm + vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338 + sudo dpkg -i *.deb + +### Released + + vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338 + version=0.12.0 + pkg_root="https://apache.bintray.com/aurora/debian-jessie/"; + for deb in \ + aurora-scheduler_${version}_amd64.deb \ + aurora-executor_${version}_amd64.deb \ + aurora-tools_${version}_amd64.deb; do + wget $pkg_root/$deb + sudo dpkg -i $deb + done + +## Initialize and start + +The scheduler and observer will automatically start when installed. However, teh replicated log +has to be initialized manually: + + sudo systemctl stop aurora-scheduler + sudo -u aurora mkdir -p /var/lib/aurora/scheduler/db + sudo -u aurora mesos-log initialize --path=/var/lib/aurora/scheduler/db + sudo systemctl start aurora-scheduler ## Create a job -echo " -task = SequentialTask( - processes = [Process(name = 'hello', cmdline = 'echo hello')], - resources = Resources(cpu = 1.0, ram = 128*MB, disk = 128*MB)) -jobs = [Service( - task = task, cluster = 'example', role = 'www-data', environment = 'prod', name = 'hello')]" > hello_world.aurora + echo " + task = SequentialTask( + processes = [Process(name = 'hello', cmdline = 'echo hello')], + resources = Resources(cpu = 0.5, ram = 128*MB, disk = 128*MB)) + jobs = [Service( + task = task, cluster = 'example', role = 'www-data', environment = 'prod', name = 'hello')]" > hello_world.aurora + + aurora job create example/www-data/prod/hello hello_world.aurora + +## Troubleshooting -aurora job create example/www-data/prod/hello hello_world.aurora +* Mesos: `/var/log/mesos` +* Aurora scheduler: `sudo journalctl -u aurora-scheduler` +* Aurora observer: `/var/log/aurora/thermos_observer.log` http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/deb/debian-jessie/Vagrantfile ---------------------------------------------------------------------- diff --git a/test/deb/debian-jessie/Vagrantfile b/test/deb/debian-jessie/Vagrantfile index 619b793..f79c337 100644 --- a/test/deb/debian-jessie/Vagrantfile +++ b/test/deb/debian-jessie/Vagrantfile @@ -1,12 +1,14 @@ Vagrant.require_version ">= 1.5.0" Vagrant.configure(2) do |config| + config.vm.define "aurora_jessie" config.vm.box = "debian/jessie64" + config.vm.hostname = "localhost" + config.vm.provider :virtualbox do |vb| vb.customize ["modifyvm", :id, "--memory", "4096"] vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] end - config.vm.define "aurora_jessie" config.vm.provision "shell", path: "provision.sh" end http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/deb/debian-jessie/provision.sh ---------------------------------------------------------------------- diff --git a/test/deb/debian-jessie/provision.sh b/test/deb/debian-jessie/provision.sh index 6970279..27f8b88 100644 --- a/test/deb/debian-jessie/provision.sh +++ b/test/deb/debian-jessie/provision.sh @@ -7,5 +7,12 @@ apt-get install -y openjdk-8-jre libsvn1 zookeeperd update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java package=mesos_0.26.0-0.2.145.debian81_amd64.deb -wget -c http://downloads.mesosphere.io/master/debian/8/$package +wget -c https://downloads.mesosphere.io/master/debian/8/$package dpkg -i $package + +# NOTE: This appears to be a missing dependency of the mesos deb package and is needed +# for the python mesos native bindings. +sudo apt-get -y install libcurl4-nss-dev + +systemctl start mesos-master +systemctl start mesos-slave http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/deb/ubuntu-trusty/README.md ---------------------------------------------------------------------- diff --git a/test/deb/ubuntu-trusty/README.md b/test/deb/ubuntu-trusty/README.md index ec34ef5..124b8f1 100644 --- a/test/deb/ubuntu-trusty/README.md +++ b/test/deb/ubuntu-trusty/README.md @@ -1,8 +1,29 @@ # Installing Aurora -## Fetch and install packages - version=0.9.0 - pkg_root="https://bintray.com/artifact/download/apache/aurora/"; +## Install packages + +Two methods are described, one for installing locally built packages, the other +for installing released packages or release candidate packages. + +### Locally built + + # Install vagrant scp + vagrant plugin install vagrant-scp + + # Scp over the newly built packages + for deb in ../../../artifacts/aurora-debian-jessie/dist/*.deb; do + vagrant scp $deb :$(basename $deb) + done + + # Install each rpm + vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338 + sudo dpkg -i *.deb + +### Released + + vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338 + version=0.12.0 + pkg_root="https://apache.bintray.com/aurora/ubuntu-trusty/"; for deb in \ aurora-scheduler_${version}_amd64.deb \ aurora-executor_${version}_amd64.deb \ @@ -11,9 +32,10 @@ sudo dpkg -i $deb done -The scheduler and observer will automatically start when installed. +## Initialize and start -## Initialize scheduler's replicated log +The scheduler and observer will automatically start when installed. However, teh replicated log +has to be initialized manually: sudo stop aurora-scheduler sudo -u aurora mkdir -p /var/lib/aurora/scheduler/db @@ -25,12 +47,14 @@ The scheduler and observer will automatically start when installed. echo " task = SequentialTask( processes = [Process(name = 'hello', cmdline = 'echo hello')], - resources = Resources(cpu = 1.0, ram = 128*MB, disk = 128*MB)) - + resources = Resources(cpu = 0.5, ram = 128*MB, disk = 128*MB)) jobs = [Service( task = task, cluster = 'example', role = 'www-data', environment = 'prod', name = 'hello')]" > hello_world.aurora + aurora job create example/www-data/prod/hello hello_world.aurora -## Logs -* scheduler: `/var/log/upstart/aurora-scheduler.log` -* observer: `/var/log/upstart/thermos.log` +## Troubleshooting + +* Mesos: `/var/log/mesos` +* Aurora scheduler: `/var/log/upstart/aurora-scheduler.log` +* Aurora observer: `/var/log/upstart/thermos.log` http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/deb/ubuntu-trusty/Vagrantfile ---------------------------------------------------------------------- diff --git a/test/deb/ubuntu-trusty/Vagrantfile b/test/deb/ubuntu-trusty/Vagrantfile index 3d241eb..ed5ec8e 100644 --- a/test/deb/ubuntu-trusty/Vagrantfile +++ b/test/deb/ubuntu-trusty/Vagrantfile @@ -1,8 +1,11 @@ Vagrant.require_version ">= 1.5.0" Vagrant.configure(2) do |config| + config.vm.define "aurora_trusty" config.vm.box = "ubuntu/trusty64" + config.vm.hostname = "localhost" + config.vm.provider :virtualbox do |vb| vb.customize ["modifyvm", :id, "--memory", "4096"] vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/deb/ubuntu-trusty/provision.sh ---------------------------------------------------------------------- diff --git a/test/deb/ubuntu-trusty/provision.sh b/test/deb/ubuntu-trusty/provision.sh index 328f441..2a9e9b6 100644 --- a/test/deb/ubuntu-trusty/provision.sh +++ b/test/deb/ubuntu-trusty/provision.sh @@ -6,7 +6,13 @@ apt-get install -y openjdk-8-jre libsvn1 zookeeperd update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -package=mesos_0.24.1-0.2.35.ubuntu1404_amd64.deb +package=mesos_0.26.0-0.2.145.ubuntu1404_amd64.deb wget -c https://downloads.mesosphere.io/master/ubuntu/14.04/$package dpkg -i $package + +# NOTE: This appears to be a missing dependency of the mesos deb package and is needed +# for the python mesos native bindings. +sudo apt-get -y install libcurl4-nss-dev + start mesos-master +start mesos-slave http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/rpm/centos-7/README.md ---------------------------------------------------------------------- diff --git a/test/rpm/centos-7/README.md b/test/rpm/centos-7/README.md index 527fbb2..eba2e6f 100644 --- a/test/rpm/centos-7/README.md +++ b/test/rpm/centos-7/README.md @@ -7,18 +7,15 @@ for installing released packages or release candidate packages. ### Locally built -#### Install vagrant scp - + # Install vagrant scp vagrant plugin install vagrant-scp -#### Then scp over the newly built packages - + # Scp over the newly built packages for rpm in ../../../artifacts/aurora-centos-7/dist/rpmbuild/RPMS/x86_64/*.rpm; do - vagrant scp $rpm aurora_centos_7:$(basename $rpm) + vagrant scp $rpm :$(basename $rpm) done -#### Install each rpm - + # Install each rpm vagrant ssh -- -L8081:localhost:8081 -L1338:localhost:1338 sudo yum install -y *.rpm @@ -37,6 +34,7 @@ for installing released packages or release candidate packages. ## Initialize and start + sudo -u aurora mkdir -p /var/lib/aurora/scheduler/db sudo -u aurora mesos-log initialize --path=/var/lib/aurora/scheduler/db sudo systemctl start aurora @@ -48,7 +46,13 @@ for installing released packages or release candidate packages. task = SequentialTask( processes = [Process(name = 'hello', cmdline = 'echo hello')], resources = Resources(cpu = 0.5, ram = 128*MB, disk = 128*MB)) - jobs = [Service( task = task, cluster = 'main', role = 'vagrant', environment = 'prod', name = 'hello')]" > hello_world.aurora + aurora job create main/vagrant/prod/hello hello_world.aurora + +## Troubleshooting + +* Mesos: `/var/log/mesos` +* Aurora scheduler: `sudo journalctl -u aurora` +* Aurora observer: `sudo journalctl -u thermos-observer` http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/rpm/centos-7/Vagrantfile ---------------------------------------------------------------------- diff --git a/test/rpm/centos-7/Vagrantfile b/test/rpm/centos-7/Vagrantfile index 1a64166..5059b84 100644 --- a/test/rpm/centos-7/Vagrantfile +++ b/test/rpm/centos-7/Vagrantfile @@ -1,12 +1,14 @@ Vagrant.require_version ">= 1.5.0" Vagrant.configure(2) do |config| + config.vm.define "aurora_centos_7" config.vm.box = "bento/centos-7.1" + config.vm.hostname = "localhost" + config.vm.provider :virtualbox do |vb| vb.customize ["modifyvm", :id, "--memory", "4096"] vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] end - config.vm.define "aurora_centos_7" config.vm.provision "shell", path: "provision.sh" end http://git-wip-us.apache.org/repos/asf/aurora-packaging/blob/7cb3829c/test/rpm/centos-7/provision.sh ---------------------------------------------------------------------- diff --git a/test/rpm/centos-7/provision.sh b/test/rpm/centos-7/provision.sh index 199cd43..e42a015 100644 --- a/test/rpm/centos-7/provision.sh +++ b/test/rpm/centos-7/provision.sh @@ -6,5 +6,7 @@ service zookeeper-server init systemctl start zookeeper-server rpm -Uvh https://repos.mesosphere.io/el/7/noarch/RPMS/mesosphere-el-repo-7-1.noarch.rpm -yum -y install mesos-0.25.0 -systemctl start mesos-slave mesos-master +yum -y install mesos-0.26.0 + +systemctl start mesos-master +systemctl start mesos-slave
