Repository: aurora Updated Branches: refs/heads/master e2ca371c9 -> c245b9b76
Update to Mesos 1.1.0. Included changes: * Handle new task states introduced in the latest Mesos release. * Prevent NullPointer exception when inspecting an empty/invalid executor config in a test. Probably this is due to a change in the Mesos protobufs. * Fix bug preventing the teardown of Vagrant boxes started by the egg build. * Increase resources for the Mesos egg builds. The build for all distribution now takes 2h in total. Full Mesos changelog: https://git-wip-us.apache.org/repos/asf?p=mesos.git;a=blob_plain;f=CHANGELOG;hb=1.1.0 Bugs closed: AURORA-1813 Reviewed at https://reviews.apache.org/r/54255/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/c245b9b7 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/c245b9b7 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/c245b9b7 Branch: refs/heads/master Commit: c245b9b76b6bcd6701117d09dfce008bdd1a8768 Parents: e2ca371 Author: Stephan Erb <[email protected]> Authored: Tue Dec 27 12:36:44 2016 +0100 Committer: Stephan Erb <[email protected]> Committed: Tue Dec 27 12:36:44 2016 +0100 ---------------------------------------------------------------------- 3rdparty/python/BUILD | 2 +- RELEASE-NOTES.md | 1 + Vagrantfile | 2 +- build-support/packer/build.sh | 2 +- build-support/python/make-mesos-native-egg | 17 +++++++++-------- build.gradle | 2 +- .../apache/aurora/scheduler/base/Conversions.java | 7 +++++++ .../executor/ExecutorSettingsLoader.java | 3 +-- 8 files changed, 22 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/3rdparty/python/BUILD ---------------------------------------------------------------------- diff --git a/3rdparty/python/BUILD b/3rdparty/python/BUILD index c572da0..7648ac8 100644 --- a/3rdparty/python/BUILD +++ b/3rdparty/python/BUILD @@ -11,7 +11,7 @@ # limitations under the License. # -MESOS_REV = '1.0.0' +MESOS_REV = '1.1.0' python_requirement_library( name = 'mesos.interface', http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index a792594..164d7a3 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -2,6 +2,7 @@ ========================= ### New/updated: +- Upgraded Mesos to 1.1.0. - Added a new flag --snapshot_hydrate_stores that controls which H2-backed stores to write fully hydrated into the Scheduler snapshot. Can lead to significantly lower snapshot times for large clusters if you set this flag to an empty list. Old behavior is preserved by default, but see http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/Vagrantfile ---------------------------------------------------------------------- diff --git a/Vagrantfile b/Vagrantfile index c20c786..d1c536b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -25,7 +25,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.hostname = "aurora.local" # See build-support/packer/README.md for instructions on updating this box. config.vm.box = "apache-aurora/dev-environment" - config.vm.box_version = "0.0.11" + config.vm.box_version = "0.0.12" config.vm.define "devcluster" do |dev| dev.vm.network :private_network, ip: "192.168.33.7", :auto_config => false http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/build-support/packer/build.sh ---------------------------------------------------------------------- diff --git a/build-support/packer/build.sh b/build-support/packer/build.sh index 2fb4723..548cf37 100644 --- a/build-support/packer/build.sh +++ b/build-support/packer/build.sh @@ -17,7 +17,7 @@ set -o errexit set -o nounset set -o verbose -readonly MESOS_VERSION=1.0.0 +readonly MESOS_VERSION=1.1.0 function remove_unused { # The default ubuntu/trusty64 image includes juju-core, which adds ~300 MB to our image. http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/build-support/python/make-mesos-native-egg ---------------------------------------------------------------------- diff --git a/build-support/python/make-mesos-native-egg b/build-support/python/make-mesos-native-egg index bcc6e85..3f5ab39 100755 --- a/build-support/python/make-mesos-native-egg +++ b/build-support/python/make-mesos-native-egg @@ -4,7 +4,8 @@ set -u -e MESOS_BASEURL=https://archive.apache.org/dist/mesos -BOX_RAM_MB=$((4 * 1024)) # 4 GiB, needed to link libmesos +BOX_RAM_MB=$((8 * 1024)) # at least 4 GiB, needed to link libmesos +BOX_CPUS=4 usage() { cat <<EOF @@ -20,8 +21,9 @@ EOF } setup_tempdir() { - TMPDIR="/tmp" + TMPDIR=$(mktemp -d -t make-mesos-eggs.XXXXX) cleanup() { + cd "$TMPDIR" if [[ -f Vagrantfile ]]; then vagrant destroy -f fi @@ -30,7 +32,6 @@ setup_tempdir() { } # TODO(ksweeney): Use atexit.sh from mesos and its superior handling of this cleanup idiom. trap cleanup EXIT - TMPDIR=$(mktemp -d -t make-mesos-eggs.XXXXX) } # Create a new vagrant box @@ -41,7 +42,8 @@ Vagrant.require_version ">= 1.5.0" Vagrant.configure("2") do |config| config.vm.box = "$box_name" config.vm.provider "virtualbox" do |vb| - vb.customize ["modifyvm", :id, "--memory", "$BOX_RAM_MB"] + vb.memory = "$BOX_RAM_MB" + vb.cpus = "$BOX_CPUS" end end EOF @@ -55,6 +57,7 @@ fetch_and_build_mesos() { tar zxvf mesos-${mesos_version}.tar.gz cd mesos-$mesos_version ./configure --disable-java --enable-optimize + export MAKEFLAGS="-j$BOX_CPUS" make find . -name '*.egg' -exec cp -v {} /vagrant \\; EOF @@ -165,9 +168,7 @@ CENTOS6_X86_64_DEPENDENCIES=( build_centos6() { local mesos_version=$1 output_basedir=$2 local python_outdir=$output_basedir/centos/6/python - mkdir -pv centos6 - pushd centos6 - VAGRANTDIR=$PWD + pushd "$TMPDIR" init_vagrant_box bento/centos-6.7 vagrant up vagrant ssh <<EOF @@ -206,7 +207,7 @@ build_centos7() { local mesos_version=$1 output_basedir=$2 local python_outdir=$output_basedir/centos/7/python mkdir -pv "$python_outdir" - pushd $TMPDIR + pushd "$TMPDIR" init_vagrant_box bento/centos-7.1 vagrant up vagrant ssh <<EOF http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 7064bca..439e0a7 100644 --- a/build.gradle +++ b/build.gradle @@ -370,7 +370,7 @@ dependencies { compile "org.apache.curator:curator-client:${curatorRev}" compile "org.apache.curator:curator-framework:${curatorRev}" compile "org.apache.curator:curator-recipes:${curatorRev}" - compile 'org.apache.mesos:mesos:1.0.0' + compile 'org.apache.mesos:mesos:1.1.0' compile "org.apache.httpcomponents:httpclient:${httpclientRev}" compile "org.apache.httpcomponents:httpcore:${httpcoreRev}" compile "org.apache.shiro:shiro-guice:${shiroRev}" http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/src/main/java/org/apache/aurora/scheduler/base/Conversions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/base/Conversions.java b/src/main/java/org/apache/aurora/scheduler/base/Conversions.java index 4a1e087..d08b6cf 100644 --- a/src/main/java/org/apache/aurora/scheduler/base/Conversions.java +++ b/src/main/java/org/apache/aurora/scheduler/base/Conversions.java @@ -64,6 +64,13 @@ public final class Conversions { .put(TaskState.TASK_KILLED, ScheduleStatus.KILLED) .put(TaskState.TASK_LOST, ScheduleStatus.LOST) .put(TaskState.TASK_ERROR, ScheduleStatus.LOST) + // Task states send to partition-aware Mesos frameworks. Aurora does not advertise the + // PARTITION_AWARE capability yet (AURORA-1814). We still map the task states to be safe. + .put(TaskState.TASK_UNREACHABLE, ScheduleStatus.LOST) + .put(TaskState.TASK_DROPPED, ScheduleStatus.LOST) + .put(TaskState.TASK_GONE, ScheduleStatus.LOST) + .put(TaskState.TASK_GONE_BY_OPERATOR, ScheduleStatus.LOST) + .put(TaskState.TASK_UNKNOWN, ScheduleStatus.LOST) .build(); /** http://git-wip-us.apache.org/repos/asf/aurora/blob/c245b9b7/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java index 7dc9eff..18ff2e2 100644 --- a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.io.CharStreams; -import com.google.protobuf.UninitializedMessageException; import com.hubspot.jackson.datatype.protobuf.ProtobufModule; import org.apache.aurora.GuavaUtils; @@ -92,7 +91,7 @@ public final class ExecutorSettingsLoader { Optional.fromNullable(m.volumeMounts).or(ImmutableList.of()), m.taskPrefix))); - } catch (UninitializedMessageException e) { + } catch (RuntimeException e) { throw new ExecutorConfigException(e); }
