aurora git commit: Improve performance of MemTaskStore queries

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 787ccfed5 -> 858552db0


Improve performance of MemTaskStore queries

Use `ArrayDeque` rather than `HashSet` for fetchTasks, and use imperative style
rather than functional.  I arrived at this result after running benchmarks with
some of the other usual suspects (`ArrayList`, `LinkedList`).

This patch also enables stack and heap profilers in jmh (more details
[here](http://hg.openjdk.java.net/codetools/jmh/file/25d8b2695bac/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_35_Profilers.java)),
providing insight into the heap impact of changes.  I started this change with a
heap profiler as the primary motivation, and ended up using it to guide this
improvement.

Reviewed at https://reviews.apache.org/r/65303/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/858552db
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/858552db
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/858552db

Branch: refs/heads/master
Commit: 858552db0f433eca7ae615a961de82fc56717e5d
Parents: 787ccfe
Author: Bill Farner 
Authored: Wed Jan 31 14:59:30 2018 -0800
Committer: Bill Farner 
Committed: Wed Jan 31 15:00:01 2018 -0800

--
 build.gradle|  1 +
 .../aurora/benchmark/TaskStoreBenchmarks.java   | 45 ++--
 .../scheduler/storage/mem/MemTaskStore.java | 75 +---
 3 files changed, 73 insertions(+), 48 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/858552db/build.gradle
--
diff --git a/build.gradle b/build.gradle
index 64af7ae..57355dc 100644
--- a/build.gradle
+++ b/build.gradle
@@ -602,6 +602,7 @@ jmh {
   jvmArgsPrepend = '-Xmx3g'
   humanOutputFile = project.file("$jmhHumanOutputPath")
   resultsFile = project.file("$buildDir/reports/jmh/results.txt")
+  profilers = ['gc', 'stack']
 }
 tasks.getByName('jmh').doLast() {
   println "Benchmark report generated: file://$jmhHumanOutputPath"

http://git-wip-us.apache.org/repos/asf/aurora/blob/858552db/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
--
diff --git a/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java 
b/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
index 9ec9865..4e4d36b 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/TaskStoreBenchmarks.java
@@ -16,7 +16,6 @@ package org.apache.aurora.benchmark;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.collect.Iterables;
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.util.Modules;
@@ -27,6 +26,7 @@ import org.apache.aurora.common.util.testing.FakeClock;
 import org.apache.aurora.scheduler.base.Query;
 import org.apache.aurora.scheduler.storage.Storage;
 import org.apache.aurora.scheduler.storage.TaskStore;
+import org.apache.aurora.scheduler.storage.entities.IJobKey;
 import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
 import org.apache.aurora.scheduler.storage.mem.MemStorageModule;
 import org.apache.aurora.scheduler.testing.FakeStatsProvider;
@@ -54,6 +54,7 @@ public class TaskStoreBenchmarks {
   @State(Scope.Thread)
   public abstract static class AbstractFetchTasksBenchmark {
 protected Storage storage;
+protected IJobKey job;
 public abstract void setUp();
 
 @Param({"1", "5", "10"})
@@ -63,6 +64,7 @@ public class TaskStoreBenchmarks {
   storage.write((Storage.MutateWork.NoResult.Quiet) storeProvider -> {
 TaskStore.Mutable taskStore = storeProvider.getUnsafeTaskStore();
 Set tasks = new Tasks.Builder().build(size);
+job = 
tasks.stream().findFirst().get().getAssignedTask().getTask().getJob();
 taskStore.saveTasks(tasks);
   });
 }
@@ -75,7 +77,7 @@ public class TaskStoreBenchmarks {
 }
   }
 
-  public static class MemFetchTasksBenchmark extends 
AbstractFetchTasksBenchmark {
+  public static class FetchAll extends AbstractFetchTasksBenchmark {
 @Setup(Level.Trial)
 @Override
 public void setUp() {
@@ -105,9 +107,42 @@ public class TaskStoreBenchmarks {
 
 @Benchmark
 public int run() {
-  // Iterate through results in case the result is lazily computed.
-  return Iterables.size(
-  storage.read(store -> 
store.getTaskStore().fetchTasks(Query.unscoped(;
+  return storage.read(store -> 
store.getTaskStore().fetchTasks(Query.unscoped())).size();
+}
+  }
+
+  public static class IndexedFetchAndFilter extends 

aurora git commit: Print command line parameters when the scheduler starts

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 710706f4e -> c4c55ff51


Print command line parameters when the scheduler starts

Realized i never added this in the command line parser change.
Note that this output differs from the original code in one important
way - it uses `toString()` on the parameter type rather than printing
the raw value from the command line.  Unfortunately jcommander does not
make that possible.  `shiro_ini_path` is one example of an arg that
would ideally print differently here.

Reviewed at https://reviews.apache.org/r/65234/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/c4c55ff5
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/c4c55ff5
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/c4c55ff5

Branch: refs/heads/master
Commit: c4c55ff518b254682a8610db2c9a53c5866e7bd6
Parents: 710706f
Author: Bill Farner 
Authored: Thu Jan 18 20:51:42 2018 -0800
Committer: Bill Farner 
Committed: Thu Jan 18 20:51:42 2018 -0800

--
 .../org/apache/aurora/scheduler/config/CommandLine.java | 12 ++--
 1 file changed, 10 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/c4c55ff5/src/main/java/org/apache/aurora/scheduler/config/CommandLine.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/config/CommandLine.java 
b/src/main/java/org/apache/aurora/scheduler/config/CommandLine.java
index 2085810..1402905 100644
--- a/src/main/java/org/apache/aurora/scheduler/config/CommandLine.java
+++ b/src/main/java/org/apache/aurora/scheduler/config/CommandLine.java
@@ -127,6 +127,16 @@ public final class CommandLine {
   CliOptions options = new CliOptions(ImmutableList.copyOf(customOptions));
   parser = prepareParser(options);
   parser.parse(args);
+
+  
LOG.info("---");
+  LOG.info("Parameters:");
+  parser.getParameters().stream()
+  .map(param ->
+  param.getLongestName() + ": " + 
param.getParameterized().get(param.getObject()))
+  .sorted()
+  .forEach(LOG::info);
+  
LOG.info("---");
+
   instance = options;
   return options;
 } catch (ParameterException e) {
@@ -136,8 +146,6 @@ public final class CommandLine {
   LOG.error(e.getMessage());
   System.exit(1);
   throw new RuntimeException(e);
-} catch (RuntimeException e) {
-  throw e;
 }
   }
 

aurora git commit: GitHub Pull Request template to discourage folks from making PRs

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 8a5dcacdc -> 710706f4e


GitHub Pull Request template to discourage folks from making PRs

Simple pull request template encouraging potential contributors
to submit via ReviewBoard instead of opening up a PR on GitHub.

Normally I wouldn't want to add something to the repo that is
platform specific, but given the fact that we have a lot of exposure
on GitHub and that there's no way to disallow PRs, it might be a
good idea to point potential contributors in the right direction
from the get go.

Reviewed at https://reviews.apache.org/r/65222/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/710706f4
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/710706f4
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/710706f4

Branch: refs/heads/master
Commit: 710706f4eac47f0b0563a0f4c70f8cd27de0c807
Parents: 8a5dcac
Author: Renan DelValle 
Authored: Thu Jan 18 19:29:40 2018 -0800
Committer: Bill Farner 
Committed: Thu Jan 18 19:29:40 2018 -0800

--
 .github/pull_request_template.md | 7 +++
 1 file changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/710706f4/.github/pull_request_template.md
--
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 000..e699df7
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,7 @@
+Dear potential contributor,
+
+Our project is not set up to take Pull Requests through GitHub. Instead we ask 
that contributions be made through Apache's ReviewBoard. Further instructions 
can be found here: https://aurora.apache.org/documentation/latest/contributing/
+
+Thank you for your understanding.
+
+- The Apache Aurora Team

aurora git commit: Added ExclusionStrategy to Gson formatter in structdump

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master f6146eb5e -> 8a5dcacdc


Added ExclusionStrategy to Gson formatter in structdump

This hides thrift metadata fields such as "__isset_bitfield"
and the "optionals" enum.
Added a FieldNamingStrategy to rename "value_" and "setName_"
fields to "value" and "key" on map formatting.

Reviewed at https://reviews.apache.org/r/65076/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/8a5dcacd
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/8a5dcacd
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/8a5dcacd

Branch: refs/heads/master
Commit: 8a5dcacdc73ab0468e1097d549bbf4e77d0096db
Parents: f6146eb
Author: Juan Manuel Fresia 
Authored: Wed Jan 17 15:02:26 2018 -0800
Committer: Bill Farner 
Committed: Wed Jan 17 15:02:26 2018 -0800

--
 .../aurora/scheduler/http/StructDump.java   | 27 +-
 .../aurora/scheduler/http/StructDumpTest.java   | 91 
 2 files changed, 117 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/8a5dcacd/src/main/java/org/apache/aurora/scheduler/http/StructDump.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/http/StructDump.java 
b/src/main/java/org/apache/aurora/scheduler/http/StructDump.java
index 1710e39..9e74301 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/StructDump.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/StructDump.java
@@ -24,6 +24,8 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
+import com.google.gson.ExclusionStrategy;
+import com.google.gson.FieldAttributes;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 
@@ -100,7 +102,30 @@ public class StructDump extends JerseyTemplateServlet {
 .map(IJobConfiguration::newBuilder));
   }
 
-  private static final Gson GSON = new 
GsonBuilder().setPrettyPrinting().create();
+  private static final Gson GSON = new GsonBuilder()
+  .setPrettyPrinting()
+  .setExclusionStrategies(new ExclusionStrategy() {
+@Override
+public boolean shouldSkipField(FieldAttributes f) {
+  if (f.getName().startsWith("_")) {
+return true;
+  }
+  return f.getDeclaredClass().getName().contains("$_Fields");
+}
+
+@Override
+public boolean shouldSkipClass(Class clazz) {
+  return false;
+}
+  })
+  .setFieldNamingStrategy(f -> {
+switch (f.getName()) {
+  case "setField_": return "key";
+  case "value_": return "value";
+  default: return f.getName();
+}
+  })
+  .create();
 
   private Response dumpEntity(String id, Quiet> work) {
 return fillTemplate(template -> {

http://git-wip-us.apache.org/repos/asf/aurora/blob/8a5dcacd/src/test/java/org/apache/aurora/scheduler/http/StructDumpTest.java
--
diff --git a/src/test/java/org/apache/aurora/scheduler/http/StructDumpTest.java 
b/src/test/java/org/apache/aurora/scheduler/http/StructDumpTest.java
new file mode 100644
index 000..8bd8319
--- /dev/null
+++ b/src/test/java/org/apache/aurora/scheduler/http/StructDumpTest.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http;
+
+import com.google.common.collect.ImmutableSet;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+
+import org.apache.aurora.gen.AssignedTask;
+import org.apache.aurora.gen.Resource;
+import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.gen.ScheduledTask;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class StructDumpTest extends AbstractJettyTest {
+
+  private static final String SLAVE_HOST = "fakehost";
+  private static final String FAKE_TASKID = "fake-task-id";
+
+  private static final 

[1/2] aurora git commit: Refactor scheduling code to split matching and assigning phases

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 5b34231ba -> 4e6242fed


http://git-wip-us.apache.org/repos/asf/aurora/blob/4e6242fe/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java
 
b/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java
index 348386e..b3ffb0d 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java
+++ 
b/src/test/java/org/apache/aurora/scheduler/preemptor/PreemptionVictimFilterTest.java
@@ -39,13 +39,13 @@ import org.apache.aurora.gen.ScheduledTask;
 import org.apache.aurora.gen.TaskConfig;
 import org.apache.aurora.gen.TaskEvent;
 import org.apache.aurora.gen.apiConstants;
-import org.apache.aurora.scheduler.TierInfo;
-import org.apache.aurora.scheduler.TierManager;
+import org.apache.aurora.scheduler.base.TaskTestUtil;
 import org.apache.aurora.scheduler.filter.SchedulingFilter;
 import org.apache.aurora.scheduler.filter.SchedulingFilter.Veto;
 import org.apache.aurora.scheduler.filter.SchedulingFilterImpl;
 import org.apache.aurora.scheduler.mesos.TaskExecutors;
 import org.apache.aurora.scheduler.offers.HostOffer;
+import 
org.apache.aurora.scheduler.preemptor.PreemptionVictimFilter.PreemptionVictimFilterImpl;
 import org.apache.aurora.scheduler.resources.ResourceBag;
 import org.apache.aurora.scheduler.resources.ResourceTestUtil;
 import org.apache.aurora.scheduler.resources.ResourceType;
@@ -67,9 +67,6 @@ import static org.apache.aurora.gen.MaintenanceMode.NONE;
 import static org.apache.aurora.gen.Resource.numCpus;
 import static org.apache.aurora.gen.Resource.ramMb;
 import static org.apache.aurora.gen.ScheduleStatus.RUNNING;
-import static org.apache.aurora.scheduler.base.TaskTestUtil.DEV_TIER;
-import static org.apache.aurora.scheduler.base.TaskTestUtil.PREFERRED_TIER;
-import static org.apache.aurora.scheduler.base.TaskTestUtil.REVOCABLE_TIER;
 import static org.apache.aurora.scheduler.filter.AttributeAggregate.empty;
 import static 
org.apache.aurora.scheduler.preemptor.PreemptionVictimFilter.PreemptionVictimFilterImpl.ORDER;
 import static 
org.apache.aurora.scheduler.preemptor.PreemptorMetrics.MISSING_ATTRIBUTES_NAME;
@@ -110,7 +107,6 @@ public class PreemptionVictimFilterTest extends 
EasyMockTest {
   private SchedulingFilter schedulingFilter;
   private FakeStatsProvider statsProvider;
   private PreemptorMetrics preemptorMetrics;
-  private TierManager tierManager;
   private FakeClock clock;
 
   @Before
@@ -119,7 +115,6 @@ public class PreemptionVictimFilterTest extends 
EasyMockTest {
 storageUtil.expectOperations();
 statsProvider = new FakeStatsProvider();
 preemptorMetrics = new PreemptorMetrics(new CachedCounters(statsProvider));
-tierManager = createMock(TierManager.class);
 clock = new FakeClock();
 ResourceType.initializeEmptyCliArgsForTest();
   }
@@ -129,12 +124,11 @@ public class PreemptionVictimFilterTest extends 
EasyMockTest {
   Optional offer,
   ScheduledTask... victims) {
 
-PreemptionVictimFilter.PreemptionVictimFilterImpl filter =
-new PreemptionVictimFilter.PreemptionVictimFilterImpl(
-schedulingFilter,
-TaskExecutors.NO_OVERHEAD_EXECUTOR,
-preemptorMetrics,
-tierManager);
+PreemptionVictimFilterImpl filter = new PreemptionVictimFilterImpl(
+schedulingFilter,
+TaskExecutors.NO_OVERHEAD_EXECUTOR,
+preemptorMetrics,
+TaskTestUtil.TIER_MANAGER);
 
 return filter.filterPreemptionVictims(
 ITaskConfig.build(pendingTask.getAssignedTask().getTask()),
@@ -145,16 +139,14 @@ public class PreemptionVictimFilterTest extends 
EasyMockTest {
   }
 
   @Test
-  public void testPreempted() throws Exception {
+  public void testPreempted() {
 setUpHost();
 
 schedulingFilter = createMock(SchedulingFilter.class);
 ScheduledTask lowPriority = makeTask(USER_A, JOB_A, TASK_ID_A);
 assignToHost(lowPriority);
-expectGetTier(lowPriority, DEV_TIER).times(2);
 
 ScheduledTask highPriority = makeTask(USER_A, JOB_A, TASK_ID_B, 100);
-expectGetTier(highPriority, DEV_TIER);
 
 expectFiltering();
 
@@ -163,7 +155,7 @@ public class PreemptionVictimFilterTest extends 
EasyMockTest {
   }
 
   @Test
-  public void testLowestPriorityPreempted() throws Exception {
+  public void testLowestPriorityPreempted() {
 setUpHost();
 
 schedulingFilter = createMock(SchedulingFilter.class);
@@ -172,10 +164,8 @@ public class PreemptionVictimFilterTest extends 
EasyMockTest {
 
 ScheduledTask lowerPriority = makeTask(USER_A, JOB_A, TASK_ID_B, 1);
 assignToHost(lowerPriority);
-expectGetTier(lowerPriority, DEV_TIER).atLeastOnce();
 
 ScheduledTask highPriority = makeTask(USER_A, JOB_A, TASK_ID_C, 100);
-

aurora git commit: Custom converter to allow the -thermos_executor_resources flag to take an empty string and parse it to an empty list

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 8e5e08ebf -> 5b34231ba


Custom converter to allow the -thermos_executor_resources
flag to take an empty string and parse it to an empty list

Fixes the issue that caused the voting to fail for the 0.19.0 Aurora packages.
Fix cribbed from: https://github.com/cbeust/jcommander/pull/422
Implemented as a custom converter as suggested here:
https://reviews.apache.org/r/64824/

Reviewed at https://reviews.apache.org/r/64934/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/5b34231b
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/5b34231b
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/5b34231b

Branch: refs/heads/master
Commit: 5b34231ba90f01a08c10bd9b819437d892910c5c
Parents: 8e5e08e
Author: Renan DelValle 
Authored: Tue Jan 9 13:27:28 2018 -0800
Committer: Bill Farner 
Committed: Tue Jan 9 13:27:28 2018 -0800

--
 .../apache/aurora/scheduler/app/AppModule.java  |  7 +-
 .../config/splitters/CommaSplitter.java | 34 ++
 .../configuration/executor/ExecutorModule.java  |  7 +-
 .../discovery/FlaggedZooKeeperConfig.java   |  4 +-
 .../http/api/security/HttpSecurityModule.java   |  4 +-
 .../scheduler/offers/OfferManagerModule.java|  7 +-
 .../scheduler/preemptor/PreemptorModule.java|  4 +-
 .../scheduling/TaskAssignerImplModule.java  |  4 +-
 .../apache/aurora/scheduler/sla/SlaModule.java  |  7 +-
 .../aurora/scheduler/state/StateModule.java |  4 +-
 .../aurora/scheduler/thrift/aop/AopModule.java  |  5 +-
 .../scheduler/config/CommandLineTest.java   | 71 
 12 files changed, 144 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/5b34231b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java 
b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
index 817a019..ffc0744 100644
--- a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java
@@ -37,6 +37,7 @@ import org.apache.aurora.scheduler.SchedulerServicesModule;
 import org.apache.aurora.scheduler.app.SchedulerMain.Options.DriverKind;
 import org.apache.aurora.scheduler.async.AsyncModule;
 import org.apache.aurora.scheduler.config.CliOptions;
+import org.apache.aurora.scheduler.config.splitters.CommaSplitter;
 import org.apache.aurora.scheduler.config.validators.PositiveNumber;
 import org.apache.aurora.scheduler.configuration.ConfigurationManager;
 import 
org.apache.aurora.scheduler.configuration.ConfigurationManager.ConfigurationManagerSettings;
@@ -85,7 +86,8 @@ public class AppModule extends AbstractModule {
 // parameters that define an arity. You will have to convert these values 
yourself..."
 
 @Parameter(names = "-allowed_container_types",
-description = "Container types that are allowed to be used by jobs.")
+description = "Container types that are allowed to be used by jobs.",
+splitter = CommaSplitter.class)
 public List<_Fields> allowedContainerTypes = 
ImmutableList.of(Container._Fields.MESOS);
 
 @Parameter(names = "-allow_docker_parameters",
@@ -95,7 +97,8 @@ public class AppModule extends AbstractModule {
 
 @Parameter(names = "-default_docker_parameters",
 description =
-"Default docker parameters for any job that does not explicitly 
declare parameters.")
+"Default docker parameters for any job that does not explicitly 
declare parameters.",
+splitter = CommaSplitter.class)
 public List defaultDockerParameters = ImmutableList.of();
 
 @Parameter(names = "-require_docker_use_executor",

http://git-wip-us.apache.org/repos/asf/aurora/blob/5b34231b/src/main/java/org/apache/aurora/scheduler/config/splitters/CommaSplitter.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/config/splitters/CommaSplitter.java 
b/src/main/java/org/apache/aurora/scheduler/config/splitters/CommaSplitter.java
new file mode 100644
index 000..d1ec962
--- /dev/null
+++ 
b/src/main/java/org/apache/aurora/scheduler/config/splitters/CommaSplitter.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * 

[2/2] aurora git commit: Add a test to detect incompatible storage changes

[wfarner]

Add a test to detect incompatible storage changes

This is intended as a safeguard against future compatibility regressions like
[AURORA-1959](https://issues.apache.org/jira/browse/AURORA-1959).

I approached this with a few goals:

  - golden files should be text-based and human-readable.  This allows for
non-opaque code reviews, and simpler remedy when it's necessary to update
the goldens (i.e. copy-pasteable)
  - guidance for schema evolution should be included directly in test failures
  - separate detection of 'what the scheduler _can_ read' and 'what the
scheduler writes'
  - reasonably-complete schema coverage with minimal manual labor.  These tests
auto-generate structs to mitigate maintenance burden of test code as
schemas evolve.

This is not a replacement for vigilance with data compatibility, but it should
at least

1. mitigate unintentional breakages in compatibility, especially for new
   contributors
2. draw code reviewer attention to compatibility changes in a patch (signaled by
   changes to golden files)

Reviewed at https://reviews.apache.org/r/64519/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/8e5e08eb
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/8e5e08eb
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/8e5e08eb

Branch: refs/heads/master
Commit: 8e5e08ebfe6d848beb4e6037d3ab67e49321ae42
Parents: f1d9caf
Author: Bill Farner 
Authored: Thu Jan 4 08:02:55 2018 -0800
Committer: Bill Farner 
Committed: Thu Jan 4 08:02:55 2018 -0800

--
 .../durability/DataCompatibilityTest.java   | 376 +++
 .../scheduler/storage/durability/Generator.java | 142 ++
 .../goldens/current/pruneJobUpdateHistory   |  12 +
 .../durability/goldens/current/removeJob|  19 +
 .../durability/goldens/current/removeJobUpdate  |  30 ++
 .../durability/goldens/current/removeLock   |  23 +
 .../durability/goldens/current/removeQuota  |   9 +
 .../durability/goldens/current/removeTasks  |  13 +
 .../durability/goldens/current/saveCronJob  | 200 
 .../durability/goldens/current/saveFrameworkId  |   9 +
 .../goldens/current/saveHostAttributes  |  37 ++
 .../goldens/current/saveJobInstanceUpdateEvent  |  39 ++
 .../durability/goldens/current/saveJobUpdate| 469 +++
 .../goldens/current/saveJobUpdateEvent  |  42 ++
 .../storage/durability/goldens/current/saveLock |  39 ++
 .../durability/goldens/current/saveQuota|  33 ++
 .../durability/goldens/current/saveTasks| 233 +
 .../read-compatible/1-pruneJobUpdateHistory |  12 +
 .../goldens/read-compatible/10-saveJobUpdate| 469 +++
 .../read-compatible/11-saveJobUpdateEvent   |  42 ++
 .../12-saveJobInstanceUpdateEvent   |  39 ++
 .../goldens/read-compatible/13-saveLock |  39 ++
 .../goldens/read-compatible/14-saveQuota|  43 ++
 .../goldens/read-compatible/15-saveTasks| 233 +
 .../goldens/read-compatible/2-removeJob |  19 +
 .../goldens/read-compatible/3-removeJobUpdate   |  30 ++
 .../goldens/read-compatible/4-removeLock|  23 +
 .../goldens/read-compatible/5-removeQuota   |   9 +
 .../goldens/read-compatible/6-removeTasks   |  13 +
 .../goldens/read-compatible/7-saveCronJob   | 200 
 .../goldens/read-compatible/8-saveFrameworkId   |   9 +
 .../read-compatible/9-saveHostAttributes|  37 ++
 32 files changed, 2942 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/8e5e08eb/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java
new file mode 100644
index 000..31f9545
--- /dev/null
+++ 
b/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java
@@ -0,0 +1,376 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.storage.durability;
+
+import java.io.File;
+import 

[1/2] aurora git commit: Add a test to detect incompatible storage changes

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master f1d9caf36 -> 8e5e08ebf


http://git-wip-us.apache.org/repos/asf/aurora/blob/8e5e08eb/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/15-saveTasks
--
diff --git 
a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/15-saveTasks
 
b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/15-saveTasks
new file mode 100644
index 000..4323031
--- /dev/null
+++ 
b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/15-saveTasks
@@ -0,0 +1,233 @@
+{
+  "6": {
+"rec": {
+  "1": {
+"set": [
+  "rec",
+  1,
+  {
+"1": {
+  "rec": {
+"1": {
+  "str": "string-value"
+},
+"2": {
+  "str": "string-value"
+},
+"3": {
+  "str": "string-value"
+},
+"4": {
+  "rec": {
+"7": {
+  "tf": 1
+},
+"11": {
+  "i32": 2
+},
+"13": {
+  "i32": 2
+},
+"17": {
+  "rec": {
+"2": {
+  "str": "string-value"
+}
+  }
+},
+"18": {
+  "tf": 1
+},
+"20": {
+  "set": [
+"rec",
+1,
+{
+  "1": {
+"str": "string-value"
+  },
+  "2": {
+"rec": {
+  "2": {
+"rec": {
+  "1": {
+"i32": 2
+  }
+}
+  }
+}
+  }
+}
+  ]
+},
+"22": {
+  "map": [
+"str",
+"str",
+1,
+{
+  "string-value": "string-value"
+}
+  ]
+},
+"23": {
+  "str": "string-value"
+},
+"25": {
+  "rec": {
+"1": {
+  "str": "string-value"
+},
+"2": {
+  "str": "string-value"
+}
+  }
+},
+"27": {
+  "set": [
+"rec",
+1,
+{
+  "1": {
+"str": "string-value"
+  },
+  "2": {
+"str": "string-value"
+  }
+}
+  ]
+},
+"28": {
+  "rec": {
+"1": {
+  "str": "string-value"
+},
+"2": {
+  "str": "string-value"
+},
+"3": {
+  "str": "string-value"
+}
+  }
+},
+"29": {
+  "rec": {
+"2": {
+  "rec": {
+"1": {
+  "str": "string-value"
+},
+"2": {
+  "lst": [
+"rec",
+1,
+{
+  "1": {
+"str": "string-value"
+  },
+  "2": {
+"str": "string-value"
+  }
+}
+  ]
+}
+  }
+}
+ 

aurora git commit: Add metadata field to Job object in DSL

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 2e1ca4288 -> f1d9caf36


Add metadata field to Job object in DSL

Bugs closed: AURORA-1898

Reviewed at https://reviews.apache.org/r/64341/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/f1d9caf3
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/f1d9caf3
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/f1d9caf3

Branch: refs/heads/master
Commit: f1d9caf36dea2dbab6ccc44b9ba08a5572d7bbc8
Parents: 2e1ca42
Author: Jing Chen 
Authored: Sun Dec 17 08:26:33 2017 -0800
Committer: Bill Farner 
Committed: Sun Dec 17 08:26:33 2017 -0800

--
 RELEASE-NOTES.md|  1 +
 docs/reference/configuration.md |  9 +
 .../python/apache/aurora/config/schema/base.py  |  6 +++
 src/main/python/apache/aurora/config/thrift.py  | 10 -
 .../apache/aurora/client/cli/test_inspect.py|  1 +
 .../python/apache/aurora/config/test_thrift.py  | 41 
 6 files changed, 67 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/RELEASE-NOTES.md
--
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 54dcc75..3053e54 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -47,6 +47,7 @@
 - The scheduler no longer uses an internal H2 database for storage.
 - There is a new Scheduler UI which, in addition to the facelift, provides the 
ability to inject your
   own custom UI components.
+- Introduce a metadata field in the Job object of the DSL, which will populate 
TaskConfig.metadata.
 
 ### Deprecations and removals:
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/docs/reference/configuration.md
--
diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
index 67d9914..725e073 100644
--- a/docs/reference/configuration.md
+++ b/docs/reference/configuration.md
@@ -357,6 +357,7 @@ Job Schema
   ```announce``` | ```Announcer``` object | Optionally enable Zookeeper 
ServerSet announcements. See [Announcer Objects] for more information.
   ```enable_hooks``` | Boolean | Whether to enable [Client 
Hooks](client-hooks.md) for this job. (Default: False)
   ```partition_policy``` | ```PartitionPolicy``` object | An optional 
partition policy that allows job owners to define how to handle partitions for 
running tasks (in partition-aware Aurora clusters)
+  ```metadata``` | list of ```Metadata``` objects | list of ```Metadata``` 
objects for user's customized metadata information.
 
 
 ### UpdateConfig Objects
@@ -410,6 +411,14 @@ Parameters for controlling a task's health checks via HTTP 
or a shell command.
 | ```reschedule```   | Boolean   | Whether or not to reschedule 
when running tasks become partitioned (Default: True)
 | ```delay_secs```   | Integer   | How long to delay transitioning 
to LOST when running tasks are partitioned. (Default: 0)
 
+### Metadata Objects
+
+Describes a piece of user metadata in a key value pair
+
+  param| type| description
+  -| ::  | ---
+  ```key```| String  | Indicate which metadata the user 
provides
+  ```value```  | String  | Provide the metadata content for 
corresponding key
 
 ### Announcer Objects
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/main/python/apache/aurora/config/schema/base.py
--
diff --git a/src/main/python/apache/aurora/config/schema/base.py 
b/src/main/python/apache/aurora/config/schema/base.py
index a466e78..3d57d6a 100644
--- a/src/main/python/apache/aurora/config/schema/base.py
+++ b/src/main/python/apache/aurora/config/schema/base.py
@@ -159,6 +159,11 @@ class PartitionPolicy(Struct):
   delay_secs = Default(Integer, 0)
 
 
+class Metadata(Struct):
+  key   = Required(String)
+  value = Required(String)
+
+
 class MesosJob(Struct):
   name  = Default(String, '{{task.name}}')
   role  = Required(String)
@@ -176,6 +181,7 @@ class MesosJob(Struct):
   update_config = Default(UpdateConfig, UpdateConfig())
 
   constraints= Map(String, String)
+  metadata   = Default(List(Metadata), [])
   service= Default(Boolean, False)
   max_task_failures  = Default(Integer, 1)
   production = Default(Boolean, False)

http://git-wip-us.apache.org/repos/asf/aurora/blob/f1d9caf3/src/main/python/apache/aurora/config/thrift.py
--
diff --git 

[1/2] aurora git commit: Add a storage recovery tool

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 6fd765bcf -> 2e1ca4288


http://git-wip-us.apache.org/repos/asf/aurora/blob/2e1ca428/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
deleted file mode 100644
index 2ad4e84..000
--- 
a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImplIT.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.storage.log;
-
-import java.util.Map;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.aurora.common.stats.Stats;
-import org.apache.aurora.common.util.testing.FakeBuildInfo;
-import org.apache.aurora.common.util.testing.FakeClock;
-import org.apache.aurora.gen.Attribute;
-import org.apache.aurora.gen.CronCollisionPolicy;
-import org.apache.aurora.gen.HostAttributes;
-import org.apache.aurora.gen.Identity;
-import org.apache.aurora.gen.InstanceTaskConfig;
-import org.apache.aurora.gen.JobConfiguration;
-import org.apache.aurora.gen.JobInstanceUpdateEvent;
-import org.apache.aurora.gen.JobKey;
-import org.apache.aurora.gen.JobUpdate;
-import org.apache.aurora.gen.JobUpdateAction;
-import org.apache.aurora.gen.JobUpdateDetails;
-import org.apache.aurora.gen.JobUpdateEvent;
-import org.apache.aurora.gen.JobUpdateInstructions;
-import org.apache.aurora.gen.JobUpdateKey;
-import org.apache.aurora.gen.JobUpdateSettings;
-import org.apache.aurora.gen.JobUpdateState;
-import org.apache.aurora.gen.JobUpdateStatus;
-import org.apache.aurora.gen.JobUpdateSummary;
-import org.apache.aurora.gen.MaintenanceMode;
-import org.apache.aurora.gen.Range;
-import org.apache.aurora.gen.storage.QuotaConfiguration;
-import org.apache.aurora.gen.storage.SchedulerMetadata;
-import org.apache.aurora.gen.storage.Snapshot;
-import org.apache.aurora.gen.storage.StoredCronJob;
-import org.apache.aurora.gen.storage.StoredJobUpdateDetails;
-import org.apache.aurora.scheduler.base.JobKeys;
-import org.apache.aurora.scheduler.base.TaskTestUtil;
-import org.apache.aurora.scheduler.resources.ResourceBag;
-import org.apache.aurora.scheduler.storage.Storage;
-import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
-import org.apache.aurora.scheduler.storage.durability.Loader;
-import org.apache.aurora.scheduler.storage.durability.Persistence.Edit;
-import org.apache.aurora.scheduler.storage.durability.ThriftBackfill;
-import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
-import org.apache.aurora.scheduler.storage.entities.IJobConfiguration;
-import org.apache.aurora.scheduler.storage.entities.IJobKey;
-import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails;
-import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey;
-import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
-import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
-import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
-import org.apache.aurora.scheduler.storage.mem.MemStorageModule;
-import org.junit.Test;
-
-import static 
org.apache.aurora.common.util.testing.FakeBuildInfo.generateBuildInfo;
-import static org.apache.aurora.scheduler.base.TaskTestUtil.THRIFT_BACKFILL;
-import static 
org.apache.aurora.scheduler.resources.ResourceManager.aggregateFromBag;
-import static 
org.apache.aurora.scheduler.storage.log.SnapshotStoreImpl.SNAPSHOT_RESTORE;
-import static 
org.apache.aurora.scheduler.storage.log.SnapshotStoreImpl.SNAPSHOT_SAVE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-public class SnapshotStoreImplIT {
-
-  private static final long NOW = 10335463456L;
-  private static final IJobKey JOB_KEY = JobKeys.from("role", "env", "job");
-
-  private Storage storage;
-  private SnapshotStoreImpl snapshotter;
-
-  private void setUpStore() {
-storage = MemStorageModule.newEmptyStorage();
-FakeClock clock = new FakeClock();
-clock.setNowMillis(NOW);
-snapshotter = new SnapshotStoreImpl(generateBuildInfo(), clock);
-Stats.flush();
-  }
-
-  @Test
-  public void testBackfill() {
-  

[2/2] aurora git commit: Add a storage recovery tool

[wfarner]

Add a storage recovery tool

This tool was originally intended as a migration path between Persistence
backends.  As it turns out, the model also works well for recovering from a
backup.

I propose we drop our current recovery mechanism to use this tool.  The existing
recovery-via-scheduler-rpc is slightly non-sensical, as it assumes a healthy
scheduler.  When an operator decides it is necessary to recover from a backup,
we should assume the scheduler state may be broken.  Furthermore, starting an
empty scheduler to bootstrap can have undesirable effects such as advertising
false state to clients and establishing a new empty framework with the master.

Testing Done:
end-to-end tests pass (and exercise recovery tool)

Reviewed at https://reviews.apache.org/r/64625/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/2e1ca428
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/2e1ca428
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/2e1ca428

Branch: refs/heads/master
Commit: 2e1ca42887bc8ea1e8c6cddebe9d1cf29268c714
Parents: 6fd765b
Author: Bill Farner 
Authored: Fri Dec 15 12:07:37 2017 -0800
Committer: Bill Farner 
Committed: Fri Dec 15 12:07:37 2017 -0800

--
 build.gradle|  13 +
 config/checkstyle/suppressions.xml  |   2 +
 .../aurora/benchmark/SnapshotBenchmarks.java|  10 +-
 .../aurora/scheduler/app/SchedulerMain.java |  12 +-
 .../aurora/scheduler/config/CliOptions.java |   6 +-
 .../discovery/ServiceDiscoveryBindings.java |   2 +-
 .../scheduler/storage/backup/BackupReader.java  |  56 
 .../scheduler/storage/backup/Recovery.java  |  35 +-
 .../storage/backup/TemporaryStorage.java|   4 +-
 .../durability/DurableStorageModule.java|  35 ++
 .../scheduler/storage/durability/Recovery.java  | 119 +++
 .../storage/durability/RecoveryTool.java| 196 +++
 .../storage/log/LogPersistenceModule.java   |  78 +
 .../scheduler/storage/log/LogStorageModule.java | 110 --
 .../scheduler/storage/log/SnapshotModule.java   |  54 +++
 .../storage/log/SnapshotStoreImpl.java  | 332 ---
 .../scheduler/storage/log/SnapshotterImpl.java  | 332 +++
 .../aurora/scheduler/app/SchedulerIT.java   |  12 +-
 .../scheduler/config/CommandLineTest.java   |   4 +-
 .../storage/durability/RecoveryTest.java| 110 ++
 .../storage/log/LogPersistenceTest.java |   6 +-
 .../storage/log/NonVolatileStorageTest.java |  12 +-
 .../storage/log/SnapshotServiceTest.java|   7 +-
 .../storage/log/SnapshotStoreImplIT.java| 189 ---
 .../storage/log/SnapshotterImplIT.java  | 189 +++
 .../sh/org/apache/aurora/e2e/test_end_to_end.sh |  37 +++
 26 files changed, 1284 insertions(+), 678 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/2e1ca428/build.gradle
--
diff --git a/build.gradle b/build.gradle
index 4674513..64af7ae 100644
--- a/build.gradle
+++ b/build.gradle
@@ -623,3 +623,16 @@ startScripts {
 unixScript.text = unixScript.text.replace('CLASSPATH=', 
"CLASSPATH=${environmentClasspathPrefix}:")
   }
 }
+
+// Include a script to run the recovery tool.
+task moreStartScripts(type: CreateStartScripts) {
+  mainClassName = 'org.apache.aurora.scheduler.storage.durability.RecoveryTool'
+  applicationName = 'recovery-tool'
+  outputDir = new File(project.buildDir, 'scripts')
+  classpath = jar.outputs.files + project.configurations.runtime
+}
+
+applicationDistribution.into('bin') {
+  from(moreStartScripts)
+  fileMode = 0755
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/2e1ca428/config/checkstyle/suppressions.xml
--
diff --git a/config/checkstyle/suppressions.xml 
b/config/checkstyle/suppressions.xml
index c4081b9..03f57c8 100644
--- a/config/checkstyle/suppressions.xml
+++ b/config/checkstyle/suppressions.xml
@@ -21,6 +21,8 @@ limitations under the License.
   
   
+  
   
   
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/2e1ca428/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
--
diff --git a/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java 
b/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
index 4f99f80..e3ed3f2 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/SnapshotBenchmarks.java
@@ -27,7 +27,7 @@ import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.common.util.Clock;
 import 

[1/4] aurora git commit: Use java.util.Optional throughout

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 5f79f7ca7 -> 6fd765bcf


http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/test/java/org/apache/aurora/scheduler/storage/AbstractCronJobStoreTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractCronJobStoreTest.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractCronJobStoreTest.java
index 3ec6e2b..889cb01 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractCronJobStoreTest.java
+++ 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractCronJobStoreTest.java
@@ -13,9 +13,9 @@
  */
 package org.apache.aurora.scheduler.storage;
 
+import java.util.Optional;
 import java.util.Set;
 
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -61,15 +61,15 @@ public abstract class AbstractCronJobStoreTest {
 
   @Test
   public void testJobStore() {
-assertNull(fetchJob(JobKeys.from("nobody", "nowhere", "noname")).orNull());
+assertNull(fetchJob(JobKeys.from("nobody", "nowhere", 
"noname")).orElse(null));
 assertEquals(ImmutableSet.of(), fetchJobs());
 
 saveAcceptedJob(JOB_A);
-assertEquals(JOB_A, fetchJob(KEY_A).orNull());
+assertEquals(JOB_A, fetchJob(KEY_A).orElse(null));
 assertEquals(ImmutableSet.of(JOB_A), fetchJobs());
 
 saveAcceptedJob(JOB_B);
-assertEquals(JOB_B, fetchJob(KEY_B).orNull());
+assertEquals(JOB_B, fetchJob(KEY_B).orElse(null));
 assertEquals(ImmutableSet.of(JOB_A, JOB_B), fetchJobs());
 
 removeJob(KEY_B);
@@ -93,13 +93,13 @@ public abstract class AbstractCronJobStoreTest {
 saveAcceptedJob(staging);
 
 assertNull(fetchJob(
-
IJobKey.build(templateConfig.getKey().newBuilder().setEnvironment("test"))).orNull());
-assertEquals(prod, fetchJob(prod.getKey()).orNull());
-assertEquals(staging, fetchJob(staging.getKey()).orNull());
+
IJobKey.build(templateConfig.getKey().newBuilder().setEnvironment("test"))).orElse(null));
+assertEquals(prod, fetchJob(prod.getKey()).orElse(null));
+assertEquals(staging, fetchJob(staging.getKey()).orElse(null));
 
 removeJob(prod.getKey());
-assertNull(fetchJob(prod.getKey()).orNull());
-assertEquals(staging, fetchJob(staging.getKey()).orNull());
+assertNull(fetchJob(prod.getKey()).orElse(null));
+assertEquals(staging, fetchJob(staging.getKey()).orElse(null));
   }
 
   @Test
@@ -127,7 +127,7 @@ public abstract class AbstractCronJobStoreTest {
 IJobConfiguration jobAUpdated =
 IJobConfiguration.build(JOB_A.newBuilder().setCronSchedule("changed"));
 saveAcceptedJob(jobAUpdated);
-assertEquals(jobAUpdated, fetchJob(KEY_A).orNull());
+assertEquals(jobAUpdated, fetchJob(KEY_A).orElse(null));
   }
 
   private static IJobConfiguration makeJob(String name) {

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
index 1510817..3a06a45 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
+++ 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
@@ -15,11 +15,11 @@
 package org.apache.aurora.scheduler.storage;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Injector;
@@ -134,8 +134,8 @@ public abstract class AbstractJobUpdateStoreTest {
 IJobUpdateDetails update1 = makeFullyPopulatedUpdate(updateId1);
 IJobUpdateDetails update2 = makeJobUpdate(updateId2);
 
-assertEquals(Optional.absent(), getUpdate(updateId1));
-assertEquals(Optional.absent(), getUpdate(updateId2));
+assertEquals(Optional.empty(), getUpdate(updateId1));
+assertEquals(Optional.empty(), getUpdate(updateId2));
 
 StorageEntityUtil.assertFullyPopulated(
 update1,
@@ -381,7 +381,7 @@ public abstract class AbstractJobUpdateStoreTest {
 JobUpdateDetails builder = makeJobUpdate(updateId).newBuilder();
 builder.getUpdate().getSummary().setMetadata(duplicatedMetadata);
 
-assertEquals(Optional.absent(), getUpdate(updateId));
+assertEquals(Optional.empty(), getUpdate(updateId));
 
 IJobUpdateDetails update = IJobUpdateDetails.build(builder);
 saveUpdate(update);

[3/4] aurora git commit: Use java.util.Optional throughout

[wfarner]

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java 
b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
index 727be58..cb288bb 100644
--- a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
+++ b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
@@ -15,12 +15,12 @@ package org.apache.aurora.scheduler.mesos;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.protobuf.ByteString;
@@ -211,7 +211,7 @@ public interface MesosTaskFactory {
   taskBuilder.setExecutor(execBuilder.build());
 } else {
   LOG.warn("Running Docker-based task without an executor.");
-  taskBuilder.setContainer(getDockerContainerInfo(dockerContainer, 
Optional.absent()))
+  taskBuilder.setContainer(getDockerContainerInfo(dockerContainer, 
Optional.empty()))
   .setCommand(CommandInfo.newBuilder().setShell(false));
 }
   } else {
@@ -276,7 +276,7 @@ public interface MesosTaskFactory {
 .addVolumes(volume));
   }
 
-  return Optional.absent();
+  return Optional.empty();
 }
 
 private ContainerInfo getDockerContainerInfo(

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java 
b/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java
index e0221f8..e378ef1 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/mesos/SchedulerDriverService.java
@@ -14,12 +14,12 @@
 package org.apache.aurora.scheduler.mesos;
 
 import java.util.Collection;
+import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.inject.Inject;
 
-import com.google.common.base.Optional;
 import com.google.common.collect.Collections2;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.AbstractIdleService;

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/mesos/VersionedDriverFactory.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/mesos/VersionedDriverFactory.java 
b/src/main/java/org/apache/aurora/scheduler/mesos/VersionedDriverFactory.java
index 8afeec1..e436080 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/mesos/VersionedDriverFactory.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/mesos/VersionedDriverFactory.java
@@ -13,7 +13,7 @@
  */
 package org.apache.aurora.scheduler.mesos;
 
-import com.google.common.base.Optional;
+import java.util.Optional;
 
 import org.apache.mesos.v1.Protos;
 import org.apache.mesos.v1.scheduler.Mesos;

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/mesos/VersionedMesosSchedulerImpl.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/mesos/VersionedMesosSchedulerImpl.java
 
b/src/main/java/org/apache/aurora/scheduler/mesos/VersionedMesosSchedulerImpl.java
index 5329de5..43c8ee4 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/mesos/VersionedMesosSchedulerImpl.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/mesos/VersionedMesosSchedulerImpl.java
@@ -13,12 +13,13 @@
  */
 package org.apache.aurora.scheduler.mesos;
 
+import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
+
 import javax.inject.Inject;
 
-import com.google.common.base.Optional;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/mesos/VersionedSchedulerDriverService.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/mesos/VersionedSchedulerDriverService.java
 
b/src/main/java/org/apache/aurora/scheduler/mesos/VersionedSchedulerDriverService.java
index 4609064..673788d 

[4/4] aurora git commit: Use java.util.Optional throughout

[wfarner]

Use java.util.Optional throughout

Reviewed at https://reviews.apache.org/r/64629/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/6fd765bc
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/6fd765bc
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/6fd765bc

Branch: refs/heads/master
Commit: 6fd765bcf8fa5dc0d6d8f2c2fcfce6609ab8fb01
Parents: 5f79f7c
Author: Bill Farner 
Authored: Fri Dec 15 12:01:49 2017 -0800
Committer: Bill Farner 
Committed: Fri Dec 15 12:01:49 2017 -0800

--
 .../net/http/handlers/TimeSeriesDataSource.java |  4 +-
 .../org/apache/aurora/common/stats/Stats.java   |  2 +-
 .../aurora/common/stats/StatsProvider.java  |  3 +-
 .../aurora/benchmark/StatusUpdateBenchmark.java |  6 +--
 .../benchmark/fakes/FakeOfferManager.java   |  6 +--
 .../benchmark/fakes/FakeStatsProvider.java  |  3 +-
 .../aurora/scheduler/SchedulerLifecycle.java|  2 +-
 .../aurora/scheduler/TaskStatusHandlerImpl.java |  8 ++--
 .../org/apache/aurora/scheduler/TaskVars.java   |  9 ++--
 .../apache/aurora/scheduler/base/JobKeys.java   |  4 +-
 .../aurora/scheduler/base/TaskTestUtil.java |  6 +--
 .../config/converters/DataAmountConverter.java  | 13 +++---
 .../config/converters/TimeAmountConverter.java  | 13 +++---
 .../configuration/ConfigurationManager.java |  2 +-
 .../executor/ExecutorSettingsLoader.java|  4 +-
 .../aurora/scheduler/cron/CronPredictor.java|  2 +-
 .../aurora/scheduler/cron/CronScheduler.java|  2 +-
 .../aurora/scheduler/cron/CrontabEntry.java |  4 +-
 .../aurora/scheduler/cron/SanitizedCronJob.java |  4 +-
 .../scheduler/cron/quartz/AuroraCronJob.java|  4 +-
 .../cron/quartz/CronJobManagerImpl.java |  5 ++-
 .../cron/quartz/CronPredictorImpl.java  |  5 +--
 .../cron/quartz/CronSchedulerImpl.java  |  5 ++-
 .../discovery/FlaggedZooKeeperConfig.java   |  6 +--
 .../scheduler/discovery/ZooKeeperConfig.java|  7 ++--
 .../aurora/scheduler/events/PubsubEvent.java|  4 +-
 .../aurora/scheduler/events/WebhookInfo.java| 13 +++---
 .../aurora/scheduler/events/WebhookModule.java  |  6 +--
 .../scheduler/filter/ConstraintMatcher.java | 10 ++---
 .../scheduler/filter/SchedulingFilter.java  |  4 +-
 .../scheduler/filter/SchedulingFilterImpl.java  | 15 +++
 .../scheduler/http/JettyServerModule.java   |  6 +--
 .../aurora/scheduler/http/LeaderRedirect.java   | 20 -
 .../scheduler/http/LeaderRedirectFilter.java|  2 +-
 .../org/apache/aurora/scheduler/http/Mname.java | 14 +++
 .../apache/aurora/scheduler/http/Quotas.java|  2 +-
 .../aurora/scheduler/http/StructDump.java   |  7 ++--
 .../http/api/security/FieldGetter.java  |  5 ++-
 .../http/api/security/FieldGetters.java |  4 +-
 .../http/api/security/IniShiroRealmModule.java  |  7 ++--
 .../api/security/Kerberos5ShiroRealmModule.java |  6 +--
 .../ShiroAuthorizingParamInterceptor.java   | 12 +++---
 .../ShiroKerberosAuthenticationFilter.java  |  5 +--
 .../http/api/security/ThriftFieldGetter.java|  4 +-
 .../aurora/scheduler/log/mesos/MesosLog.java|  4 +-
 .../mesos/CommandLineDriverSettingsModule.java  |  8 ++--
 .../aurora/scheduler/mesos/DriverFactory.java   |  2 +-
 .../scheduler/mesos/DriverFactoryImpl.java  |  4 +-
 .../aurora/scheduler/mesos/DriverSettings.java  |  3 +-
 .../scheduler/mesos/LibMesosLoadingModule.java  |  4 +-
 .../scheduler/mesos/MesosCallbackHandler.java   |  9 ++--
 .../scheduler/mesos/MesosTaskFactory.java   |  6 +--
 .../scheduler/mesos/SchedulerDriverService.java |  2 +-
 .../scheduler/mesos/VersionedDriverFactory.java |  2 +-
 .../mesos/VersionedMesosSchedulerImpl.java  |  3 +-
 .../mesos/VersionedSchedulerDriverService.java  |  6 +--
 .../aurora/scheduler/offers/HostOffer.java  |  4 +-
 .../aurora/scheduler/offers/HostOffers.java | 10 ++---
 .../aurora/scheduler/offers/OfferManager.java   |  3 +-
 .../scheduler/offers/OfferManagerImpl.java  |  3 +-
 .../scheduler/offers/OfferOrderBuilder.java |  2 +-
 .../aurora/scheduler/preemptor/BiCache.java |  4 +-
 .../preemptor/PendingTaskProcessor.java |  4 +-
 .../preemptor/PreemptionVictimFilter.java   | 15 +++
 .../aurora/scheduler/preemptor/Preemptor.java   |  6 +--
 .../scheduler/preemptor/PreemptorMetrics.java   |  2 +-
 .../scheduler/preemptor/PreemptorModule.java|  4 +-
 .../scheduler/quota/QuotaCheckResult.java   |  5 ++-
 .../aurora/scheduler/quota/QuotaManager.java| 18 
 .../reconciliation/TaskReconciler.java  |  4 +-
 .../scheduler/reconciliation/TaskTimeout.java   |  2 +-
 .../scheduling/FirstFitOfferSelector.java   |  5 ++-
 .../scheduler/scheduling/OfferSelector.java |  2 +-
 .../scheduling/RescheduleCalculator.java|  4 +-
 

[2/4] aurora git commit: Use java.util.Optional throughout

[wfarner]

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java 
b/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
index 61d855e..aa736a6 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
@@ -14,10 +14,10 @@
 package org.apache.aurora.scheduler.updater;
 
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 import com.google.common.base.MoreObjects;
-import com.google.common.base.Optional;
 
 import org.apache.aurora.scheduler.updater.StateEvaluator.Failure;
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java 
b/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java
index 06a7695..7d87ddf 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/StateEvaluator.java
@@ -13,7 +13,7 @@
  */
 package org.apache.aurora.scheduler.updater;
 
-import com.google.common.base.Optional;
+import java.util.Optional;
 
 import static java.util.Objects.requireNonNull;
 
@@ -47,7 +47,7 @@ interface StateEvaluator {
*/
   Result evaluate(T actualState);
 
-  Optional NO_FAILURE = Optional.absent();
+  Optional NO_FAILURE = Optional.empty();
 
   enum Result {
 EVALUATE_ON_STATE_CHANGE(Optional.of(InstanceAction.AWAIT_STATE_CHANGE), 
NO_FAILURE),
@@ -56,8 +56,8 @@ interface StateEvaluator {
 KILL_TASK_WITH_RESERVATION_AND_EVALUATE_ON_STATE_CHANGE(
 Optional.of(InstanceAction.KILL_TASK_AND_RESERVE), NO_FAILURE),
 EVALUATE_AFTER_MIN_RUNNING_MS(Optional.of(InstanceAction.WATCH_TASK), 
NO_FAILURE),
-SUCCEEDED(Optional.absent(), NO_FAILURE),
-FAILED_TERMINATED(Optional.absent(), Optional.of(Failure.EXITED));
+SUCCEEDED(Optional.empty(), NO_FAILURE),
+FAILED_TERMINATED(Optional.empty(), Optional.of(Failure.EXITED));
 
 private final Optional action;
 private final Optional failure;

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/updater/UpdateAgentReserver.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/updater/UpdateAgentReserver.java 
b/src/main/java/org/apache/aurora/scheduler/updater/UpdateAgentReserver.java
index fca4b4e..b6909a6 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/UpdateAgentReserver.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/UpdateAgentReserver.java
@@ -13,9 +13,9 @@
  */
 package org.apache.aurora.scheduler.updater;
 
+import java.util.Optional;
 import java.util.Set;
 
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
 
@@ -133,7 +133,7 @@ public interface UpdateAgentReserver {
 
 @Override
 public Optional getAgent(IInstanceKey key) {
-  return Optional.absent();
+  return Optional.empty();
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/aurora/blob/6fd765bc/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java 
b/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
index b408a65..3992aa7 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/UpdateFactory.java
@@ -14,10 +14,10 @@
 package org.apache.aurora.scheduler.updater;
 
 import java.io.Serializable;
+import java.util.Optional;
 import java.util.Set;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
 import com.google.common.collect.DiscreteDomain;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -98,7 +98,7 @@ interface UpdateFactory {
 if (rollingForward) {
   desiredStateConfig = desiredInstances.contains(instanceId)
   ? Optional.of(instructions.getDesiredState().getTask())
-  : Optional.absent();
+  : Optional.empty();
 } else {
   desiredStateConfig = getConfig(instanceId, 
instructions.getInitialState());
 }
@@ -152,7 +152,7 @@ interface UpdateFactory {
 }
   }
 
-  return Optional.absent();
+  return Optional.empty();
 }
   }
 

[2/2] aurora git commit: Recover snapshots via the Op stream

[wfarner]

nal QuotaStore.Mutable writeBehindQuotaStore;
-  private final AttributeStore.Mutable writeBehindAttributeStore;
-  private final JobUpdateStore.Mutable writeBehindJobUpdateStore;
   private final ReentrantLock writeLock;
   private final ThriftBackfill thriftBackfill;
 
-  private final WriteAheadStorage writeAheadStorage;
+  private final WriteRecorder writeRecorder;
 
-  // TODO(wfarner): It should be possible to remove this flag now, since all 
call stacks when
-  // recovering are controlled at this layer (they're all calls to Mutable 
store implementations).
-  // The more involved change is changing SnapshotStore to accept a Mutable 
store provider to
-  // avoid a call to Storage.write() when we replay a Snapshot.
-  private boolean recovered = false;
   private TransactionRecorder transaction = null;
 
   private final SlidingStats writerWaitStats = new 
SlidingStats("storage_write_lock_wait", "ns");
 
-  private final Map<Op._Fields, Consumer> transactionReplayActions;
-
   @Inject
   DurableStorage(
   Persistence persistence,
@@ -147,12 +116,6 @@ public class DurableStorage implements NonVolatileStorage {
 // we write directly to the writeBehind stores since we are replaying 
what's already persisted.
 // After that, all writes must succeed in Persistence before they may be 
considered successful.
 this.writeBehindStorage = requireNonNull(delegateStorage);
-this.writeBehindSchedulerStore = requireNonNull(schedulerStore);
-this.writeBehindJobStore = requireNonNull(jobStore);
-this.writeBehindTaskStore = requireNonNull(taskStore);
-this.writeBehindQuotaStore = requireNonNull(quotaStore);
-this.writeBehindAttributeStore = requireNonNull(attributeStore);
-this.writeBehindJobUpdateStore = requireNonNull(jobUpdateStore);
 this.writeLock = requireNonNull(writeLock);
 this.thriftBackfill = requireNonNull(thriftBackfill);
 TransactionManager transactionManager = new TransactionManager() {
@@ -166,7 +129,7 @@ public class DurableStorage implements NonVolatileStorage {
 transaction.add(op);
   }
 };
-this.writeAheadStorage = new WriteAheadStorage(
+this.writeRecorder = new WriteRecorder(
 transactionManager,
 schedulerStore,
 jobStore,
@@ -174,81 +137,8 @@ public class DurableStorage implements NonVolatileStorage {
 quotaStore,
 attributeStore,
 jobUpdateStore,
-LoggerFactory.getLogger(WriteAheadStorage.class),
+LoggerFactory.getLogger(WriteRecorder.class),
 eventSink);
-
-this.transactionReplayActions = buildTransactionReplayActions();
-  }
-
-  @VisibleForTesting
-  final Map<Op._Fields, Consumer> buildTransactionReplayActions() {
-return ImmutableMap.<Op._Fields, Consumer>builder()
-.put(
-Op._Fields.SAVE_FRAMEWORK_ID,
-op -> 
writeBehindSchedulerStore.saveFrameworkId(op.getSaveFrameworkId().getId()))
-.put(Op._Fields.SAVE_CRON_JOB, op -> {
-  SaveCronJob cronJob = op.getSaveCronJob();
-  writeBehindJobStore.saveAcceptedJob(
-  thriftBackfill.backfillJobConfiguration(cronJob.getJobConfig()));
-})
-.put(
-Op._Fields.REMOVE_JOB,
-op -> 
writeBehindJobStore.removeJob(IJobKey.build(op.getRemoveJob().getJobKey(
-.put(
-Op._Fields.SAVE_TASKS,
-op -> writeBehindTaskStore.saveTasks(
-thriftBackfill.backfillTasks(op.getSaveTasks().getTasks(
-.put(
-Op._Fields.REMOVE_TASKS,
-op -> 
writeBehindTaskStore.deleteTasks(op.getRemoveTasks().getTaskIds()))
-.put(Op._Fields.SAVE_QUOTA, op -> {
-  SaveQuota saveQuota = op.getSaveQuota();
-  writeBehindQuotaStore.saveQuota(
-  saveQuota.getRole(),
-  ThriftBackfill.backfillResourceAggregate(saveQuota.getQuota()));
-})
-.put(
-Op._Fields.REMOVE_QUOTA,
-op -> 
writeBehindQuotaStore.removeQuota(op.getRemoveQuota().getRole()))
-.put(Op._Fields.SAVE_HOST_ATTRIBUTES, op -> {
-  HostAttributes attributes = 
op.getSaveHostAttributes().getHostAttributes();
-  // Prior to commit 5cf760b, the store would persist maintenance mode 
changes for
-  // unknown hosts.  5cf760b began rejecting these, but the storage 
may still
-  // contain entries with a null slave ID.
-  if (attributes.isSetSlaveId()) {
-
writeBehindAttributeStore.saveHostAttributes(IHostAttributes.build(attributes));
-  } else {
-LOG.info("Dropping host attributes with no agent ID: " + 
attributes);
-  }
-})
-.put(
-Op._Fields.SAVE_LOCK, // TODO(jly): Deprecated, remove in 0.21. 
See AURORA-1959.
-op -> { /* no-op */ })
-.put(
-Op._Fields.REMOVE_LOCK, // TODO(jly): Deprecated, remove in 0

[1/2] aurora git commit: Recover snapshots via the Op stream

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 4489dc345 -> 5f79f7ca7


http://git-wip-us.apache.org/repos/asf/aurora/blob/5f79f7ca/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java 
b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java
new file mode 100644
index 000..b30de88
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.storage.log;
+
+import javax.inject.Inject;
+
+import com.google.common.util.concurrent.AbstractScheduledService;
+
+import org.apache.aurora.codec.ThriftBinaryCodec.CodingException;
+import org.apache.aurora.common.inject.TimedInterceptor.Timed;
+import org.apache.aurora.common.quantity.Amount;
+import org.apache.aurora.common.quantity.Time;
+import org.apache.aurora.gen.storage.Snapshot;
+import org.apache.aurora.scheduler.log.Log.Stream.InvalidPositionException;
+import org.apache.aurora.scheduler.log.Log.Stream.StreamAccessException;
+import org.apache.aurora.scheduler.storage.SnapshotStore;
+import org.apache.aurora.scheduler.storage.Snapshotter;
+import org.apache.aurora.scheduler.storage.Storage;
+import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
+import org.apache.aurora.scheduler.storage.Storage.StorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A {@link SnapshotStore} that snapshots to the log, and automatically 
snapshots on
+ * a fixed interval.
+ */
+class SnapshotService extends AbstractScheduledService implements 
SnapshotStore {
+  private static final Logger LOG = 
LoggerFactory.getLogger(SnapshotService.class);
+
+  private final Storage storage;
+  private final LogPersistence log;
+  private final Snapshotter snapshotter;
+  private final Amount snapshotInterval;
+
+  @Inject
+  SnapshotService(Storage storage, LogPersistence log, Snapshotter 
snapshotter, Settings settings) {
+this.storage = requireNonNull(storage);
+this.log = requireNonNull(log);
+this.snapshotter = requireNonNull(snapshotter);
+this.snapshotInterval = settings.getSnapshotInterval();
+  }
+
+  @Override
+  protected void runOneIteration() {
+snapshot();
+  }
+
+  @Timed("scheduler_log_snapshot")
+  @Override
+  public void snapshot() throws StorageException {
+try {
+  LOG.info("Creating snapshot");
+
+  // It's important to perform snapshot creation in a write lock to ensure 
all upstream callers
+  // are correctly synchronized (e.g. during backup creation).
+  storage.write((NoResult.Quiet) stores -> {
+Snapshot snapshot = snapshotter.from(stores);
+LOG.info("Saving snapshot");
+snapshotWith(snapshot);
+
+LOG.info("Snapshot complete."
++ " host attrs: " + snapshot.getHostAttributesSize()
++ ", cron jobs: " + snapshot.getCronJobsSize()
++ ", quota confs: " + snapshot.getQuotaConfigurationsSize()
++ ", tasks: " + snapshot.getTasksSize()
++ ", updates: " + snapshot.getJobUpdateDetailsSize());
+  });
+} catch (CodingException e) {
+  throw new StorageException("Failed to encode a snapshot", e);
+} catch (InvalidPositionException e) {
+  throw new StorageException("Saved snapshot but failed to truncate 
entries preceding it", e);
+} catch (StreamAccessException e) {
+  throw new StorageException("Failed to create a snapshot", e);
+}
+  }
+
+  @Timed("scheduler_log_snapshot_persist")
+  @Override
+  public void snapshotWith(Snapshot snapshot)
+  throws CodingException, InvalidPositionException, StreamAccessException {
+
+log.persist(snapshot);
+  }
+
+  @Override
+  protected Scheduler scheduler() {
+return Scheduler.newFixedDelaySchedule(
+snapshotInterval.getValue(),
+snapshotInterval.getValue(),
+snapshotInterval.getUnit().getTimeUnit());
+  }
+
+  /**
+   * Configuration settings for log persistence.
+   */
+  public static class Settings {
+private final Amount snapshotInterval;
+
+Settings(Amount snapshotInterval) {
+  this.snapshotInterval = 

aurora git commit: Deprecated Ops re-added, perform no-op instead of throwing an exception.

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 6c897e520 -> 4f0299b2a


Deprecated Ops re-added, perform no-op instead of throwing an exception.

Bugs closed: AURORA-1959

Reviewed at https://reviews.apache.org/r/64459/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/4f0299b2
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/4f0299b2
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/4f0299b2

Branch: refs/heads/master
Commit: 4f0299b2ada81932feb5c06fd223c0fdbd303c18
Parents: 6c897e5
Author: Jordan Ly 
Authored: Fri Dec 8 18:32:06 2017 -0800
Committer: Bill Farner 
Committed: Fri Dec 8 18:32:06 2017 -0800

--
 .../thrift/org/apache/aurora/gen/api.thrift | 21 
 .../thrift/org/apache/aurora/gen/storage.thrift | 14 +++--
 .../storage/durability/DurableStorage.java  |  6 ++
 .../storage/durability/DurableStorageTest.java  |  8 
 4 files changed, 47 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/4f0299b2/api/src/main/thrift/org/apache/aurora/gen/api.thrift
--
diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift 
b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
index c9f4210..ef754e3 100644
--- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift
+++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
@@ -115,6 +115,27 @@ struct JobKey {
   3: string name
 }
 
+// TODO(jly): Deprecated, remove in 0.21. See AURORA-1959.
+/** A unique lock key. */
+union LockKey {
+  1: JobKey job
+}
+
+// TODO(jly): Deprecated, remove in 0.21. See AURORA-1959.
+/** A generic lock struct to facilitate context specific resource/operation 
serialization. */
+struct Lock {
+  /** ID of the lock - unique per storage */
+  1: LockKey key
+  /** UUID - facilitating soft lock authorization */
+  2: string token
+  /** Lock creator */
+  3: string user
+  /** Lock creation timestamp in milliseconds */
+  4: i64 timestampMs
+  /** Optional message to record with the lock */
+  5: optional string message
+}
+
 /** A unique identifier for the active task within a job. */
 struct InstanceKey {
   /** Key identifying the job. */

http://git-wip-us.apache.org/repos/asf/aurora/blob/4f0299b2/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
--
diff --git a/api/src/main/thrift/org/apache/aurora/gen/storage.thrift 
b/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
index 2210497..b79e204 100644
--- a/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
+++ b/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
@@ -28,6 +28,16 @@ struct SaveCronJob {
   2: api.JobConfiguration jobConfig
 }
 
+// TODO(jly): Deprecated, remove in 0.21. See AURORA-1959.
+struct SaveLock {
+  1: api.Lock lock
+}
+
+// TODO(jly): Deprecated, remove in 0.21. See AURORA-1959.
+struct RemoveLock {
+  1: api.LockKey lockKey
+}
+
 struct RemoveJob {
   2: api.JobKey jobKey
 }
@@ -92,8 +102,8 @@ union Op {
   9: RemoveQuota removeQuota
   10: SaveHostAttributes saveHostAttributes
   // 11: removed
-  // 12: deleted
-  // 13: deleted
+  12: SaveLock saveLock // TODO(jly): Deprecated, remove in 0.21. See 
AURORA-1959.
+  13: RemoveLock removeLock // TODO(jly): Deprecated, remove in 0.21. See 
AURORA-1959.
   14: SaveJobUpdate saveJobUpdate
   15: SaveJobUpdateEvent saveJobUpdateEvent
   16: SaveJobInstanceUpdateEvent saveJobInstanceUpdateEvent

http://git-wip-us.apache.org/repos/asf/aurora/blob/4f0299b2/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java
 
b/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java
index 85b2113..6a7c0ad 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java
@@ -221,6 +221,12 @@ public class DurableStorage implements NonVolatileStorage {
 LOG.info("Dropping host attributes with no agent ID: " + 
attributes);
   }
 })
+.put(
+Op._Fields.SAVE_LOCK, // TODO(jly): Deprecated, remove in 0.21. 
See AURORA-1959.
+op -> { /* no-op */ })
+.put(
+Op._Fields.REMOVE_LOCK, // TODO(jly): Deprecated, remove in 0.21. 
See AURORA-1959.
+op -> { /* no-op */ })
 .put(Op._Fields.SAVE_JOB_UPDATE, op ->
   writeBehindJobUpdateStore.saveJobUpdate(
   

[aurora-packaging] Git Push Summary

[wfarner]

Repository: aurora-packaging
Updated Branches:
  refs/heads/0.19.x [created] de2ad96bd

aurora git commit: Remove hack for guice error logging

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 103f794ed -> 6c897e520


Remove hack for guice error logging

With `103f794ed126e135f2fe0ff1bde04a4093413521`, this hack is no longer needed.

Reviewed at https://reviews.apache.org/r/64426/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/6c897e52
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/6c897e52
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/6c897e52

Branch: refs/heads/master
Commit: 6c897e5207db67b222d9c693baeb94de381d6c6f
Parents: 103f794
Author: Bill Farner 
Authored: Thu Dec 7 10:35:45 2017 -0800
Committer: Bill Farner 
Committed: Thu Dec 7 10:35:45 2017 -0800

--
 .../aurora/scheduler/app/SchedulerMain.java | 27 +---
 1 file changed, 1 insertion(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/6c897e52/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java 
b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
index 7ffcf4f..2bf7e7b 100644
--- a/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
+++ b/src/main/java/org/apache/aurora/scheduler/app/SchedulerMain.java
@@ -28,12 +28,9 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.net.HostAndPort;
 import com.google.inject.AbstractModule;
-import com.google.inject.CreationException;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Module;
-import com.google.inject.ProvisionException;
-import com.google.inject.spi.Message;
 import com.google.inject.util.Modules;
 
 import org.apache.aurora.GuavaUtils.ServiceManagerIface;
@@ -204,29 +201,7 @@ public class SchedulerMain {
 Thread.setDefaultUncaughtExceptionHandler((t, e) -> {
   uncaughtExceptions.incrementAndGet();
 
-  if (e instanceof CreationException) {
-try {
-  LOG.error("Uncaught exception from " + t + ":" + e, e);
-} catch (RuntimeException ex) {
-  LOG.warn("Using fallback printer for guice CreationException");
-
-  // Special handling for guice creation exceptions, which break in 
guice 3 when java 8
-  // lambdas are used.  Remove this once using guice >=4.0.
-  CreationException creationException = (CreationException) e;
-  for (Message m : creationException.getErrorMessages()) {
-LOG.error(m.getMessage());
-LOG.error("  source: " + m.getSource());
-  }
-}
-  } else if (e instanceof ProvisionException) {
-// More special handling for guice 3 + java 8.  Remove this once using 
guice >=4.0.
-ProvisionException pe = (ProvisionException) e;
-for (Message message : pe.getErrorMessages()) {
-  LOG.error(message.getMessage());
-}
-  } else {
-LOG.error("Uncaught exception from " + t + ":" + e, e);
-  }
+  LOG.error("Uncaught exception from " + t + ":" + e, e);
 });
 
 Module module = Modules.combine(

aurora git commit: Update to guice 4.1.0, switch from jersey to resteasy

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 2754636c6 -> 103f794ed


Update to guice 4.1.0, switch from jersey to resteasy

Upgrading guice was the original goal with this patch, which pulled along
several other dependencies.  Guice 3
[suffers from obscure errors](https://github.com/google/guice/issues/757)
when creating binding error messages with java 8 and lambdas.  This has been a
frequent source of frustration since we first upgraded to java 8 mid-2015.

I've gone spelunking down this path numerous times, and frequently hit a wall
with jersey.  We needed to upgrade jersey-guice, to upgrade jersey, to upgrade
guice.  jersey introduced their own dependency injection (HK2) in jersey 2.0,
which complicated matters.  There have been some promising developments since
(hk2-guice 
[bridge](https://javaee.github.io/hk2/guice-bridge.html#bi-directional-hk2-guice-bridge),
2.26 [abstracted HK2](https://jersey.github.io/release-notes/2.26.html), and
several [projects](https://github.com/Squarespace/jersey2-guice) have emerged
to solve the issue).  Unfortunately, each avenue failed with some combination
of not working well with our application design, or i just plain couldn't get
it working.  I began to look beyond jersey.

This left restlet and resteasy as the most common alternatives.  I balked early
at restlet due to their guice integration being
[apparently](https://github.com/restlet/restlet-framework-java/commits/master/modules/org.restlet.ext.guice)
[dormant](https://stackoverflow.com/questions/8227583/what-is-the-status-of-org-restlet-ext-guice).

Fortunately i achieved some early wins with resteasy!  Migrating was
straightforward with a small patch based on some examples.

However, i hit a hurdle with shiro-guice.  It
[needed to be updated](https://issues.apache.org/jira/browse/SHIRO-493) to work
with guice 4, and there were some necessary API changes.  No big deal, just the
`filterConfig()` nesting you see in this patch.  This revealed a deeper issue
with binding custom `Filter`s with `ShiroWebModule`.  Previously,
`ShiroWebModule` would effectively only `bind()` keys
[they 
define](https://github.com/apache/shiro/blob/f326fd3814f672464deb11c3dadadb27af618eec/support/guice/src/main/java/org/apache/shiro/guice/web/ShiroWebModule.java#L65-L86),
 allowing the API user to `bind()` custom keys.  The 
[patch](https://github.com/apache/shiro/commit/f2dfa7ff39c9870e7b9856ceca8690c5398080fa#diff-359a7b20d3b7fdcc0ffce11ad57d6e1c)
 to support guice 4 changed that, and `bind()` will be 
[called](https://github.com/apache/shiro/commit/f2dfa7ff39c9870e7b9856ceca8690c5398080fa#diff-359a7b20d3b7fdcc0ffce11ad57d6e1cR183)
on these custom keys.  In our case, this caused a duplicate binding.

The simplest workaround to this was to avoid `bind()`ing the custom
`afterAuthFilter` key, and use the custom type as the key type (e.g.
`Key.get(CountingFilter.class)` rather than `Key.get(Filter.class)`).

Lastly, `GuiceResteasyBootstrapServletContextListener` does not integrate with
`GuiceServletContextListener` in the way `GuiceFilter`
[demands](https://github.com/google/guice/blob/e7bef34ef6379735e2a58df1b23f351bb7e30e44/extensions/servlet/src/com/google/inject/servlet/ServletModule.java#L322-L323),
which necessitated the passing of `ServletContext` you see in this patch.

I can't say i'm happy with the outcome here, but i am overall happier given
that guice is upgraded.

Reviewed at https://reviews.apache.org/r/64362/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/103f794e
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/103f794e
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/103f794e

Branch: refs/heads/master
Commit: 103f794ed126e135f2fe0ff1bde04a4093413521
Parents: 2754636
Author: Bill Farner 
Authored: Thu Dec 7 09:50:22 2017 -0800
Committer: Bill Farner 
Committed: Thu Dec 7 09:50:22 2017 -0800

--
 build.gradle| 27 +---
 .../scheduler/http/JettyServerModule.java   | 46 +++--
 .../aurora/scheduler/http/api/ApiModule.java|  4 --
 .../http/api/security/HttpSecurityModule.java   | 44 
 .../scheduler/http/AbstractJettyTest.java   |  8 +--
 .../aurora/scheduler/http/api/ApiBetaTest.java  |  7 +-
 .../apache/aurora/scheduler/http/api/ApiIT.java |  6 +-
 .../http/api/security/HttpSecurityIT.java   | 72 +++-
 .../ShiroKerberosAuthenticationFilterTest.java  |  6 +-
 ...berosPermissiveAuthenticationFilterTest.java |  6 +-
 10 files changed, 131 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/103f794e/build.gradle
--
diff --git a/build.gradle b/build.gradle
index 

aurora git commit: Remove H2-related end-to-end test case

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master e74fb3240 -> 2754636c6


Remove H2-related end-to-end test case

This test still exercises kerberos functionality through the `snapshot_as`
function, so the h2 test case can safely be considered redundant.

Reviewed at https://reviews.apache.org/r/64377/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/2754636c
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/2754636c
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/2754636c

Branch: refs/heads/master
Commit: 2754636c674a9912475edafe01669602125fbee3
Parents: e74fb32
Author: Bill Farner 
Authored: Wed Dec 6 08:58:16 2017 -0800
Committer: Bill Farner 
Committed: Wed Dec 6 08:58:16 2017 -0800

--
 .../aurora/e2e/test_kerberos_end_to_end.sh  | 20 
 1 file changed, 20 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/2754636c/src/test/sh/org/apache/aurora/e2e/test_kerberos_end_to_end.sh
--
diff --git a/src/test/sh/org/apache/aurora/e2e/test_kerberos_end_to_end.sh 
b/src/test/sh/org/apache/aurora/e2e/test_kerberos_end_to_end.sh
index 2a7a66d..646c213 100755
--- a/src/test/sh/org/apache/aurora/e2e/test_kerberos_end_to_end.sh
+++ b/src/test/sh/org/apache/aurora/e2e/test_kerberos_end_to_end.sh
@@ -63,16 +63,6 @@ function snapshot_as {
   kdestroy
 }
 
-readonly H2_RESPONSE_OUTFILE="h2console-response.%s.json"
-function h2console_as {
-  local principal=$1
-  kinit -k -t "testdir/${principal}.keytab" $principal
-  curl -u : --negotiate -w '%{http_code}\n' \
--o $(printf $H2_RESPONSE_OUTFILE $principal) \
--s "http://$SCHEDULER_HOSTNAME:8081/h2console/;
-  kdestroy
-}
-
 function setup {
   cat >> $KRB5_CONFIG <

aurora git commit: Clean up some lint rules

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 89338dd7f -> a0628efd1


Clean up some lint rules

Reviewed at https://reviews.apache.org/r/64287/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/a0628efd
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/a0628efd
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/a0628efd

Branch: refs/heads/master
Commit: a0628efd1773bd1a650de7e4d8c94ffdf0313651
Parents: 89338dd
Author: Bill Farner 
Authored: Mon Dec 4 08:08:43 2017 -0800
Committer: Bill Farner 
Committed: Mon Dec 4 08:08:43 2017 -0800

--
 config/pmd/main.xml |  7 -
 config/spotbugs/excludeFilter.xml   | 31 
 .../aurora/scheduler/http/Utilization.java  |  7 +++--
 .../aurora/scheduler/http/api/ApiBeta.java  |  2 +-
 .../api/security/AuthorizeHeaderTokenTest.java  |  2 +-
 5 files changed, 6 insertions(+), 43 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/a0628efd/config/pmd/main.xml
--
diff --git a/config/pmd/main.xml b/config/pmd/main.xml
index b90c221..9c3f578 100644
--- a/config/pmd/main.xml
+++ b/config/pmd/main.xml
@@ -23,10 +23,6 @@ limitations under the License.
   
 
   
-
-
-
 
 
 
@@ -40,9 +36,6 @@ limitations under the License.
 
 
-
-
-
   
 
   

http://git-wip-us.apache.org/repos/asf/aurora/blob/a0628efd/config/spotbugs/excludeFilter.xml
--
diff --git a/config/spotbugs/excludeFilter.xml 
b/config/spotbugs/excludeFilter.xml
index 51790cc..799ccdc 100644
--- a/config/spotbugs/excludeFilter.xml
+++ b/config/spotbugs/excludeFilter.xml
@@ -21,12 +21,6 @@ limitations under the License.
   
   
   
-  
-  
-  
-  
-  
-  
 
 
   
@@ -38,18 +32,6 @@ limitations under the License.
 
   
 
-  
-  
-
-
-  
-  
-  
-
-  
-
   
   
@@ -80,10 +62,6 @@ limitations under the License.
 
   
   
-
-
-  
-  
 
 
   
@@ -113,15 +91,6 @@ limitations under the License.
   
 
   
-
-
-  
-  
-
-
-  
-
-  
 
 
 

http://git-wip-us.apache.org/repos/asf/aurora/blob/a0628efd/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/http/Utilization.java 
b/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
index efacd8c..d4dfd1b 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/Utilization.java
@@ -178,12 +178,13 @@ public class Utilization {
   }
 
   private MetricType getTypeByName(String name) throws WebApplicationException 
{
-MetricType type = MetricType.valueOf(name.toUpperCase(Locale.ENGLISH));
-if (type == null) {
+try {
+  return MetricType.valueOf(name.toUpperCase(Locale.ENGLISH));
+} catch (IllegalArgumentException e) {
   throw new WebApplicationException(
+  e,
   Response.status(Status.BAD_REQUEST).entity("Invalid metric 
type.").build());
 }
-return type;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/aurora/blob/a0628efd/src/main/java/org/apache/aurora/scheduler/http/api/ApiBeta.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/ApiBeta.java 
b/src/main/java/org/apache/aurora/scheduler/http/api/ApiBeta.java
index f923067..2414d37 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/api/ApiBeta.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/api/ApiBeta.java
@@ -107,7 +107,7 @@ public class ApiBeta {
 return params.toArray();
   }
 
-  private Method getApiMethod(String name, Class[] parameterTypes) {
+  private Method getApiMethod(String name, Class... parameterTypes) {
 try {
   return Iface.class.getMethod(name, parameterTypes);
 } catch (NoSuchMethodException e) {

http://git-wip-us.apache.org/repos/asf/aurora/blob/a0628efd/src/test/java/org/apache/aurora/scheduler/http/api/security/AuthorizeHeaderTokenTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/http/api/security/AuthorizeHeaderTokenTest.java
 
b/src/test/java/org/apache/aurora/scheduler/http/api/security/AuthorizeHeaderTokenTest.java
index 9e956de..89dc447 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/http/api/security/AuthorizeHeaderTokenTest.java
+++ 

aurora git commit: Remove redundant transaction recorder

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master cea43db9d -> 5e008ff7f


Remove redundant transaction recorder

Reviewed at https://reviews.apache.org/r/64283/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/5e008ff7
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/5e008ff7
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/5e008ff7

Branch: refs/heads/master
Commit: 5e008ff7fe7f5701e1baf2051c2873f655ca7aed
Parents: cea43db
Author: Bill Farner 
Authored: Sun Dec 3 06:50:45 2017 -0800
Committer: Bill Farner 
Committed: Sun Dec 3 06:50:45 2017 -0800

--
 .../scheduler/storage/log/LogPersistence.java   |   5 +-
 .../scheduler/storage/log/StreamManager.java|   9 +-
 .../storage/log/StreamManagerImpl.java  | 133 ++---
 .../storage/log/StreamTransaction.java  |  40 -
 .../scheduler/storage/log/LogManagerTest.java   | 149 +--
 5 files changed, 21 insertions(+), 315 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/5e008ff7/src/main/java/org/apache/aurora/scheduler/storage/log/LogPersistence.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/log/LogPersistence.java 
b/src/main/java/org/apache/aurora/scheduler/storage/log/LogPersistence.java
index a0a6b6c..e70e605 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogPersistence.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogPersistence.java
@@ -18,6 +18,7 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
@@ -99,10 +100,8 @@ class LogPersistence implements Persistence, 
DistributedSnapshotStore {
 
   @Override
   public void persist(Stream mutations) throws PersistenceException {
-StreamTransaction transaction = streamManager.startTransaction();
-mutations.forEach(transaction::add);
 try {
-  transaction.commit();
+  streamManager.commit(mutations.collect(Collectors.toList()));
 } catch (CodingException e) {
   throw new PersistenceException(e);
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/5e008ff7/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManager.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManager.java 
b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManager.java
index 18da32d..73602cb 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManager.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManager.java
@@ -14,8 +14,10 @@
 package org.apache.aurora.scheduler.storage.log;
 
 import java.util.Iterator;
+import java.util.List;
 
 import org.apache.aurora.gen.storage.LogEntry;
+import org.apache.aurora.gen.storage.Op;
 import org.apache.aurora.gen.storage.Snapshot;
 import org.apache.aurora.scheduler.log.Log;
 
@@ -52,12 +54,11 @@ public interface StreamManager {
   void truncateBefore(Log.Position position);
 
   /**
-   * Starts a transaction that can be used to commit a series of ops to the 
log stream atomically.
+   * Saves operations to the log stream.
*
-   * @return StreamTransaction A transaction manager to handle batching up 
commits to the
-   *underlying stream.
+   * @param mutations Operations to save.
*/
-  StreamTransaction startTransaction();
+  void commit(List mutations);
 
   /**
* Adds a snapshot to the log and if successful, truncates the log entries 
preceding the

http://git-wip-us.apache.org/repos/asf/aurora/blob/5e008ff7/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java 
b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
index c5b107f..9eb309a 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java
@@ -15,33 +15,24 @@ package org.apache.aurora.scheduler.storage.log;
 
 import java.util.Arrays;
 import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.annotation.Nullable;
 import javax.inject.Inject;
 
-import 

[1/4] aurora git commit: Extract a storage Persistence layer

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master de8b37549 -> cea43db9d


http://git-wip-us.apache.org/repos/asf/aurora/blob/cea43db9/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java 
b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
deleted file mode 100644
index 3c056c9..000
--- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java
+++ /dev/null
@@ -1,897 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.storage.log;
-
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantLock;
-
-import com.google.common.base.Function;
-import com.google.common.base.Functions;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-import org.apache.aurora.codec.ThriftBinaryCodec;
-import org.apache.aurora.codec.ThriftBinaryCodec.CodingException;
-import org.apache.aurora.common.quantity.Amount;
-import org.apache.aurora.common.quantity.Data;
-import org.apache.aurora.common.quantity.Time;
-import org.apache.aurora.common.testing.easymock.EasyMockTest;
-import org.apache.aurora.gen.AssignedTask;
-import org.apache.aurora.gen.Attribute;
-import org.apache.aurora.gen.HostAttributes;
-import org.apache.aurora.gen.InstanceTaskConfig;
-import org.apache.aurora.gen.JobConfiguration;
-import org.apache.aurora.gen.JobInstanceUpdateEvent;
-import org.apache.aurora.gen.JobUpdate;
-import org.apache.aurora.gen.JobUpdateAction;
-import org.apache.aurora.gen.JobUpdateEvent;
-import org.apache.aurora.gen.JobUpdateInstructions;
-import org.apache.aurora.gen.JobUpdateKey;
-import org.apache.aurora.gen.JobUpdateSettings;
-import org.apache.aurora.gen.JobUpdateStatus;
-import org.apache.aurora.gen.JobUpdateSummary;
-import org.apache.aurora.gen.MaintenanceMode;
-import org.apache.aurora.gen.Range;
-import org.apache.aurora.gen.ResourceAggregate;
-import org.apache.aurora.gen.ScheduleStatus;
-import org.apache.aurora.gen.ScheduledTask;
-import org.apache.aurora.gen.TaskConfig;
-import org.apache.aurora.gen.storage.DeduplicatedSnapshot;
-import org.apache.aurora.gen.storage.LogEntry;
-import org.apache.aurora.gen.storage.Op;
-import org.apache.aurora.gen.storage.PruneJobUpdateHistory;
-import org.apache.aurora.gen.storage.RemoveJob;
-import org.apache.aurora.gen.storage.RemoveJobUpdates;
-import org.apache.aurora.gen.storage.RemoveQuota;
-import org.apache.aurora.gen.storage.RemoveTasks;
-import org.apache.aurora.gen.storage.SaveCronJob;
-import org.apache.aurora.gen.storage.SaveFrameworkId;
-import org.apache.aurora.gen.storage.SaveHostAttributes;
-import org.apache.aurora.gen.storage.SaveJobInstanceUpdateEvent;
-import org.apache.aurora.gen.storage.SaveJobUpdate;
-import org.apache.aurora.gen.storage.SaveJobUpdateEvent;
-import org.apache.aurora.gen.storage.SaveQuota;
-import org.apache.aurora.gen.storage.SaveTasks;
-import org.apache.aurora.gen.storage.Snapshot;
-import org.apache.aurora.gen.storage.Transaction;
-import org.apache.aurora.gen.storage.storageConstants;
-import org.apache.aurora.scheduler.base.JobKeys;
-import org.apache.aurora.scheduler.base.TaskTestUtil;
-import org.apache.aurora.scheduler.base.Tasks;
-import org.apache.aurora.scheduler.events.EventSink;
-import org.apache.aurora.scheduler.events.PubsubEvent;
-import org.apache.aurora.scheduler.log.Log;
-import org.apache.aurora.scheduler.log.Log.Entry;
-import org.apache.aurora.scheduler.log.Log.Position;
-import org.apache.aurora.scheduler.log.Log.Stream;
-import org.apache.aurora.scheduler.resources.ResourceTestUtil;
-import org.apache.aurora.scheduler.storage.AttributeStore;
-import org.apache.aurora.scheduler.storage.SnapshotStore;
-import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider;
-import org.apache.aurora.scheduler.storage.Storage.MutateWork;
-import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
-import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult.Quiet;
-import 

[4/4] aurora git commit: Extract a storage Persistence layer

[wfarner]

ain/java/org/apache/aurora/scheduler/storage/DistributedSnapshotStore.java
@@ -15,18 +15,25 @@ package org.apache.aurora.scheduler.storage;
 
 import org.apache.aurora.codec.ThriftBinaryCodec.CodingException;
 import org.apache.aurora.gen.storage.Snapshot;
+import org.apache.aurora.scheduler.storage.Storage.StorageException;
 
 /**
  * A distributed snapshot store that supports persisting globally-visible 
snapshots.
  */
 public interface DistributedSnapshotStore {
+
+  /**
+   * Clean up the underlying storage by optimizing internal data structures. 
Does not change
+   * externally-visible state but might not run concurrently with write 
operations.
+   */
+  void snapshot() throws StorageException;
+
   /**
-   * Writes a snapshot to the distributed storage system.
-   * TODO(William Farner): Currently we're hiding some exceptions (which 
happen to be
-   * RuntimeExceptions).  Clean these up to be checked, and throw another 
exception type here.
+   * Identical to {@link #snapshot()}, using a custom {@link Snapshot} rather 
than an
+   * internally-generated one based on the current state.
*
* @param snapshot Snapshot to write.
* @throws CodingException If the snapshot could not be serialized.
*/
-  void persist(Snapshot snapshot) throws CodingException;
+  void snapshotWith(Snapshot snapshot) throws CodingException;
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/cea43db9/src/main/java/org/apache/aurora/scheduler/storage/Storage.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/Storage.java 
b/src/main/java/org/apache/aurora/scheduler/storage/Storage.java
index 7d325b6..c9ea1de 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/Storage.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/Storage.java
@@ -196,10 +196,6 @@ public interface Storage {
* Executes the unit of read-only {@code work}.  The consistency model 
creates the possibility
* for a reader to read uncommitted state from a concurrent writer.
* 
-   * TODO(wfarner): Update this documentation once all stores are backed by
-   * {@link org.apache.aurora.scheduler.storage.db.DbStorage}, as the 
concurrency behavior will then
-   * be dictated by the {@link 
org.mybatis.guice.transactional.Transactional#isolation()} used.
-   * 
* TODO(wfarner): This method no longer needs to exist now that there is no 
global locking for
* reads.  We could instead directly inject the individual stores where they 
are used, as long
* as the stores have a layer to replicate what is currently done by
@@ -253,12 +249,6 @@ public interface Storage {
 void start(MutateWork.NoResult.Quiet initializationLogic) throws 
StorageException;
 
 /**
- * Clean up the underlying storage by optimizing internal data structures. 
Does not change
- * externally-visible state but might not run concurrently with write 
operations.
- */
-void snapshot() throws StorageException;
-
-/**
  * Prepares the underlying storage system for clean shutdown.
  */
 void stop();

http://git-wip-us.apache.org/repos/asf/aurora/blob/cea43db9/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java 
b/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java
index 6cd5b2b..3a62f02 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/backup/Recovery.java
@@ -197,7 +197,7 @@ public interface Recovery {
   void commit() {
 primaryStorage.write((NoResult.Quiet) storeProvider -> {
   try {
-distributedStore.persist(tempStorage.toSnapshot());
+distributedStore.snapshotWith(tempStorage.toSnapshot());
 shutDownNow.execute();
   } catch (CodingException e) {
 throw new IllegalStateException("Failed to encode snapshot.", e);

http://git-wip-us.apache.org/repos/asf/aurora/blob/cea43db9/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
 
b/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
index 3000796..18296b0 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/storage/backup/TemporaryStorage.java
@@ -27,9 +27,9 @@ import org.apache.aurora.scheduler.base.Tasks;
 import org.apache.aurora.scheduler.storage.SnapshotStore;
 import org.apache.aurora.scheduler.storage.Storage;
 import org.apache.aurora.scheduler.storage.Storage.Mut

[2/4] aurora git commit: Extract a storage Persistence layer

[wfarner]

http://git-wip-us.apache.org/repos/asf/aurora/blob/cea43db9/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java
new file mode 100644
index 000..07912b6
--- /dev/null
+++ 
b/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java
@@ -0,0 +1,781 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.storage.durability;
+
+import java.util.EnumSet;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.aurora.common.testing.easymock.EasyMockTest;
+import org.apache.aurora.gen.AssignedTask;
+import org.apache.aurora.gen.Attribute;
+import org.apache.aurora.gen.HostAttributes;
+import org.apache.aurora.gen.InstanceTaskConfig;
+import org.apache.aurora.gen.JobConfiguration;
+import org.apache.aurora.gen.JobInstanceUpdateEvent;
+import org.apache.aurora.gen.JobUpdate;
+import org.apache.aurora.gen.JobUpdateAction;
+import org.apache.aurora.gen.JobUpdateEvent;
+import org.apache.aurora.gen.JobUpdateInstructions;
+import org.apache.aurora.gen.JobUpdateKey;
+import org.apache.aurora.gen.JobUpdateSettings;
+import org.apache.aurora.gen.JobUpdateStatus;
+import org.apache.aurora.gen.JobUpdateSummary;
+import org.apache.aurora.gen.MaintenanceMode;
+import org.apache.aurora.gen.Range;
+import org.apache.aurora.gen.ResourceAggregate;
+import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.gen.ScheduledTask;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.storage.Op;
+import org.apache.aurora.gen.storage.PruneJobUpdateHistory;
+import org.apache.aurora.gen.storage.RemoveJob;
+import org.apache.aurora.gen.storage.RemoveJobUpdates;
+import org.apache.aurora.gen.storage.RemoveQuota;
+import org.apache.aurora.gen.storage.RemoveTasks;
+import org.apache.aurora.gen.storage.SaveCronJob;
+import org.apache.aurora.gen.storage.SaveFrameworkId;
+import org.apache.aurora.gen.storage.SaveHostAttributes;
+import org.apache.aurora.gen.storage.SaveJobInstanceUpdateEvent;
+import org.apache.aurora.gen.storage.SaveJobUpdate;
+import org.apache.aurora.gen.storage.SaveJobUpdateEvent;
+import org.apache.aurora.gen.storage.SaveQuota;
+import org.apache.aurora.gen.storage.SaveTasks;
+import org.apache.aurora.scheduler.base.JobKeys;
+import org.apache.aurora.scheduler.base.TaskTestUtil;
+import org.apache.aurora.scheduler.base.Tasks;
+import org.apache.aurora.scheduler.events.EventSink;
+import org.apache.aurora.scheduler.events.PubsubEvent;
+import org.apache.aurora.scheduler.resources.ResourceTestUtil;
+import org.apache.aurora.scheduler.storage.AttributeStore;
+import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider;
+import org.apache.aurora.scheduler.storage.Storage.MutateWork;
+import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
+import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult.Quiet;
+import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
+import org.apache.aurora.scheduler.storage.entities.IJobConfiguration;
+import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
+import org.apache.aurora.scheduler.storage.entities.IJobKey;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey;
+import org.apache.aurora.scheduler.storage.entities.IResourceAggregate;
+import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.testing.StorageTestUtil;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.easymock.IAnswer;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.aurora.gen.Resource.diskMb;
+import static 

[3/4] aurora git commit: Extract a storage Persistence layer

[wfarner]

cal database instead of an 
in-memory data structure.
- * If we die after such a failure, then another instance can read and apply 
the logged op
- * erroneously.
- *
- * This implementation leverages a local transaction to handle this:
- * 
- *   start local transaction
- *   perform op locally (uncommitted!)
- *   write op to log
- *   commit local transaction
- *   *checkpoint
- * 
- *
- * If the op fails to apply to local storage we will never write the op to 
the log and if the op
- * fails to apply to the log, it'll throw and abort the local storage 
transaction as well.
- */
-public class LogStorage implements NonVolatileStorage, 
DistributedSnapshotStore {
-
-  /**
-   * A service that can schedule an action to be executed periodically.
-   */
-  @VisibleForTesting
-  interface SchedulingService {
-
-/**
- * Schedules an action to execute periodically.
- *
- * @param interval The time period to wait until running the {@code 
action} again.
- * @param action The action to execute periodically.
- */
-void doEvery(Amount<Long, Time> interval, Runnable action);
-  }
-
-  /**
-   * A maintainer for context about open transactions. Assumes that an 
external entity is
-   * responsible for opening and closing transactions.
-   */
-  interface TransactionManager {
-
-/**
- * Checks whether there is an open transaction.
- *
- * @return {@code true} if there is an open transaction, {@code false} 
otherwise.
- */
-boolean hasActiveTransaction();
-
-/**
- * Adds an operation to the existing transaction.
- *
- * @param op Operation to include in the existing transaction.
- */
-void log(Op op);
-  }
-
-  private static class ScheduledExecutorSchedulingService implements 
SchedulingService {
-private final ScheduledExecutorService scheduledExecutor;
-
-ScheduledExecutorSchedulingService(ShutdownRegistry shutdownRegistry,
-Amount<Long, Time> shutdownGracePeriod) {
-  scheduledExecutor = 
AsyncUtil.singleThreadLoggingScheduledExecutor("LogStorage-%d", LOG);
-  shutdownRegistry.addAction(() -> 
MoreExecutors.shutdownAndAwaitTermination(
-  scheduledExecutor,
-  shutdownGracePeriod.getValue(),
-  shutdownGracePeriod.getUnit().getTimeUnit()));
-}
-
-@Override
-public void doEvery(Amount<Long, Time> interval, Runnable action) {
-  requireNonNull(interval);
-  requireNonNull(action);
-
-  long delay = interval.getValue();
-  TimeUnit timeUnit = interval.getUnit().getTimeUnit();
-  scheduledExecutor.scheduleWithFixedDelay(action, delay, delay, timeUnit);
-}
-  }
-
-  private static final Logger LOG = LoggerFactory.getLogger(LogStorage.class);
-
-  private final LogManager logManager;
-  private final SchedulingService schedulingService;
-  private final SnapshotStore snapshotStore;
-  private final Amount<Long, Time> snapshotInterval;
-  private final Storage writeBehindStorage;
-  private final SchedulerStore.Mutable writeBehindSchedulerStore;
-  private final CronJobStore.Mutable writeBehindJobStore;
-  private final TaskStore.Mutable writeBehindTaskStore;
-  private final QuotaStore.Mutable writeBehindQuotaStore;
-  private final AttributeStore.Mutable writeBehindAttributeStore;
-  private final JobUpdateStore.Mutable writeBehindJobUpdateStore;
-  private final ReentrantLock writeLock;
-  private final ThriftBackfill thriftBackfill;
-
-  private StreamManager streamManager;
-  private final WriteAheadStorage writeAheadStorage;
-
-  // TODO(wfarner): It should be possible to remove this flag now, since all 
call stacks when
-  // recovering are controlled at this layer (they're all calls to Mutable 
store implementations).
-  // The more involved change is changing SnapshotStore to accept a Mutable 
store provider to
-  // avoid a call to Storage.write() when we replay a Snapshot.
-  private boolean recovered = false;
-  private StreamTransaction transaction = null;
-
-  private final SlidingStats writerWaitStats =
-  new SlidingStats("log_storage_write_lock_wait", "ns");
-
-  private final Map<LogEntry._Fields, Consumer> 
logEntryReplayActions;
-  private final Map<Op._Fields, Consumer> transactionReplayActions;
-
-  @Inject
-  LogStorage(
-  LogManager logManager,
-  ShutdownRegistry shutdownRegistry,
-  Settings settings,
-  SnapshotStore snapshotStore,
-  @Volatile Storage storage,
-  @Volatile SchedulerStore.Mutable schedulerStore,
-  @Volatile CronJobStore.Mutable jobStore,
-  @Volatile TaskStore.Mutable taskStore,
-  @Volatile QuotaStore.Mutable quotaStore,
-  @Volatile AttributeStore.Mutable attributeStore,
-  @Volatile JobUpdateStore.Mutable jobUpdateStore,
-  EventSink eventSink,
-  ReentrantLock writeLock,
-  ThriftBackfill thriftBackfill) {
-
-this(logManager,
-new ScheduledExecutorSchedu

[1/2] aurora git commit: Add RemoveJobUpdates log Op, slim JobUpdateStore API

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 80139da46 -> 284f40f5e


http://git-wip-us.apache.org/repos/asf/aurora/blob/284f40f5/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
index 9fab7ae..1510817 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
+++ 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
@@ -15,16 +15,13 @@
 package org.apache.aurora.scheduler.storage;
 
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
 import com.google.inject.Injector;
 
 import org.apache.aurora.gen.InstanceTaskConfig;
@@ -46,48 +43,37 @@ import org.apache.aurora.gen.TaskConfig;
 import org.apache.aurora.scheduler.base.JobKeys;
 import org.apache.aurora.scheduler.base.TaskTestUtil;
 import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
-import org.apache.aurora.scheduler.storage.Storage.StorageException;
 import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
 import org.apache.aurora.scheduler.storage.entities.IJobKey;
-import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateInstructions;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateQuery;
-import org.apache.aurora.scheduler.storage.entities.IJobUpdateState;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary;
 import org.apache.aurora.scheduler.storage.testing.StorageEntityUtil;
-import org.apache.aurora.scheduler.testing.FakeStatsProvider;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLBACK_FAILED;
 import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLED_BACK;
 import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLING_BACK;
 import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATED;
 import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATING;
 import static org.apache.aurora.gen.JobUpdateStatus.ABORTED;
 import static org.apache.aurora.gen.JobUpdateStatus.ERROR;
-import static org.apache.aurora.gen.JobUpdateStatus.FAILED;
 import static org.apache.aurora.gen.JobUpdateStatus.ROLLED_BACK;
 import static org.apache.aurora.gen.JobUpdateStatus.ROLLING_BACK;
 import static org.apache.aurora.gen.JobUpdateStatus.ROLLING_FORWARD;
-import static org.apache.aurora.gen.JobUpdateStatus.ROLL_BACK_PAUSED;
 import static org.apache.aurora.gen.JobUpdateStatus.ROLL_FORWARD_PAUSED;
 import static org.apache.aurora.gen.Resource.diskMb;
 import static org.apache.aurora.gen.Resource.numCpus;
 import static org.apache.aurora.gen.Resource.ramMb;
-import static org.apache.aurora.scheduler.storage.Util.jobUpdateActionStatName;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 
 public abstract class AbstractJobUpdateStoreTest {
 
   private static final IJobKey JOB = JobKeys.from("testRole", "testEnv", 
"job");
-  private static final IJobUpdateKey UPDATE1 =
-  IJobUpdateKey.build(new JobUpdateKey(JOB.newBuilder(), "update1"));
   private static final long CREATED_MS = 111L;
   private static final IJobUpdateEvent FIRST_EVENT =
   makeJobUpdateEvent(ROLLING_FORWARD, CREATED_MS);
@@ -95,13 +81,11 @@ public abstract class AbstractJobUpdateStoreTest {
   ImmutableSet.of(new Metadata("k1", "v1"), new Metadata("k2", "v2"), new 
Metadata("k3", "v3"));
 
   private Storage storage;
-  private FakeStatsProvider stats;
 
   @Before
   public void setUp() throws Exception {
 Injector injector = createStorageInjector();
 storage = injector.getInstance(Storage.class);
-stats = injector.getInstance(FakeStatsProvider.class);
   }
 
   protected abstract Injector createStorageInjector();
@@ -111,9 +95,18 @@ public abstract class AbstractJobUpdateStoreTest {
 truncateUpdates();
   }
 
-  private static IJobUpdate makeFullyPopulatedUpdate(IJobUpdateKey key) {
-JobUpdate builder = makeJobUpdate(key).newBuilder();
-JobUpdateInstructions instructions = builder.getInstructions();
+  private static IJobUpdateDetails makeFullyPopulatedUpdate(IJobUpdateKey key) 
{
+  

[2/2] aurora git commit: Add RemoveJobUpdates log Op, slim JobUpdateStore API

[wfarner]

storage.entities.IScheduledTask;
@@ -274,8 +272,13 @@ public interface QuotaManager {
   
.from(storeProvider.getTaskStore().fetchTasks(Query.roleScoped(role).active()))
   .transform(IScheduledTask::getAssignedTask);
 
-  Map<IJobKey, IJobUpdateInstructions> updates = Maps.newHashMap(
-  fetchActiveJobUpdates(storeProvider.getJobUpdateStore(), role));
+  // Relies on the invariant of at-most-one active update per job.
+  Map<IJobKey, IJobUpdateInstructions> updates = 
storeProvider.getJobUpdateStore()
+  .fetchJobUpdates(updateQuery(role))
+  .stream()
+  .collect(Collectors.toMap(
+  u -> u.getUpdate().getSummary().getKey().getJob(),
+  u -> u.getUpdate().getInstructions()));
 
   // Mix in a requested job update (if present) to correctly calculate 
consumption.
   // This would be an update that is not saved in the store yet (i.e. the 
one quota is
@@ -398,20 +401,6 @@ public interface QuotaManager {
   };
 }
 
-private static Map<IJobKey, IJobUpdateInstructions> fetchActiveJobUpdates(
-final JobUpdateStore jobUpdateStore,
-String role) {
-
-  Function<IJobUpdateSummary, IJobUpdate> fetchUpdate =
-  summary -> jobUpdateStore.fetchJobUpdate(summary.getKey()).get();
-
-  return Maps.transformValues(
-  
FluentIterable.from(jobUpdateStore.fetchJobUpdateSummaries(updateQuery(role)))
-  .transform(fetchUpdate)
-  .uniqueIndex(UPDATE_TO_JOB_KEY),
-  IJobUpdate::getInstructions);
-}
-
 @VisibleForTesting
 static IJobUpdateQuery updateQuery(String role) {
   return IJobUpdateQuery.build(new JobUpdateQuery()
@@ -474,9 +463,6 @@ public interface QuotaManager {
   return addAll(Iterables.transform(tasks, QUOTA_RESOURCES));
 }
 
-private static final Function<IJobUpdate, IJobKey> UPDATE_TO_JOB_KEY =
-input -> input.getSummary().getKey().getJob();
-
 private static int getUpdateInstanceCount(Set ranges) {
   int instanceCount = 0;
   for (IRange range : ranges) {

http://git-wip-us.apache.org/repos/asf/aurora/blob/284f40f5/src/main/java/org/apache/aurora/scheduler/storage/JobUpdateStore.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/JobUpdateStore.java 
b/src/main/java/org/apache/aurora/scheduler/storage/JobUpdateStore.java
index b3d906b..6b91d97 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/JobUpdateStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/JobUpdateStore.java
@@ -19,15 +19,14 @@ import java.util.Set;
 
 import com.google.common.base.Optional;
 
+import org.apache.aurora.gen.JobUpdateQuery;
 import org.apache.aurora.gen.JobUpdateStatus;
 import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
-import org.apache.aurora.scheduler.storage.entities.IJobUpdateInstructions;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateQuery;
-import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary;
 
 import static org.apache.aurora.gen.JobUpdateStatus.ABORTED;
 import static org.apache.aurora.gen.JobUpdateStatus.ERROR;
@@ -48,21 +47,15 @@ public interface JobUpdateStore {
   ERROR
   );
 
-  /**
-   * Fetches a read-only view of job update summaries.
-   *
-   * @param query Query to identify job update summaries with.
-   * @return A read-only view of job update summaries.
-   */
-  List fetchJobUpdateSummaries(IJobUpdateQuery query);
+  IJobUpdateQuery MATCH_ALL = IJobUpdateQuery.build(new JobUpdateQuery());
 
   /**
* Fetches a read-only view of job update details matching the {@code query}.
*
* @param query Query to identify job update details with.
-   * @return A read-only list view of job update details matching the query.
+   * @return A read-only view of job update details matching the query.
*/
-  List fetchJobUpdateDetails(IJobUpdateQuery query);
+  List fetchJobUpdates(IJobUpdateQuery query);
 
   /**
* Fetches a read-only view of job update details.
@@ -70,57 +63,12 @@ public interface JobUpdateStore {
* @param key Update identifier.
* @return A read-only view of job update details.
*/
-  Optional fetchJobUpdateDetails(IJobUpdateKey key);
-
-  /**
-   * Fetches a read-only view of a job update.
-   *
-   * @param key Update identifier.
-   * @return A read-only view of job update.
-   */
-  Optional fetchJobUpdate(IJobUpdateKey key);
-
-  /**
-   * Fetches a read-only view of the instructions for a job update.
-   *
-   * @param key Update identifier.
-   * @return A rea

[2/3] aurora git commit: Enable custom offer scoring modules for task assignment

[wfarner]

http://git-wip-us.apache.org/repos/asf/aurora/blob/80139da4/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelector.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelector.java
 
b/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelector.java
new file mode 100644
index 000..ee65bab
--- /dev/null
+++ 
b/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelector.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.scheduling;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Iterables;
+
+import org.apache.aurora.scheduler.HostOffer;
+import org.apache.aurora.scheduler.filter.SchedulingFilter.ResourceRequest;
+
+public class FirstFitOfferSelector implements OfferSelector {
+
+  @Override
+  public Optional select(Iterable offers, 
ResourceRequest resourceRequest) {
+
+return Optional.fromNullable(Iterables.getFirst(offers, null));
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/80139da4/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelectorModule.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelectorModule.java
 
b/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelectorModule.java
new file mode 100644
index 000..4d36487
--- /dev/null
+++ 
b/src/main/java/org/apache/aurora/scheduler/scheduling/FirstFitOfferSelectorModule.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.scheduling;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Singleton;
+
+public class FirstFitOfferSelectorModule extends AbstractModule {
+
+  @Override
+  protected void configure() {
+bind(OfferSelector.class).to(FirstFitOfferSelector.class);
+bind(FirstFitOfferSelector.class).in(Singleton.class);
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/80139da4/src/main/java/org/apache/aurora/scheduler/scheduling/OfferSelector.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/scheduling/OfferSelector.java 
b/src/main/java/org/apache/aurora/scheduler/scheduling/OfferSelector.java
new file mode 100644
index 000..c95b980
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/scheduling/OfferSelector.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.scheduling;
+
+import com.google.common.base.Optional;
+
+import org.apache.aurora.scheduler.HostOffer;
+import org.apache.aurora.scheduler.filter.SchedulingFilter.ResourceRequest;
+
+/**
+ * Injected into {@link TaskAssignerImpl}, this class scores the offers 
available and returns an
+ * option containing the offer to use.
+ */
+public interface OfferSelector {
+
+  /**
+   * Score offers that fit within the given {@link ResourceRequest} and return 
an option containing
+   * the offer to use for assignment.
+   *
+   * @param offers A stream of offers that match the given {@link 
ResourceRequest}.
+   * @param resourceRequest The {@link ResourceRequest} for the task to assign.
+   * @return An 

[3/3] aurora git commit: Enable custom offer scoring modules for task assignment

[wfarner]

Enable custom offer scoring modules for task assignment

Major portions of the refactor:

* Refactor `OfferManager` to do filtering of offers (added `getMatching` and
  `getAllMatching` methods) as opposed to TaskAssigner
* Refactor `TaskAssigner`, allow for injection of custom "scoring" class
  through `OfferRanker` interface

And some minor things as well:

* Moved `TaskAssignerImpl`, `TaskSchedulerImpl`, and `HostOffers` into their own
  upper-level classes
* Moved `TaskAssigner` to the `scheduling` package and out of the `state` 
package
* Renamed some methods in `OfferManager` to avoid code stutter
* Renaming of some classes (e.g. `FirstFitTaskAssigner` -> `TaskAssignerImpl`)
* And a slew of others

Reviewed at https://reviews.apache.org/r/63973/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/80139da4
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/80139da4
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/80139da4

Branch: refs/heads/master
Commit: 80139da4624916e406c7e80c4ea2d286d4d859c3
Parents: 21af250
Author: Jordan Ly 
Authored: Tue Nov 28 11:02:14 2017 -0800
Committer: Bill Farner 
Committed: Tue Nov 28 11:02:14 2017 -0800

--
 RELEASE-NOTES.md|   3 +
 docs/reference/scheduler-configuration.md   |   2 +-
 .../org/apache/aurora/benchmark/Offers.java |   2 +-
 .../aurora/benchmark/SchedulingBenchmarks.java  |  16 +-
 .../benchmark/fakes/FakeOfferManager.java   |  31 +-
 .../apache/aurora/scheduler/app/AppModule.java  |   4 +-
 .../aurora/scheduler/config/CliOptions.java |   6 +-
 .../scheduler/filter/AttributeAggregate.java|  38 +-
 .../scheduler/filter/SchedulingFilter.java  |   5 +
 .../scheduler/filter/SchedulingFilterImpl.java  |   2 +-
 .../apache/aurora/scheduler/http/Offers.java|   2 +-
 .../scheduler/mesos/MesosCallbackHandler.java   |  12 +-
 .../aurora/scheduler/offers/HostOffers.java | 253 +
 .../aurora/scheduler/offers/OfferManager.java   | 404 ++
 .../scheduler/offers/OfferManagerImpl.java  | 246 +
 .../scheduler/offers/OfferManagerModule.java| 211 
 .../aurora/scheduler/offers/OfferSettings.java  |   7 +-
 .../aurora/scheduler/offers/OffersModule.java   | 211 
 .../preemptor/PendingTaskProcessor.java |   2 +-
 .../aurora/scheduler/preemptor/Preemptor.java   |   2 +-
 .../scheduling/FirstFitOfferSelector.java   |  29 +
 .../scheduling/FirstFitOfferSelectorModule.java |  26 +
 .../scheduler/scheduling/OfferSelector.java |  36 ++
 .../scheduler/scheduling/SchedulingModule.java  |   4 +-
 .../scheduler/scheduling/TaskAssigner.java  |  46 ++
 .../scheduler/scheduling/TaskAssignerImpl.java  | 284 ++
 .../scheduling/TaskAssignerImplModule.java  |  59 ++
 .../scheduler/scheduling/TaskScheduler.java | 191 ---
 .../scheduler/scheduling/TaskSchedulerImpl.java | 207 +++
 .../state/FirstFitTaskAssignerModule.java   |  31 --
 .../aurora/scheduler/state/StateModule.java |   3 +-
 .../aurora/scheduler/state/TaskAssigner.java| 338 
 .../scheduler/stats/AsyncStatsModule.java   |   2 +-
 .../scheduler/config/CommandLineTest.java   |   4 +-
 .../aurora/scheduler/http/OffersTest.java   |   4 +-
 .../mesos/MesosCallbackHandlerTest.java |  20 +-
 .../scheduler/offers/OfferManagerImplTest.java  | 381 +
 .../preemptor/PendingTaskProcessorTest.java |   2 +-
 .../scheduler/preemptor/PreemptorImplTest.java  |   2 +-
 .../preemptor/PreemptorModuleTest.java  |   2 +-
 .../scheduling/FirstFitOfferSelectorTest.java   |  66 +++
 .../scheduling/TaskAssignerImplTest.java| 374 +
 .../scheduling/TaskSchedulerImplTest.java   |   2 -
 .../state/FirstFitTaskAssignerTest.java | 539 ---
 .../scheduler/stats/AsyncStatsModuleTest.java   |   2 +-
 45 files changed, 2244 insertions(+), 1869 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/80139da4/RELEASE-NOTES.md
--
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 2d3c423..54dcc75 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -12,6 +12,9 @@
   a production cluster. For that reason, the functionality is behind a new 
flag `-partition_aware`
   that is disabled by default. When Mesos support is improved and the new 
behavior is vetted in
   production clusters, we'll enable this by default.
+- Added the ability to "score" offers for a given scheduling assignment via 
the `OfferSelector`
+  interface. The default implementation is first fit, but cluster operators 
can inject a custom
+  scoring algorithm through the `-offer_selector_modules` 

[1/3] aurora git commit: Enable custom offer scoring modules for task assignment

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 21af250c9 -> 80139da46


http://git-wip-us.apache.org/repos/asf/aurora/blob/80139da4/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java 
b/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java
index 6b18296..ff80baa 100644
--- a/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/offers/OfferManagerImplTest.java
@@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 
+import org.apache.aurora.common.collections.Pair;
 import org.apache.aurora.common.quantity.Amount;
 import org.apache.aurora.common.quantity.Time;
 import org.apache.aurora.common.testing.easymock.EasyMockTest;
@@ -32,9 +33,12 @@ import org.apache.aurora.scheduler.base.TaskGroupKey;
 import org.apache.aurora.scheduler.base.Tasks;
 import org.apache.aurora.scheduler.events.PubsubEvent.DriverDisconnected;
 import org.apache.aurora.scheduler.events.PubsubEvent.HostAttributesChanged;
+import org.apache.aurora.scheduler.filter.SchedulingFilter;
+import org.apache.aurora.scheduler.filter.SchedulingFilter.ResourceRequest;
+import org.apache.aurora.scheduler.filter.SchedulingFilter.UnusedResource;
 import org.apache.aurora.scheduler.mesos.Driver;
 import org.apache.aurora.scheduler.offers.Deferment.Noop;
-import org.apache.aurora.scheduler.offers.OfferManager.OfferManagerImpl;
+import org.apache.aurora.scheduler.resources.ResourceBag;
 import org.apache.aurora.scheduler.resources.ResourceType;
 import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
 import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
@@ -54,11 +58,13 @@ import static 
org.apache.aurora.gen.MaintenanceMode.DRAINING;
 import static org.apache.aurora.gen.MaintenanceMode.NONE;
 import static org.apache.aurora.scheduler.base.TaskTestUtil.JOB;
 import static org.apache.aurora.scheduler.base.TaskTestUtil.makeTask;
-import static 
org.apache.aurora.scheduler.offers.OfferManager.OfferManagerImpl.GLOBALLY_BANNED_OFFERS;
-import static 
org.apache.aurora.scheduler.offers.OfferManager.OfferManagerImpl.OFFER_ACCEPT_RACES;
-import static 
org.apache.aurora.scheduler.offers.OfferManager.OfferManagerImpl.OFFER_CANCEL_FAILURES;
-import static 
org.apache.aurora.scheduler.offers.OfferManager.OfferManagerImpl.OUTSTANDING_OFFERS;
-import static 
org.apache.aurora.scheduler.offers.OfferManager.OfferManagerImpl.STATICALLY_BANNED_OFFERS;
+import static org.apache.aurora.scheduler.filter.AttributeAggregate.empty;
+import static 
org.apache.aurora.scheduler.offers.OfferManagerImpl.GLOBALLY_BANNED_OFFERS;
+import static 
org.apache.aurora.scheduler.offers.OfferManagerImpl.OFFER_ACCEPT_RACES;
+import static 
org.apache.aurora.scheduler.offers.OfferManagerImpl.OFFER_CANCEL_FAILURES;
+import static 
org.apache.aurora.scheduler.offers.OfferManagerImpl.OUTSTANDING_OFFERS;
+import static 
org.apache.aurora.scheduler.offers.OfferManagerImpl.STATICALLY_BANNED_OFFERS;
+import static 
org.apache.aurora.scheduler.offers.OfferManagerImpl.VETO_EVALUATED_OFFERS;
 import static 
org.apache.aurora.scheduler.resources.ResourceTestUtil.mesosRange;
 import static 
org.apache.aurora.scheduler.resources.ResourceTestUtil.mesosScalar;
 import static org.apache.aurora.scheduler.resources.ResourceTestUtil.offer;
@@ -66,6 +72,8 @@ import static 
org.apache.aurora.scheduler.resources.ResourceType.CPUS;
 import static org.apache.aurora.scheduler.resources.ResourceType.DISK_MB;
 import static org.apache.aurora.scheduler.resources.ResourceType.PORTS;
 import static org.apache.aurora.scheduler.resources.ResourceType.RAM_MB;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -89,21 +97,25 @@ public class OfferManagerImplTest extends EasyMockTest {
   private static final String HOST_C = "HOST_C";
   private static final HostOffer OFFER_C = new HostOffer(
   Offers.makeOffer("OFFER_C", HOST_C),
-  IHostAttributes.build(new HostAttributes().setMode(NONE)));
+  IHostAttributes.build(new 
HostAttributes().setMode(NONE).setHost(HOST_C)));
   private static final int PORT = 1000;
   private static final Protos.Offer MESOS_OFFER = offer(mesosRange(PORTS, 
PORT));
   private static final IScheduledTask TASK = makeTask("id", JOB);
+  private static final ResourceRequest EMPTY_REQUEST = new ResourceRequest(
+  TASK.getAssignedTask().getTask(),
+  ResourceBag.EMPTY,
+  empty());
   private static final TaskGroupKey GROUP_KEY = 
TaskGroupKey.from(TASK.getAssignedTask().getTask());
   

aurora git commit: Remove task level resource fields from thrift interface

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 0f3dc939e -> 21af250c9


Remove task level resource fields from thrift interface

Bugs closed: AURORA-1707

Reviewed at https://reviews.apache.org/r/60942/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/21af250c
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/21af250c
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/21af250c

Branch: refs/heads/master
Commit: 21af250c9e781d7d4a121c74f561cbda36245040
Parents: 0f3dc93
Author: Nicolás Donatucci 
Authored: Mon Nov 27 17:01:03 2017 -0800
Committer: Bill Farner 
Committed: Mon Nov 27 17:01:03 2017 -0800

--
 RELEASE-NOTES.md|  1 +
 .../thrift/org/apache/aurora/gen/api.thrift |  8 ---
 .../aurora/scheduler/base/TaskTestUtil.java |  4 --
 .../apache/aurora/client/cli/diff_formatter.py  |  5 +-
 .../python/apache/aurora/client/cli/jobs.py |  6 +-
 .../apache/aurora/scheduler/http/TestUtils.java | 13 -
 .../aurora/scheduler/thrift/Fixtures.java   |  4 --
 .../updater/UpdateAgentReserverImplTest.java|  8 ++-
 .../apache/aurora/client/cli/test_status.py | 59 +---
 .../python/apache/aurora/client/cli/util.py |  8 ++-
 .../apache/aurora/config/test_resources.py  | 11 +---
 11 files changed, 64 insertions(+), 63 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/RELEASE-NOTES.md
--
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index e622a8d..2d3c423 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -17,6 +17,7 @@
 
 - Removed the ability to recover from SQL-based backups and snapshots.  An 
0.20.0 scheduler
   will not be able to recover backups or replicated log data created prior to 
0.19.0.
+- Removed task level resource fields (`numCpus`, `ramMb`, `diskMb`, 
`requestedPorts`).
 
 0.19.0
 ==

http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/api/src/main/thrift/org/apache/aurora/gen/api.thrift
--
diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift 
b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
index 2978f6d..c9f4210 100644
--- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift
+++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
@@ -232,12 +232,6 @@ struct TaskConfig {
  /** contains the role component of JobKey */
  17: Identity owner
   7: bool isService
-  // TODO(maxim): Deprecated. See AURORA-1707.
-  8: double numCpus
-  // TODO(maxim): Deprecated. See AURORA-1707.
-  9: i64 ramMb
-  // TODO(maxim): Deprecated. See AURORA-1707.
- 10: i64 diskMb
  11: i32 priority
  13: i32 maxTaskFailures
  // TODO(mnurolahzade): Deprecated. See AURORA-1708.
@@ -249,8 +243,6 @@ struct TaskConfig {
  32: set resources
 
  20: set constraints
- /** a list of named ports this task requests */
- 21: set requestedPorts
  /** Resources to retrieve with Mesos Fetcher */
  33: optional set mesosFetcherUris
  /**

http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java 
b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
index dd64a5b..5fe7b9b 100644
--- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
+++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java
@@ -116,9 +116,6 @@ public final class TaskTestUtil {
 .setJob(job.newBuilder())
 .setOwner(new Identity().setUser(job.getRole() + "-user"))
 .setIsService(true)
-.setNumCpus(1.0)
-.setRamMb(1024)
-.setDiskMb(1024)
 .setPriority(1)
 .setMaxTaskFailures(-1)
 .setProduction(true)
@@ -132,7 +129,6 @@ public final class TaskTestUtil {
 new Constraint(
 "limitConstraint",
 TaskConstraint.limit(new LimitConstraint(5)
-.setRequestedPorts(ImmutableSet.of("http"))
 .setTaskLinks(ImmutableMap.of("http", "link", "admin", "otherLink"))
 .setContactEmail("f...@bar.com")
 .setMetadata(ImmutableSet.of(new Metadata("key", "value")))

http://git-wip-us.apache.org/repos/asf/aurora/blob/21af250c/src/main/python/apache/aurora/client/cli/diff_formatter.py
--
diff --git a/src/main/python/apache/aurora/client/cli/diff_formatter.py 
b/src/main/python/apache/aurora/client/cli/diff_formatter.py
index 7871777..2f66e29 100644
--- 

aurora git commit: Fix flaky MesosCallbackHandlerTest

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 89b0fa9b2 -> 46b1112ee


Fix flaky MesosCallbackHandlerTest

In the same vein as: https://reviews.apache.org/r/63760/

Fix a flaky test that uses `Thread.sleep` by injecting a fake Executor.

Reviewed at https://reviews.apache.org/r/63763/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/46b1112e
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/46b1112e
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/46b1112e

Branch: refs/heads/master
Commit: 46b1112eeef341ac1bb1f42bf81491ee7bd626f4
Parents: 89b0fa9
Author: Jordan Ly 
Authored: Tue Nov 14 17:43:58 2017 -0800
Committer: Bill Farner 
Committed: Tue Nov 14 17:43:58 2017 -0800

--
 .../mesos/MesosCallbackHandlerTest.java | 56 +++-
 1 file changed, 32 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/46b1112e/src/test/java/org/apache/aurora/scheduler/mesos/MesosCallbackHandlerTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/mesos/MesosCallbackHandlerTest.java 
b/src/test/java/org/apache/aurora/scheduler/mesos/MesosCallbackHandlerTest.java
index 8f8b86d..45ae6bb 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/mesos/MesosCallbackHandlerTest.java
+++ 
b/src/test/java/org/apache/aurora/scheduler/mesos/MesosCallbackHandlerTest.java
@@ -13,8 +13,10 @@
  */
 package org.apache.aurora.scheduler.mesos;
 
+import java.util.LinkedList;
+import java.util.NoSuchElementException;
+import java.util.Queue;
 import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
@@ -56,6 +58,7 @@ import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.strictMock;
 import static org.easymock.EasyMock.verify;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 public class MesosCallbackHandlerTest extends EasyMockTest {
   private static final Protos.AgentID AGENT_ID =
@@ -298,7 +301,6 @@ public class MesosCallbackHandlerTest extends EasyMockTest {
 control.replay();
 handler.handleOffers(ImmutableList.of(HOST_OFFER.getOffer()));
 assertEquals(1L, statsProvider.getLongValue("scheduler_resource_offers"));
-
   }
 
   @Test
@@ -331,8 +333,8 @@ public class MesosCallbackHandlerTest extends EasyMockTest {
 // We want to observe the order of the offerManager calls to we create a 
strict mock.
 offerManager = strictMock(OfferManager.class);
 
-long delayInMilliseconds = 50;
-createHandler(false, new DelayExecutor(delayInMilliseconds));
+FakeScheduledThreadPoolExecutor fakeExecutor = new 
FakeScheduledThreadPoolExecutor();
+createHandler(false, fakeExecutor);
 
 expect(offerManager.cancelOffer(OFFER_ID)).andReturn(false);
 offerManager.banOffer(OFFER_ID);
@@ -353,10 +355,12 @@ public class MesosCallbackHandlerTest extends 
EasyMockTest {
 // Eventually, we unban the offer.
 handler.handleRescind(OFFER_ID);
 
-// 2 commands executed (addOffer and unbanOffer) so we wait the length of 
3.
-Thread.sleep(delayInMilliseconds * 3);
+// 2 commands executed (addOffer and unbanOffer).
+fakeExecutor.advance();
+fakeExecutor.advance();
 
 assertEquals(1L, statsProvider.getLongValue("offers_rescinded"));
+assertTrue(fakeExecutor.isEmpty());
 verify(offerManager);
   }
 
@@ -563,28 +567,32 @@ public class MesosCallbackHandlerTest extends 
EasyMockTest {
   }
 
   /**
-   * Test executor that will execute commands after waiting {@code 
delayTimeInMilliseconds}.
+   * Test executor that will execute commands when {@code advance} is called.
*/
-  private static final class DelayExecutor implements Executor {
-private final Executor executor = Executors.newSingleThreadExecutor();
-private final long delayTimeInMilliseconds;
-
-DelayExecutor(long delayTimeInMilliseconds) {
-  this.delayTimeInMilliseconds = delayTimeInMilliseconds;
-}
+  private static class FakeScheduledThreadPoolExecutor implements Executor {
+private final Queue workQueue = new LinkedList<>();
 
 @Override
 public void execute(Runnable command) {
-  executor.execute(() -> {
-try {
-  Thread.sleep(delayTimeInMilliseconds);
-} catch (InterruptedException e) {
-  // Do not interrupt this thread.
-  throw new RuntimeException("DelayExecutor sleep interrupted.", e);
-}
-
-command.run();
-  });
+  workQueue.add(command);
+}
+
+/**
+ * Returns whether or not the work queue is empty.
+ */
+public boolean 

aurora git commit: Use transition method and fix documentation in Webhooks

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 4fecf1f59 -> 89b0fa9b2


Use transition method and fix documentation in Webhooks

Reviewed at https://reviews.apache.org/r/63705/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/89b0fa9b
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/89b0fa9b
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/89b0fa9b

Branch: refs/heads/master
Commit: 89b0fa9b2cbe3f7dc0e514bf32f3077eb62f9778
Parents: 4fecf1f
Author: Jordan Ly 
Authored: Tue Nov 14 17:42:50 2017 -0800
Committer: Bill Farner 
Committed: Tue Nov 14 17:42:50 2017 -0800

--
 .../java/org/apache/aurora/scheduler/events/Webhook.java | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/89b0fa9b/src/main/java/org/apache/aurora/scheduler/events/Webhook.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/events/Webhook.java 
b/src/main/java/org/apache/aurora/scheduler/events/Webhook.java
index 2af8118..95e3f52 100644
--- a/src/main/java/org/apache/aurora/scheduler/events/Webhook.java
+++ b/src/main/java/org/apache/aurora/scheduler/events/Webhook.java
@@ -92,10 +92,10 @@ public class Webhook extends AbstractIdleService implements 
EventSubscriber {
   @Subscribe
   public void taskChangedState(TaskStateChange stateChange) {
 LOG.debug("Got an event: {}", stateChange);
-// Old state is not present because a scheduler just failed over. In that 
case we do not want to
-// resend the entire state. This check also ensures that only whitelisted 
statuses will be sent
-// to the configured endpoint.
-if (stateChange.getOldState().isPresent() && 
isWhitelisted.apply(stateChange.getNewState())) {
+// Ensure that this state change event is a transition, and not an event 
from when the scheduler
+// first initializes. In that case we do not want to resend the entire 
state. This check also
+// ensures that only whitelisted statuses will be sent to the configured 
endpoint.
+if (stateChange.isTransition() && 
isWhitelisted.apply(stateChange.getNewState())) {
   attemptsCounter.incrementAndGet();
   try {
 // We don't care about the response body, so only listen for the HTTP 
status code.

[1/2] aurora git commit: Remove LockStore

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master fb64df240 -> 4fecf1f59


http://git-wip-us.apache.org/repos/asf/aurora/blob/4fecf1f5/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
index 5c82bcf..9fab7ae 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
+++ 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractJobUpdateStoreTest.java
@@ -21,7 +21,6 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.google.common.base.Optional;
-import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -41,12 +40,9 @@ import org.apache.aurora.gen.JobUpdateSettings;
 import org.apache.aurora.gen.JobUpdateState;
 import org.apache.aurora.gen.JobUpdateStatus;
 import org.apache.aurora.gen.JobUpdateSummary;
-import org.apache.aurora.gen.Lock;
-import org.apache.aurora.gen.LockKey;
 import org.apache.aurora.gen.Metadata;
 import org.apache.aurora.gen.Range;
 import org.apache.aurora.gen.TaskConfig;
-import org.apache.aurora.gen.storage.StoredJobUpdateDetails;
 import org.apache.aurora.scheduler.base.JobKeys;
 import org.apache.aurora.scheduler.base.TaskTestUtil;
 import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
@@ -61,8 +57,6 @@ import 
org.apache.aurora.scheduler.storage.entities.IJobUpdateKey;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateQuery;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateState;
 import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary;
-import org.apache.aurora.scheduler.storage.entities.ILock;
-import org.apache.aurora.scheduler.storage.entities.ILockKey;
 import org.apache.aurora.scheduler.storage.testing.StorageEntityUtil;
 import org.apache.aurora.scheduler.testing.FakeStatsProvider;
 import org.junit.After;
@@ -156,16 +150,16 @@ public abstract class AbstractJobUpdateStoreTest {
 StorageEntityUtil.getField(IJobUpdateSummary.class, "state"),
 StorageEntityUtil.getField(Range.class, "first"),
 StorageEntityUtil.getField(Range.class, "last"));
-saveUpdate(update1, Optional.of("lock1"));
+saveUpdate(update1);
 assertUpdate(update1);
 
-saveUpdate(update2, Optional.absent());
+saveUpdate(update2);
 assertUpdate(update2);
 
 // Colliding update keys should be forbidden.
 IJobUpdate update3 = makeJobUpdate(updateId2);
 try {
-  saveUpdate(update3, Optional.absent());
+  saveUpdate(update3);
   fail("Update ID collision should not be allowed");
 } catch (StorageException e) {
   // Expected.
@@ -194,7 +188,7 @@ public abstract class AbstractJobUpdateStoreTest {
 StorageEntityUtil.getField(IJobUpdateSummary.class, "state"),
 StorageEntityUtil.getField(Range.class, "first"),
 StorageEntityUtil.getField(Range.class, "last"));
-saveUpdate(update, Optional.of("lock1"));
+saveUpdate(update);
 assertUpdate(update);
   }
 
@@ -204,7 +198,7 @@ public abstract class AbstractJobUpdateStoreTest {
 builder.getInstructions().unsetInitialState();
 
 // Save with null initial state instances.
-saveUpdate(IJobUpdate.build(builder), Optional.of("lock"));
+saveUpdate(IJobUpdate.build(builder));
 
 builder.getInstructions().setInitialState(ImmutableSet.of());
 assertUpdate(IJobUpdate.build(builder));
@@ -216,7 +210,7 @@ public abstract class AbstractJobUpdateStoreTest {
 builder.getInstructions().unsetDesiredState();
 
 // Save with null desired state instances.
-saveUpdate(IJobUpdate.build(builder), Optional.of("lock"));
+saveUpdate(IJobUpdate.build(builder));
 
 assertUpdate(IJobUpdate.build(builder));
   }
@@ -227,7 +221,7 @@ public abstract class AbstractJobUpdateStoreTest {
 builder.getInstructions().unsetInitialState();
 builder.getInstructions().unsetDesiredState();
 
-saveUpdate(IJobUpdate.build(builder), Optional.of("lock"));
+saveUpdate(IJobUpdate.build(builder));
   }
 
   @Test(expected = NullPointerException.class)
@@ -236,7 +230,7 @@ public abstract class AbstractJobUpdateStoreTest {
 builder.getInstructions().getInitialState().add(
 new InstanceTaskConfig(null, ImmutableSet.of()));
 
-saveUpdate(IJobUpdate.build(builder), Optional.of("lock"));
+saveUpdate(IJobUpdate.build(builder));
   }
 
   @Test(expected = IllegalArgumentException.class)
@@ -247,7 +241,7 @@ public abstract class AbstractJobUpdateStoreTest {
 TaskTestUtil.makeConfig(TaskTestUtil.JOB).newBuilder(),
 ImmutableSet.of()));
 
-

[2/2] aurora git commit: Remove LockStore

[wfarner]

Remove LockStore

Reviewed at https://reviews.apache.org/r/63744/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/4fecf1f5
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/4fecf1f5
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/4fecf1f5

Branch: refs/heads/master
Commit: 4fecf1f594e09a5ed6909df49faeee51e5007f8e
Parents: fb64df2
Author: Bill Farner 
Authored: Mon Nov 13 16:35:36 2017 -0800
Committer: Bill Farner 
Committed: Mon Nov 13 16:35:36 2017 -0800

--
 .../thrift/org/apache/aurora/gen/api.thrift |  19 --
 .../thrift/org/apache/aurora/gen/storage.thrift |  19 +-
 .../org/apache/aurora/benchmark/JobUpdates.java |  13 +-
 .../aurora/benchmark/ThriftApiBenchmarks.java   |   2 -
 .../aurora/benchmark/UpdateStoreBenchmarks.java |   3 -
 .../ShiroAuthorizingParamInterceptor.java   |  10 -
 .../aurora/scheduler/state/LockManager.java |  50 -
 .../aurora/scheduler/state/LockManagerImpl.java |  93 -
 .../aurora/scheduler/state/StateModule.java |   3 -
 .../scheduler/storage/JobUpdateStore.java   |   9 +-
 .../aurora/scheduler/storage/LockStore.java |  61 --
 .../aurora/scheduler/storage/Storage.java   |   2 -
 .../scheduler/storage/log/LogStorage.java   |  20 +-
 .../storage/log/SnapshotStoreImpl.java  |  37 +---
 .../storage/log/WriteAheadStorage.java  |  56 +-
 .../storage/mem/MemJobUpdateStore.java  |  81 ++--
 .../scheduler/storage/mem/MemLockStore.java |  72 ---
 .../scheduler/storage/mem/MemStorage.java   |   7 -
 .../scheduler/storage/mem/MemStorageModule.java |   2 -
 .../thrift/SchedulerThriftInterface.java|  24 +--
 .../updater/JobUpdateControllerImpl.java|  19 +-
 .../scheduler/http/AbstractJettyTest.java   |   2 -
 .../scheduler/state/LockManagerImplTest.java|  99 -
 .../storage/AbstractJobUpdateStoreTest.java | 165 +--
 .../storage/AbstractLockStoreTest.java  | 200 ---
 .../scheduler/storage/backup/RecoveryTest.java  |   1 -
 .../scheduler/storage/log/LogStorageTest.java   |  80 +---
 .../storage/log/SnapshotStoreImplIT.java|  11 +-
 .../storage/log/WriteAheadStorageTest.java  |   8 -
 .../scheduler/storage/mem/MemLockStoreTest.java |  24 ---
 .../storage/testing/StorageTestUtil.java|   5 -
 .../aurora/scheduler/thrift/Fixtures.java   |   3 -
 .../thrift/SchedulerThriftInterfaceTest.java|  77 +--
 .../aurora/scheduler/updater/JobUpdaterIT.java  | 183 ++---
 .../aurora/client/api/test_scheduler_client.py  |   6 +-
 35 files changed, 126 insertions(+), 1340 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/4fecf1f5/api/src/main/thrift/org/apache/aurora/gen/api.thrift
--
diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift 
b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
index c869493..1d36926 100644
--- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift
+++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift
@@ -115,25 +115,6 @@ struct JobKey {
   3: string name
 }
 
-/** A unique lock key. */
-union LockKey {
-  1: JobKey job
-}
-
-/** A generic lock struct to facilitate context specific resource/operation 
serialization. */
-struct Lock {
-  /** ID of the lock - unique per storage */
-  1: LockKey key
-  /** UUID - facilitating soft lock authorization */
-  2: string token
-  /** Lock creator */
-  3: string user
-  /** Lock creation timestamp in milliseconds */
-  4: i64 timestampMs
-  /** Optional message to record with the lock */
-  5: optional string message
-}
-
 /** A unique identifier for the active task within a job. */
 struct InstanceKey {
   /** Key identifying the job. */

http://git-wip-us.apache.org/repos/asf/aurora/blob/4fecf1f5/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
--
diff --git a/api/src/main/thrift/org/apache/aurora/gen/storage.thrift 
b/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
index 74983ba..c692a5f 100644
--- a/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
+++ b/api/src/main/thrift/org/apache/aurora/gen/storage.thrift
@@ -28,14 +28,6 @@ struct SaveCronJob {
   2: api.JobConfiguration jobConfig
 }
 
-struct SaveLock {
-  1: api.Lock lock
-}
-
-struct RemoveLock {
-  1: api.LockKey lockKey
-}
-
 struct RemoveJob {
   2: api.JobKey jobKey
 }
@@ -63,13 +55,12 @@ struct SaveHostAttributes {
 
 struct SaveJobUpdate {
   1: api.JobUpdate jobUpdate
-  2: string lockToken
+  // 2: deleted
 }
 
 struct StoredJobUpdateDetails {
   1: api.JobUpdateDetails details
-  /** ID of the lock 

aurora git commit: Make testTaskChangedWithOldStateError more robust

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master bd1218867 -> fb64df240


Make testTaskChangedWithOldStateError more robust

Reviewed at https://reviews.apache.org/r/63760/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/fb64df24
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/fb64df24
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/fb64df24

Branch: refs/heads/master
Commit: fb64df2402327a487ab3ce14554e150c917bb6a5
Parents: bd12188
Author: Bill Farner 
Authored: Mon Nov 13 14:46:45 2017 -0800
Committer: Bill Farner 
Committed: Mon Nov 13 14:46:45 2017 -0800

--
 .../aurora/scheduler/events/WebhookTest.java   | 17 +++--
 1 file changed, 11 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/fb64df24/src/test/java/org/apache/aurora/scheduler/events/WebhookTest.java
--
diff --git a/src/test/java/org/apache/aurora/scheduler/events/WebhookTest.java 
b/src/test/java/org/apache/aurora/scheduler/events/WebhookTest.java
index 2906b08..adeff89 100644
--- a/src/test/java/org/apache/aurora/scheduler/events/WebhookTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/events/WebhookTest.java
@@ -19,6 +19,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -180,16 +181,20 @@ public class WebhookTest {
 
   @Test
   public void testTaskChangedWithOldStateError() throws Exception {
-// We have a special handler here to cause a TimeoutException to trigger 
`onThrowable`
+// We have a special handler here to trigger `onThrowable` on the client.
 jettyServer.setHandler(new AbstractHandler() {
   @Override
   public void handle(String target, Request baseRequest, 
HttpServletRequest request,
  HttpServletResponse response) throws IOException, 
ServletException {
-try {
-  Thread.sleep(TIMEOUT + 100);
-} catch (InterruptedException e) {
-  // Should never get here.
-}
+
+Stream.of(jettyServer.getConnectors())
+.forEach(c -> {
+  try {
+c.stop();
+  } catch (Exception e) {
+throw new RuntimeException(e);
+  }
+});
   }
 });
 jettyServer.start();

aurora git commit: Add a workaround for test_du_diskcollector failing on macOS

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 942760466 -> bd1218867


Add a workaround for test_du_diskcollector failing on macOS

Bugs closed: AURORA-1956

Reviewed at https://reviews.apache.org/r/63746/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/bd121886
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/bd121886
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/bd121886

Branch: refs/heads/master
Commit: bd12188674c15e743b17acaa4491f7db6cba61e4
Parents: 9427604
Author: Bill Farner <wfar...@apache.org>
Authored: Mon Nov 13 13:44:39 2017 -0800
Committer: Bill Farner <wfar...@apache.org>
Committed: Mon Nov 13 13:44:39 2017 -0800

--
 build-support/python/make-pycharm-virtualenv| 20 ++--
 .../apache/thermos/monitoring/test_disk.py  | 24 +---
 2 files changed, 29 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/bd121886/build-support/python/make-pycharm-virtualenv
--
diff --git a/build-support/python/make-pycharm-virtualenv 
b/build-support/python/make-pycharm-virtualenv
index 0f422d6..b1b575a 100755
--- a/build-support/python/make-pycharm-virtualenv
+++ b/build-support/python/make-pycharm-virtualenv
@@ -29,13 +29,19 @@ pushd "$BUILDROOT"
   # TODO(John Sirois): Either get this info from the current pants install 
itself instead of
   # using a parallel ephemeral install like we do here (slow), or else invert 
things and
   # allow user-control of the pytest lib versions within reason.
-  rm -rf "$VENV_DIR"
-  ./build-support/virtualenv "$VENV_DIR"
-  source $VENV_DIR/bin/activate
-python -m pip install pantsbuild.pants==$(./pants --version)
-pytest_requirement=$(python -m pip freeze | grep pytest==)
-pytest_cov_requirement=$(python -m pip freeze | grep pytest-cov==)
-  deactivate
+
+  #rm -rf "$VENV_DIR"
+  #./build-support/virtualenv "$VENV_DIR"
+  #source $VENV_DIR/bin/activate
+  #  python -m pip install pantsbuild.pants==$(./pants --version)
+  #  pytest_requirement=$(python -m pip freeze | grep pytest==)
+  #  pytest_cov_requirement=$(python -m pip freeze | grep pytest-cov==)
+  #deactivate
+  # TODO(wfarner): The above attempt to fetch pytest versions from pants 
stopped
+  # working with commit 0c177058.  As a workaround, manually specify the 
versions
+  # for now.
+  pytest_requirement='pytest==3.2.3'
+  pytest_cov_requirement='pytest-cov==2.5.1'
 
   rm -rf "$VENV_DIR"
   ./build-support/virtualenv "$VENV_DIR"

http://git-wip-us.apache.org/repos/asf/aurora/blob/bd121886/src/test/python/apache/thermos/monitoring/test_disk.py
--
diff --git a/src/test/python/apache/thermos/monitoring/test_disk.py 
b/src/test/python/apache/thermos/monitoring/test_disk.py
index e57467c..362393b 100644
--- a/src/test/python/apache/thermos/monitoring/test_disk.py
+++ b/src/test/python/apache/thermos/monitoring/test_disk.py
@@ -14,7 +14,9 @@
 
 import os
 from tempfile import mkstemp
+from unittest import TestCase
 
+from twitter.common import dirutil
 from twitter.common.dirutil import safe_mkdtemp
 from twitter.common.quantity import Amount, Data
 
@@ -29,6 +31,11 @@ def make_file(size, dir):
   _, filename = mkstemp(dir=dir)
   with open(filename, 'w') as f:
 f.write('0' * int(size.as_(Data.BYTES)))
+
+# Workaround for AURORA-1956.  On macOS 10.13 with APFS, st_blocks is not
+# consistent with st_size.
+while dirutil.safe_size(filename) < int(size.as_(Data.BYTES)):
+  f.write('0' * 1024)
   return filename
 
 
@@ -52,13 +59,14 @@ def _run_collector_tests(collector, target, wait):
   assert TEST_AMOUNT_SUM.as_(Data.BYTES) > collector.value >= 
TEST_AMOUNT_2.as_(Data.BYTES)
 
 
-def test_du_diskcollector():
-  target = safe_mkdtemp()
-  collector = DiskCollector(target)
+class TestDiskCollector(TestCase):
+  def test_du_diskcollector(self):
+target = safe_mkdtemp()
+collector = DiskCollector(target)
 
-  def wait():
-collector.sample()
-if collector._thread is not None:
-  collector._thread.event.wait()
+def wait():
+  collector.sample()
+  if collector._thread is not None:
+collector._thread.event.wait()
 
-  _run_collector_tests(collector, target, wait)
+_run_collector_tests(collector, target, wait)

[3/5] aurora git commit: Remove the internal SQL database

[wfarner]

http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java 
b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java
deleted file mode 100644
index cda55c5..000
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.storage.db;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.aurora.common.collections.Pair;
-import org.apache.aurora.scheduler.storage.db.views.DbTaskConfig;
-import org.apache.aurora.scheduler.storage.entities.IConstraint;
-import org.apache.aurora.scheduler.storage.entities.IDockerContainer;
-import org.apache.aurora.scheduler.storage.entities.IDockerParameter;
-import org.apache.aurora.scheduler.storage.entities.IJobKey;
-import org.apache.aurora.scheduler.storage.entities.ILimitConstraint;
-import org.apache.aurora.scheduler.storage.entities.IMesosFetcherURI;
-import org.apache.aurora.scheduler.storage.entities.IMetadata;
-import org.apache.aurora.scheduler.storage.entities.ITaskConfig;
-import org.apache.aurora.scheduler.storage.entities.IValueConstraint;
-import org.apache.aurora.scheduler.storage.entities.IVolume;
-import org.apache.ibatis.annotations.Param;
-
-/**
- * MyBatis mapper for task config objects.
- */
-interface TaskConfigMapper extends GarbageCollectedTableMapper {
-
-  /**
-   * Inserts fields from a task config into the {@code task_configs} table.
-   *
-   * @param config Configuration to insert.
-   * @param result Container for auto-generated ID of the inserted row.
-   */
-  void insert(
-  @Param("config") ITaskConfig config,
-  @Param("result") InsertResult result);
-
-  /**
-   * Gets all task config rows referenced by a job.
-   *
-   * @param job Job to look up.
-   * @return Task config row container.
-   */
-  List selectConfigsByJob(IJobKey job);
-
-  /**
-   * Inserts the constraint association within an {@link ITaskConfig}.
-   *
-   * @param configId Task config ID.
-   * @param constraint Constraint to insert.
-   * @param result Container for auto-generated ID of the inserted row.
-   */
-  void insertConstraint(
-  @Param("configId") long configId,
-  @Param("constraint") IConstraint constraint,
-  @Param("result") InsertResult result);
-
-  /**
-   * Inserts the limit constraint association within an {@link IConstraint}.
-   *
-   * @param constraintId Constraint ID.
-   * @param constraint Constraint to insert.
-   */
-  void insertLimitConstraint(
-  @Param("constraintId") long constraintId,
-  @Param("constraint") ILimitConstraint constraint);
-
-  /**
-   * Inserts the value constraint association within an {@link IConstraint}.
-   *
-   * @param constraintId Constraint ID.
-   * @param constraint Constraint to insert.
-   * @param result Container for auto-generated ID of the inserted row.
-   */
-  void insertValueConstraint(
-  @Param("constraintId") long constraintId,
-  @Param("constraint") IValueConstraint constraint,
-  @Param("result") InsertResult result);
-
-  /**
-   * Inserts the values association within an {@link IValueConstraint}.
-   *
-   * @param valueConstraintId Value constraint ID.
-   * @param values Values to insert.
-   */
-  void insertValueConstraintValues(
-  @Param("valueConstraintId") long valueConstraintId,
-  @Param("values") Set values);
-
-  /**
-   * Inserts the requested ports association within an {@link ITaskConfig}.
-   *
-   * @param configId Task config ID.
-   * @param ports Port names to insert.
-   */
-  void insertRequestedPorts(
-  @Param("configId") long configId,
-  @Param("ports") Set ports);
-
-  /**
-   * Inserts the task links association within an {@link ITaskConfig}.
-   *
-   * @param configId Task config ID.
-   * @param links Task links to insert.
-   */
-  void insertTaskLinks(@Param("configId") long configId, @Param("links") 
Map links);
-
-  /**
-   * Inserts the container association within an {@link ITaskConfig}.
-   *
-   * @param configId Task config ID.
-   * @param container Container to insert.
-   */
-  void insertContainer(
-  @Param("configId") long 

[2/5] aurora git commit: Remove the internal SQL database

[wfarner]

http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/views/Pairs.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/db/views/Pairs.java 
b/src/main/java/org/apache/aurora/scheduler/storage/db/views/Pairs.java
deleted file mode 100644
index 106b7af..000
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/Pairs.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.storage.db.views;
-
-import java.util.Map;
-
-import com.google.common.collect.ImmutableMap;
-
-import org.apache.aurora.common.collections.Pair;
-
-/**
- * Utility class for translating collections of {@link Pair} to and from maps.
- */
-public final class Pairs {
-
-  private Pairs() {
-// Utility class.
-  }
-
-  public static  Map toMap(Iterable> pairs) {
-ImmutableMap.Builder map = ImmutableMap.builder();
-for (Pair pair : pairs) {
-  map.put(pair.getFirst(), pair.getSecond());
-}
-return map.build();
-  }
-}

http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java 
b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java
index 3258879..6462b80 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotStoreImpl.java
@@ -13,27 +13,19 @@
  */
 package org.apache.aurora.scheduler.storage.log;
 
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
 import java.util.Arrays;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import javax.inject.Inject;
-import javax.sql.DataSource;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
 import com.google.common.base.Optional;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Lists;
-import com.google.inject.Injector;
 
 import org.apache.aurora.common.inject.TimedInterceptor.Timed;
 import org.apache.aurora.common.stats.SlidingStats;
@@ -57,11 +49,6 @@ import org.apache.aurora.scheduler.storage.Storage;
 import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider;
 import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
 import org.apache.aurora.scheduler.storage.Storage.Volatile;
-import org.apache.aurora.scheduler.storage.db.DbModule;
-import org.apache.aurora.scheduler.storage.db.DbStorage;
-import org.apache.aurora.scheduler.storage.db.DbUtil;
-import org.apache.aurora.scheduler.storage.db.EnumBackfill;
-import org.apache.aurora.scheduler.storage.db.MigrationManager;
 import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
 import org.apache.aurora.scheduler.storage.entities.IJobConfiguration;
 import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
@@ -88,12 +75,6 @@ public class SnapshotStoreImpl implements 
SnapshotStore {
 
   private static final Logger LOG = 
LoggerFactory.getLogger(SnapshotStoreImpl.class);
 
-  /**
-   * Number of rows to run in a single batch during dbsnapshot restore.
-   */
-  private static final int DB_BATCH_SIZE = 20;
-
-  private static final String DB_SCRIPT_FIELD = "dbscript";
   private static final String LOCK_FIELD = "locks";
   private static final String HOST_ATTRIBUTES_FIELD = "hosts";
   private static final String QUOTA_FIELD = "quota";
@@ -110,73 +91,6 @@ public class SnapshotStoreImpl implements 
SnapshotStore {
   }
 
   private final Iterable snapshotFields = Arrays.asList(
-  // Order is critical here. The DB snapshot should always be tried first 
to ensure
-  // graceful migration to DBTaskStore. Otherwise, there is a direct risk 
of losing the cluster.
-  // The following scenario illustrates how that can happen:
-  // - Dbsnapshot:ON, DBTaskStore:OFF
-  // - Scheduler is updated 

[4/5] aurora git commit: Remove the internal SQL database

[wfarner]

afeTaskStore() {
-return taskStore;
-  }
-
-  @Override
-  public LockStore.Mutable getLockStore() {
-return lockStore;
-  }
-
-  @Override
-  public QuotaStore.Mutable getQuotaStore() {
-return quotaStore;
-  }
-
-  @Override
-  public AttributeStore.Mutable getAttributeStore() {
-return attributeStore;
-  }
-
-  @Override
-  public JobUpdateStore.Mutable getJobUpdateStore() {
-return jobUpdateStore;
-  }
-};
-this.statsProvider = requireNonNull(statsProvider);
-  }
-
-  @SuppressWarnings("unchecked")
-  public  T getUnsafeStoreAccess() {
-return (T) 
sessionFactory.getConfiguration().getEnvironment().getDataSource();
-  }
-
-  @Timed("db_storage_read_operation")
-  @Override
-  @Transactional
-  public <T, E extends Exception> T read(Work<T, E> work) throws 
StorageException, E {
-try {
-  return work.apply(storeProvider);
-} catch (PersistenceException e) {
-  throw new StorageException(e.getMessage(), e);
-}
-  }
-
-  @Transactional
-  <T, E extends Exception> T transactionedWrite(MutateWork<T, E> work) throws 
E {
-return work.apply(storeProvider);
-  }
-
-  @Timed("db_storage_write_operation")
-  @Override
-  public <T, E extends Exception> T write(MutateWork<T, E> work) throws 
StorageException, E {
-// NOTE: Async work is intentionally executed regardless of whether the 
transaction succeeded.
-// Doing otherwise runs the risk of cross-talk between transactions and 
losing async tasks
-// due to failure of an unrelated transaction.  This matches behavior 
prior to the
-// introduction of DbStorage, but should be revisited.
-// TODO(wfarner): Consider revisiting to execute async work only when the 
transaction is
-// successful.
-return gatedWorkQueue.closeDuring((GatedOperation<T, E>) () -> {
-  try {
-return transactionedWrite(work);
-  } catch (PersistenceException e) {
-throw new StorageException(e.getMessage(), e);
-  }
-});
-  }
-
-  @Override
-  public void prepare() {
-startAsync().awaitRunning();
-  }
-
-  private static void addMappedStatement(Configuration configuration, String 
name, String sql) {
-configuration.addMappedStatement(
-new Builder(configuration, name, new StaticSqlSource(configuration, 
sql), UPDATE).build());
-  }
-
-  /**
-   * Creates the SQL schema during service start-up.
-   * Note: This design assumes a volatile database engine.
-   */
-  @Override
-  @Transactional
-  protected void startUp() throws IOException {
-Configuration configuration = sessionFactory.getConfiguration();
-String createStatementName = "create_tables";
-configuration.setMapUnderscoreToCamelCase(true);
-
-// The ReuseExecutor will cache jdbc Statements with equivalent SQL, 
improving performance
-// slightly when redundant queries are made.
-configuration.setDefaultExecutorType(ExecutorType.REUSE);
-
-addMappedStatement(
-configuration,
-createStatementName,
-CharStreams.toString(new InputStreamReader(
-DbStorage.class.getResourceAsStream("schema.sql"),
-StandardCharsets.UTF_8)));
-
-try (SqlSession session = sessionFactory.openSession()) {
-  session.update(createStatementName);
-}
-
-enumBackfill.backfill();
-
-createPoolMetrics();
-  }
-
-  @Override
-  protected void shutDown() {
-// noop
-  }
-
-  private void createPoolMetrics() {
-DataSource dataSource = 
sessionFactory.getConfiguration().getEnvironment().getDataSource();
-// Should not fail because we specify a PoolDataSource in DbModule
-PoolState poolState = ((PooledDataSource) dataSource).getPoolState();
-
-createPoolGauge("requests", poolState::getRequestCount);
-createPoolGauge("average_request_time_ms", 
poolState::getAverageRequestTime);
-createPoolGauge("average_wait_time_ms", poolState::getAverageWaitTime);
-createPoolGauge("connections_had_to_wait", poolState::getHadToWaitCount);
-createPoolGauge("bad_connections", poolState::getBadConnectionCount);
-createPoolGauge("claimed_overdue_connections", 
poolState::getClaimedOverdueConnectionCount);
-createPoolGauge("average_overdue_checkout_time_ms", 
poolState::getAverageOverdueCheckoutTime);
-createPoolGauge("average_checkout_time_ms", 
poolState::getAverageCheckoutTime);
-createPoolGauge("idle_connections", poolState::getIdleConnectionCount);
-createPoolGauge("active_connections", poolState::getActiveConnectionCount);
-  }
-
-  private void createPoolGauge(String name, Supplier gauge) {
-String prefix = "db_storage_mybatis_connection_pool_";
-statsProvider.makeGauge(prefix + name, gauge);
-  }
-}

http://git-wip-us.apache.org/

[1/5] aurora git commit: Remove the internal SQL database

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master e0624b27b -> 942760466


http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/test/java/org/apache/aurora/scheduler/async/GatingDelayExecutorTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/async/GatingDelayExecutorTest.java 
b/src/test/java/org/apache/aurora/scheduler/async/GatingDelayExecutorTest.java
deleted file mode 100644
index 55b9a8b..000
--- 
a/src/test/java/org/apache/aurora/scheduler/async/GatingDelayExecutorTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.aurora.scheduler.async;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ScheduledExecutorService;
-
-import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-import org.apache.aurora.common.quantity.Amount;
-import org.apache.aurora.common.quantity.Time;
-import org.apache.aurora.common.testing.easymock.EasyMockTest;
-import org.easymock.EasyMock;
-import org.easymock.IExpectationSetters;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.easymock.EasyMock.expectLastCall;
-import static org.junit.Assert.assertEquals;
-
-public class GatingDelayExecutorTest extends EasyMockTest {
-
-  private static final Amount ONE_SECOND = Amount.of(1L, 
Time.SECONDS);
-
-  private ScheduledExecutorService gatedExecutor;
-  private Runnable runnable;
-  private GatingDelayExecutor gatingExecutor;
-
-  @Before
-  public void setUp() {
-gatedExecutor = createMock(ScheduledExecutorService.class);
-runnable = createMock(Runnable.class);
-gatingExecutor = new GatingDelayExecutor(gatedExecutor);
-  }
-
-  @Test
-  public void testGateOpen() {
-gatedExecutor.execute(runnable);
-
-control.replay();
-
-// The gate was not closed, so the work is executed immediately.
-gatingExecutor.execute(runnable);
-  }
-
-  private IExpectationSetters invokeWorkWhenSubmitted() {
-return expectLastCall().andAnswer(() -> {
-  ((Runnable) EasyMock.getCurrentArguments()[0]).run();
-  return null;
-});
-  }
-
-  @Test
-  public void testGateIsThreadSpecific() throws InterruptedException {
-gatedExecutor.execute(runnable);
-
-control.replay();
-
-CountDownLatch gateClosed = new CountDownLatch(1);
-CountDownLatch unblock = new CountDownLatch(1);
-Runnable closer = () -> gatingExecutor.closeDuring(() -> {
-  gateClosed.countDown();
-  try {
-unblock.await();
-  } catch (InterruptedException e) {
-throw Throwables.propagate(e);
-  }
-  return "hi";
-});
-new ThreadFactoryBuilder()
-.setDaemon(true)
-.setNameFormat("GateTest")
-.build()
-.newThread(closer)
-.start();
-
-gateClosed.await();
-gatingExecutor.execute(runnable);
-assertQueueSize(0);
-unblock.countDown();
-  }
-
-  private void assertQueueSize(int size) {
-assertEquals(size, gatingExecutor.getQueueSize());
-  }
-
-  @Test
-  public void testReentrantClose() {
-gatedExecutor.execute(runnable);
-expectLastCall().times(3);
-
-control.replay();
-
-gatingExecutor.execute(runnable);
-assertQueueSize(0);
-
-String result = gatingExecutor.closeDuring(() -> {
-  gatingExecutor.execute(runnable);
-  assertQueueSize(1);
-
-  String result1 = gatingExecutor.closeDuring(() -> {
-gatingExecutor.execute(runnable);
-assertQueueSize(2);
-return "hello";
-  });
-  assertEquals("hello", result1);
-
-  return "hi";
-});
-assertEquals("hi", result);
-assertQueueSize(0);
-  }
-
-  @Test
-  public void testExecute() {
-gatedExecutor.execute(runnable);
-invokeWorkWhenSubmitted();
-runnable.run();
-expectLastCall();
-
-control.replay();
-
-gatingExecutor.execute(runnable);
-  }
-
-  @Test
-  public void testExecuteAfterDelay() {
-gatedExecutor.schedule(
-runnable,
-ONE_SECOND.getValue().longValue(),
-ONE_SECOND.getUnit().getTimeUnit());
-invokeWorkWhenSubmitted();
-runnable.run();
-
-control.replay();
-
-gatingExecutor.execute(runnable, ONE_SECOND);
-  }
-}

http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/test/java/org/apache/aurora/scheduler/http/H2ConsoleModuleIT.java

[5/5] aurora git commit: Remove the internal SQL database

[wfarner]

offStrategy.calculateBackoffMs(retryInMs.get()));
-  executor.execute(this, Amount.of(retryInMs.get(), Time.MILLISECONDS));
+  executor.schedule(this, retryInMs.get(), TimeUnit.MILLISECONDS);
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/reconciliation/TaskTimeout.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/reconciliation/TaskTimeout.java 
b/src/main/java/org/apache/aurora/scheduler/reconciliation/TaskTimeout.java
index 8e9a0d3..9910e77 100644
--- a/src/main/java/org/apache/aurora/scheduler/reconciliation/TaskTimeout.java
+++ b/src/main/java/org/apache/aurora/scheduler/reconciliation/TaskTimeout.java
@@ -15,6 +15,8 @@ package org.apache.aurora.scheduler.reconciliation;
 
 import java.util.EnumSet;
 import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.inject.Inject;
@@ -29,7 +31,6 @@ import org.apache.aurora.common.quantity.Time;
 import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.gen.ScheduleStatus;
 import org.apache.aurora.scheduler.async.AsyncModule.AsyncExecutor;
-import org.apache.aurora.scheduler.async.DelayExecutor;
 import org.apache.aurora.scheduler.events.PubsubEvent.EventSubscriber;
 import org.apache.aurora.scheduler.events.PubsubEvent.TaskStateChange;
 import org.apache.aurora.scheduler.state.StateChangeResult;
@@ -65,7 +66,7 @@ class TaskTimeout extends AbstractIdleService implements 
EventSubscriber {
   ScheduleStatus.KILLING,
   ScheduleStatus.DRAINING);
 
-  private final DelayExecutor executor;
+  private final ScheduledExecutorService executor;
   private final Storage storage;
   private final StateManager stateManager;
   private final Amount<Long, Time> timeout;
@@ -73,7 +74,7 @@ class TaskTimeout extends AbstractIdleService implements 
EventSubscriber {
 
   @Inject
   TaskTimeout(
-  @AsyncExecutor DelayExecutor executor,
+  @AsyncExecutor ScheduledExecutorService executor,
   Storage storage,
   StateManager stateManager,
   Amount<Long, Time> timeout,
@@ -140,7 +141,7 @@ class TaskTimeout extends AbstractIdleService implements 
EventSubscriber {
 LOG.debug("Retrying timeout of task {} in {}", taskId, 
NOT_STARTED_RETRY);
 // TODO(wfarner): This execution should not wait for a transaction, 
but a second executor
 // would be weird.
-executor.execute(this, NOT_STARTED_RETRY);
+executor.schedule(this, NOT_STARTED_RETRY.as(Time.MILLISECONDS), 
TimeUnit.MILLISECONDS);
   }
 }
   }
@@ -148,9 +149,10 @@ class TaskTimeout extends AbstractIdleService implements 
EventSubscriber {
   @Subscribe
   public void recordStateChange(TaskStateChange change) {
 if (isTransient(change.getNewState())) {
-  executor.execute(
+  executor.schedule(
   new TimedOutTaskHandler(change.getTaskId(), change.getNewState()),
-  timeout);
+  timeout.as(Time.MILLISECONDS),
+  TimeUnit.MILLISECONDS);
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/scheduling/TaskGroups.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/scheduling/TaskGroups.java 
b/src/main/java/org/apache/aurora/scheduler/scheduling/TaskGroups.java
index 2d3492d..b9987e4 100644
--- a/src/main/java/org/apache/aurora/scheduler/scheduling/TaskGroups.java
+++ b/src/main/java/org/apache/aurora/scheduler/scheduling/TaskGroups.java
@@ -19,6 +19,8 @@ import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.inject.Inject;
@@ -39,7 +41,6 @@ import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.common.util.BackoffStrategy;
 import org.apache.aurora.scheduler.BatchWorker;
 import org.apache.aurora.scheduler.async.AsyncModule.AsyncExecutor;
-import org.apache.aurora.scheduler.async.DelayExecutor;
 import org.apache.aurora.scheduler.base.TaskGroupKey;
 import org.apache.aurora.scheduler.base.Tasks;
 import org.apache.aurora.scheduler.events.PubsubEvent.EventSubscriber;
@@ -73,7 +74,7 @@ public class TaskGroups implements EventSubscriber {
   static final String SCHEDULE_ATTEMPTS_BLOCKS = "schedule_attempts_blocks";
 
   private final ConcurrentMap<TaskGroupKey, TaskGroup> groups = 
Maps.newConcurrentMap();
-  private final DelayExecutor executor;
+  private final ScheduledExecutorService executor;
   private final TaskGroupsSettings settings;
   priv

svn commit: r1814961 [12/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentatio

[wfarner]

Propchange: 
aurora/site/source/documentation/0.19.0/images/presentations/10_08_2015_sla_aware_maintenance_for_operators_thumb.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/source/documentation/0.19.0/images/runningtask.png
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.19.0/images/runningtask.png?rev=1814961=auto
==
Binary file - no diff available.

Propchange: aurora/site/source/documentation/0.19.0/images/runningtask.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/source/documentation/0.19.0/images/stderr.png
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.19.0/images/stderr.png?rev=1814961=auto
==
Binary file - no diff available.

Propchange: aurora/site/source/documentation/0.19.0/images/stderr.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/source/documentation/0.19.0/images/stdout.png
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.19.0/images/stdout.png?rev=1814961=auto
==
Binary file - no diff available.

Propchange: aurora/site/source/documentation/0.19.0/images/stdout.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/source/documentation/0.19.0/images/storage_hierarchy.png
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.19.0/images/storage_hierarchy.png?rev=1814961=auto
==
Binary file - no diff available.

Propchange: aurora/site/source/documentation/0.19.0/images/storage_hierarchy.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/source/documentation/0.19.0/index.html.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.19.0/index.html.md?rev=1814961=auto
==
--- aurora/site/source/documentation/0.19.0/index.html.md (added)
+++ aurora/site/source/documentation/0.19.0/index.html.md Sat Nov 11 16:49:46 
2017
@@ -0,0 +1,79 @@
+## Introduction
+
+Apache Aurora is a service scheduler that runs on top of Apache Mesos, 
enabling you to run
+long-running services, cron jobs, and ad-hoc jobs that take advantage of 
Apache Mesos' scalability,
+fault-tolerance, and resource isolation.
+
+We encourage you to ask questions on the [Aurora user 
list](http://aurora.apache.org/community/) or
+the `#aurora` IRC channel on `irc.freenode.net`.
+
+
+## Getting Started
+Information for everyone new to Apache Aurora.
+
+ * [Aurora System Overview](getting-started/overview/)
+ * [Hello World Tutorial](getting-started/tutorial/)
+ * [Local cluster with Vagrant](getting-started/vagrant/)
+
+## Features
+Description of important Aurora features.
+
+ * [Containers](features/containers/)
+ * [Cron Jobs](features/cron-jobs/)
+ * [Custom Executors](features/custom-executors/)
+ * [Job Updates](features/job-updates/)
+ * [Multitenancy](features/multitenancy/)
+ * [Resource Isolation](features/resource-isolation/)
+ * [Scheduling Constraints](features/constraints/)
+ * [Services](features/services/)
+ * [Service Discovery](features/service-discovery/)
+ * [SLA Metrics](features/sla-metrics/)
+ * [Webhooks](features/webhooks/)
+
+## Operators
+For those that wish to manage and fine-tune an Aurora cluster.
+
+ * [Installation](operations/installation/)
+ * [Configuration](operations/configuration/)
+ * [Upgrades](operations/upgrades/)
+ * [Troubleshooting](operations/troubleshooting/)
+ * [Monitoring](operations/monitoring/)
+ * [Security](operations/security/)
+ * [Storage](operations/storage/)
+ * [Backup](operations/backup-restore/)
+
+## Reference
+The complete reference of commands, configuration options, and scheduler 
internals.
+
+ * [Task lifecycle](reference/task-lifecycle/)
+ * Configuration (`.aurora` files)
+- [Configuration Reference](reference/configuration/)
+- [Configuration Tutorial](reference/configuration-tutorial/)
+- [Configuration Best Practices](reference/configuration-best-practices/)
+- [Configuration Templating](reference/configuration-templating/)
+ * Aurora Client
+- [Client Commands](reference/client-commands/)
+- [Client Hooks](reference/client-hooks/)
+- [Client Cluster Configuration](reference/client-cluster-configuration/)
+ * [Scheduler Configuration](reference/scheduler-configuration/)
+ * 

svn commit: r1814961 [4/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Added: aurora/site/publish/documentation/0.19.0/features/services/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/features/services/index.html?rev=1814961=auto
==
--- aurora/site/publish/documentation/0.19.0/features/services/index.html 
(added)
+++ aurora/site/publish/documentation/0.19.0/features/services/index.html Sat 
Nov 11 16:49:46 2017
@@ -0,0 +1,248 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Long-running Services
+
+Jobs that are always restart on completion, whether successful or 
unsuccessful,
+are called services. This is useful for long-running processes
+such as webservices that should always be running, unless stopped 
explicitly.
+
+Service Specification
+
+A job is identified as a service by the presence of the flag
+`service=True in the Job object.
+The Service alias can be used as shorthand for Job 
with service=True.
+
+Example (available in the Vagrant 
environment):
+$ cat 
/vagrant/examples/jobs/hello_world.aurora
+hello = Process(
+  name = 'hello',
+  cmdline = """
+while true; do
+  echo hello world
+  sleep 10
+done
+  """)
+
+task = SequentialTask(
+  processes = [hello],
+  resources = Resources(cpu = 1.0, ram = 128*MB, disk = 128*MB)
+)
+
+jobs = [
+  Service(
+task = task,
+cluster = 'devcluster',
+role = 'www-data',
+environment = 'prod',
+name = 'hello'
+  )
+]
+
+
+Jobs without the service bit set only restart up to 
max_task_failures times and only if they
+terminated unsuccessfully either due to human error or machine failure (see the
+Job 
object for details).
+
+Ports
+
+In order to be useful, most services have to bind to one or more ports. 
Aurora enables this
+usecase via the thermos.ports
 namespace that
+allows to request arbitrarily named ports:
+nginx = Process(
+  name = 'nginx',
+  cmdline = './run_nginx.sh -port {{thermos.ports[http]}}'
+)
+
+
+When this process is included in a job, the job will be allocated a port, 
and the command line
+will be replaced with something like:
+./run_nginx.sh -port 42816
+
+
+Where 42816 happens to be the allocated port.
+
+For details on how to enable clients to discover this dynamically assigned 
port, see our
+Service Discovery documentation.
+
+Health Checking
+
+Typically, the Thermos executor monitors processes within a task only by 
liveness of the forked
+process. In addition to that, Aurora has support for rudimentary health 
checking: Either via HTTP
+via custom shell scripts.
+
+For example, simply by requesting a health port, a process can 
request to be health checked
+via repeated calls to the /health endpoint:
+nginx = Process(
+  name = 'nginx',
+  cmdline = './run_nginx.sh -port {{thermos.ports[health]}}'
+)
+
+
+Please see the
+configuration 
reference
+for configuration options for this feature.
+
+Starting with the 0.17.0 release, job updates rely only on task 
health-checks by introducing
+a min_consecutive_successes parameter on the HealthCheckConfig 
object. This parameter represents
+the number of successful health checks needed before a task is moved into the 
RUNNING state. Tasks
+that do not have enough successful health checks within the first 
n attempts, are moved to the
+FAILED state, where n = 
ceil(initial_interval_secs/interval_secs) + max_consecutive_failures +
+min_consecutive_successes. In order to accommodate variability during 
task warm up, initial_interval_secs
+will act as a grace period. Any health-check failures during the first 
m attempts are ignored and
+do not count towards 

svn commit: r1814961 [10/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentatio

[wfarner]

Added: 
aurora/site/publish/documentation/0.19.0/reference/task-lifecycle/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/reference/task-lifecycle/index.html?rev=1814961=auto
==
--- 
aurora/site/publish/documentation/0.19.0/reference/task-lifecycle/index.html 
(added)
+++ 
aurora/site/publish/documentation/0.19.0/reference/task-lifecycle/index.html 
Sat Nov 11 16:49:46 2017
@@ -0,0 +1,292 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Task Lifecycle
+
+When Aurora reads a configuration file and finds a Job 
definition, it:
+
+
+ Evaluates the Job definition.
+ Splits the Job into its constituent Tasks.
+ Sends those Tasks to the scheduler.
+ The scheduler puts the Tasks into PENDING 
state, starting each
+Tasks life cycle.
+
+
+
+
+Please note, a couple of task states described below are missing from
+this state diagram.
+
+PENDING to RUNNING states
+
+When a Task is in the PENDING state, the 
scheduler constantly
+searches for machines satisfying that Tasks resource 
request
+requirements (RAM, disk space, CPU time) while maintaining configuration
+constraints such as a Task must run on machines  dedicated 
 to a
+particular role or attribute limit constraints such as at most 2
+Tasks from the same Job may run on each rack. 
When the scheduler
+finds a suitable match, it assigns the Task to a machine and puts 
the
+Task into the ASSIGNED state.
+
+From the ASSIGNED state, the scheduler sends an RPC to the 
agent
+machine containing Task configuration, which the agent uses to 
spawn
+an executor responsible for the Tasks lifecycle. When the 
scheduler
+receives an acknowledgment that the machine has accepted the Task,
+the Task goes into STARTING state.
+
+STARTING state initializes a Task sandbox. When 
the sandbox is fully
+initialized, Thermos begins to invoke Processes. Also, the agent
+machine sends an update to the scheduler that the Task is
+in RUNNING state, only after the task satisfies the liveness 
requirements.
+See Health Checking for 
more details
+for how to configure health checks.
+
+RUNNING to terminal states
+
+There are various ways that an active Task can transition into 
a terminal
+state. By definition, it can never leave this state. However, depending on
+nature of the termination and the originating Job definition
+(e.g. service, max_task_failures), a replacement 
Task might be
+scheduled.
+
+Natural Termination: FINISHED, 
FAILED
+
+A RUNNING Task can terminate without direct user 
interaction. For
+example, it may be a finite computation that finishes, even something as
+simple as echo hello world., or it could be an exceptional 
condition in
+a long-lived service. If the Task is successful (its underlying
+processes have succeeded with exit status 0 or finished without
+reaching failure limits) it moves into FINISHED state. If it 
finished
+after reaching a set of failure limits, it goes into FAILED 
state.
+
+A terminated TASK which is subject to rescheduling will be 
temporarily
+THROTTLED, if it is considered to be flapping. A task is 
flapping, if its
+previous invocation was terminated after less than 5 minutes (scheduler
+default). The time penalty a task has to remain in the THROTTLED 
state,
+before it is eligible for rescheduling, increases with each consecutive
+failure.
+
+Forceful Termination: 
KILLING, RESTARTING
+
+You can terminate a Task by issuing an aurora job 
kill command, which
+moves it into KILLING state. The scheduler then sends the agent a
+request to terminate the Task. If the scheduler receives a 
successful

svn commit: r1814961 [2/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Added: 
aurora/site/publish/documentation/0.19.0/development/db-migration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/development/db-migration/index.html?rev=1814961=auto
==
--- 
aurora/site/publish/documentation/0.19.0/development/db-migration/index.html 
(added)
+++ 
aurora/site/publish/documentation/0.19.0/development/db-migration/index.html 
Sat Nov 11 16:49:46 2017
@@ -0,0 +1,175 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+DB Migrations
+
+Changes to the DB schema should be made in the form of migrations. This 
ensures that all changes
+are applied correctly after a DB dump from a previous version is restored.
+
+DB migrations are managed through a system built on top of
+http://www.mybatis.org/migrations/;>MyBatis Migrations. The 
migrations are run automatically when
+a snapshot is restored, no manual interaction is required by cluster 
operators.
+
+Upgrades
+
+When adding or altering tables or changing data, in addition to making to 
change in
+schema.sql,
 a new
+migration class should be created under the 
org.apache.aurora.scheduler.storage.db.migration
+package. The class should implement the https://github.com/mybatis/migrations/blob/master/src/main/java/org/apache/ibatis/migration/MigrationScript.java;>MigrationScript
+interface (see https://github.com/apache/aurora/blob/rel/0.19.0/src/test/java/org/apache/aurora/scheduler/storage/db/testmigration/V001_TestMigration.java;>V001_TestMigration
+as an example). The upgrade and downgrade scripts are defined in this class. 
When restoring a
+snapshot the list of migrations on the classpath is compared to the list of 
applied changes in the
+DB. Any changes that have not yet been applied are executed and their 
downgrade script is stored
+alongside the changelog entry in the database to faciliate downgrades in the 
event of a rollback.
+
+Downgrades
+
+If, while running migrations, a rollback is detected, i.e. a change exists 
in the DB changelog that
+does not exist on the classpath, the downgrade script associated with each 
affected change is
+applied.
+
+Baselines
+
+After enough time has passed (at least 1 official release), it should be 
safe to baseline migrations
+if desired. This can be accomplished by ensuring the changes from migrations 
have been applied to
+schema.sql
 and then
+removing the corresponding migration classes and adding a migration to remove 
the changelog entries.
+
+
+
+  
+
+   
+  
+
+ Quick Links
+ 
+   Downloads
+Mailing Lists
+   http://issues.apache.org/jira/browse/AURORA;>Issue Tracking
+   How 
To Contribute 
+ 
+ 
+ The ASF
+  
+http://www.apache.org/licenses/;>License
+http://www.apache.org/foundation/sponsorship.html;>Sponsorship  
+http://www.apache.org/foundation/thanks.html;>Thanks
+http://www.apache.org/security/;>Security
+  
+ 
+ 
+2014-2017 http://www.apache.org/;>Apache Software Foundation. Licensed under 
the http://www.apache.org/licenses/;>Apache License v2.0. The https://www.flickr.com/photos/trondk/12706051375/;>Aurora Borealis IX 
photo displayed on the homepage is available under a https://creativecommons.org/licenses/by-nc-nd/2.0/;>Creative Commons 
BY-NC-ND 2.0 license. Apache, Apache Aurora, and the Apache feather logo 
are trademarks of The Apache 

svn commit: r1814961 [8/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Added: 
aurora/site/publish/documentation/0.19.0/reference/configuration-best-practices/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/reference/configuration-best-practices/index.html?rev=1814961=auto
==
--- 
aurora/site/publish/documentation/0.19.0/reference/configuration-best-practices/index.html
 (added)
+++ 
aurora/site/publish/documentation/0.19.0/reference/configuration-best-practices/index.html
 Sat Nov 11 16:49:46 2017
@@ -0,0 +1,321 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Aurora Configuration Best 
Practices
+
+Use As Few .aurora Files As 
Possible
+
+When creating your .aurora configuration, try to keep all 
versions of
+a particular job within the same .aurora file. For example, if you
+have separate jobs for cluster1, cluster1 staging, 
cluster1
+testing, andcluster2, keep them as close together as possible.
+
+Constructs shared across multiple jobs owned by your team (e.g.
+team-level defaults or structural templates) can be split into separate
+.aurorafiles and included via the include 
directive.
+
+Avoid Boilerplate
+
+If you see repetition or find yourself copy and pasting any parts of
+your configuration, its likely an opportunity for templating. Take the
+example below:
+
+redundant.aurora contains:
+download = Process(
+  name = 'download',
+  cmdline = 'wget http://www.python.org/ftp/python/2.7.3/Python-2.7.3.tar.bz2',
+  max_failures = 5,
+  min_duration = 1)
+
+unpack = Process(
+  name = 'unpack',
+  cmdline = 'rm -rf Python-2.7.3  tar xzf Python-2.7.3.tar.bz2',
+  max_failures = 5,
+  min_duration = 1)
+
+build = Process(
+  name = 'build',
+  cmdline = 'pushd Python-2.7.3  ./configure  make 
 popd',
+  max_failures = 1)
+
+email = Process(
+  name = 'email',
+  cmdline = 'echo Success | mail feyn...@tmc.com',
+  max_failures = 5,
+  min_duration = 1)
+
+build_python = Task(
+  name = 'build_python',
+  processes = [download, unpack, build, email],
+  constraints = [Constraint(order = ['download', 'unpack', 'build', 'email'])])
+
+
+As youll notice, theres a lot of repetition in the 
Process
+definitions. For example, almost every process sets a max_failures
+limit to 5 and a min_duration to 1. This is an opportunity for 
factoring
+into a common process template.
+
+Furthermore, the Python version is repeated everywhere. This can be
+bound via structural templating as described in the Advanced Binding
+section.
+
+less_redundant.aurora contains:
+class Python(Struct):
+  version = Required(String)
+  base = Default(String, 'Python-{{version}}')
+  package = Default(String, '{{base}}.tar.bz2')
+
+ReliableProcess = Process(
+  max_failures = 5,
+  min_duration = 1)
+
+download = ReliableProcess(
+  name = 'download',
+  cmdline = 'wget 
http://www.python.org/ftp/python/{{python.version}}/{{python.package}}')
+
+unpack = ReliableProcess(
+  name = 'unpack',
+  cmdline = 'rm -rf {{python.base}}  tar xzf {{python.package}}')
+
+build = ReliableProcess(
+  name = 'build',
+  cmdline = 'pushd {{python.base}}  ./configure  make 
 popd',
+  max_failures = 1)
+
+email = ReliableProcess(
+  name = 'email',
+  cmdline = 'echo Success | mail {{role}}@foocorp.com')
+
+build_python = SequentialTask(
+  name = 'build_python',
+  processes = [download, unpack, build, email]).bind(python = Python(version = 
"2.7.3"))
+
+
+Thermos Uses bash, But 
Thermos Is Not bash
+
+Bad
+
+Many tiny Processes makes for harder to manage configurations.
+copy = Process(
+  name = 'copy',
+  cmdline = 'rcp user@my_machine:my_application .'
+ )
+
+ unpack = 

svn commit: r1814961 [6/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Added: 
aurora/site/publish/documentation/0.19.0/operations/configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/operations/configuration/index.html?rev=1814961=auto
==
--- 
aurora/site/publish/documentation/0.19.0/operations/configuration/index.html 
(added)
+++ 
aurora/site/publish/documentation/0.19.0/operations/configuration/index.html 
Sat Nov 11 16:49:46 2017
@@ -0,0 +1,505 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Scheduler Configuration
+
+The Aurora scheduler can take a variety of configuration options through 
command-line arguments.
+Examples are available under examples/scheduler/. For a list of 
available Aurora flags and their
+documentation, see Scheduler Configuration 
Reference.
+
+A Note on Configuration
+
+Like Mesos, Aurora uses command-line flags for runtime configuration. As 
such the Aurora
+configuration file is typically a scheduler.sh 
shell script of the form.
+#!/bin/bash
+AURORA_HOME=/usr/local/aurora-scheduler
+
+# Flags controlling the 
JVM.
+JAVA_OPTS=(
+  -Xmx2g
+  -Xms2g
+  # GC tuning, etc.
+)
+
+# Flags controlling the 
scheduler.
+AURORA_FLAGS=(
+  # Port for client RPCs and 
the web UI
+  -http_port=8081
+  # Log configuration, 
etc.
+)
+
+# Environment variables 
controlling libmesos
+export JAVA_HOME=...
+export GLOG_v=1
+export LIBPROCESS_PORT=8083
+export LIBPROCESS_IP=192.168.33.7
+
+JAVA_OPTS="${JAVA_OPTS[*]}" exec "$AURORA_HOME/bin/aurora-scheduler" "${AURORA_FLAGS[@]}"
+
+
+That way Auroras current flags are visible in ps and in 
the /vars admin endpoint.
+
+JVM Configuration
+
+JVM settings are dependent on your environment and cluster size. They might 
require
+custom tuning. As a starting point, we recommend:
+
+
+Ensure the initial (-Xms) and maximum (-Xmx) 
heap size are idential to prevent heap resizing
+at runtime.
+Either -XX:+UseConcMarkSweepGC or -XX:+UseG1GC 
-XX:+UseStringDeduplication are
+sane defaults for the garbage collector.
+-Djava.net.preferIPv4Stack=true makes sense in most cases as 
well.
+
+
+Network Configuration
+
+By default, Aurora binds to all interfaces and auto-discovers its hostname. 
To reduce ambiguity
+it helps to hardcode them though:
+-http_port=8081
+-ip=192.168.33.7
+-hostname="aurora1.us-east1.example.org"
+
+
+Two environment variables control the ip and port for the communication 
with the Mesos master
+and for the replicated log used by Aurora:
+export LIBPROCESS_PORT=8083
+export LIBPROCESS_IP=192.168.33.7
+
+
+It is important that those can be reached from all Mesos master and Aurora 
scheduler instances.
+
+Replicated Log Configuration
+
+Aurora schedulers use ZooKeeper to discover log replicas and elect a 
leader. Only one scheduler is
+leader at a given time - the other schedulers follow log writes and prepare to 
take over as leader
+but do not communicate with the Mesos master. Either 3 or 5 schedulers are 
recommended in a
+production deployment depending on failure tolerance and they must have 
persistent storage.
+
+Below is a summary of scheduler storage configuration flags that either 
dont have default values
+or require attention before deploying in a production environment.
+
+-native_log_quorum_size
+
+Defines the Mesos replicated log quorum size. In a cluster with 
N schedulers, the flag
+-native_log_quorum_size should be set to floor(N/2) + 
1. So in a cluster with 1 scheduler
+it should be set to 1, in a cluster with 3 it should be set to 
2, and in a cluster of 5 it
+should be set to 3.
+
+
+
+Number of schedulers (N)
+-native_log_quorum_size 

svn commit: r1814961 [13/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentatio

[wfarner]

Added: aurora/site/source/documentation/0.19.0/reference/client-commands.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.19.0/reference/client-commands.md?rev=1814961=auto
==
--- aurora/site/source/documentation/0.19.0/reference/client-commands.md (added)
+++ aurora/site/source/documentation/0.19.0/reference/client-commands.md Sat 
Nov 11 16:49:46 2017
@@ -0,0 +1,339 @@
+Aurora Client Commands
+==
+
+- [Introduction](#introduction)
+- [Cluster Configuration](#cluster-configuration)
+- [Job Keys](#job-keys)
+- [Modifying Aurora Client Commands](#modifying-aurora-client-commands)
+- [Regular Jobs](#regular-jobs)
+- [Creating and Running a Job](#creating-and-running-a-job)
+- [Running a Command On a Running Job](#running-a-command-on-a-running-job)
+- [Killing a Job](#killing-a-job)
+- [Adding Instances](#adding-instances)
+- [Updating a Job](#updating-a-job)
+- [Coordinated job updates](#coordinated-job-updates)
+- [Renaming a Job](#renaming-a-job)
+- [Restarting Jobs](#restarting-jobs)
+- [Cron Jobs](#cron-jobs)
+- [Comparing Jobs](#comparing-jobs)
+- [Viewing/Examining Jobs](#viewingexamining-jobs)
+- [Listing Jobs](#listing-jobs)
+- [Inspecting a Job](#inspecting-a-job)
+- [Versions](#versions)
+- [Checking Your Quota](#checking-your-quota)
+- [Finding a Job on Web UI](#finding-a-job-on-web-ui)
+- [Getting Job Status](#getting-job-status)
+- [Opening the Web UI](#opening-the-web-ui)
+- [SSHing to a Specific Task Machine](#sshing-to-a-specific-task-machine)
+- [SCPing with Specific Task Machines](#scping-with-specific-task-machines)
+- [Templating Command Arguments](#templating-command-arguments)
+
+Introduction
+
+
+Once you have written an `.aurora` configuration file that describes
+your Job and its parameters and functionality, you interact with Aurora
+using Aurora Client commands. This document describes all of these commands
+and how and when to use them. All Aurora Client commands start with
+`aurora`, followed by the name of the specific command and its
+arguments.
+
+*Job keys* are a very common argument to Aurora commands, as well as the
+gateway to useful information about a Job. Before using Aurora, you
+should read the next section which describes them in detail. The section
+after that briefly describes how you can modify the behavior of certain
+Aurora Client commands, linking to a detailed document about how to do
+that.
+
+This is followed by the Regular Jobs section, which describes the basic
+Client commands for creating, running, and manipulating Aurora Jobs.
+After that are sections on Comparing Jobs and Viewing/Examining Jobs. In
+other words, various commands for getting information and metadata about
+Aurora Jobs.
+
+Cluster Configuration
+-
+
+The client must be able to find a configuration file that specifies available 
clusters. This file
+declares shorthand names for clusters, which are in turn referenced by job 
configuration files
+and client commands.
+
+The client will load at most two configuration files, making both of their 
defined clusters
+available. The first is intended to be a system-installed cluster, using the 
path specified in
+the environment variable `AURORA_CONFIG_ROOT`, defaulting to 
`/etc/aurora/clusters.json` if the
+environment variable is not set. The second is a user-installed file, located 
at
+`~/.aurora/clusters.json`.
+
+For more details on cluster configuration see the
+[Client Cluster Configuration](../client-cluster-configuration/) documentation.
+
+Job Keys
+
+
+A job key is a unique system-wide identifier for an Aurora-managed
+Job, for example `cluster1/web-team/test/experiment204`. It is a 4-tuple
+consisting of, in order, *cluster*, *role*, *environment*, and
+*jobname*, separated by /s. Cluster is the name of an Aurora
+cluster. Role is the Unix service account under which the Job
+runs. Environment is a namespace component like `devel`, `test`,
+`prod`, or `stagingN.` Jobname is the Job's name.
+
+The combination of all four values uniquely specifies the Job. If any
+one value is different from that of another job key, the two job keys
+refer to different Jobs. For example, job key
+`cluster1/tyg/prod/workhorse` is different from
+`cluster1/tyg/prod/workcamel` is different from
+`cluster2/tyg/prod/workhorse` is different from
+`cluster2/foo/prod/workhorse` is different from
+`cluster1/tyg/test/workhorse.`
+
+Role names are user accounts existing on the agent machines. If you don't know 
what accounts
+are available, contact your sysadmin.
+
+Environment names are namespaces; you can count on `prod`, `devel` and `test` 
existing.
+
+Modifying Aurora Client Commands
+
+
+For certain Aurora Client commands, you can define hook methods that run
+either before or after an 

svn commit: r1814961 [7/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Added: aurora/site/publish/documentation/0.19.0/operations/storage/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/operations/storage/index.html?rev=1814961=auto
==
--- aurora/site/publish/documentation/0.19.0/operations/storage/index.html 
(added)
+++ aurora/site/publish/documentation/0.19.0/operations/storage/index.html Sat 
Nov 11 16:49:46 2017
@@ -0,0 +1,245 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Aurora Scheduler Storage
+
+
+Overview
+Storage Semantics
+
+
+Reads, writes, modifications
+Read lifecycle
+Write lifecycle
+Atomicity, consistency and 
isolation
+Population on restart
+
+
+
+Overview
+
+Aurora scheduler maintains data that need to be persisted to survive 
failovers and restarts.
+For example:
+
+
+Task configurations and scheduled task instances
+Job update configurations and update progress
+Production resource quotas
+Mesos resource offer host attributes
+
+
+Aurora solves its persistence needs by leveraging the
+http://mesos.apache.org/documentation/latest/replicated-log-internals/;>Mesos
 implementation of a Paxos replicated log
+https://ramcloud.stanford.edu/~ongaro/userstudy/paxos.pdf;>[1]
+http://en.wikipedia.org/wiki/State_machine_replication;>[2] with 
a key-value
+https://github.com/google/leveldb;>LevelDB storage as persistence 
media.
+
+Conceptually, it can be represented by the following major components:
+
+
+Volatile storage: in-memory cache of all available data. Implemented via 
in-memory
+http://www.h2database.com/html/main.html;>H2 Database and 
accessed via
+http://mybatis.github.io/mybatis-3/;>MyBatis.
+Log manager: interface between Aurora storage and Mesos replicated log. 
The default schema format
+is https://github.com/apache/thrift;>thrift. Data is stored in 
serialized binary form.
+Snapshot manager: all data is periodically persisted in Mesos replicated 
log in a single snapshot.
+This helps establishing periodic recovery checkpoints and speeds up volatile 
storage recovery on
+restart.
+Backup manager: as a precaution, snapshots are periodically written out 
into backup files.
+This solves a disaster recovery problem
+in case of a complete loss or corruption of Mesos log files.
+
+
+
+
+Storage Semantics
+
+Implementation details of the Aurora storage system. Understanding those 
can sometimes be useful
+when investigating performance issues.
+
+Reads, writes, modifications
+
+All services in Aurora access data via a set of predefined store interfaces 
(aka stores) logically
+grouped by the type of data they serve. Every interface defines a specific set 
of operations allowed
+on the data thus abstracting out the storage access and the actual persistence 
implementation. The
+latter is especially important in view of a general immutability of persisted 
data. With the Mesos
+replicated log as the underlying persistence solution, data can be read and 
written easily but not
+modified. All modifications are simulated by saving new versions of modified 
objects. This feature
+and general performance considerations justify the existence of the volatile 
in-memory store.
+
+Read lifecycle
+
+There are two types of reads available in Aurora: consistent and 
weakly-consistent. The difference
+is explained below.
+
+All reads are served from the volatile storage making reads generally cheap 
storage operations
+from the performance standpoint. The majority of the volatile stores are 
represented by the
+in-memory H2 database. This allows for rich schema definitions, queries and 
relationships that
+key-value storage 

svn commit: r1814961 [9/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Added: 
aurora/site/publish/documentation/0.19.0/reference/configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/reference/configuration/index.html?rev=1814961=auto
==
--- aurora/site/publish/documentation/0.19.0/reference/configuration/index.html 
(added)
+++ aurora/site/publish/documentation/0.19.0/reference/configuration/index.html 
Sat Nov 11 16:49:46 2017
@@ -0,0 +1,1291 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Aurora Configuration Reference
+
+Dont know where to start? The Aurora configuration schema is very
+powerful, and configurations can become quite complex for advanced use
+cases.
+
+For examples of simple configurations to get something up and running
+quickly, check out the Tutorial. 
When you feel comfortable with the basics, move
+on to the Configuration Tutorial for 
more in-depth coverage of
+configuration design.
+
+
+Process Schema
+
+
+Process Objects
+
+Task Schema
+
+
+Task Object
+Constraint Object
+Resource Object
+
+Job Schema
+
+
+Job Objects
+UpdateConfig Objects
+HealthCheckConfig Objects
+Announcer Objects
+Container Objects
+LifecycleConfig Objects
+
+Specifying Scheduling 
Constraints
+Template Namespaces
+
+
+mesos Namespace
+thermos Namespace
+
+
+
+Process Schema
+
+Process objects consist of required name and 
cmdline attributes. You can customize Process
+behavior with its optional attributes. Remember, Processes are handled by 
Thermos.
+
+Process Objects
+
+
+
+Attribute Name
+Type
+Description
+
+
+
+name
+String
+Process name (Required)
+
+
+cmdline
+String
+Command line (Required)
+
+
+max_failures
+Integer
+Maximum process failures (Default: 1)
+
+
+daemon
+Boolean
+When True, this is a daemon process. (Default: False)
+
+
+ephemeral
+Boolean
+When True, this is an ephemeral process. (Default: False)
+
+
+min_duration
+Integer
+Minimum duration between process restarts in seconds. (Default: 5)
+
+
+final
+Boolean
+When True, this process is a finalizing one that should run last. 
(Default: False)
+
+
+logger
+Logger
+Struct defining the log behavior for the process. (Default: Empty)
+
+
+
+name
+
+The name is any valid UNIX filename string (specifically no
+slashes, NULLs or leading periods). Within a Task object, each Process name
+must be unique.
+
+cmdline
+
+The command line run by the process. The command line is invoked in a bash
+subshell, so can involve fully-blown bash scripts. However, nothing is
+supplied for command-line arguments so $* is unspecified.
+
+max_failures
+
+The maximum number of failures (non-zero exit statuses) this process can
+have before being marked permanently failed and not retried. If a
+process permanently fails, Thermos looks at the failure limit of the task
+containing the process (usually 1) to determine if the task has
+failed as well.
+
+Setting max_failures to 0 makes the process retry
+indefinitely until it achieves a successful (zero) exit status.
+It retries at most once every min_duration seconds to prevent
+an effective denial of service attack on the coordinating Thermos 
scheduler.
+
+daemon
+
+By default, Thermos processes are non-daemon. If daemon is set 
to True, a
+successful (zero) exit status does not prevent future process runs.
+Instead, the process reinvokes after min_duration seconds.
+However, the maximum failure limit still applies. A combination of
+daemon=True and max_failures=0 causes a process to 
retry
+indefinitely regardless of exit status. This should be avoided
+for very short-lived processes because of the accumulation of
+checkpointed 

svn commit: r1814961 [5/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Added: 
aurora/site/publish/documentation/0.19.0/getting-started/tutorial/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.19.0/getting-started/tutorial/index.html?rev=1814961=auto
==
--- 
aurora/site/publish/documentation/0.19.0/getting-started/tutorial/index.html 
(added)
+++ 
aurora/site/publish/documentation/0.19.0/getting-started/tutorial/index.html 
Sat Nov 11 16:49:46 2017
@@ -0,0 +1,399 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.19.0
+  (latest)
+  
+  
+0.18.1
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Aurora Tutorial
+
+This tutorial shows how to use the Aurora scheduler to run (and 
printf-debug)
+a hello world program on Mesos. This is the recommended document for new 
Aurora users
+to start getting up to speed on the system.
+
+
+Prerequisite
+The Script
+Aurora Configuration
+Creating the Job
+Watching the Job Run
+Cleanup
+Next Steps
+
+
+Prerequisite
+
+This tutorial assumes you are running Aurora locally 
using Vagrant.
+However, in general the instructions are also applicable to any other
+Aurora installation.
+
+Unless otherwise stated, all commands are to be run from the root of the 
aurora
+repository clone.
+
+The Script
+
+Our hello world application is a simple Python script that 
loops
+forever, displaying the time every few seconds. Copy the code below and
+put it in a file named hello_world.py in the root of your Aurora 
repository clone
+(Note: this directory is the same as /vagrant inside the Vagrant 
VMs).
+
+The script has an intentional bug, which we will explain later on.
+
+
+import time
+
+def main():
+  SLEEP_DELAY = 10
+  # Python experts - ignore 
this blatant bug.
+  for i in xrang(100):
+print("Hello 
world! The time is now: %s. Sleeping for %d secs" % (
+  time.asctime(), SLEEP_DELAY))
+time.sleep(SLEEP_DELAY)
+
+if __name__ == "__main__":
+  main()
+
+
+Aurora Configuration
+
+Once we have our script/program, we need to create a configuration
+file that tells Aurora how to manage and launch our Job. Save the below
+code in the file hello_world.aurora.
+
+
+pkg_path = '/vagrant/hello_world.py'
+
+# we use a trick here to make 
the configuration change with
+# the contents of the file, 
for simplicity.  in a normal setting, packages would be
+# versioned, and the version 
number would be changed in the configuration.
+import hashlib
+with open(pkg_path, 'rb') as f:
+  pkg_checksum = hashlib.md5(f.read()).hexdigest()
+
+# copy hello_world.py into the 
local sandbox
+install = Process(
+  name = 'fetch_package',
+  cmdline = 'cp %s .  echo 
%s  
chmod +x hello_world.py' % (pkg_path, pkg_checksum))
+
+# run the script
+hello_world = Process(
+  name = 'hello_world',
+  cmdline = 'python -u 
hello_world.py')
+
+# describe the task
+hello_world_task = SequentialTask(
+  processes = [install, hello_world],
+  resources = Resources(cpu = 1, ram = 
1*MB, disk=8*MB)
 )
+
+jobs = [
+  Service(cluster = 
'devcluster',
+  environment = 'devel',
+  role = 'www-data',
+  name = 'hello_world',
+  task = hello_world_task)
+]
+
+
+There is a lot going on in that configuration file:
+
+
+From a big picture viewpoint, it first defines two
+Processes. Then it defines a Task that runs the two Processes in the
+order specified in the Task definition, as well as specifying what
+computational and memory resources are available for them.  Finally,
+it defines a Job that will schedule the Task on available and suitable
+machines. This Job is the sole member of a list of Jobs; you can
+specify more than one Job in a config file.
+At the Process level, it specifies how to get your 

svn commit: r1814961 [11/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentatio

[wfarner]

Added: aurora/site/source/documentation/0.19.0/features/custom-executors.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.19.0/features/custom-executors.md?rev=1814961=auto
==
--- aurora/site/source/documentation/0.19.0/features/custom-executors.md (added)
+++ aurora/site/source/documentation/0.19.0/features/custom-executors.md Sat 
Nov 11 16:49:46 2017
@@ -0,0 +1,153 @@
+Custom Executors
+
+
+If the need arises to use a Mesos executor other than the Thermos executor, 
the scheduler can be
+configured to utilize a custom executor by specifying the 
`-custom_executor_config` flag.
+The flag must be set to the path of a valid executor configuration file.
+
+The configuration file must be a valid **JSON array** and contain, at minimum,
+one executor configuration including the name, command and resources fields and
+must be pointed to by the `-custom_executor_config` flag when the scheduler is
+started.
+
+### Array Entry
+
+Property | Description
+---  | -
+executor (required)  | Description of executor.
+task_prefix (required) ) | Prefix given to tasks launched with this executor's 
configuration.
+volume_mounts (optional) | Volumes to be mounted in container running executor.
+
+ executor
+
+Property | Description
+---  | -
+name (required)  | Name of the executor.
+command (required)   | How to run the executor.
+resources (required) | Overhead to use for each executor instance.
+
+ command
+
+Property | Description
+---  | -
+value (required) | The command to execute.
+arguments (optional) | A list of arguments to pass to the command.
+uris (optional)  | List of resources to download into the task sandbox.
+shell (optional) | Run executor via shell.
+
+A note on the command property (from 
[mesos.proto](https://github.com/apache/mesos/blob/master/include/mesos/mesos.proto)):
+
+```
+1) If 'shell == true', the command will be launched via shell
+   (i.e., /bin/sh -c 'value'). The 'value' specified will be
+   treated as the shell command. The 'arguments' will be ignored.
+2) If 'shell == false', the command will be launched by passing
+   arguments to an executable. The 'value' specified will be
+   treated as the filename of the executable. The 'arguments'
+   will be treated as the arguments to the executable. This is
+   similar to how POSIX exec families launch processes (i.e.,
+   execlp(value, arguments(0), arguments(1), ...)).
+```
+
+# uris (list)
+* Follows the [Mesos Fetcher 
schema](http://mesos.apache.org/documentation/latest/fetcher/)
+
+Property | Description
+---  | -
+value (required) | Path to the resource needed in the sandbox.
+executable (optional)| Change resource to be executable via chmod.
+extract (optional)   | Extract files from packed or compressed archives 
into the sandbox.
+cache (optional) | Use caching mechanism provided by Mesos for 
resources.
+
+ resources (list)
+
+Property | Description
+---  | -
+name (required)  | Name of the resource: cpus or mem.
+type (required)  | Type of resource. Should always be SCALAR.
+scalar (required)| Value in float for cpus or int for mem (in MBs)
+
+### volume_mounts (list)
+
+Property | Description
+---  | -
+host_path (required) | Host path to mount inside the container.
+container_path (required)| Path inside the container where `host_path` 
will be mounted.
+mode (required)  | Mode in which to mount the volume, Read-Write 
(RW) or Read-Only (RO).
+
+A sample configuration is as follows:
+
+```json
+[
+{
+  "executor": {
+"name": "myExecutor",
+"command": {
+  "value": "myExecutor.a",
+  "shell": "false",
+  "arguments": [
+"localhost:2181",
+"-verbose",
+"-config myConfiguration.config"
+  ],
+  "uris": [
+{
+  "value": "/dist/myExecutor.a",
+  "executable": true,
+  "extract": false,
+  "cache": true
+},
+{
+  "value": "/home/user/myConfiguration.config",
+  "executable": false,
+  "extract": false,
+  "cache": false
+}
+  ]
+},
+"resources": [
+  {
+"name": "cpus",
+"type": "SCALAR",
+"scalar": {
+  "value": 1.00
+}
+  },
+  {
+"name": 

svn commit: r1814961 [1/14] - in /aurora/site: publish/blog/ publish/blog/aurora-0-19-0-released/ publish/documentation/0.19.0/ publish/documentation/0.19.0/additional-resources/ publish/documentation

[wfarner]

Author: wfarner
Date: Sat Nov 11 16:49:46 2017
New Revision: 1814961

URL: http://svn.apache.org/viewvc?rev=1814961=rev
Log:
Add files missing from 0.19.0 publish

Added:
aurora/site/publish/blog/aurora-0-19-0-released/
aurora/site/publish/blog/aurora-0-19-0-released/index.html
aurora/site/publish/documentation/0.19.0/
aurora/site/publish/documentation/0.19.0/additional-resources/
aurora/site/publish/documentation/0.19.0/additional-resources/presentations/

aurora/site/publish/documentation/0.19.0/additional-resources/presentations/index.html
aurora/site/publish/documentation/0.19.0/additional-resources/tools/

aurora/site/publish/documentation/0.19.0/additional-resources/tools/index.html
aurora/site/publish/documentation/0.19.0/contributing/
aurora/site/publish/documentation/0.19.0/contributing/index.html
aurora/site/publish/documentation/0.19.0/development/
aurora/site/publish/documentation/0.19.0/development/client/
aurora/site/publish/documentation/0.19.0/development/client/index.html
aurora/site/publish/documentation/0.19.0/development/committers-guide/

aurora/site/publish/documentation/0.19.0/development/committers-guide/index.html
aurora/site/publish/documentation/0.19.0/development/db-migration/
aurora/site/publish/documentation/0.19.0/development/db-migration/index.html
aurora/site/publish/documentation/0.19.0/development/design/
aurora/site/publish/documentation/0.19.0/development/design-documents/

aurora/site/publish/documentation/0.19.0/development/design-documents/index.html
aurora/site/publish/documentation/0.19.0/development/design/command-hooks/

aurora/site/publish/documentation/0.19.0/development/design/command-hooks/index.html
aurora/site/publish/documentation/0.19.0/development/scheduler/
aurora/site/publish/documentation/0.19.0/development/scheduler/index.html
aurora/site/publish/documentation/0.19.0/development/thermos/
aurora/site/publish/documentation/0.19.0/development/thermos/index.html
aurora/site/publish/documentation/0.19.0/development/thrift/
aurora/site/publish/documentation/0.19.0/development/thrift/index.html
aurora/site/publish/documentation/0.19.0/development/ui/
aurora/site/publish/documentation/0.19.0/development/ui/index.html
aurora/site/publish/documentation/0.19.0/features/
aurora/site/publish/documentation/0.19.0/features/constraints/
aurora/site/publish/documentation/0.19.0/features/constraints/index.html
aurora/site/publish/documentation/0.19.0/features/containers/
aurora/site/publish/documentation/0.19.0/features/containers/index.html
aurora/site/publish/documentation/0.19.0/features/cron-jobs/
aurora/site/publish/documentation/0.19.0/features/cron-jobs/index.html
aurora/site/publish/documentation/0.19.0/features/custom-executors/

aurora/site/publish/documentation/0.19.0/features/custom-executors/index.html
aurora/site/publish/documentation/0.19.0/features/job-updates/
aurora/site/publish/documentation/0.19.0/features/job-updates/index.html
aurora/site/publish/documentation/0.19.0/features/mesos-fetcher/
aurora/site/publish/documentation/0.19.0/features/mesos-fetcher/index.html
aurora/site/publish/documentation/0.19.0/features/multitenancy/
aurora/site/publish/documentation/0.19.0/features/multitenancy/index.html
aurora/site/publish/documentation/0.19.0/features/resource-isolation/

aurora/site/publish/documentation/0.19.0/features/resource-isolation/index.html
aurora/site/publish/documentation/0.19.0/features/service-discovery/

aurora/site/publish/documentation/0.19.0/features/service-discovery/index.html
aurora/site/publish/documentation/0.19.0/features/services/
aurora/site/publish/documentation/0.19.0/features/services/index.html
aurora/site/publish/documentation/0.19.0/features/sla-metrics/
aurora/site/publish/documentation/0.19.0/features/sla-metrics/index.html
aurora/site/publish/documentation/0.19.0/features/webhooks/
aurora/site/publish/documentation/0.19.0/features/webhooks/index.html
aurora/site/publish/documentation/0.19.0/getting-started/
aurora/site/publish/documentation/0.19.0/getting-started/overview/
aurora/site/publish/documentation/0.19.0/getting-started/overview/index.html
aurora/site/publish/documentation/0.19.0/getting-started/tutorial/
aurora/site/publish/documentation/0.19.0/getting-started/tutorial/index.html
aurora/site/publish/documentation/0.19.0/getting-started/vagrant/
aurora/site/publish/documentation/0.19.0/getting-started/vagrant/index.html
aurora/site/publish/documentation/0.19.0/images/
aurora/site/publish/documentation/0.19.0/images/CPUavailability.png   (with 
props)
aurora/site/publish/documentation/0.19.0/images/CompletedTasks.png   (with 
props)
aurora/site/publish/documentation/0.19.0/images/HelloWorldJob.png   (with 
props)
aurora/site/publish

svn commit: r1814960 [3/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: 
aurora/site/publish/documentation/0.14.0/development/design-documents/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/design-documents/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.14.0/development/design-documents/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.14.0/development/design-documents/index.html
 Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html
 Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/scheduler/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/scheduler/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.14.0/development/scheduler/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/scheduler/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/thermos/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/thermos/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.14.0/development/thermos/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/thermos/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.14.0/development/thrift/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/thrift/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.14.0/development/thrift/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/thrift/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.14.0/development/ui/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/ui/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.14.0/development/ui/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/ui/index.html Sat Nov 
11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/features/constraints/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/features/constraints/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.14.0/features/constraints/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/features/constraints/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/features/containers/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/features/containers/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.14.0/features/containers/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/features/containers/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 

svn commit: r1814960 [7/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: 
aurora/site/publish/documentation/latest/features/multitenancy/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/latest/features/multitenancy/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/latest/features/multitenancy/index.html 
(original)
+++ aurora/site/publish/documentation/latest/features/multitenancy/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   
@@ -129,8 +133,9 @@ assignment of quot
 sysadmin.
 
 The environment component in the job key, serves as a namespace. The values 
for
-environment are validated in the client and the scheduler so as to allow any 
of devel, test,
-production, and any value matching the regular expression 
staging[0-9]*.
+environment are validated in the scheduler. By default allowing any of 
devel, test,
+production, and any value matching the regular expression 
staging[0-9]*. This validation can be
+changed to allow any arbitrary regular expression by setting the scheduler 
option allowed_job_environments.
 
 None of the values imply any difference in the scheduling behavior. 
Conventionally, the
 environment is set so as to indicate a certain level of 
stability in the behavior of the job

Modified: 
aurora/site/publish/documentation/latest/features/resource-isolation/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/latest/features/resource-isolation/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/latest/features/resource-isolation/index.html 
(original)
+++ 
aurora/site/publish/documentation/latest/features/resource-isolation/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/latest/features/service-discovery/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/latest/features/service-discovery/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/latest/features/service-discovery/index.html 
(original)
+++ 
aurora/site/publish/documentation/latest/features/service-discovery/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/latest/features/services/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/latest/features/services/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/latest/features/services/index.html 
(original)
+++ aurora/site/publish/documentation/latest/features/services/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/latest/features/sla-metrics/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/latest/features/sla-metrics/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/latest/features/sla-metrics/index.html 
(original)
+++ aurora/site/publish/documentation/latest/features/sla-metrics/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   
@@ -174,7 +178,7 @@ relevant to uptime calculations. By appl
 transition records, we can build a deterministic downtime trace for every 
given service instance.
 
 A task going through a state transition carries one of three possible SLA 
meanings
-(see https://github.com/apache/aurora/blob/rel/0.18.1/src/main/java/org/apache/aurora/scheduler/sla/SlaAlgorithm.java;>SlaAlgorithm.java
 for
+(see https://github.com/apache/aurora/blob/rel/0.19.0/src/main/java/org/apache/aurora/scheduler/sla/SlaAlgorithm.java;>SlaAlgorithm.java
 for
 sla-to-task-state mapping):
 
 
@@ -224,7 +228,7 @@ metric that helps track the dependency o
 Per job - sla_job_key_mtta_ms
 Per cluster - sla_cluster_mtta_ms
 Per instance size (small, medium, large, x-large, xx-large). Size are 
defined in:
-https://github.com/apache/aurora/blob/rel/0.18.1/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java;>ResourceBag.java
+https://github.com/apache/aurora/blob/rel/0.19.0/src/main/java/org/apache/aurora/scheduler/resources/ResourceBag.java;>ResourceBag.java
 
 
 By CPU:
@@ -266,7 +270,7 @@ for a task.
 Per job - 

svn commit: r1814960 [8/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: aurora/site/publish/sitemap.xml
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/sitemap.xml?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/sitemap.xml (original)
+++ aurora/site/publish/sitemap.xml Sat Nov 11 16:32:40 2017
@@ -2,2302 +2,2494 @@
 http://www.sitemaps.org/schemas/sitemap/0.9;>
   
 http://aurora.apache.org/community/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/downloads/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/documentation/0.6.0-incubating/storage/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/configuration-reference/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/design/command-hooks/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/documentation/0.6.0-incubating/sla/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/client-commands/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/configuration-tutorial/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/cron-jobs/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/documentation/0.6.0-incubating/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/clientv2/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/developing-aurora-scheduler/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/scheduler-storage/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/user-guide/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/client-cluster-configuration/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/developing-aurora-client/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/tutorial/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/contributing/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/resource-isolation/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/documentation/0.6.0-incubating/hooks/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/storage-config/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/deploying-aurora-scheduler/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/documentation/0.6.0-incubating/vagrant/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/monitoring/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/test-resource-generation/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.6.0-incubating/committers/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/documentation/0.7.0-incubating/storage/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.7.0-incubating/configuration-reference/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.7.0-incubating/design/command-hooks/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 http://aurora.apache.org/documentation/0.7.0-incubating/sla/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 
http://aurora.apache.org/documentation/0.7.0-incubating/client-commands/
-2017-11-01T00:00:00+00:00
+2017-11-11T00:00:00+00:00
   
   
 

svn commit: r1814960 [2/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: aurora/site/publish/documentation/0.11.0/build-system/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/build-system/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.11.0/build-system/index.html (original)
+++ aurora/site/publish/documentation/0.11.0/build-system/index.html Sat Nov 11 
16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html
 Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.11.0/client-commands/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/client-commands/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.11.0/client-commands/index.html 
(original)
+++ aurora/site/publish/documentation/0.11.0/client-commands/index.html Sat Nov 
11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.11.0/committers/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/committers/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.11.0/committers/index.html (original)
+++ aurora/site/publish/documentation/0.11.0/committers/index.html Sat Nov 11 
16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/configuration-reference/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/configuration-reference/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.11.0/configuration-reference/index.html 
(original)
+++ aurora/site/publish/documentation/0.11.0/configuration-reference/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html 
(original)
+++ aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.11.0/cron-jobs/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/cron-jobs/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.11.0/cron-jobs/index.html (original)
+++ aurora/site/publish/documentation/0.11.0/cron-jobs/index.html Sat Nov 11 
16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/deploying-aurora-scheduler/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/deploying-aurora-scheduler/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.11.0/deploying-aurora-scheduler/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.11.0/deploying-aurora-scheduler/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/design/command-hooks/index.html
URL: 

svn commit: r1814960 [5/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: 
aurora/site/publish/documentation/0.18.0/operations/security/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/security/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.18.0/operations/security/index.html 
(original)
+++ aurora/site/publish/documentation/0.18.0/operations/security/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.18.0/operations/storage/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/storage/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.18.0/operations/storage/index.html 
(original)
+++ aurora/site/publish/documentation/0.18.0/operations/storage/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/operations/troubleshooting/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/troubleshooting/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.18.0/operations/troubleshooting/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.18.0/operations/troubleshooting/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/operations/upgrades/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/upgrades/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.18.0/operations/upgrades/index.html 
(original)
+++ aurora/site/publish/documentation/0.18.0/operations/upgrades/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/reference/client-cluster-configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/reference/client-cluster-configuration/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.18.0/reference/client-cluster-configuration/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.18.0/reference/client-cluster-configuration/index.html
 Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/reference/client-commands/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/reference/client-commands/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.18.0/reference/client-commands/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.18.0/reference/client-commands/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/reference/client-hooks/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/reference/client-hooks/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.18.0/reference/client-hooks/index.html 
(original)
+++ aurora/site/publish/documentation/0.18.0/reference/client-hooks/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/reference/configuration-best-practices/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/reference/configuration-best-practices/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.18.0/reference/configuration-best-practices/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.18.0/reference/configuration-best-practices/index.html
 Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+

svn commit: r1814960 [9/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md?rev=1814960=1814959=1814960=diff
==
--- aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md (original)
+++ aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md Sat Nov 11 
16:32:40 2017
@@ -11,6 +11,6 @@ tags: Release
 The latest Apache Aurora release, 0.18.1, is now available for
 [download](http://aurora.apache.org/downloads/). This is a patch release to
 update the Shiro library to version 1.2.5.
-  
+
 Full release notes are available in the release
 
[CHANGELOG](https://git-wip-us.apache.org/repos/asf?p=aurora.git=CHANGELOG=rel/0.18.1).

Modified: 
aurora/site/source/documentation/latest/development/committers-guide.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/latest/development/committers-guide.md?rev=1814960=1814959=1814960=diff
==
--- aurora/site/source/documentation/latest/development/committers-guide.md 
(original)
+++ aurora/site/source/documentation/latest/development/committers-guide.md Sat 
Nov 11 16:32:40 2017
@@ -100,3 +100,6 @@ git pre-receive hook explicitly forbiddi
 6. Update the draft email created fom the `release` script in step #5 to 
include the Apache ID's for
 all binding votes and send the [RESULT][VOTE] email to the dev@ mailing list.
 
+7. Update the [Aurora Website](http://aurora.apache.org/) by following the
+[instructions](https://svn.apache.org/repos/asf/aurora/site/README.md) on the 
ASF Aurora SVN repo.
+Remember to add a blog post under source/blog and regenerate the site before 
committing.

Modified: aurora/site/source/documentation/latest/development/db-migration.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/latest/development/db-migration.md?rev=1814960=1814959=1814960=diff
==
--- aurora/site/source/documentation/latest/development/db-migration.md 
(original)
+++ aurora/site/source/documentation/latest/development/db-migration.md Sat Nov 
11 16:32:40 2017
@@ -14,7 +14,7 @@ When adding or altering tables or changi
 
[schema.sql](../../src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql),
 a new
 migration class should be created under the 
org.apache.aurora.scheduler.storage.db.migration
 package. The class should implement the 
[MigrationScript](https://github.com/mybatis/migrations/blob/master/src/main/java/org/apache/ibatis/migration/MigrationScript.java)
-interface (see 
[V001_TestMigration](https://github.com/apache/aurora/blob/rel/0.18.1/src/test/java/org/apache/aurora/scheduler/storage/db/testmigration/V001_TestMigration.java)
+interface (see 
[V001_TestMigration](https://github.com/apache/aurora/blob/rel/0.19.0/src/test/java/org/apache/aurora/scheduler/storage/db/testmigration/V001_TestMigration.java)
 as an example). The upgrade and downgrade scripts are defined in this class. 
When restoring a
 snapshot the list of migrations on the classpath is compared to the list of 
applied changes in the
 DB. Any changes that have not yet been applied are executed and their 
downgrade script is stored

Modified: aurora/site/source/documentation/latest/development/thrift.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/latest/development/thrift.md?rev=1814960=1814959=1814960=diff
==
--- aurora/site/source/documentation/latest/development/thrift.md (original)
+++ aurora/site/source/documentation/latest/development/thrift.md Sat Nov 11 
16:32:40 2017
@@ -6,7 +6,7 @@ client/server RPC protocol as well as fo
 correctly handling additions and renames of the existing members, field 
removals must be done
 carefully to ensure backwards compatibility and provide predictable 
deprecation cycle. This
 document describes general guidelines for making Thrift schema changes to the 
existing fields in
-[api.thrift](https://github.com/apache/aurora/blob/rel/0.18.1/api/src/main/thrift/org/apache/aurora/gen/api.thrift).
+[api.thrift](https://github.com/apache/aurora/blob/rel/0.19.0/api/src/main/thrift/org/apache/aurora/gen/api.thrift).
 
 It is highly recommended to go through the
 [Thrift: The Missing 
Guide](http://diwakergupta.github.io/thrift-missing-guide/) first to refresh on
@@ -33,7 +33,7 @@ communicate with scheduler/client from v
 * Add a new field as an eventual replacement of the old one and implement a 
dual read/write
 anywhere the old field is used. If a thrift struct is mapped in the DB store 
make sure both columns
 are marked as `NOT NULL`
-* Check 
[storage.thrift](https://github.com/apache/aurora/blob/rel/0.18.1/api/src/main/thrift/org/apache/aurora/gen/storage.thrift)
 to see if
+* Check 

svn commit: r1814960 [6/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: aurora/site/publish/documentation/0.6.0-incubating/storage/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.6.0-incubating/storage/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.6.0-incubating/storage/index.html 
(original)
+++ aurora/site/publish/documentation/0.6.0-incubating/storage/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.6.0-incubating/test-resource-generation/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.6.0-incubating/test-resource-generation/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.6.0-incubating/test-resource-generation/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.6.0-incubating/test-resource-generation/index.html
 Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.6.0-incubating/tutorial/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.6.0-incubating/tutorial/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.6.0-incubating/tutorial/index.html 
(original)
+++ aurora/site/publish/documentation/0.6.0-incubating/tutorial/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.6.0-incubating/user-guide/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.6.0-incubating/user-guide/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.6.0-incubating/user-guide/index.html 
(original)
+++ aurora/site/publish/documentation/0.6.0-incubating/user-guide/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.6.0-incubating/vagrant/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.6.0-incubating/vagrant/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.6.0-incubating/vagrant/index.html 
(original)
+++ aurora/site/publish/documentation/0.6.0-incubating/vagrant/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.7.0-incubating/client-cluster-configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.7.0-incubating/client-cluster-configuration/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.7.0-incubating/client-cluster-configuration/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.7.0-incubating/client-cluster-configuration/index.html
 Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.7.0-incubating/client-commands/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.7.0-incubating/client-commands/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.7.0-incubating/client-commands/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.7.0-incubating/client-commands/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.7.0-incubating/committers/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.7.0-incubating/committers/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.7.0-incubating/committers/index.html 
(original)
+++ aurora/site/publish/documentation/0.7.0-incubating/committers/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

svn commit: r1814960 [4/9] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/2015-upcoming-apache-aurora-meetups/ publish/blog/aurora-0-10-0-released/ publish/blog/aurora-0-11-0-released/ p

[wfarner]

Modified: aurora/site/publish/documentation/0.16.0/features/services/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/services/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.16.0/features/services/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/services/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.16.0/features/webhooks/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/webhooks/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.16.0/features/webhooks/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/webhooks/index.html Sat 
Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/getting-started/overview/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/getting-started/overview/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.16.0/getting-started/overview/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.16.0/getting-started/overview/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/getting-started/tutorial/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/getting-started/tutorial/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.16.0/getting-started/tutorial/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.16.0/getting-started/tutorial/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/getting-started/vagrant/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/getting-started/vagrant/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.16.0/getting-started/vagrant/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/getting-started/vagrant/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.16.0/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/index.html?rev=1814960=1814959=1814960=diff
==
--- aurora/site/publish/documentation/0.16.0/index.html (original)
+++ aurora/site/publish/documentation/0.16.0/index.html Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/operations/backup-restore/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/operations/backup-restore/index.html?rev=1814960=1814959=1814960=diff
==
--- 
aurora/site/publish/documentation/0.16.0/operations/backup-restore/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.16.0/operations/backup-restore/index.html 
Sat Nov 11 16:32:40 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.19.0
+  (latest)
+  
   
 0.18.1
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/operations/configuration/index.html
URL: 

[aurora] Git Push Summary

[wfarner]

Repository: aurora
Updated Tags:  refs/tags/rel/0.19.0 [created] 3fd641700

svn commit: r23056 - /release/aurora/0.19.0/

[wfarner]

Author: wfarner
Date: Sat Nov 11 05:02:32 2017
New Revision: 23056

Log:
aurora-0.19.0 release

Added:
release/aurora/0.19.0/

svn commit: r23057 - in /release/aurora/0.19.0: apache-aurora-0.19.0.tar.gz apache-aurora-0.19.0.tar.gz.asc apache-aurora-0.19.0.tar.gz.md5 apache-aurora-0.19.0.tar.gz.sha512

[wfarner]

Author: wfarner
Date: Sat Nov 11 05:02:39 2017
New Revision: 23057

Log:
aurora-0.19.0 release

Added:
release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz   (with props)
release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.asc
release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.md5
release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.sha512

Added: release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz
==
Binary file - no diff available.

Propchange: release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz
--
svn:mime-type = application/octet-stream

Added: release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.asc
==
--- release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.asc (added)
+++ release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.asc Sat Nov 11 05:02:39 
2017
@@ -0,0 +1,16 @@
+-BEGIN PGP SIGNATURE-
+
+iQIcBAABCAAGBQJaBoRrAAoJEFYhfmc+mb7zK8cP/ROFNhBjMPwAG7RkPg6JoCag
+30RYr6n9SC3mTYrpmD26dNpJPG/DMXFZ7v06Q4Prd1DOccGzMG43MxDC96ZIsXwq
+euEx6Zd9Dvpa4tmXzkUM1skrJV5M/WVVvMkpF+M3wfEoGivSZjuXanxTTfpcvfoc
+Q/v02G1qVAB4ZcHanO91k8zec5ugG6l0C4Pk4WaBOXt3i8DNbnPpWzCUZxFafwVl
+kC9ftzh4v3dEJyRyLJD70aMnOHPUl/D2BixQU8wAKpAbOJj9jxjYeX0jLD0uev1G
+RLG8dODBc2X6ooT4XHbY3F136zNaVLhLYIrFSuLgA44bNd8tPl+iGapZ4n5af8ki
+5zuVs/R8mFjFZGUDmfomVyZ0O/lNDbpZr9oefK5qoygTpwnJQZ4E2NGlu18+myHT
+McDaYOmOGBFjd6cAwfmtg1tYsUd9T4b+NsFf1QQmWJIgilYnrKlyTqhijlKAS0Bm
+ua+p3D9CC1Fo2MpVymN/YnxOw41caLhVVK4k7yT0n/MtgkQn54P0TWM4DJ1E9dzz
+GksJ+YedIDcFSG1hB6TWtvGCKykAlT2jN0l8W/j8Jp4KEbn5AtzN7VJfTCHJipRe
+uQcArTOnJMNfy7QxyWrOuzCH0QYca/alAjJzUvtARMamxh0MyhbhwCWGAFwHBcAI
+SU7/qiQG5Pjhkloe08tt
+=hAYU
+-END PGP SIGNATURE-

Added: release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.md5
==
--- release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.md5 (added)
+++ release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.md5 Sat Nov 11 05:02:39 
2017
@@ -0,0 +1 @@
+apache-aurora-0.19.0.tar.gz: CE B1 24 EF B6 E5 C6 6D  94 76 3B 7A 43 1C 44 44

Added: release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.sha512
==
--- release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.sha512 (added)
+++ release/aurora/0.19.0/apache-aurora-0.19.0.tar.gz.sha512 Sat Nov 11 
05:02:39 2017
@@ -0,0 +1 @@
+04d8159dcdcb655647099fed451c2a1bf4e5e78f6cf2eb83f065442511c723741675d038bdaf5355d7f5b7f9d1376990c2a61a9b3214c29b7820b454ed4922c6
  apache-aurora-0.19.0.tar.gz

aurora git commit: Add a test for storage durability

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 773d2d6ab -> 9b9b2ee7b


Add a test for storage durability

Reviewed at https://reviews.apache.org/r/63670/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/9b9b2ee7
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/9b9b2ee7
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/9b9b2ee7

Branch: refs/heads/master
Commit: 9b9b2ee7b52e5299494828a0fa7fc793c9364de7
Parents: 773d2d6
Author: Bill Farner 
Authored: Thu Nov 9 09:26:50 2017 -0800
Committer: Bill Farner 
Committed: Thu Nov 9 09:26:50 2017 -0800

--
 .../org/apache/aurora/scheduler/log/Log.java|   2 +-
 .../aurora/scheduler/log/mesos/MesosLog.java|   6 -
 .../aurora/scheduler/app/SchedulerIT.java   |  20 +--
 .../scheduler/log/mesos/MesosLogTest.java   |  14 --
 .../aurora/scheduler/storage/log/FakeLog.java   |  68 
 .../storage/log/NonVolatileStorageTest.java | 158 +++
 6 files changed, 228 insertions(+), 40 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/9b9b2ee7/src/main/java/org/apache/aurora/scheduler/log/Log.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/log/Log.java 
b/src/main/java/org/apache/aurora/scheduler/log/Log.java
index dc77eb4..2a76144 100644
--- a/src/main/java/org/apache/aurora/scheduler/log/Log.java
+++ b/src/main/java/org/apache/aurora/scheduler/log/Log.java
@@ -28,7 +28,7 @@ public interface Log {
   /**
* An opaque ordered handle to a log entry's position in the log stream.
*/
-  interface Position extends Comparable {
+  interface Position {
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/aurora/blob/9b9b2ee7/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
--
diff --git a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java 
b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
index 21855e1..bcdd459 100644
--- a/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
+++ b/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLog.java
@@ -376,12 +376,6 @@ public class MesosLog implements 
org.apache.aurora.scheduler.log.Log {
   Log.Position unwrap() {
 return underlying;
   }
-
-  @Override
-  public int compareTo(Position o) {
-Preconditions.checkArgument(o instanceof LogPosition);
-return underlying.compareTo(((LogPosition) o).underlying);
-  }
 }
 
 private static class LogEntry implements 
org.apache.aurora.scheduler.log.Log.Entry {

http://git-wip-us.apache.org/repos/asf/aurora/blob/9b9b2ee7/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java
--
diff --git a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java 
b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java
index 67a0d5a..4929ecd 100644
--- a/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java
+++ b/src/test/java/org/apache/aurora/scheduler/app/SchedulerIT.java
@@ -22,7 +22,6 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
 import com.google.common.base.Optional;
@@ -272,23 +271,6 @@ public class SchedulerIT extends BaseZooKeeperTest {
 }).get();
   }
 
-  private final AtomicInteger curPosition = new AtomicInteger();
-  private static class IntPosition implements Position {
-private final int pos;
-
-IntPosition(int pos) {
-  this.pos = pos;
-}
-
-@Override
-public int compareTo(Position position) {
-  return pos - ((IntPosition) position).pos;
-}
-  }
-  private IntPosition nextPosition() {
-return new IntPosition(curPosition.incrementAndGet());
-  }
-
   private Iterable toEntries(LogEntry... entries) {
 return Iterables.transform(Arrays.asList(entries),
 entry -> {
@@ -337,7 +319,7 @@ public class SchedulerIT extends BaseZooKeeperTest {
 expect(log.open()).andReturn(logStream);
 
expect(logStream.readAll()).andReturn(recoveredEntries.iterator()).anyTimes();
 streamMatcher.expectTransaction(Op.saveFrameworkId(new 
SaveFrameworkId(FRAMEWORK_ID)))
-.andReturn(nextPosition());
+.andReturn(new Position() { });
 
 CountDownLatch driverStarted = new CountDownLatch(1);
 expect(driver.start()).andAnswer(() -> {

[aurora] Git Push Summary

[wfarner]

Repository: aurora
Updated Tags:  refs/tags/rel/0.19.0-rc0 [created] 4b431bcca

[2/2] aurora git commit: Updating CHANGELOG for 0.19.0 release.

[wfarner]

Updating CHANGELOG for 0.19.0 release.


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/dd6da803
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/dd6da803
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/dd6da803

Branch: refs/heads/master
Commit: dd6da8036510be686fcba4e5ac67395a696d45a5
Parents: d41d335
Author: Bill Farner 
Authored: Tue Nov 7 20:47:30 2017 -0800
Committer: Bill Farner 
Committed: Tue Nov 7 20:47:30 2017 -0800

--
 CHANGELOG | 28 
 1 file changed, 28 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/dd6da803/CHANGELOG
--
diff --git a/CHANGELOG b/CHANGELOG
index ef3e36e..40b0096 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,31 @@
+Aurora 0.19.0
+
+## Bug
+* [AURORA-1953] - Scheduler livelock during startup
+* [AURORA-1944] - Aurora is unable to elect leader after losing ZK for an 
extended period of time
+* [AURORA-1950] - Exceptions in LogStorage.prepare() cause scheduler die 
and stay in DEAD state
+* [AURORA-1955] - thermos should exit on irrecoverable errors to avoid 
zombies
+* [AURORA-1943] - PreemptionVictimFilter does not handle varied 
ResourceTypes
+* [AURORA-1939] - Thermos landing (host) page reports incorrect CPU rates 
when it is busy
+* [AURORA-1945] - Rescinds received but not processed in time before offer 
accept
+
+## Story
+* [AURORA-319] - Allow job environments other than prod, devel, test or 
staging
+* [AURORA-1947] - Improve in-process ZooKeeper support for local 
development
+
+## Sub-task
+* [AURORA-1773] - Asynchronously call webhook endpoint not to block 
EventBus
+
+## Task
+* [AURORA-1951] - UI build broken in gradle 4.2
+* [AURORA-1937] - Add metrics for status updates before switching to V1 
Mesos Driver implementaion
+* [AURORA-1940] - aurora job restart request should be retryable
+* [AURORA-1934] - Add a whitelist for TaskStateChange events in Aurora 
Scheduler WebHooks
+* [AURORA-1931] - Configurable wait period for graceful shutdown
+* [AURORA-1669] - Kill twitter/commons ZK libs when Curator replacements 
are vetted
+* [AURORA-1897] - Remove task length restrictions.
+
+
 Aurora 0.18.0
 

 ## Bug

[1/2] aurora git commit: Incrementing snapshot version to 0.20.0-SNAPSHOT.

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master d41d3357d -> 5dfe51c27


Incrementing snapshot version to 0.20.0-SNAPSHOT.


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/5dfe51c2
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/5dfe51c2
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/5dfe51c2

Branch: refs/heads/master
Commit: 5dfe51c27579a9a5bdb1ae35511af8e96a9bfcc9
Parents: dd6da80
Author: Bill Farner 
Authored: Tue Nov 7 20:47:30 2017 -0800
Committer: Bill Farner 
Committed: Tue Nov 7 20:47:30 2017 -0800

--
 .auroraversion | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/5dfe51c2/.auroraversion
--
diff --git a/.auroraversion b/.auroraversion
index aa2ab03..8bec379 100644
--- a/.auroraversion
+++ b/.auroraversion
@@ -1 +1 @@
-0.19.0-SNAPSHOT
+0.20.0-SNAPSHOT

svn commit: r23006 - in /dev/aurora/0.19.0-rc0: apache-aurora-0.19.0-rc0.tar.gz apache-aurora-0.19.0-rc0.tar.gz.asc apache-aurora-0.19.0-rc0.tar.gz.md5 apache-aurora-0.19.0-rc0.tar.gz.sha512

[wfarner]

Author: wfarner
Date: Wed Nov  8 04:47:59 2017
New Revision: 23006

Log:
aurora-0.19.0 release candidate rel/0.19.0-rc0

Added:
dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz   (with props)
dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.asc
dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.md5
dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.sha512

Added: dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz
==
Binary file - no diff available.

Propchange: dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz
--
svn:mime-type = application/octet-stream

Added: dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.asc
==
--- dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.asc (added)
+++ dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.asc Wed Nov  8 
04:47:59 2017
@@ -0,0 +1,16 @@
+-BEGIN PGP SIGNATURE-
+
+iQIcBAABCAAGBQJaAox0AAoJEFYhfmc+mb7z6nIQAICNiSMlwtsar4mnX9WsOMem
+nmrBn3R7SmKdk8/BNDVUZPPgVlkoUb2vZQuE5/GVB5Nm1suMgK2OqMsAkwU/eKYV
+XQvZpzpc3ctDdO05wKxweQtci9solrQzkVnBA6aYZvDR2ZElYTp++xcZAXFGfeTq
+akLLnGGIuxwMBO1TpnoiBdFKlYypRPieVxrXPwHiVdyic5vgkk679ZKMQUPpuVu1
+MW+8efRlCqMt+EPjk3v6CvXPBt8UWn+GFgZT3eXcdTLbCR0wDylOQe3AIN4qnNvd
+iJhty9Lyz6rkRnz1oHCYEf1cBHsZMSFcoW1+0MeX8yH5psNucP//Zzwn4gMArKJ2
+0vOEu/vgzuGbpFexF9zVkTL3X2pjIyROK9gwN7xD2/qijwpp3Tbt8B5MCY7X8AFY
+p39FjoeNo4Okm6Ebj9XCoEQ0gq7xqpDb7yV7tJ8ctrNxjfKaByYxH3uvEsoNs8SR
+MsL2e9xChFbHdcE+UAOpuC7596SvYaFy1rx/4XerRbp1iJRHyz0CyssmO7fI/1pW
+wgC7+M8GDnhrYMvuaFnc3BBFkfMRglAQSbPD0n/xWhyrsmS425Lzny9ztKUpC/FI
+5LdvOVT++eb7LMu5oGDY0P1Y8lXXChHlcJj3Ebm8vPts8pEowLaAHGSpWXSoC8Jq
+gi4j4c7ORcFaZkIiiRxr
+=6TPd
+-END PGP SIGNATURE-

Added: dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.md5
==
--- dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.md5 (added)
+++ dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.md5 Wed Nov  8 
04:47:59 2017
@@ -0,0 +1,2 @@
+apache-aurora-0.19.0-rc0.tar.gz: A2 5E B7 5D 90 D0 45 E1  BE 49 A1 1E 3E 4C 62
+ 14

Added: dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.sha512
==
--- dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.sha512 (added)
+++ dev/aurora/0.19.0-rc0/apache-aurora-0.19.0-rc0.tar.gz.sha512 Wed Nov  8 
04:47:59 2017
@@ -0,0 +1 @@
+6a6839e95bcceee1ffee3c7a38cada4634ca3b5f3887e99958c8667e439db3af5536995dd00cc54e15214ea3aab9094348ae91d09bd3dcdc405e761489ab3ca8
  apache-aurora-0.19.0-rc0.tar.gz

svn commit: r23005 - /dev/aurora/0.19.0-rc0/

[wfarner]

Author: wfarner
Date: Wed Nov  8 04:47:49 2017
New Revision: 23005

Log:
aurora-0.19.0 release candidate rel/0.19.0-rc0

Added:
dev/aurora/0.19.0-rc0/

aurora git commit: Update release notes in preparation for 0.19.0 release

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 0e3e0dd71 -> d41d3357d


Update release notes in preparation for 0.19.0 release


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/d41d3357
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/d41d3357
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/d41d3357

Branch: refs/heads/master
Commit: d41d3357da9972c01ba1f6c6d3f040551928fe64
Parents: 0e3e0dd
Author: Bill Farner 
Authored: Tue Nov 7 20:47:04 2017 -0800
Committer: Bill Farner 
Committed: Tue Nov 7 20:47:04 2017 -0800

--
 RELEASE-NOTES.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/d41d3357/RELEASE-NOTES.md
--
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 5122c49..1a3cbec 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -1,5 +1,5 @@
-0.19.0 (unreleased)
-===
+0.19.0
+==
 
 ### New/updated:
 

aurora git commit: Use a pair of fields for caching offer resources rather than a Cache

[wfarner]

getOfferResources(offer, 
true)));
+this.nonRevocableResources =
+Suppliers.memoize(() -> bagFromMesosResources(getOfferResources(offer, 
false)));
   }
 
   private static boolean offerHasCpuAndMem(Offer offer) {
@@ -79,8 +76,8 @@ public class HostOffer {
 return nonZeroCpuAndMem;
   }
 
-  public ResourceBag getResourceBag(TierInfo tierInfo) {
-return resourceBagCache.getUnchecked(tierInfo);
+  public ResourceBag getResourceBag(boolean revocable) {
+return revocable ? revocableResources.get() : nonRevocableResources.get();
   }
 
   public Optional getUnavailabilityStart() {

http://git-wip-us.apache.org/repos/asf/aurora/blob/0e3e0dd7/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java 
b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
index 34ed820..727be58 100644
--- a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
+++ b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java
@@ -27,7 +27,6 @@ import com.google.protobuf.ByteString;
 
 import org.apache.aurora.Protobufs;
 import org.apache.aurora.codec.ThriftBinaryCodec;
-import org.apache.aurora.scheduler.TierManager;
 import org.apache.aurora.scheduler.base.JobKeys;
 import org.apache.aurora.scheduler.base.SchedulerException;
 import org.apache.aurora.scheduler.base.Tasks;
@@ -77,7 +76,7 @@ public interface MesosTaskFactory {
* @return A new task.
* @throws SchedulerException If the task could not be encoded.
*/
-  TaskInfo createFrom(IAssignedTask task, Offer offer) throws 
SchedulerException;
+  TaskInfo createFrom(IAssignedTask task, Offer offer, boolean revocable) 
throws SchedulerException;
 
   // TODO(wfarner): Move this class to its own file to reduce visibility to 
package private.
   class MesosTaskFactoryImpl implements MesosTaskFactory {
@@ -100,17 +99,11 @@ public interface MesosTaskFactory {
 static final String TIER_LABEL = AURORA_LABEL_PREFIX + ".tier";
 
 private final ExecutorSettings executorSettings;
-private final TierManager tierManager;
 private final IServerInfo serverInfo;
 
 @Inject
-MesosTaskFactoryImpl(
-ExecutorSettings executorSettings,
-TierManager tierManager,
-IServerInfo serverInfo) {
-
+MesosTaskFactoryImpl(ExecutorSettings executorSettings, IServerInfo 
serverInfo) {
   this.executorSettings = requireNonNull(executorSettings);
-  this.tierManager = requireNonNull(tierManager);
   this.serverInfo = requireNonNull(serverInfo);
 }
 
@@ -151,7 +144,9 @@ public interface MesosTaskFactory {
 }
 
 @Override
-public TaskInfo createFrom(IAssignedTask task, Offer offer) throws 
SchedulerException {
+public TaskInfo createFrom(IAssignedTask task, Offer offer, boolean 
revocable)
+throws SchedulerException {
+
   requireNonNull(task);
   requireNonNull(offer);
 
@@ -171,7 +166,7 @@ public interface MesosTaskFactory {
 offer,
 task,
 executorOverhead,
-tierManager.getTier(task.getTask()));
+revocable);
   } catch (ResourceManager.InsufficientResourcesException e) {
 throw new SchedulerException(e);
   }

http://git-wip-us.apache.org/repos/asf/aurora/blob/0e3e0dd7/src/main/java/org/apache/aurora/scheduler/resources/AcceptedOffer.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/resources/AcceptedOffer.java 
b/src/main/java/org/apache/aurora/scheduler/resources/AcceptedOffer.java
index 291d5c9..e8ea3f2 100644
--- a/src/main/java/org/apache/aurora/scheduler/resources/AcceptedOffer.java
+++ b/src/main/java/org/apache/aurora/scheduler/resources/AcceptedOffer.java
@@ -20,7 +20,6 @@ import java.util.stream.StreamSupport;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 
-import org.apache.aurora.scheduler.TierInfo;
 import org.apache.aurora.scheduler.storage.entities.IAssignedTask;
 import org.apache.mesos.v1.Protos.Offer;
 import org.apache.mesos.v1.Protos.Resource;
@@ -63,7 +62,7 @@ public final class AcceptedOffer {
   Offer offer,
   IAssignedTask task,
   ResourceBag executorOverhead,
-  TierInfo tierInfo) throws ResourceManager.InsufficientResourcesException 
{
+  boolean revocable) throws ResourceManager.InsufficientResourcesException 
{
 
 ImmutableList.Builder taskResources = ImmutableList.builder();
 ImmutableList.Builder executorResources = 
ImmutableList.builder();
@@ -73,13 +72,13 @@ public final class AcceptedOffer {
 .forEach(entry -> {
   ResourceType type = entry.getKey();
   Iterable offerResources = StreamSupport
-  .stream(getOfferResources(offer, tierInfo, 
entry.getKey()

aurora git commit: Fixed issue where saving attributes are not being persisted to log

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 06d25fd4d -> d10165544


Fixed issue where saving attributes are not being persisted to log

A bug was introduced when the old `MemAttributeStore` was revived. Previously,
the `saveHostAttributes` method did not return anything. However, after
migrating to the DB stores, the signature of the interface was changed to return
a `boolean` if the save modified the previous attributes. The new changes
accidentally inverted the order. The `AbstractAttributeStoreTest` did not test
for this scenario so it went unnoticed.

Reviewed at https://reviews.apache.org/r/63521/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/d1016554
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/d1016554
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/d1016554

Branch: refs/heads/master
Commit: d1016554408a0b4d6dc6327de60bffb5db260ea5
Parents: 06d25fd
Author: Jordan Ly 
Authored: Thu Nov 2 14:49:10 2017 -0700
Committer: Bill Farner 
Committed: Thu Nov 2 14:49:10 2017 -0700

--
 .../storage/mem/MemAttributeStore.java  |  2 +-
 .../storage/AbstractAttributeStoreTest.java | 40 +---
 2 files changed, 28 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/d1016554/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java 
b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java
index 483af19..f1a5309 100644
--- 
a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java
+++ 
b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemAttributeStore.java
@@ -48,7 +48,7 @@ class MemAttributeStore implements AttributeStore.Mutable {
 IHostAttributes previous = hostAttributes.put(
 attributes.getHost(),
 merge(attributes, 
Optional.fromNullable(hostAttributes.get(attributes.getHost();
-return attributes.equals(previous);
+return !attributes.equals(previous);
   }
 
   private IHostAttributes merge(IHostAttributes newAttributes, 
Optional previous) {

http://git-wip-us.apache.org/repos/asf/aurora/blob/d1016554/src/test/java/org/apache/aurora/scheduler/storage/AbstractAttributeStoreTest.java
--
diff --git 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractAttributeStoreTest.java
 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractAttributeStoreTest.java
index 34db54b..1ad4680 100644
--- 
a/src/test/java/org/apache/aurora/scheduler/storage/AbstractAttributeStoreTest.java
+++ 
b/src/test/java/org/apache/aurora/scheduler/storage/AbstractAttributeStoreTest.java
@@ -33,6 +33,8 @@ import org.junit.Test;
 
 import static org.apache.aurora.gen.MaintenanceMode.DRAINED;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 public abstract class AbstractAttributeStoreTest {
 
@@ -64,27 +66,39 @@ public abstract class AbstractAttributeStoreTest {
   protected abstract Storage createStorage();
 
   @Test
+  public void testSaveAttributes() {
+assertEquals(Optional.absent(), read(HOST_A));
+
+// Initial save returns true since it changes previous attributes
+assertTrue(insert(HOST_A_ATTRS));
+assertEquals(Optional.of(HOST_A_ATTRS), read(HOST_A));
+
+// Second save returns false since it does not change previous attributes
+assertFalse(insert(HOST_A_ATTRS));
+  }
+
+  @Test
   public void testCrud() {
 assertEquals(Optional.absent(), read(HOST_A));
 assertEquals(ImmutableSet.of(), readAll());
 
-insert(HOST_A_ATTRS);
+assertTrue(insert(HOST_A_ATTRS));
 assertEquals(Optional.of(HOST_A_ATTRS), read(HOST_A));
 assertEquals(ImmutableSet.of(HOST_A_ATTRS), readAll());
 
-insert(HOST_B_ATTRS);
-insert(HOST_B_ATTRS);  // Double insert should be allowed.
+assertTrue(insert(HOST_B_ATTRS));
+assertFalse(insert(HOST_B_ATTRS));  // Double insert should be allowed.
 assertEquals(Optional.of(HOST_B_ATTRS), read(HOST_B));
 assertEquals(ImmutableSet.of(HOST_A_ATTRS, HOST_B_ATTRS), readAll());
 
 IHostAttributes updatedA = IHostAttributes.build(
 HOST_A_ATTRS.newBuilder().setAttributes(ImmutableSet.of(ATTR1, 
ATTR3)));
-insert(updatedA);
+assertTrue(insert(updatedA));
 assertEquals(Optional.of(updatedA), read(HOST_A));
 assertEquals(ImmutableSet.of(updatedA, HOST_B_ATTRS), readAll());
 
 IHostAttributes updatedMode = 

svn commit: r1813982 [19/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Added: aurora/site/source/documentation/0.18.1/operations/configuration.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.18.1/operations/configuration.md?rev=1813982=auto
==
--- aurora/site/source/documentation/0.18.1/operations/configuration.md (added)
+++ aurora/site/source/documentation/0.18.1/operations/configuration.md Wed Nov 
 1 18:39:52 2017
@@ -0,0 +1,339 @@
+# Scheduler Configuration
+
+The Aurora scheduler can take a variety of configuration options through 
command-line arguments.
+Examples are available under `examples/scheduler/`. For a list of available 
Aurora flags and their
+documentation, see [Scheduler Configuration 
Reference](../../reference/scheduler-configuration/).
+
+
+## A Note on Configuration
+Like Mesos, Aurora uses command-line flags for runtime configuration. As such 
the Aurora
+"configuration file" is typically a `scheduler.sh` shell script of the form.
+
+#!/bin/bash
+AURORA_HOME=/usr/local/aurora-scheduler
+
+# Flags controlling the JVM.
+JAVA_OPTS=(
+  -Xmx2g
+  -Xms2g
+  # GC tuning, etc.
+)
+
+# Flags controlling the scheduler.
+AURORA_FLAGS=(
+  # Port for client RPCs and the web UI
+  -http_port=8081
+  # Log configuration, etc.
+)
+
+# Environment variables controlling libmesos
+export JAVA_HOME=...
+export GLOG_v=1
+export LIBPROCESS_PORT=8083
+export LIBPROCESS_IP=192.168.33.7
+
+JAVA_OPTS="${JAVA_OPTS[*]}" exec "$AURORA_HOME/bin/aurora-scheduler" 
"${AURORA_FLAGS[@]}"
+
+That way Aurora's current flags are visible in `ps` and in the `/vars` admin 
endpoint.
+
+
+## JVM Configuration
+
+JVM settings are dependent on your environment and cluster size. They might 
require
+custom tuning. As a starting point, we recommend:
+
+* Ensure the initial (`-Xms`) and maximum (`-Xmx`) heap size are idential to 
prevent heap resizing
+  at runtime.
+* Either `-XX:+UseConcMarkSweepGC` or `-XX:+UseG1GC 
-XX:+UseStringDeduplication` are
+  sane defaults for the garbage collector.
+* `-Djava.net.preferIPv4Stack=true` makes sense in most cases as well.
+
+
+## Network Configuration
+
+By default, Aurora binds to all interfaces and auto-discovers its hostname. To 
reduce ambiguity
+it helps to hardcode them though:
+
+-http_port=8081
+-ip=192.168.33.7
+-hostname="aurora1.us-east1.example.org"
+
+Two environment variables control the ip and port for the communication with 
the Mesos master
+and for the replicated log used by Aurora:
+
+export LIBPROCESS_PORT=8083
+export LIBPROCESS_IP=192.168.33.7
+
+It is important that those can be reached from all Mesos master and Aurora 
scheduler instances.
+
+
+## Replicated Log Configuration
+
+Aurora schedulers use ZooKeeper to discover log replicas and elect a leader. 
Only one scheduler is
+leader at a given time - the other schedulers follow log writes and prepare to 
take over as leader
+but do not communicate with the Mesos master. Either 3 or 5 schedulers are 
recommended in a
+production deployment depending on failure tolerance and they must have 
persistent storage.
+
+Below is a summary of scheduler storage configuration flags that either don't 
have default values
+or require attention before deploying in a production environment.
+
+### `-native_log_quorum_size`
+Defines the Mesos replicated log quorum size. In a cluster with `N` 
schedulers, the flag
+`-native_log_quorum_size` should be set to `floor(N/2) + 1`. So in a cluster 
with 1 scheduler
+it should be set to `1`, in a cluster with 3 it should be set to `2`, and in a 
cluster of 5 it
+should be set to `3`.
+
+  Number of schedulers (N) | ```-native_log_quorum_size``` setting 
(```floor(N/2) + 1```)
+   | 
-
+  1| 1
+  3| 2
+  5| 3
+  7| 4
+
+*Incorrectly setting this flag will cause data corruption to occur!*
+
+### `-native_log_file_path`
+Location of the Mesos replicated log files. For optimal and consistent 
performance, consider
+allocating a dedicated disk (preferably SSD) for the replicated log. Ensure 
that this disk is not
+used by anything else (e.g. no process logging) and in particular that it is a 
real disk
+and not just a partition.
+
+Even when a dedicated disk is used, switching from `CFQ` to `deadline` I/O 
scheduler of Linux kernel
+can furthermore help with storage performance in Aurora ([see this ticket for 
details](https://issues.apache.org/jira/browse/AURORA-1211)).
+
+### `-native_log_zk_group_path`
+ZooKeeper path used for Mesos replicated log quorum discovery.
+
+See 
[code](https://github.com/apache/aurora/blob/rel/0.18.1/src/main/java/org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule.java)
 for
+other available Mesos replicated log configuration options and 

svn commit: r1813982 [21/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Added: 
aurora/site/source/documentation/0.18.1/reference/observer-configuration.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.18.1/reference/observer-configuration.md?rev=1813982=auto
==
--- aurora/site/source/documentation/0.18.1/reference/observer-configuration.md 
(added)
+++ aurora/site/source/documentation/0.18.1/reference/observer-configuration.md 
Wed Nov  1 18:39:52 2017
@@ -0,0 +1,89 @@
+# Observer Configuration Reference
+
+The Aurora/Thermos observer can take a variety of configuration options 
through command-line arguments.
+A list of the available options can be seen by running `thermos_observer 
--long-help`.
+
+Please refer to the [Operator Configuration 
Guide](../../operations/configuration/) for details on how
+to properly set the most important options.
+
+```
+$ thermos_observer.pex --long-help
+Options:
+  -h, --help, --short-help
+show this help message and exit.
+  --long-help   show options from all registered modules, not just the
+__main__ module.
+  --mesos-root=MESOS_ROOT
+The mesos root directory to search for Thermos
+executor sandboxes [default: /var/lib/mesos]
+  --ip=IP   The IP address the observer will bind to. [default:
+0.0.0.0]
+  --port=PORT   The port on which the observer should listen.
+[default: 1338]
+  --polling_interval_secs=POLLING_INTERVAL_SECS
+The number of seconds between observer refresh
+attempts. [default: 5]
+  --task_process_collection_interval_secs=TASK_PROCESS_COLLECTION_INTERVAL_SECS
+The number of seconds between per task process
+resource collections. [default: 20]
+  --task_disk_collection_interval_secs=TASK_DISK_COLLECTION_INTERVAL_SECS
+The number of seconds between per task disk resource
+collections. [default: 60]
+
+  From module twitter.common.app:
+--app_daemonize Daemonize this application. [default: False]
+--app_profile_output=FILENAME
+Dump the profiling output to a binary profiling
+format. [default: None]
+--app_daemon_stderr=TWITTER_COMMON_APP_DAEMON_STDERR
+Direct this app's stderr to this file if daemonized.
+[default: /dev/null]
+--app_debug Print extra debugging information during application
+initialization. [default: False]
+--app_rc_filename   Print the filename for the rc file and quit. [default:
+False]
+--app_daemon_stdout=TWITTER_COMMON_APP_DAEMON_STDOUT
+Direct this app's stdout to this file if daemonized.
+[default: /dev/null]
+--app_profiling Run profiler on the code while it runs.  Note this can
+cause slowdowns. [default: False]
+--app_ignore_rc_file
+Ignore default arguments from the rc file. [default:
+False]
+--app_pidfile=TWITTER_COMMON_APP_PIDFILE
+The pidfile to use if --app_daemonize is specified.
+[default: None]
+
+  From module twitter.common.log.options:
+--log_to_stdout=[scheme:]LEVEL
+OBSOLETE - legacy flag, use --log_to_stderr instead.
+[default: ERROR]
+--log_to_stderr=[scheme:]LEVEL
+The level at which logging to stderr [default: ERROR].
+Takes either LEVEL or scheme:LEVEL, where LEVEL is one
+of ['INFO', 'NONE', 'WARN', 'ERROR', 'DEBUG', 'FATAL']
+and scheme is one of ['google', 'plain'].
+--log_to_disk=[scheme:]LEVEL
+The level at which logging to disk [default: INFO].
+Takes either LEVEL or scheme:LEVEL, where LEVEL is one
+of ['INFO', 'NONE', 'WARN', 'ERROR', 'DEBUG', 'FATAL']
+and scheme is one of ['google', 'plain'].
+--log_dir=DIR   The directory into which log files will be generated
+[default: /var/tmp].
+--log_simpleWrite a single log file rather than one log file per
+log level [default: False].
+--log_to_scribe=[scheme:]LEVEL
+The level at which logging to scribe [default: NONE].
+Takes either LEVEL or scheme:LEVEL, where LEVEL is one
+of ['INFO', 'NONE', 'WARN', 'ERROR', 'DEBUG', 'FATAL']
+and scheme is one of ['google', 'plain'].
+--scribe_category=CATEGORY
+The category used 

svn commit: r1813982 [18/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Added: aurora/site/source/documentation/0.18.1/features/resource-isolation.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.18.1/features/resource-isolation.md?rev=1813982=auto
==
--- aurora/site/source/documentation/0.18.1/features/resource-isolation.md 
(added)
+++ aurora/site/source/documentation/0.18.1/features/resource-isolation.md Wed 
Nov  1 18:39:52 2017
@@ -0,0 +1,181 @@
+Resources Isolation and Sizing
+==
+
+This document assumes Aurora and Mesos have been configured
+using our [recommended resource isolation 
settings](../../operations/configuration/#resource-isolation).
+
+- [Isolation](#isolation)
+- [Sizing](#sizing)
+- [Oversubscription](#oversubscription)
+
+
+Isolation
+-
+
+Aurora is a multi-tenant system; a single software instance runs on a
+server, serving multiple clients/tenants. To share resources among
+tenants, it leverages Mesos for isolation of:
+
+* CPU
+* GPU
+* memory
+* disk space
+* ports
+
+CPU is a soft limit, and handled differently from memory and disk space.
+Too low a CPU value results in throttling your application and
+slowing it down. Memory and disk space are both hard limits; when your
+application goes over these values, it's killed.
+
+### CPU Isolation
+
+Mesos can be configured to use a quota based CPU scheduler (the *Completely*
+*Fair Scheduler*) to provide consistent and predictable performance.
+This is effectively a guarantee of resources -- you receive at least what
+you requested, but also no more than you've requested.
+
+The scheduler gives applications a CPU quota for every 100 ms interval.
+When an application uses its quota for an interval, it is throttled for
+the rest of the 100 ms. Usage resets for each interval and unused
+quota does not carry over.
+
+For example, an application specifying 4.0 CPU has access to 400 ms of
+CPU time every 100 ms. This CPU quota can be used in different ways,
+depending on the application and available resources. Consider the
+scenarios shown in this diagram.
+
+![CPU Availability](../images/CPUavailability.png)
+
+* *Scenario A*: the application can use up to 4 cores continuously for
+every 100 ms interval. It is never throttled and starts processing
+new requests immediately.
+
+* *Scenario B* : the application uses up to 8 cores (depending on
+availability) but is throttled after 50 ms. The CPU quota resets at the
+start of each new 100 ms interval.
+
+* *Scenario C* : is like Scenario A, but there is a garbage collection
+event in the second interval that consumes all CPU quota. The
+application throttles for the remaining 75 ms of that interval and
+cannot service requests until the next interval. In this example, the
+garbage collection finished in one interval but, depending on how much
+garbage needs collecting, it may take more than one interval and further
+delay service of requests.
+
+*Technical Note*: Mesos considers logical cores, also known as
+hyperthreading or SMT cores, as the unit of CPU.
+
+### Memory Isolation
+
+Mesos uses dedicated memory allocation. Your application always has
+access to the amount of memory specified in your configuration. The
+application's memory use is defined as the sum of the resident set size
+(RSS) of all processes in a shard. Each shard is considered
+independently.
+
+In other words, say you specified a memory size of 10GB. Each shard
+would receive 10GB of memory. If an individual shard's memory demands
+exceed 10GB, that shard is killed, but the other shards continue
+working.
+
+*Technical note*: Total memory size is not enforced at allocation time,
+so your application can request more than its allocation without getting
+an ENOMEM. However, it will be killed shortly after.
+
+### Disk Space
+
+Disk space used by your application is defined as the sum of the files'
+disk space in your application's directory, including the `stdout` and
+`stderr` logged from your application. Each shard is considered
+independently. You should use off-node storage for your application's
+data whenever possible.
+
+In other words, say you specified disk space size of 100MB. Each shard
+would receive 100MB of disk space. If an individual shard's disk space
+demands exceed 100MB, that shard is killed, but the other shards
+continue working.
+
+After your application finishes running, its allocated disk space is
+reclaimed. Thus, your job's final action should move any disk content
+that you want to keep, such as logs, to your home file system or other
+less transitory storage. Disk reclamation takes place an undefined
+period after the application finish time; until then, the disk contents
+are still available but you shouldn't count on them being so.
+
+*Technical note* : Disk space is not enforced at write so your
+application can write above its quota without getting an ENOSPC, but it
+will be killed shortly after. This is subject to change.

svn commit: r1813982 [6/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Added: 
aurora/site/publish/documentation/0.18.1/development/design/command-hooks/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/development/design/command-hooks/index.html?rev=1813982=auto
==
--- 
aurora/site/publish/documentation/0.18.1/development/design/command-hooks/index.html
 (added)
+++ 
aurora/site/publish/documentation/0.18.1/development/design/command-hooks/index.html
 Wed Nov  1 18:39:52 2017
@@ -0,0 +1,241 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.18.1
+  (latest)
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Command Hooks for the Aurora 
Client
+
+Introduction/Motivation
+
+Weve got hooks in the client that surround API calls. These are
+pretty awkward, because they dont correlate with user actions. For
+example, suppose we wanted a policy that said users werent allowed to
+kill all instances of a production job at once.
+
+Right now, all that we could hook would be the killJob api 
call. But
+kill (at least in newer versions of the client) normally runs in
+batches. If a user called killall, what we would see on the API level
+is a series of killJob calls, each of which specified a batch of
+instances. We woudnt be able to distinguish between really killing
+all instances of a job (which is forbidden under this policy), and
+carefully killing in batches (which is permitted.) In each case, the
+hook would just see a series of API calls, and couldnt find out what
+the actual command being executed was!
+
+For most policy enforcement, what we really want to be able to do is
+look at and vet the commands that a user is performing, not the API
+calls that the client uses to implement those commands.
+
+So I propose that we add a new kind of hooks, which surround noun/verb
+commands. A hook will register itself to handle a collection of (noun,
+verb) pairs. Whenever any of those noun/verb commands are invoked, the
+hooks methods will be called around the execution of the verb. A
+pre-hook will have the ability to reject a command, preventing the
+verb from being executed.
+
+Registering Hooks
+
+These hooks will be registered via configuration plugins. A configuration 
plugin
+can register hooks using an API. Hooks registered this way are, effectively,
+hardwired into the client executable.
+
+The order of execution of hooks is unspecified: they may be called in
+any order. There is no way to guarantee that one hook will execute
+before some other hook.
+
+Global Hooks
+
+Commands registered by the python call are called global hooks,
+because they will run for all configurations, whether or not they
+specify any hooks in the configuration file.
+
+In the implementation, hooks are registered in the module
+apache.aurora.client.cli.command_hooks, using the class
+GlobalCommandHookRegistry. A global hook can be registered by 
calling
+GlobalCommandHookRegistry.register_command_hook in a 
configuration plugin.
+
+The API
+class CommandHook(object)
+  @property
+  def name(self):
+"""Returns a name for the hook."
+
+  def get_nouns(self):
+"""Return the nouns that have verbs that should 
invoke this hook."""
+
+  def get_verbs(self, noun):
+"""Return the verbs for a particular noun that 
should invoke his hook."""
+
+  @abstractmethod
+  def pre_command(self, noun, verb, context, commandline):
+"""Execute a hook before invoking a 
verb.
+* noun: the noun being 
invoked.
+* verb: the verb being 
invoked.
+* context: the context object that will 
be used to invoke the verb.
+  The options object will be initialized 
before calling the hook
+* commandline: the original argv 
collection used to invoke 

svn commit: r1813982 [11/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Added: 
aurora/site/publish/documentation/0.18.1/reference/client-commands/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/reference/client-commands/index.html?rev=1813982=auto
==
--- 
aurora/site/publish/documentation/0.18.1/reference/client-commands/index.html 
(added)
+++ 
aurora/site/publish/documentation/0.18.1/reference/client-commands/index.html 
Wed Nov  1 18:39:52 2017
@@ -0,0 +1,481 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.18.1
+  (latest)
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Aurora Client Commands
+
+
+Introduction
+Cluster Configuration
+Job Keys
+Modifying Aurora Client 
Commands
+Regular Jobs
+
+
+Creating and Running a Job
+Running a Command On a 
Running Job
+Killing a Job
+Adding Instances
+Updating a Job
+
+
+Coordinated job updates
+
+Renaming a Job
+Restarting Jobs
+
+Cron Jobs
+Comparing Jobs
+Viewing/Examining Jobs
+
+
+Listing Jobs
+Inspecting a Job
+Versions
+Checking Your Quota
+Finding a Job on Web UI
+Getting Job Status
+Opening the Web UI
+SSHing to a Specific Task 
Machine
+SCPing with Specific Task 
Machines
+Templating Command 
Arguments
+
+
+
+Introduction
+
+Once you have written an .aurora configuration file that 
describes
+your Job and its parameters and functionality, you interact with Aurora
+using Aurora Client commands. This document describes all of these commands
+and how and when to use them. All Aurora Client commands start with
+aurora, followed by the name of the specific command and its
+arguments.
+
+Job keys are a very common argument to Aurora commands, as well as 
the
+gateway to useful information about a Job. Before using Aurora, you
+should read the next section which describes them in detail. The section
+after that briefly describes how you can modify the behavior of certain
+Aurora Client commands, linking to a detailed document about how to do
+that.
+
+This is followed by the Regular Jobs section, which describes the basic
+Client commands for creating, running, and manipulating Aurora Jobs.
+After that are sections on Comparing Jobs and Viewing/Examining Jobs. In
+other words, various commands for getting information and metadata about
+Aurora Jobs.
+
+Cluster Configuration
+
+The client must be able to find a configuration file that specifies 
available clusters. This file
+declares shorthand names for clusters, which are in turn referenced by job 
configuration files
+and client commands.
+
+The client will load at most two configuration files, making both of their 
defined clusters
+available. The first is intended to be a system-installed cluster, using the 
path specified in
+the environment variable AURORA_CONFIG_ROOT, defaulting to 
/etc/aurora/clusters.json if the
+environment variable is not set. The second is a user-installed file, located 
at
+~/.aurora/clusters.json.
+
+For more details on cluster configuration see the
+Client Cluster Configuration 
documentation.
+
+Job Keys
+
+A job key is a unique system-wide identifier for an Aurora-managed
+Job, for example cluster1/web-team/test/experiment204. It is a 
4-tuple
+consisting of, in order, cluster, role, 
environment, and
+jobname, separated by /s. Cluster is the name of an Aurora
+cluster. Role is the Unix service account under which the Job
+runs. Environment is a namespace component like devel, 
test,
+prod, or stagingN. Jobname is the Jobs 
name.
+
+The combination of all four values uniquely specifies the Job. If any
+one value is different from that of another job key, the two job keys
+refer to different Jobs. For example, job key
+cluster1/tyg/prod/workhorse is different from

svn commit: r1813982 [5/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Modified: 
aurora/site/publish/documentation/0.18.0/getting-started/overview/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/getting-started/overview/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.18.0/getting-started/overview/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.18.0/getting-started/overview/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/getting-started/tutorial/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/getting-started/tutorial/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.18.0/getting-started/tutorial/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.18.0/getting-started/tutorial/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/getting-started/vagrant/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/getting-started/vagrant/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.18.0/getting-started/vagrant/index.html 
(original)
+++ aurora/site/publish/documentation/0.18.0/getting-started/vagrant/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.18.0/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.18.0/index.html (original)
+++ aurora/site/publish/documentation/0.18.0/index.html Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/operations/backup-restore/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/backup-restore/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.18.0/operations/backup-restore/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.18.0/operations/backup-restore/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/operations/configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/configuration/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.18.0/operations/configuration/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.18.0/operations/configuration/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/operations/installation/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/installation/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.18.0/operations/installation/index.html 
(original)
+++ aurora/site/publish/documentation/0.18.0/operations/installation/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/operations/monitoring/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.0/operations/monitoring/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.18.0/operations/monitoring/index.html 
(original)
+++ aurora/site/publish/documentation/0.18.0/operations/monitoring/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.18.0/operations/security/index.html
URL: 

svn commit: r1813982 [8/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Added: aurora/site/publish/documentation/0.18.1/features/sla-metrics/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/features/sla-metrics/index.html?rev=1813982=auto
==
--- aurora/site/publish/documentation/0.18.1/features/sla-metrics/index.html 
(added)
+++ aurora/site/publish/documentation/0.18.1/features/sla-metrics/index.html 
Wed Nov  1 18:39:52 2017
@@ -0,0 +1,376 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.18.1
+  (latest)
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Aurora SLA Measurement
+
+
+Overview
+Metric Details
+
+
+Platform Uptime
+Job Uptime
+Median Time To Assigned 
(MTTA)
+Median Time To Starting 
(MTTS)
+Median Time To Running 
(MTTR)
+
+Limitations
+
+
+Overview
+
+The primary goal of the feature is collection and monitoring of Aurora job 
SLA (Service Level
+Agreements) metrics that defining a contractual relationship between the 
Aurora/Mesos platform
+and hosted services.
+
+The Aurora SLA feature is by default only enabled for service (non-cron)
+production jobs (production=True in your 
.aurora config). It can be enabled for
+non-production services by an operator via the scheduler command line flag 
-sla_non_prod_metrics.
+
+Counters that track SLA measurements are computed periodically within the 
scheduler.
+The individual instance metrics are refreshed every minute (configurable via
+sla_stat_refresh_interval). The instance counters are 
subsequently aggregated by
+relevant grouping types before exporting to scheduler /vars 
endpoint (when using vagrant
+that would be http://192.168.33.7:8081/vars)
+
+Metric Details
+
+Platform Uptime
+
+Aggregate amount of time a job spends in a non-runnable state due to 
platform unavailability
+or scheduling delays. This metric tracks Aurora/Mesos uptime performance and 
reflects on any
+system-caused downtime events (tasks LOST or DRAINED). Any user-initiated task 
kills/restarts
+will not degrade this metric.
+
+Collection scope:
+
+
+Per job - sla_job_key_platform_uptime_percent
+Per cluster - sla_cluster_platform_uptime_percent
+
+
+Units: percent
+
+A fault in the task environment may cause the Aurora/Mesos to have 
different views on the task state
+or lose track of the task existence. In such cases, the service task is marked 
as LOST and
+rescheduled by Aurora. For example, this may happen when the task stays in 
ASSIGNED or STARTING
+for too long or the Mesos agent becomes unhealthy (or disappears completely). 
The time between
+task entering LOST and its replacement reaching RUNNING state is counted 
towards platform downtime.
+
+Another example of a platform downtime event is the administrator-requested 
task rescheduling. This
+happens during planned Mesos agent maintenance when all agent tasks are marked 
as DRAINED and
+rescheduled elsewhere.
+
+To accurately calculate Platform Uptime, we must separate platform incurred 
downtime from user
+actions that put a service instance in a non-operational state. It is simpler 
to isolate
+user-incurred downtime and treat all other downtime as platform incurred.
+
+Currently, a user can cause a healthy service (task) downtime in only two 
ways: via killTasks
+or restartShards RPCs. For both, their affected tasks leave an 
audit state transition trail
+relevant to uptime calculations. By applying a special SLA 
meaning to exposed task state
+transition records, we can build a deterministic downtime trace for every 
given service instance.
+
+A task going through a state transition carries one of three possible SLA 
meanings
+(see 

svn commit: r1813982 [2/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Modified: aurora/site/publish/documentation/0.10.0/vagrant/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.10.0/vagrant/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.10.0/vagrant/index.html (original)
+++ aurora/site/publish/documentation/0.10.0/vagrant/index.html Wed Nov  1 
18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.11.0/build-system/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/build-system/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.11.0/build-system/index.html (original)
+++ aurora/site/publish/documentation/0.11.0/build-system/index.html Wed Nov  1 
18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.11.0/client-cluster-configuration/index.html
 Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.11.0/client-commands/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/client-commands/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.11.0/client-commands/index.html 
(original)
+++ aurora/site/publish/documentation/0.11.0/client-commands/index.html Wed Nov 
 1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.11.0/committers/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/committers/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.11.0/committers/index.html (original)
+++ aurora/site/publish/documentation/0.11.0/committers/index.html Wed Nov  1 
18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/configuration-reference/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/configuration-reference/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.11.0/configuration-reference/index.html 
(original)
+++ aurora/site/publish/documentation/0.11.0/configuration-reference/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html 
(original)
+++ aurora/site/publish/documentation/0.11.0/configuration-tutorial/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.11.0/cron-jobs/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/cron-jobs/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.11.0/cron-jobs/index.html (original)
+++ aurora/site/publish/documentation/0.11.0/cron-jobs/index.html Wed Nov  1 
18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.11.0/deploying-aurora-scheduler/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.11.0/deploying-aurora-scheduler/index.html?rev=1813982=1813981=1813982=diff

svn commit: r1813982 [17/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Added: aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md?rev=1813982=auto
==
--- aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md (added)
+++ aurora/site/source/blog/2017-11-01-aurora-0-18-1-released.md Wed Nov  1 
18:39:52 2017
@@ -0,0 +1,16 @@
+---
+layout: post
+title: 0.18.1 Released
+permalink: /blog/aurora-0-18-1-released/
+published: true
+post_author:
+  display_name: Bill Farner
+tags: Release
+---
+
+The latest Apache Aurora release, 0.18.1, is now available for
+[download](http://aurora.apache.org/downloads/). This is a patch release to
+update the Shiro library to version 1.2.5.
+  
+Full release notes are available in the release
+[CHANGELOG](https://git-wip-us.apache.org/repos/asf?p=aurora.git=CHANGELOG=rel/0.18.1).

Added: 
aurora/site/source/documentation/0.18.1/additional-resources/presentations.md
URL: 
http://svn.apache.org/viewvc/aurora/site/source/documentation/0.18.1/additional-resources/presentations.md?rev=1813982=auto
==
--- 
aurora/site/source/documentation/0.18.1/additional-resources/presentations.md 
(added)
+++ 
aurora/site/source/documentation/0.18.1/additional-resources/presentations.md 
Wed Nov  1 18:39:52 2017
@@ -0,0 +1,80 @@
+# Apache Aurora Presentations
+Video and slides from presentations and panel discussions about Apache Aurora.
+
+_(Listed in date descending order)_
+
+
+
+   
+   
+   https://www.youtube.com/watch?v=q5iIqhaCJ_o;>Mesos  Aurora on a 
Small Scale (Video)
+   Presented by Florian Pfeiffer
+   October 8, 2015 at http://events.linuxfoundation.org/events/archive/2015/mesoscon-europe;>#MesosCon
 Europe 2015
+   
+   
+   
+   https://www.youtube.com/watch?v=tZ0-SISvCis;>SLA Aware Maintenance for 
Operators (Video)
+   Presented by Joe Smith
+   October 8, 2015 at http://events.linuxfoundation.org/events/archive/2015/mesoscon-europe;>#MesosCon
 Europe 2015
+   
+   
+   
+   https://www.youtube.com/watch?v=y1hi7K1lPkk;>Shipping Code with Aurora 
(Video)
+   Presented by Bill Farner
+   August 20, 2015 at http://events.linuxfoundation.org/events/archive/2015/mesoscon;>#MesosCon 
2015
+   
+   
+   
+   https://www.youtube.com/watch?v=nNrh-gdu9m4;>Twitter’s Production 
Scale: Mesos and Aurora Operations (Video)
+   Presented by Joe Smith
+   August 20, 2015 at http://events.linuxfoundation.org/events/archive/2015/mesoscon;>#MesosCon 
2015
+   
+   
+   
+   https://www.youtube.com/watch?v=yXkOgnyK4Hw;>>From Monolith to 
Microservices w/ Aurora (Video)
+   Presented by Thanos Baskous, Tony Dong, Dobromir Montauk
+   April 30, 2015 at http://www.meetup.com/Bay-Area-Apache-Aurora-Users-Group/events/221219480/;>Bay
 Area Apache Aurora Users Group
+   
+   
+   
+   https://www.youtube.com/watch?v=1XYJGX_qZVU;>Aurora + Mesos in Practice 
at Twitter (Video)
+   Presented by Bill Farner
+   March 07, 2015 at http://www.bigeng.io/aurora-mesos-in-practice-at-twitter;>Bigcommerce 
TechTalk
+   
+   
+   
+   http://www.slideshare.net/zembutsu/apache-aurora-introduction-and-tutorial-osc15tk;>Apache
 Auroraの始めかた (Slides)
+   Presented by Masahito Zembutsu
+   February 28, 2015 at http://www.ospn.jp/osc2015-spring/;>Open Source Conference 2015 Tokyo 
Spring
+   
+   
+   
+   https://www.youtube.com/watch?v=2Jsj0zFdRlg;>Apache Aurora Adopters Panel 
(Video)
+   Panelists Ben Staffin, Josh Adams, Bill Farner, Berk 
Demir
+   February 19, 2015 at http://www.meetup.com/Bay-Area-Mesos-User-Group/events/220279080/;>Bay 
Area Mesos Users Group
+   
+   
+   
+   https://www.youtube.com/watch?v=E4lxX6epM_U;>Operating Apache Aurora and 
Mesos at Twitter (Video)
+   Presented by Joe Smith
+   February 19, 2015 at http://www.meetup.com/Bay-Area-Mesos-User-Group/events/220279080/;>Bay 
Area Mesos Users Group
+   
+   
+   
+   https://www.youtube.com/watch?v=ZZXtXLvTXAE;>Apache Aurora and Mesos at 
TellApart (Video)
+   Presented by Steve Niemitz
+   February 19, 2015 at http://www.meetup.com/Bay-Area-Mesos-User-Group/events/220279080/;>Bay 
Area Mesos Users Group
+   
+   
+   
+   https://www.youtube.com/watch?v=Dsc5CPhKs4o;>Past, Present, and Future of 
the Aurora Scheduler (Video)
+   Presented by Bill Farner
+  

svn commit: r1813982 [3/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Modified: 
aurora/site/publish/documentation/0.14.0/development/committers-guide/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/committers-guide/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.14.0/development/committers-guide/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.14.0/development/committers-guide/index.html
 Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/db-migration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/db-migration/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.14.0/development/db-migration/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.14.0/development/db-migration/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/design-documents/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/design-documents/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.14.0/development/design-documents/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.14.0/development/design-documents/index.html
 Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html
 (original)
+++ 
aurora/site/publish/documentation/0.14.0/development/design/command-hooks/index.html
 Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/scheduler/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/scheduler/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.14.0/development/scheduler/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/scheduler/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.14.0/development/thermos/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/thermos/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.14.0/development/thermos/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/thermos/index.html Wed 
Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.14.0/development/thrift/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/thrift/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.14.0/development/thrift/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/thrift/index.html Wed 
Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.14.0/development/ui/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.14.0/development/ui/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.14.0/development/ui/index.html 
(original)
+++ aurora/site/publish/documentation/0.14.0/development/ui/index.html Wed Nov  
1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 

svn commit: r1813982 [13/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Added: 
aurora/site/publish/documentation/0.18.1/reference/scheduler-configuration/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/reference/scheduler-configuration/index.html?rev=1813982=auto
==
--- 
aurora/site/publish/documentation/0.18.1/reference/scheduler-configuration/index.html
 (added)
+++ 
aurora/site/publish/documentation/0.18.1/reference/scheduler-configuration/index.html
 Wed Nov  1 18:39:52 2017
@@ -0,0 +1,404 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.18.1
+  (latest)
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Scheduler Configuration 
Reference
+
+The Aurora scheduler can take a variety of configuration options through 
command-line arguments.
+A list of the available options can be seen by running aurora-scheduler 
-help.
+
+Please refer to the Operator 
Configuration Guide for details on how
+to properly set the most important options.
+$ aurora-scheduler -help
+-
+-h or -help to print this help message
+
+Required flags:
+-backup_dir [not null]
+Directory to store backups under. Will be created if it does not exist.
+-cluster_name [not null]
+Name to identify the cluster being served.
+-db_max_active_connection_count [must be  0]
+Max number of connections to use with database via MyBatis
+-db_max_idle_connection_count [must be  0]
+Max number of idle connections to the database via MyBatis
+-framework_authentication_file
+Properties file which contains framework credentials to authenticate with 
Mesosmaster. Must contain the properties 'aurora_authentication_principal' and 
'aurora_authentication_secret'.
+-ip
+The ip address to listen. If not set, the scheduler will listen on all 
interfaces.
+-mesos_master_address [not null]
+Address for the mesos master, can be a socket address or zookeeper path.
+-mesos_role
+The Mesos role this framework will register as. The default is to left 
this empty, and the framework will register without any role and only receive 
unreserved resources in offer.
+-serverset_path [not null, must be non-empty]
+ZooKeeper ServerSet path to register at.
+-shiro_after_auth_filter
+Fully qualified class name of the servlet filter to be applied after the 
shiro auth filters are applied.
+-thermos_executor_path
+Path to the thermos executor entry point.
+-tier_config [file must be readable]
+Configuration file defining supported task tiers, task traits and 
behaviors.
+-webhook_config [file must exist, file must be readable]
+Path to webhook configuration file.
+-zk_endpoints [must have at least 1 item]
+Endpoint specification for the ZooKeeper servers.
+
+Optional flags:
+-allow_container_volumes (default false)
+Allow passing in volumes in the job. Enabling this could pose a privilege 
escalation threat.
+-allow_docker_parameters (default false)
+Allow to pass docker container parameters in the job.
+-allow_gpu_resource (default false)
+Allow jobs to request Mesos GPU resource.
+-allowed_container_types (default [MESOS])
+Container types that are allowed to be used by jobs.
+-async_slot_stat_update_interval (default (1, mins))
+Interval on which to try to update open slot stats.
+-async_task_stat_update_interval (default (1, hrs))
+Interval on which to try to update resource consumption stats.
+-async_worker_threads (default 8)
+The number of worker threads to process async task operations with.
+-backup_interval (default (1, hrs))
+Minimum interval on which to write a storage backup.
+-cron_scheduler_num_threads (default 10)
+Number 

svn commit: r1813982 [9/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Added: aurora/site/publish/documentation/0.18.1/images/RunningJob.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/RunningJob.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/RunningJob.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/ScheduledJobs.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/ScheduledJobs.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/ScheduledJobs.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/TaskBreakdown.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/TaskBreakdown.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/TaskBreakdown.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/aurora_hierarchy.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/aurora_hierarchy.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/aurora_hierarchy.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/aurora_logo.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/aurora_logo.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/aurora_logo.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/components.odg
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/components.odg?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/components.odg
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/components.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/components.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/components.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/debug-client-test.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/debug-client-test.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: 
aurora/site/publish/documentation/0.18.1/images/debug-client-test.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/debugging-client-test.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/debugging-client-test.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: 
aurora/site/publish/documentation/0.18.1/images/debugging-client-test.png
--
svn:mime-type = application/octet-stream

Added: aurora/site/publish/documentation/0.18.1/images/killedtask.png
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/images/killedtask.png?rev=1813982=auto
==
Binary file - no diff available.

Propchange: aurora/site/publish/documentation/0.18.1/images/killedtask.png

svn commit: r1813982 [16/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Modified: aurora/site/publish/sitemap.xml
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/sitemap.xml?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/sitemap.xml (original)
+++ aurora/site/publish/sitemap.xml Wed Nov  1 18:39:52 2017
@@ -1,2111 +1,2303 @@
 
 http://www.sitemaps.org/schemas/sitemap/0.9;>
   
-http://aurora.apache.org/blog/aurora-0-6-0-incubating-released/
-2017-06-21T00:00:00+00:00
+http://aurora.apache.org/community/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-7-0-incubating-released/
-2017-06-21T00:00:00+00:00
+http://aurora.apache.org/downloads/
+2017-11-01T00:00:00+00:00
   
   
-
http://aurora.apache.org/blog/2015-upcoming-apache-aurora-meetups/
-2017-06-21T00:00:00+00:00
+http://aurora.apache.org/documentation/0.6.0-incubating/storage/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-8-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/configuration-reference/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-9-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/design/command-hooks/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-at-mesoscon-seattle/
-2017-06-21T00:00:00+00:00
+http://aurora.apache.org/documentation/0.6.0-incubating/sla/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-10-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/client-commands/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-11-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/configuration-tutorial/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-12-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/cron-jobs/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-13-0-released/
-2017-06-21T00:00:00+00:00
+http://aurora.apache.org/documentation/0.6.0-incubating/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-14-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/clientv2/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-15-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/developing-aurora-scheduler/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-16-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/scheduler-storage/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-17-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/user-guide/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/aurora-0-18-0-released/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/client-cluster-configuration/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/blog/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/developing-aurora-client/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/community/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/tutorial/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/documentation/0.10.0/build-system/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/contributing/
+2017-11-01T00:00:00+00:00
   
   
-
http://aurora.apache.org/documentation/0.10.0/client-cluster-configuration/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/resource-isolation/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/documentation/0.10.0/client-commands/
-2017-06-21T00:00:00+00:00
+http://aurora.apache.org/documentation/0.6.0-incubating/hooks/
+2017-11-01T00:00:00+00:00
   
   
-http://aurora.apache.org/documentation/0.10.0/committers/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/storage-config/
+2017-11-01T00:00:00+00:00
   
   
-
http://aurora.apache.org/documentation/0.10.0/configuration-reference/
-2017-06-21T00:00:00+00:00
+
http://aurora.apache.org/documentation/0.6.0-incubating/deploying-aurora-scheduler/
+2017-11-01T00:00:00+00:00
   
   
-

svn commit: r1813982 [12/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docum

[wfarner]

Added: 
aurora/site/publish/documentation/0.18.1/reference/configuration-tutorial/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.18.1/reference/configuration-tutorial/index.html?rev=1813982=auto
==
--- 
aurora/site/publish/documentation/0.18.1/reference/configuration-tutorial/index.html
 (added)
+++ 
aurora/site/publish/documentation/0.18.1/reference/configuration-tutorial/index.html
 Wed Nov  1 18:39:52 2017
@@ -0,0 +1,674 @@
+
+
+  
+
+
+   Apache Aurora
+https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css;>
+
+   
+   
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-45879646-1']);
+ _gaq.push(['_setDomainName', 'apache.org']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+   var ga = document.createElement('script'); ga.type = 
'text/javascript'; ga.async = true;
+   ga.src = ('https:' == document.location.protocol ? 
'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+   var s = document.getElementsByTagName('script')[0]; 
s.parentNode.insertBefore(ga, s);
+ })();
+   
+  
+  
+
+  
+
+
+
+  Documentation
+  Community
+  Downloads
+  Blog
+
+
+  
+
+   
+
+  
+
+Documentation
+
+  
+0.18.1
+  (latest)
+  
+  
+0.18.0
+  
+  
+0.17.0
+  
+  
+0.16.0
+  
+  
+0.15.0
+  
+  
+0.14.0
+  
+  
+0.13.0
+  
+  
+0.12.0
+  
+  
+0.11.0
+  
+  
+0.10.0
+  
+  
+0.9.0
+  
+  
+0.8.0
+  
+  
+0.7.0-incubating
+  
+  
+0.6.0-incubating
+  
+  
+0.5.0-incubating
+  
+
+
+Aurora Configuration Tutorial
+
+How to write Aurora configuration files, including feature descriptions
+and best practices. When writing a configuration file, make use of
+aurora job inspect. It takes the same job key and configuration 
file
+arguments as aurora job create or aurora update 
start. It first ensures the
+configuration parses, then outputs it in human-readable form.
+
+You should read this after going through the general Aurora Tutorial.
+
+
+The Basics
+
+
+Use Bottom-To-Top Object 
Ordering
+
+An Example Configuration 
File
+Defining Process Objects
+Getting Your Code Into The 
Sandbox
+Defining Task Objects
+
+
+SequentialTask:
 Running Processes in Parallel or Sequentially
+SimpleTask
+Combining tasks
+
+Defining Job Objects
+The jobs List
+Basic Examples
+
+
+The Basics
+
+To run a job on Aurora, you must specify a configuration file that tells
+Aurora what it needs to know to schedule the job, what Mesos needs to
+run the tasks the job is made up of, and what Thermos needs to run the
+processes that make up the tasks. This file must have
+a.aurora suffix.
+
+A configuration file defines a collection of objects, along with parameter
+values for their attributes. An Aurora configuration file contains the
+following three types of objects:
+
+
+Job
+Task
+Process
+
+
+A configuration also specifies a list of Job objects assigned
+to the variable jobs.
+
+
+jobs (list of defined Jobs to run)
+
+
+The .aurora file format is just Python. However, 
Job, Task,
+Process, and other classes are defined by a type-checked 
dictionary
+templating library called Pystachio, a powerful tool for
+configuration specification and reuse. Pystachio objects are tailored
+via {{}} surrounded templates.
+
+When writing your .aurora file, you may use any Pystachio 
datatypes, as
+well as any objects shown in the Aurora 
Configuration
+Reference, without import statements - the
+Aurora config loader injects them automatically. Other than that, an 
.aurora
+file works like any other Python script.
+
+Aurora Configuration Reference
+has a full reference of all Aurora/Thermos defined Pystachio objects.
+
+Use Bottom-To-Top Object 
Ordering
+
+A well-structured configuration starts with structural templates (if
+any). Structural templates encapsulate in their attributes all the
+differences between Jobs in the configuration that are not directly
+manipulated at the Job level, but typically at the 
Process or Task
+level. For example, if certain processes are invoked with slightly
+different settings or input.
+
+After structural templates, define, in order, Processes, 
Tasks, and
+Jobs.
+
+Structural template names should be UpperCamelCased and their
+instantiations are typically UPPER_SNAKE_CASED. Process, 
Task,
+and Job names are typically lower_snake_cased. 
Indentation is typically 2
+spaces.
+
+An Example Configuration File
+
+The following is a typical configuration file. Dont worry if there 
are
+parts you dont understand yet, but you may want to refer back to this
+as you read about its individual parts. Note that names 

svn commit: r1813982 [4/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Modified: 
aurora/site/publish/documentation/0.16.0/features/job-updates/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/job-updates/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.16.0/features/job-updates/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/job-updates/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/features/mesos-fetcher/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/mesos-fetcher/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.16.0/features/mesos-fetcher/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/mesos-fetcher/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/features/multitenancy/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/multitenancy/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.16.0/features/multitenancy/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/multitenancy/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/features/resource-isolation/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/resource-isolation/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.16.0/features/resource-isolation/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.16.0/features/resource-isolation/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/features/service-discovery/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/service-discovery/index.html?rev=1813982=1813981=1813982=diff
==
--- 
aurora/site/publish/documentation/0.16.0/features/service-discovery/index.html 
(original)
+++ 
aurora/site/publish/documentation/0.16.0/features/service-discovery/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.16.0/features/services/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/services/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.16.0/features/services/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/services/index.html Wed 
Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 
aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/sla-metrics/index.html 
Wed Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: aurora/site/publish/documentation/0.16.0/features/webhooks/index.html
URL: 
http://svn.apache.org/viewvc/aurora/site/publish/documentation/0.16.0/features/webhooks/index.html?rev=1813982=1813981=1813982=diff
==
--- aurora/site/publish/documentation/0.16.0/features/webhooks/index.html 
(original)
+++ aurora/site/publish/documentation/0.16.0/features/webhooks/index.html Wed 
Nov  1 18:39:52 2017
@@ -41,10 +41,14 @@
 Documentation
 
+  
+0.18.1
+  (latest)
+  
   
 0.18.0
-  (latest)
   
   

Modified: 

svn commit: r1813982 [1/21] - in /aurora/site: data/ publish/ publish/blog/ publish/blog/aurora-0-18-1-released/ publish/documentation/0.10.0/ publish/documentation/0.10.0/build-system/ publish/docume

[wfarner]

Author: wfarner
Date: Wed Nov  1 18:39:52 2017
New Revision: 1813982

URL: http://svn.apache.org/viewvc?rev=1813982=rev
Log:
Update website for Aurora 0.18.1

Added:
aurora/site/publish/blog/aurora-0-18-1-released/
aurora/site/publish/blog/aurora-0-18-1-released/index.html
aurora/site/publish/documentation/0.18.1/
aurora/site/publish/documentation/0.18.1/additional-resources/
aurora/site/publish/documentation/0.18.1/additional-resources/presentations/

aurora/site/publish/documentation/0.18.1/additional-resources/presentations/index.html
aurora/site/publish/documentation/0.18.1/additional-resources/tools/

aurora/site/publish/documentation/0.18.1/additional-resources/tools/index.html
aurora/site/publish/documentation/0.18.1/contributing/
aurora/site/publish/documentation/0.18.1/contributing/index.html
aurora/site/publish/documentation/0.18.1/development/
aurora/site/publish/documentation/0.18.1/development/client/
aurora/site/publish/documentation/0.18.1/development/client/index.html
aurora/site/publish/documentation/0.18.1/development/committers-guide/

aurora/site/publish/documentation/0.18.1/development/committers-guide/index.html
aurora/site/publish/documentation/0.18.1/development/db-migration/
aurora/site/publish/documentation/0.18.1/development/db-migration/index.html
aurora/site/publish/documentation/0.18.1/development/design/
aurora/site/publish/documentation/0.18.1/development/design-documents/

aurora/site/publish/documentation/0.18.1/development/design-documents/index.html
aurora/site/publish/documentation/0.18.1/development/design/command-hooks/

aurora/site/publish/documentation/0.18.1/development/design/command-hooks/index.html
aurora/site/publish/documentation/0.18.1/development/scheduler/
aurora/site/publish/documentation/0.18.1/development/scheduler/index.html
aurora/site/publish/documentation/0.18.1/development/thermos/
aurora/site/publish/documentation/0.18.1/development/thermos/index.html
aurora/site/publish/documentation/0.18.1/development/thrift/
aurora/site/publish/documentation/0.18.1/development/thrift/index.html
aurora/site/publish/documentation/0.18.1/development/ui/
aurora/site/publish/documentation/0.18.1/development/ui/index.html
aurora/site/publish/documentation/0.18.1/features/
aurora/site/publish/documentation/0.18.1/features/constraints/
aurora/site/publish/documentation/0.18.1/features/constraints/index.html
aurora/site/publish/documentation/0.18.1/features/containers/
aurora/site/publish/documentation/0.18.1/features/containers/index.html
aurora/site/publish/documentation/0.18.1/features/cron-jobs/
aurora/site/publish/documentation/0.18.1/features/cron-jobs/index.html
aurora/site/publish/documentation/0.18.1/features/custom-executors/

aurora/site/publish/documentation/0.18.1/features/custom-executors/index.html
aurora/site/publish/documentation/0.18.1/features/job-updates/
aurora/site/publish/documentation/0.18.1/features/job-updates/index.html
aurora/site/publish/documentation/0.18.1/features/mesos-fetcher/
aurora/site/publish/documentation/0.18.1/features/mesos-fetcher/index.html
aurora/site/publish/documentation/0.18.1/features/multitenancy/
aurora/site/publish/documentation/0.18.1/features/multitenancy/index.html
aurora/site/publish/documentation/0.18.1/features/resource-isolation/

aurora/site/publish/documentation/0.18.1/features/resource-isolation/index.html
aurora/site/publish/documentation/0.18.1/features/service-discovery/

aurora/site/publish/documentation/0.18.1/features/service-discovery/index.html
aurora/site/publish/documentation/0.18.1/features/services/
aurora/site/publish/documentation/0.18.1/features/services/index.html
aurora/site/publish/documentation/0.18.1/features/sla-metrics/
aurora/site/publish/documentation/0.18.1/features/sla-metrics/index.html
aurora/site/publish/documentation/0.18.1/features/webhooks/
aurora/site/publish/documentation/0.18.1/features/webhooks/index.html
aurora/site/publish/documentation/0.18.1/getting-started/
aurora/site/publish/documentation/0.18.1/getting-started/overview/
aurora/site/publish/documentation/0.18.1/getting-started/overview/index.html
aurora/site/publish/documentation/0.18.1/getting-started/tutorial/
aurora/site/publish/documentation/0.18.1/getting-started/tutorial/index.html
aurora/site/publish/documentation/0.18.1/getting-started/vagrant/
aurora/site/publish/documentation/0.18.1/getting-started/vagrant/index.html
aurora/site/publish/documentation/0.18.1/images/
aurora/site/publish/documentation/0.18.1/images/CPUavailability.png   (with 
props)
aurora/site/publish/documentation/0.18.1/images/CompletedTasks.png   (with 
props)
aurora/site/publish/documentation/0.18.1/images/HelloWorldJob.png   (with 
props)
aurora/site/publish/documentation

[aurora] Git Push Summary

[wfarner]

Repository: aurora
Updated Tags:  refs/tags/rel/0.18.1 [created] 1e6b4443a

svn commit: r22814 - in /release/aurora/0.18.1: apache-aurora-0.18.1.tar.gz apache-aurora-0.18.1.tar.gz.asc apache-aurora-0.18.1.tar.gz.md5 apache-aurora-0.18.1.tar.gz.sha

[wfarner]

Author: wfarner
Date: Wed Nov  1 17:53:50 2017
New Revision: 22814

Log:
aurora-0.18.1 release

Added:
release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz   (with props)
release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.asc
release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.md5
release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.sha

Added: release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz
==
Binary file - no diff available.

Propchange: release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz
--
svn:mime-type = application/octet-stream

Added: release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.asc
==
--- release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.asc (added)
+++ release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.asc Wed Nov  1 17:53:50 
2017
@@ -0,0 +1,16 @@
+-BEGIN PGP SIGNATURE-
+
+iQIcBAABCAAGBQJZ+golAAoJEFYhfmc+mb7zNxkP/RZq90efPvLJQlLWmpNaEPDo
+1Ob1fKkPwUUrY3oqUMqIYmv0oK/OBLLrf6F74Er6D+6fkO4vSanXEZW8muWmDezc
+QXihaDiH5FIrUUrISwCl2XkLbL7BEo3HGcc8fOJ/F8OlSBQaM+Ke8W7ImdnSoxa9
+rIv7kkvdVu6q3RCC8zfr+RA+W5q/4YRWt6gXyW5v4+ybtso9WSrUIscrcfBYpb6A
++uuKV2wuzrd67vj0lOv2HUfvlqBKUYTjcHwGjtxhD+3W53QO4Ddobd/H9tANij1h
+2KLZw2/sb67GplcbudcQdF1MRmaxK71zj8LjonjLk0z2EuYXISU5oQeNX6y1LzlF
+nZhRjPQ8255BNGjqDarzGVh+7pnKEtB0gI+EFpugxp14WgftaNxNtjPtN/CvAdcj
+Br/DAVS30KlVII85DjanIJJitCH816hLvHuAPWXFM9mS7DaDCnGRz8zzwwNQleNr
+zcwpHGzA7Iz25nrTbZGHqCxdhZVjb8gu+sbxemQIzC1MAVfLqin2XhipF6u6rkHe
+HI16+tH7f7DKFlpvls9qP6o/QcmHIUBYCt6v8z4GdhpZhM9OgUSQmgYmN/UPXAeB
+Cy6Xum+fFVW6OHiwX2TLJNm4eJRdZIPfXBColQjgqPvipxGcQ3Gz7ixFy8Xze2DK
+mId3Pim6F7IxHe2eUEOs
+=nWAL
+-END PGP SIGNATURE-

Added: release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.md5
==
--- release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.md5 (added)
+++ release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.md5 Wed Nov  1 17:53:50 
2017
@@ -0,0 +1 @@
+apache-aurora-0.18.1.tar.gz: FB 66 61 7A 68 26 68 4D  7B 8A D0 10 1D F7 2B 79

Added: release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.sha
==
--- release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.sha (added)
+++ release/aurora/0.18.1/apache-aurora-0.18.1.tar.gz.sha Wed Nov  1 17:53:50 
2017
@@ -0,0 +1 @@
+8e172fcf8b670f0a3feff6f9a59b29e0c5861fcd  apache-aurora-0.18.1.tar.gz

svn commit: r22813 - /release/aurora/0.18.1/

[wfarner]

Author: wfarner
Date: Wed Nov  1 17:53:37 2017
New Revision: 22813

Log:
aurora-0.18.1 release

Added:
release/aurora/0.18.1/

aurora git commit: Refactor staticallyBannedOffers into a LRU cache

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master 8af269f52 -> d106b4ecc


Refactor staticallyBannedOffers into a LRU cache

Using the new `hold_offers_forever` option, it is possible for the
`staticallyBannedOffers` to grow very large in size as we never release
offers.
1. The current behavior of `staticallyBannedOffers` is (kinda) preserved.
   Entries will no longer be removed when the offer is used, but they will be
   removed within `maxOfferHoldTime`. This means cluster operators will not
   have to think about the new `offer_static_ban_cache_max_size` if they aren't
   affected by the memory leak now.
2. Cluster operators that use Aurora as a single framework and hold offers
   indefinitely can cap the size of the cache to avoid the memory leak.
3. Using an LRU cache greatly benefits quickly recurring crons and job updates.

Reviewed at https://reviews.apache.org/r/63199/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/d106b4ec
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/d106b4ec
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/d106b4ec

Branch: refs/heads/master
Commit: d106b4ecc9537b8e844c4edc2210b9fe1853ccc4
Parents: 8af269f
Author: Jordan Ly 
Authored: Tue Oct 31 10:20:27 2017 -0700
Committer: Bill Farner 
Committed: Tue Oct 31 10:20:27 2017 -0700

--
 .../aurora/benchmark/SchedulingBenchmarks.java  |  7 ++-
 .../config/validators/NotNegativeNumber.java| 26 ++
 .../aurora/scheduler/offers/OfferManager.java   | 47 ++---
 .../aurora/scheduler/offers/OfferSettings.java  | 54 
 .../aurora/scheduler/offers/OffersModule.java   | 33 +++-
 .../scheduler/config/CommandLineTest.java   |  2 +
 .../scheduler/offers/OfferManagerImplTest.java  | 31 +++
 7 files changed, 159 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/d106b4ec/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
--
diff --git a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java 
b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
index e0ec793..456e780 100644
--- a/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
+++ b/src/jmh/java/org/apache/aurora/benchmark/SchedulingBenchmarks.java
@@ -40,6 +40,7 @@ import org.apache.aurora.common.quantity.Time;
 import org.apache.aurora.common.stats.StatsProvider;
 import org.apache.aurora.common.util.Clock;
 import org.apache.aurora.common.util.testing.FakeClock;
+import org.apache.aurora.common.util.testing.FakeTicker;
 import org.apache.aurora.gen.ServerInfo;
 import org.apache.aurora.scheduler.HostOffer;
 import org.apache.aurora.scheduler.TaskIdGenerator;
@@ -153,7 +154,11 @@ public class SchedulingBenchmarks {
   bind(OfferManager.class).to(OfferManager.OfferManagerImpl.class);
   bind(OfferManager.OfferManagerImpl.class).in(Singleton.class);
   bind(OfferSettings.class).toInstance(
-  new OfferSettings(NO_DELAY, 
ImmutableList.of(OfferOrder.RANDOM)));
+  new OfferSettings(NO_DELAY,
+  ImmutableList.of(OfferOrder.RANDOM),
+  Amount.of(Long.MAX_VALUE, Time.SECONDS),
+  Long.MAX_VALUE,
+  new FakeTicker()));
   bind(BiCache.BiCacheSettings.class).toInstance(
   new BiCache.BiCacheSettings(DELAY_FOREVER, ""));
   
bind(TaskScheduler.class).to(TaskScheduler.TaskSchedulerImpl.class);

http://git-wip-us.apache.org/repos/asf/aurora/blob/d106b4ec/src/main/java/org/apache/aurora/scheduler/config/validators/NotNegativeNumber.java
--
diff --git 
a/src/main/java/org/apache/aurora/scheduler/config/validators/NotNegativeNumber.java
 
b/src/main/java/org/apache/aurora/scheduler/config/validators/NotNegativeNumber.java
new file mode 100644
index 000..fd0320e
--- /dev/null
+++ 
b/src/main/java/org/apache/aurora/scheduler/config/validators/NotNegativeNumber.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

aurora git commit: Add support for generating patch RCs from non-master branches

[wfarner]

Repository: aurora
Updated Branches:
  refs/heads/master bc190641f -> 3fe5d5907


Add support for generating patch RCs from non-master branches

Reviewed at https://reviews.apache.org/r/63401/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/3fe5d590
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/3fe5d590
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/3fe5d590

Branch: refs/heads/master
Commit: 3fe5d5907f93995de67e63bfb11846216285337d
Parents: bc19064
Author: Bill Farner 
Authored: Mon Oct 30 08:03:44 2017 -0700
Committer: Bill Farner 
Committed: Mon Oct 30 08:03:44 2017 -0700

--
 build-support/release/release-candidate | 33 ++--
 1 file changed, 26 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/aurora/blob/3fe5d590/build-support/release/release-candidate
--
diff --git a/build-support/release/release-candidate 
b/build-support/release/release-candidate
index 1a03d5e..a982cf7 100755
--- a/build-support/release/release-candidate
+++ b/build-support/release/release-candidate
@@ -95,12 +95,16 @@ fi
 git fetch --all -q
 git fetch --tags -q
 
+starting_git_branch=$(git rev-parse --abbrev-ref HEAD)
+
 # Verify that this is a clean repository
 if [[ -n "`git status --porcelain`" ]]; then
   echo "ERROR: Please run from a clean git repository."
   exit 1
-elif [[ "`git rev-parse --abbrev-ref HEAD`" != "master" ]]; then
-  echo "ERROR: This script must be run from master."
+# A patch release may be performed from the patched git SHA, but all other 
releases
+# should be from master.
+elif [[ "$starting_git_branch" != "master" && "$increment_level" != 'patch' 
]]; then
+  echo "ERROR: This script must be run from master when not creating a patch 
release"
   exit 1
 fi
 
@@ -111,7 +115,10 @@ fi
 
 # Calculate the new version string
 current_version=$(cat .auroraversion | tr '[a-z]' '[A-Z]')
-if ! [[ $current_version =~ .*-SNAPSHOT ]]; then
+# When patching a release, we may be cherry-picking on the released git tag,
+# which would not contain a -SNAPSHOT version identifier.  Other releases,
+# however, should start from a git tree with a -SNAPSHOT version.
+if ! [[ $current_version =~ .*-SNAPSHOT || "$increment_level" == 'patch' ]]; 
then
   echo "ERROR: .auroraversion is required to contain 'SNAPSHOT', it is 
${current_version}"
   exit 1
 else
@@ -139,6 +146,13 @@ else
   new_snapshot_version="${new_master_version}-SNAPSHOT"
 fi
 
+if [[ "$increment_level" == 'patch' && "$starting_git_branch" != 'master' ]]; 
then
+  echo 'NOTE: This release candidate is patching a non-master branch.'
+  echo '.auroraversion will be treated as the patched release, rather'
+  echo 'than the to-be-released version.'
+  current_version="$new_master_version"
+fi
+
 # Add the rc tag to the current version
 rc_version="${current_version}-rc${rc_number}"
 rc_version_tag="rel/${rc_version}"
@@ -159,7 +173,7 @@ function print_reset_instructions {
 cat <

svn commit: r22724 - /dev/aurora/0.18.1-rc1/apache-aurora-0.18.1-rc1.tar.gz.sha512

[wfarner]

Author: wfarner
Date: Sun Oct 29 23:29:15 2017
New Revision: 22724

Log:
Adding sha512 signature for 0.18.1-rc1

Added:
dev/aurora/0.18.1-rc1/apache-aurora-0.18.1-rc1.tar.gz.sha512

Added: dev/aurora/0.18.1-rc1/apache-aurora-0.18.1-rc1.tar.gz.sha512
==
--- dev/aurora/0.18.1-rc1/apache-aurora-0.18.1-rc1.tar.gz.sha512 (added)
+++ dev/aurora/0.18.1-rc1/apache-aurora-0.18.1-rc1.tar.gz.sha512 Sun Oct 29 
23:29:15 2017
@@ -0,0 +1 @@
+1973d9b07cd55069479096997a99ad8556c183e3cf51a680b58df167fcaf98e214728b5b45ab2d3544946e6ab42204f79af5a71f3aac3984da95de77a8bc4265
  dist/apache-aurora-0.18.1-rc1.tar.gz

svn commit: r22722 - /dev/aurora/0.18.1-rc1/

[wfarner]

Author: wfarner
Date: Sun Oct 29 17:30:47 2017
New Revision: 22722

Log:
aurora-0.18.1 release candidate rel/0.18.1-rc1

Added:
dev/aurora/0.18.1-rc1/