Repository: aurora Updated Branches: refs/heads/master 83025f471 -> 34be63158
Introduce structs to enable specifying custom SLA. Add `SlaPolicy` and `HostMaintenanceRequest` structs to the thrift definition and introduce a new `HostMaintenanceStore` for tracking maintenance requests. These changes will be used in https://reviews.apache.org/r/66716 for implementing custom SLA and scheduler driven maintenance. This RB splits the storage related changes from https://reviews.apache.org/r/66716 for better rollback story. Tested rollback on the vagrant. Testing Done: ./build-support/jenkins/build.sh Bugs closed: AURORA-1977 Reviewed at https://reviews.apache.org/r/67141/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/34be6315 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/34be6315 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/34be6315 Branch: refs/heads/master Commit: 34be631589ebf899e663b698dc76511eb1b9ad8a Parents: 83025f4 Author: Santhosh Kumar Shanmugham <[email protected]> Authored: Mon May 21 16:40:16 2018 -0700 Committer: Santhosh Kumar <[email protected]> Committed: Mon May 21 16:40:16 2018 -0700 ---------------------------------------------------------------------- .../thrift/org/apache/aurora/gen/api.thrift | 40 +++++ .../thrift/org/apache/aurora/gen/storage.thrift | 11 ++ .../scheduler/storage/HostMaintenanceStore.java | 61 +++++++ .../aurora/scheduler/storage/Storage.java | 2 + .../storage/durability/DurableStorage.java | 3 + .../scheduler/storage/durability/Loader.java | 13 ++ .../storage/durability/WriteRecorder.java | 49 +++++- .../scheduler/storage/log/SnapshotService.java | 3 +- .../scheduler/storage/log/SnapshotterImpl.java | 27 +++- .../storage/mem/MemHostMaintenanceStore.java | 55 +++++++ .../scheduler/storage/mem/MemStorage.java | 9 +- .../scheduler/storage/mem/MemStorageModule.java | 2 + .../python/apache/aurora/config/schema/base.py | 18 +++ src/main/python/apache/aurora/config/thrift.py | 41 +++++ .../apache/aurora/executor/executor_vars.py | 2 +- .../aurora/scheduler/base/TaskTestUtil.java | 14 +- .../AbstractHostMaintenanceStoreTest.java | 160 +++++++++++++++++++ .../scheduler/storage/backup/RecoveryTest.java | 1 + .../durability/DataCompatibilityTest.java | 4 + .../storage/durability/DurableStorageTest.java | 57 +++++++ .../storage/durability/WriteRecorderTest.java | 4 + .../storage/log/SnapshotterImplIT.java | 16 +- .../mem/MemHostMaintenanceStoreTest.java | 25 +++ .../storage/testing/StorageTestUtil.java | 6 + .../apache/aurora/client/cli/test_inspect.py | 4 +- .../python/apache/aurora/config/test_thrift.py | 74 ++++++++- .../current/removeHostMaintenanceRequest | 9 ++ .../durability/goldens/current/saveCronJob | 14 ++ .../goldens/current/saveHostMaintenanceRequest | 33 ++++ .../durability/goldens/current/saveJobUpdate | 28 ++++ .../durability/goldens/current/saveTasks | 14 ++ .../16-saveHostMaintenanceRequest | 30 ++++ .../17-removeHostMaintenanceRequest | 9 ++ 33 files changed, 828 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/api/src/main/thrift/org/apache/aurora/gen/api.thrift ---------------------------------------------------------------------- diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift index ef754e3..ff48000 100644 --- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift +++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift @@ -245,6 +245,37 @@ struct PartitionPolicy { 2: optional i64 delaySecs } +/** SLA requirements expressed as the percentage of instances to be RUNNING every durationSecs */ +struct PercentageSlaPolicy { + /* The percentage of active instances required every `durationSecs`. */ + 1: double percentage + /** Minimum time duration a task needs to be `RUNNING` to be treated as active */ + 2: i64 durationSecs +} + +/** SLA requirements expressed as the number of instances to be RUNNING every durationSecs */ +struct CountSlaPolicy { + /** The number of active instances required every `durationSecs` */ + 1: i64 count + /** Minimum time duration a task needs to be `RUNNING` to be treated as active */ + 2: i64 durationSecs +} + +/** SLA requirements to be delegated to an external coordinator */ +struct CoordinatorSlaPolicy { + /** URL for the coordinator service that needs to be contacted for SLA checks */ + 1: string coordinatorUrl + /** Field in the Coordinator response json indicating if the action is allowed or not */ + 2: string statusKey +} + +/** SLA requirements expressed in one of the many types */ +union SlaPolicy { + 1: PercentageSlaPolicy percentageSlaPolicy + 2: CountSlaPolicy countSlaPolicy + 3: CoordinatorSlaPolicy coordinatorSlaPolicy +} + /** Description of the tasks contained within a job. */ struct TaskConfig { /** Job task belongs to. */ @@ -279,6 +310,8 @@ struct TaskConfig { 27: optional set<Metadata> metadata /** Policy for how to deal with task partitions */ 34: optional PartitionPolicy partitionPolicy + /** SLA requirements to be met during maintenance */ + 35: optional SlaPolicy slaPolicy // This field is deliberately placed at the end to work around a bug in the immutable wrapper // code generator. See AURORA-1185 for details. @@ -867,6 +900,13 @@ struct JobUpdateQuery { 7: i32 limit } +struct HostMaintenanceRequest { + 1: string host + 2: SlaPolicy defaultSlaPolicy + 3: i64 timeoutSecs + 4: i64 createdTimestampMs +} + struct ListBackupsResult { 1: set<string> backups } http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/api/src/main/thrift/org/apache/aurora/gen/storage.thrift ---------------------------------------------------------------------- diff --git a/api/src/main/thrift/org/apache/aurora/gen/storage.thrift b/api/src/main/thrift/org/apache/aurora/gen/storage.thrift index b79e204..9933fdd 100644 --- a/api/src/main/thrift/org/apache/aurora/gen/storage.thrift +++ b/api/src/main/thrift/org/apache/aurora/gen/storage.thrift @@ -92,6 +92,14 @@ struct PruneJobUpdateHistory { 2: i64 historyPruneThresholdMs } +struct SaveHostMaintenanceRequest { + 1: api.HostMaintenanceRequest hostMaintenanceRequest +} + +struct RemoveHostMaintenanceRequest { + 1: string host +} + union Op { 1: SaveFrameworkId saveFrameworkId 2: SaveCronJob saveCronJob @@ -109,6 +117,8 @@ union Op { 16: SaveJobInstanceUpdateEvent saveJobInstanceUpdateEvent 17: PruneJobUpdateHistory pruneJobUpdateHistory 18: RemoveJobUpdates removeJobUpdate + 19: SaveHostMaintenanceRequest saveHostMaintenanceRequest + 20: RemoveHostMaintenanceRequest removeHostMaintenanceRequest } // The current schema version ID. This should be incremented each time the @@ -152,6 +162,7 @@ struct Snapshot { 10: set<StoredJobUpdateDetails> jobUpdateDetails //11: removed //12: removed + 13: set<api.HostMaintenanceRequest> hostMaintenanceRequests } // A message header that calls out the number of expected FrameChunks to follow to form a complete http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/HostMaintenanceStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/HostMaintenanceStore.java b/src/main/java/org/apache/aurora/scheduler/storage/HostMaintenanceStore.java new file mode 100644 index 0000000..14d18f9 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/storage/HostMaintenanceStore.java @@ -0,0 +1,61 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.storage; + +import java.util.Optional; +import java.util.Set; + +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; + +/** + * Stores maintenance requests for hosts. + */ +public interface HostMaintenanceStore { + + /** + * Returns the maintenance request for the given host if exists. + * {@link Optional#empty()} otherwise. + * @param host Name of the host + * @return {@link Optional} of {@link IHostMaintenanceRequest} + */ + Optional<IHostMaintenanceRequest> getHostMaintenanceRequest(String host); + + /** + * Returns all {@link IHostMaintenanceRequest}s currently in storage. + * @return {@link Set} of {@link IHostMaintenanceRequest}s + */ + Set<IHostMaintenanceRequest> getHostMaintenanceRequests(); + + /** + * Provides write operations to the {@link HostMaintenanceStore}. + */ + interface Mutable extends HostMaintenanceStore { + /** + * Deletes all attributes in the store. + */ + void deleteHostMaintenanceRequests(); + + /** + * Saves the maintenance request for the given host. + * @param hostMaintenanceRequest {@link IHostMaintenanceRequest} + */ + void saveHostMaintenanceRequest(IHostMaintenanceRequest hostMaintenanceRequest); + + /** + * Removes the maintenance request for the given host if one exists. + * @param host Name of the host + */ + void removeHostMaintenanceRequest(String host); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/Storage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/Storage.java b/src/main/java/org/apache/aurora/scheduler/storage/Storage.java index da5534f..4e269b6 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/Storage.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/Storage.java @@ -42,6 +42,7 @@ public interface Storage { QuotaStore getQuotaStore(); AttributeStore getAttributeStore(); JobUpdateStore getJobUpdateStore(); + HostMaintenanceStore getHostMaintenanceStore(); } /** @@ -70,6 +71,7 @@ public interface Storage { QuotaStore.Mutable getQuotaStore(); AttributeStore.Mutable getAttributeStore(); JobUpdateStore.Mutable getJobUpdateStore(); + HostMaintenanceStore.Mutable getHostMaintenanceStore(); } /** http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java index f1fdc27..6776edf 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/durability/DurableStorage.java @@ -25,6 +25,7 @@ import org.apache.aurora.scheduler.base.SchedulerException; import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.storage.AttributeStore; import org.apache.aurora.scheduler.storage.CronJobStore; +import org.apache.aurora.scheduler.storage.HostMaintenanceStore; import org.apache.aurora.scheduler.storage.JobUpdateStore; import org.apache.aurora.scheduler.storage.QuotaStore; import org.apache.aurora.scheduler.storage.SchedulerStore; @@ -106,6 +107,7 @@ public class DurableStorage implements NonVolatileStorage { @Volatile QuotaStore.Mutable quotaStore, @Volatile AttributeStore.Mutable attributeStore, @Volatile JobUpdateStore.Mutable jobUpdateStore, + @Volatile HostMaintenanceStore.Mutable hostMaintenanceStore, EventSink eventSink, ReentrantLock writeLock, ThriftBackfill thriftBackfill) { @@ -137,6 +139,7 @@ public class DurableStorage implements NonVolatileStorage { quotaStore, attributeStore, jobUpdateStore, + hostMaintenanceStore, LoggerFactory.getLogger(WriteRecorder.class), eventSink); } http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/durability/Loader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/durability/Loader.java b/src/main/java/org/apache/aurora/scheduler/storage/durability/Loader.java index 10864f1..cbe8f02 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/durability/Loader.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/durability/Loader.java @@ -23,6 +23,7 @@ import org.apache.aurora.gen.storage.SaveQuota; import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider; import org.apache.aurora.scheduler.storage.durability.Persistence.Edit; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent; import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent; @@ -61,6 +62,7 @@ public final class Loader { stores.getQuotaStore().deleteQuotas(); stores.getAttributeStore().deleteHostAttributes(); stores.getJobUpdateStore().deleteAllUpdates(); + stores.getHostMaintenanceStore().deleteHostMaintenanceRequests(); return; } @@ -143,6 +145,17 @@ public final class Loader { IJobUpdateKey.setFromBuilders(op.getRemoveJobUpdate().getKeys())); break; + case SAVE_HOST_MAINTENANCE_REQUEST: + stores.getHostMaintenanceStore().saveHostMaintenanceRequest( + IHostMaintenanceRequest + .build(op.getSaveHostMaintenanceRequest().getHostMaintenanceRequest())); + break; + + case REMOVE_HOST_MAINTENANCE_REQUEST: + stores.getHostMaintenanceStore().removeHostMaintenanceRequest( + op.getRemoveHostMaintenanceRequest().getHost()); + break; + default: throw new IllegalArgumentException("Unrecognized op type " + op.getSetField()); } http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/durability/WriteRecorder.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/durability/WriteRecorder.java b/src/main/java/org/apache/aurora/scheduler/storage/durability/WriteRecorder.java index 8d70cae..b31c1a0 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/durability/WriteRecorder.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/durability/WriteRecorder.java @@ -24,12 +24,14 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import org.apache.aurora.gen.storage.Op; +import org.apache.aurora.gen.storage.RemoveHostMaintenanceRequest; import org.apache.aurora.gen.storage.RemoveJob; import org.apache.aurora.gen.storage.RemoveQuota; import org.apache.aurora.gen.storage.RemoveTasks; import org.apache.aurora.gen.storage.SaveCronJob; import org.apache.aurora.gen.storage.SaveFrameworkId; import org.apache.aurora.gen.storage.SaveHostAttributes; +import org.apache.aurora.gen.storage.SaveHostMaintenanceRequest; import org.apache.aurora.gen.storage.SaveJobInstanceUpdateEvent; import org.apache.aurora.gen.storage.SaveJobUpdate; import org.apache.aurora.gen.storage.SaveJobUpdateEvent; @@ -40,6 +42,7 @@ import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; import org.apache.aurora.scheduler.storage.AttributeStore; import org.apache.aurora.scheduler.storage.CronJobStore; +import org.apache.aurora.scheduler.storage.HostMaintenanceStore; import org.apache.aurora.scheduler.storage.JobUpdateStore; import org.apache.aurora.scheduler.storage.QuotaStore; import org.apache.aurora.scheduler.storage.SchedulerStore; @@ -47,6 +50,7 @@ import org.apache.aurora.scheduler.storage.Storage.MutableStoreProvider; import org.apache.aurora.scheduler.storage.TaskStore; import org.apache.aurora.scheduler.storage.durability.DurableStorage.TransactionManager; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent; import org.apache.aurora.scheduler.storage.entities.IJobKey; @@ -73,7 +77,8 @@ public class WriteRecorder implements TaskStore.Mutable, QuotaStore.Mutable, AttributeStore.Mutable, - JobUpdateStore.Mutable { + JobUpdateStore.Mutable, + HostMaintenanceStore.Mutable { private final TransactionManager transactionManager; private final SchedulerStore.Mutable schedulerStore; @@ -82,6 +87,7 @@ public class WriteRecorder implements private final QuotaStore.Mutable quotaStore; private final AttributeStore.Mutable attributeStore; private final JobUpdateStore.Mutable jobUpdateStore; + private final HostMaintenanceStore.Mutable hostMaintenanceStore; private final Logger log; private final EventSink eventSink; @@ -104,6 +110,7 @@ public class WriteRecorder implements QuotaStore.Mutable quotaStore, AttributeStore.Mutable attributeStore, JobUpdateStore.Mutable jobUpdateStore, + HostMaintenanceStore.Mutable hostMaintenanceStore, Logger log, EventSink eventSink) { @@ -114,6 +121,7 @@ public class WriteRecorder implements this.quotaStore = requireNonNull(quotaStore); this.attributeStore = requireNonNull(attributeStore); this.jobUpdateStore = requireNonNull(jobUpdateStore); + this.hostMaintenanceStore = requireNonNull(hostMaintenanceStore); this.log = requireNonNull(log); this.eventSink = requireNonNull(eventSink); } @@ -243,6 +251,24 @@ public class WriteRecorder implements } @Override + public void saveHostMaintenanceRequest(IHostMaintenanceRequest hostMaintenanceRequest) { + requireNonNull(hostMaintenanceRequest); + + write(Op.saveHostMaintenanceRequest( + new SaveHostMaintenanceRequest(hostMaintenanceRequest.newBuilder()))); + this.hostMaintenanceStore.saveHostMaintenanceRequest(hostMaintenanceRequest); + } + + @Override + public void removeHostMaintenanceRequest(String host) { + requireNonNull(host); + + write(Op.removeHostMaintenanceRequest( + new RemoveHostMaintenanceRequest(host))); + this.hostMaintenanceStore.removeHostMaintenanceRequest(host); + } + + @Override public void deleteAllTasks() { throw new UnsupportedOperationException( "Unsupported since casual storage users should never be doing this."); @@ -255,6 +281,12 @@ public class WriteRecorder implements } @Override + public void deleteHostMaintenanceRequests() { + throw new UnsupportedOperationException( + "Unsupported since casual storage users should never be doing this."); + } + + @Override public void deleteJobs() { throw new UnsupportedOperationException( "Unsupported since casual storage users should never be doing this."); @@ -308,6 +340,11 @@ public class WriteRecorder implements } @Override + public HostMaintenanceStore.Mutable getHostMaintenanceStore() { + return this; + } + + @Override public Optional<String> fetchFrameworkId() { return this.schedulerStore.fetchFrameworkId(); } @@ -366,4 +403,14 @@ public class WriteRecorder implements public Optional<IJobUpdateDetails> fetchJobUpdate(IJobUpdateKey key) { return this.jobUpdateStore.fetchJobUpdate(key); } + + @Override + public Optional<IHostMaintenanceRequest> getHostMaintenanceRequest(String host) { + return this.hostMaintenanceStore.getHostMaintenanceRequest(host); + } + + @Override + public Set<IHostMaintenanceRequest> getHostMaintenanceRequests() { + return this.hostMaintenanceStore.getHostMaintenanceRequests(); + } } http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java index b30de88..751ea55 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotService.java @@ -77,7 +77,8 @@ class SnapshotService extends AbstractScheduledService implements SnapshotStore + ", cron jobs: " + snapshot.getCronJobsSize() + ", quota confs: " + snapshot.getQuotaConfigurationsSize() + ", tasks: " + snapshot.getTasksSize() - + ", updates: " + snapshot.getJobUpdateDetailsSize()); + + ", updates: " + snapshot.getJobUpdateDetailsSize() + + ", host maintenance requests: " + snapshot.getHostMaintenanceRequestsSize()); }); } catch (CodingException e) { throw new StorageException("Failed to encode a snapshot", e); http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotterImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotterImpl.java b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotterImpl.java index 4b52be0..e761238 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotterImpl.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/SnapshotterImpl.java @@ -18,7 +18,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; - import javax.inject.Inject; import com.google.common.annotations.VisibleForTesting; @@ -39,6 +38,7 @@ import org.apache.aurora.gen.storage.QuotaConfiguration; import org.apache.aurora.gen.storage.SaveCronJob; import org.apache.aurora.gen.storage.SaveFrameworkId; import org.apache.aurora.gen.storage.SaveHostAttributes; +import org.apache.aurora.gen.storage.SaveHostMaintenanceRequest; import org.apache.aurora.gen.storage.SaveJobInstanceUpdateEvent; import org.apache.aurora.gen.storage.SaveJobUpdate; import org.apache.aurora.gen.storage.SaveJobUpdateEvent; @@ -53,6 +53,7 @@ import org.apache.aurora.scheduler.storage.JobUpdateStore; import org.apache.aurora.scheduler.storage.Snapshotter; import org.apache.aurora.scheduler.storage.Storage.StoreProvider; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IResourceAggregate; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; @@ -80,6 +81,7 @@ public class SnapshotterImpl implements Snapshotter { private static final String CRON_FIELD = "crons"; private static final String JOB_UPDATE_FIELD = "job_updates"; private static final String SCHEDULER_METADATA_FIELD = "scheduler_metadata"; + private static final String HOST_MAINTENANCE_REQUESTS_FIELDS = "host_maintenance_requests"; @VisibleForTesting Set<String> snapshotFieldNames() { @@ -257,6 +259,29 @@ public class SnapshotterImpl implements Snapshotter { } return Stream.empty(); } + }, + new SnapshotField() { + @Override + String getName() { + return HOST_MAINTENANCE_REQUESTS_FIELDS; + } + + @Override + void saveToSnapshot(StoreProvider storeProvider, Snapshot snapshot) { + snapshot.setHostMaintenanceRequests( + IHostMaintenanceRequest.toBuildersSet( + storeProvider.getHostMaintenanceStore().getHostMaintenanceRequests())); + } + + @Override + Stream<Op> doStreamFrom(Snapshot snapshot) { + if (snapshot.getHostMaintenanceRequestsSize() > 0) { + return snapshot.getHostMaintenanceRequests().stream() + .map(request -> Op.saveHostMaintenanceRequest( + new SaveHostMaintenanceRequest().setHostMaintenanceRequest(request))); + } + return Stream.empty(); + } } ); http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStore.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStore.java new file mode 100644 index 0000000..c8d96f2 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStore.java @@ -0,0 +1,55 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.storage.mem; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + +import org.apache.aurora.scheduler.storage.HostMaintenanceStore; +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; + +public class MemHostMaintenanceStore implements HostMaintenanceStore.Mutable { + + private final Map<String, IHostMaintenanceRequest> hostMaintenanceRequests = + Maps.newConcurrentMap(); + + @Override + public Optional<IHostMaintenanceRequest> getHostMaintenanceRequest(String host) { + return Optional.ofNullable(hostMaintenanceRequests.get(host)); + } + + @Override + public Set<IHostMaintenanceRequest> getHostMaintenanceRequests() { + return ImmutableSet.copyOf(hostMaintenanceRequests.values()); + } + + @Override + public void deleteHostMaintenanceRequests() { + hostMaintenanceRequests.clear(); + } + + @Override + public void saveHostMaintenanceRequest(IHostMaintenanceRequest hostMaintenanceRequest) { + hostMaintenanceRequests.put(hostMaintenanceRequest.getHost(), hostMaintenanceRequest); + } + + @Override + public void removeHostMaintenanceRequest(String host) { + hostMaintenanceRequests.remove(host); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java index 9f324b0..6037c24 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorage.java @@ -18,6 +18,7 @@ import javax.inject.Inject; import org.apache.aurora.common.inject.TimedInterceptor.Timed; import org.apache.aurora.scheduler.storage.AttributeStore; import org.apache.aurora.scheduler.storage.CronJobStore; +import org.apache.aurora.scheduler.storage.HostMaintenanceStore; import org.apache.aurora.scheduler.storage.JobUpdateStore; import org.apache.aurora.scheduler.storage.QuotaStore; import org.apache.aurora.scheduler.storage.SchedulerStore; @@ -37,7 +38,8 @@ public class MemStorage implements Storage { @Volatile final TaskStore.Mutable taskStore, @Volatile final QuotaStore.Mutable quotaStore, @Volatile final AttributeStore.Mutable attributeStore, - @Volatile final JobUpdateStore.Mutable updateStore) { + @Volatile final JobUpdateStore.Mutable updateStore, + @Volatile final HostMaintenanceStore.Mutable hostMaintenanceStore) { storeProvider = new MutableStoreProvider() { @Override @@ -74,6 +76,11 @@ public class MemStorage implements Storage { public JobUpdateStore.Mutable getJobUpdateStore() { return updateStore; } + + @Override + public HostMaintenanceStore.Mutable getHostMaintenanceStore() { + return hostMaintenanceStore; + } }; } http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java index edcea09..f1650e7 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/mem/MemStorageModule.java @@ -29,6 +29,7 @@ import org.apache.aurora.common.quantity.Time; import org.apache.aurora.common.stats.StatsProvider; import org.apache.aurora.scheduler.storage.AttributeStore; import org.apache.aurora.scheduler.storage.CronJobStore; +import org.apache.aurora.scheduler.storage.HostMaintenanceStore; import org.apache.aurora.scheduler.storage.JobUpdateStore; import org.apache.aurora.scheduler.storage.QuotaStore; import org.apache.aurora.scheduler.storage.SchedulerStore; @@ -76,6 +77,7 @@ public final class MemStorageModule extends PrivateModule { bindStore(QuotaStore.Mutable.class, MemQuotaStore.class); bindStore(SchedulerStore.Mutable.class, MemSchedulerStore.class); bindStore(JobUpdateStore.Mutable.class, MemJobUpdateStore.class); + bindStore(HostMaintenanceStore.Mutable.class, MemHostMaintenanceStore.class); Key<Storage> storageKey = keyFactory.create(Storage.class); bind(storageKey).to(MemStorage.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/python/apache/aurora/config/schema/base.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/schema/base.py b/src/main/python/apache/aurora/config/schema/base.py index a629bcd..7baded7 100644 --- a/src/main/python/apache/aurora/config/schema/base.py +++ b/src/main/python/apache/aurora/config/schema/base.py @@ -21,6 +21,7 @@ from apache.thermos.config.schema import * from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME + # TODO(wickman) Bind {{mesos.instance}} to %shard_id% class MesosContext(Struct): # The instance id (i.e. replica id, shard id) in the context of a task @@ -168,6 +169,22 @@ class ExecutorConfig(Struct): name = Default(String, AURORA_EXECUTOR_NAME) data = Default(String, "") + +class PercentageSlaPolicy(Struct): + percentage = Required(Float) + duration_secs = Required(Integer) + + +class CountSlaPolicy(Struct): + count = Required(Integer) + duration_secs = Required(Integer) + + +class CoordinatorSlaPolicy(Struct): + coordinator_url = Required(String) + status_key = Default(String, "drain") + + class MesosJob(Struct): name = Default(String, '{{task.name}}') role = Required(String) @@ -199,6 +216,7 @@ class MesosJob(Struct): enable_hooks = Default(Boolean, False) # enable client API hooks; from env python-list 'hooks' partition_policy = PartitionPolicy + sla_policy = Choice([CoordinatorSlaPolicy, CountSlaPolicy, PercentageSlaPolicy]) # Specifying a `Container` with a `docker` property for Docker jobs is deprecated, instead just # specify the value of the container property to be a `Docker` container directly. http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/python/apache/aurora/config/thrift.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/config/thrift.py b/src/main/python/apache/aurora/config/thrift.py index 6d2dde6..2ffa5b6 100644 --- a/src/main/python/apache/aurora/config/thrift.py +++ b/src/main/python/apache/aurora/config/thrift.py @@ -20,7 +20,10 @@ from twitter.common.lang import Compatibility from apache.aurora.config.schema.base import AppcImage as PystachioAppcImage from apache.aurora.config.schema.base import Container as PystachioContainer +from apache.aurora.config.schema.base import CoordinatorSlaPolicy as PystachioCoordinatorSlaPolicy +from apache.aurora.config.schema.base import CountSlaPolicy as PystachioCountSlaPolicy from apache.aurora.config.schema.base import DockerImage as PystachioDockerImage +from apache.aurora.config.schema.base import PercentageSlaPolicy as PystachioPercentageSlaPolicy from apache.aurora.config.schema.base import ( Docker, HealthCheckConfig, @@ -35,6 +38,8 @@ from gen.apache.aurora.api.ttypes import ( AppcImage, Constraint, Container, + CoordinatorSlaPolicy, + CountSlaPolicy, CronCollisionPolicy, DockerContainer, DockerImage, @@ -49,7 +54,9 @@ from gen.apache.aurora.api.ttypes import ( Metadata, Mode, PartitionPolicy, + PercentageSlaPolicy, Resource, + SlaPolicy, TaskConfig, TaskConstraint, ValueConstraint, @@ -244,6 +251,37 @@ THERMOS_PORT_SCOPE_REF = Ref.from_address('thermos.ports') THERMOS_TASK_ID_REF = Ref.from_address('thermos.task_id') +def create_sla_policy(sla_policy): + unwrapped = sla_policy.unwrap() + if isinstance(unwrapped, PystachioPercentageSlaPolicy): + return SlaPolicy( + percentageSlaPolicy=PercentageSlaPolicy( + fully_interpolated(unwrapped.percentage()), + fully_interpolated(unwrapped.duration_secs()), + ), + countSlaPolicy=None, + coordinatorSlaPolicy=None + ) + elif isinstance(unwrapped, PystachioCountSlaPolicy): + return SlaPolicy( + percentageSlaPolicy=None, + countSlaPolicy=CountSlaPolicy( + fully_interpolated(unwrapped.count()), + fully_interpolated(unwrapped.duration_secs()), + ), + coordinatorSlaPolicy=None + ) + elif isinstance(unwrapped, PystachioCoordinatorSlaPolicy): + return SlaPolicy( + percentageSlaPolicy=None, + countSlaPolicy=None, + coordinatorSlaPolicy=CoordinatorSlaPolicy( + fully_interpolated(unwrapped.coordinator_url()), + fully_interpolated(unwrapped.status_key()), + ) + ) + + def convert(job, metadata=frozenset(), ports=frozenset()): """Convert a Pystachio MesosJob to an Aurora Thrift JobConfiguration.""" @@ -274,6 +312,9 @@ def convert(job, metadata=frozenset(), ports=frozenset()): fully_interpolated(job.partition_policy().reschedule()), fully_interpolated(job.partition_policy().delay_secs())) + if job.has_sla_policy(): + task.slaPolicy = create_sla_policy(job.sla_policy()) + # Add metadata to a task, to display in the scheduler UI. metadata_set = frozenset() if job.has_metadata(): http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/main/python/apache/aurora/executor/executor_vars.py ---------------------------------------------------------------------- diff --git a/src/main/python/apache/aurora/executor/executor_vars.py b/src/main/python/apache/aurora/executor/executor_vars.py index 561f945..f5676ea 100644 --- a/src/main/python/apache/aurora/executor/executor_vars.py +++ b/src/main/python/apache/aurora/executor/executor_vars.py @@ -62,7 +62,7 @@ class ExecutorVars(Observable, ExceptionalThread): def aggregate_memory(cls, process, attribute='pss'): try: return sum(getattr(mmap, attribute) for mmap in process.memory_maps()) - except (psutil.Error, AttributeError): + except (OSError, psutil.Error, AttributeError): # psutil on OS X does not support get_memory_maps return 0 http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/base/TaskTestUtil.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/test/java/org/apache/aurora/scheduler/base/TaskTestUtil.java index 778148a..8c1f5ce 100644 --- a/src/test/java/org/apache/aurora/scheduler/base/TaskTestUtil.java +++ b/src/test/java/org/apache/aurora/scheduler/base/TaskTestUtil.java @@ -32,9 +32,11 @@ import org.apache.aurora.gen.LimitConstraint; import org.apache.aurora.gen.MesosFetcherURI; import org.apache.aurora.gen.Metadata; import org.apache.aurora.gen.PartitionPolicy; +import org.apache.aurora.gen.PercentageSlaPolicy; import org.apache.aurora.gen.Resource; import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.gen.ScheduledTask; +import org.apache.aurora.gen.SlaPolicy; import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.gen.TaskConstraint; import org.apache.aurora.gen.TaskEvent; @@ -113,15 +115,21 @@ public final class TaskTestUtil { } public static ITaskConfig makeConfig(IJobKey job) { + return makeConfig(job, true); + } + + public static ITaskConfig makeConfig(IJobKey job, boolean prod) { return ITaskConfig.build(new TaskConfig() .setJob(job.newBuilder()) .setOwner(new Identity().setUser(job.getRole() + "-user")) .setIsService(true) .setPriority(1) .setMaxTaskFailures(-1) - .setProduction(true) + .setProduction(prod) .setTier(PROD_TIER_NAME) .setPartitionPolicy(new PartitionPolicy().setDelaySecs(5).setReschedule(true)) + .setSlaPolicy(SlaPolicy.percentageSlaPolicy( + new PercentageSlaPolicy().setPercentage(95.0).setDurationSecs(1800))) .setConstraints(ImmutableSet.of( new Constraint( "valueConstraint", @@ -159,6 +167,10 @@ public final class TaskTestUtil { return makeTask(id, makeConfig(job), instanceId, Optional.empty()); } + public static IScheduledTask makeTask(String id, IJobKey job, int instanceId, boolean prod) { + return makeTask(id, makeConfig(job, prod), instanceId, Optional.empty()); + } + public static IScheduledTask makeTask(String id, IJobKey job, int instanceId, String agentId) { return makeTask(id, makeConfig(job), instanceId, Optional.of(agentId)); } http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/AbstractHostMaintenanceStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/AbstractHostMaintenanceStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/AbstractHostMaintenanceStoreTest.java new file mode 100644 index 0000000..e95955c --- /dev/null +++ b/src/test/java/org/apache/aurora/scheduler/storage/AbstractHostMaintenanceStoreTest.java @@ -0,0 +1,160 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.storage; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import com.google.common.collect.ImmutableSet; + +import org.apache.aurora.gen.CountSlaPolicy; +import org.apache.aurora.gen.HostMaintenanceRequest; +import org.apache.aurora.gen.PercentageSlaPolicy; +import org.apache.aurora.gen.SlaPolicy; +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public abstract class AbstractHostMaintenanceStoreTest { + + private static final String HOST_A = "hostA"; + private static final String HOST_B = "hostB"; + + private static final SlaPolicy PERCENTAGE_SLA_POLICY = SlaPolicy.percentageSlaPolicy( + new PercentageSlaPolicy() + .setPercentage(95) + .setDurationSecs(1800) + ); + + private static final SlaPolicy COUNT_SLA_POLICY = SlaPolicy.countSlaPolicy( + new CountSlaPolicy() + .setCount(3) + .setDurationSecs(1800) + ); + + private static final IHostMaintenanceRequest HOST_A_MAINTENANCE_REQUEST = + IHostMaintenanceRequest.build(new HostMaintenanceRequest() + .setHost(HOST_A) + .setDefaultSlaPolicy(PERCENTAGE_SLA_POLICY) + .setCreatedTimestampMs(0) + .setTimeoutSecs(1800) + ); + + private static final IHostMaintenanceRequest HOST_B_MAINTENANCE_REQUEST = + IHostMaintenanceRequest.build(new HostMaintenanceRequest() + .setHost(HOST_B) + .setDefaultSlaPolicy(COUNT_SLA_POLICY) + .setCreatedTimestampMs(0) + .setTimeoutSecs(1800) + ); + + private Storage storage; + + @Before + public void setUp() { + storage = createStorage(); + } + + protected abstract Storage createStorage(); + + @Test + public void testReadHostMaintenanceRequestNonExistant() { + assertEquals(Optional.empty(), read(HOST_A)); + } + + @Test + public void testReadHostMaintenanceRequest() { + assertEquals(Optional.empty(), read(HOST_A)); + + insert(HOST_A_MAINTENANCE_REQUEST); + assertEquals(Optional.of(HOST_A_MAINTENANCE_REQUEST), read(HOST_A)); + } + + @Test + public void testReadAllHostMaintenanceRequest() { + assertEquals(Collections.emptySet(), readAll()); + + insert(HOST_A_MAINTENANCE_REQUEST); + assertEquals(ImmutableSet.of(HOST_A_MAINTENANCE_REQUEST), readAll()); + + insert(HOST_B_MAINTENANCE_REQUEST); + assertEquals( + ImmutableSet.of(HOST_A_MAINTENANCE_REQUEST, HOST_B_MAINTENANCE_REQUEST), readAll()); + } + + @Test + public void testSaveHostMaintenanceRequest() { + insert(HOST_A_MAINTENANCE_REQUEST); + assertEquals(Optional.of(HOST_A_MAINTENANCE_REQUEST), read(HOST_A)); + } + + @Test + public void testSaveHostMaintenanceRequestUpdates() { + insert(HOST_A_MAINTENANCE_REQUEST); + insert(HOST_B_MAINTENANCE_REQUEST); + assertEquals( + ImmutableSet.of(HOST_A_MAINTENANCE_REQUEST, HOST_B_MAINTENANCE_REQUEST), readAll()); + + IHostMaintenanceRequest updatedA = IHostMaintenanceRequest.build( + HOST_A_MAINTENANCE_REQUEST.newBuilder().setTimeoutSecs(0)); + + insert(updatedA); + assertEquals( + ImmutableSet.of(updatedA, HOST_B_MAINTENANCE_REQUEST), readAll()); + } + + @Test + public void testRemoveHostMaintenanceRequestNotExistant() { + assertEquals(Collections.emptySet(), readAll()); + + assertEquals(Optional.empty(), read(HOST_A)); + } + + @Test + public void testRemoveHostMaintenanceRequest() { + assertEquals(Optional.empty(), read(HOST_A)); + + insert(HOST_A_MAINTENANCE_REQUEST); + assertEquals(Optional.of(HOST_A_MAINTENANCE_REQUEST), read(HOST_A)); + + truncate(HOST_A); + assertEquals(Optional.empty(), read(HOST_A)); + } + + private void insert(IHostMaintenanceRequest hostMaintenanceRequest) { + storage.write( + store -> { + store.getHostMaintenanceStore().saveHostMaintenanceRequest(hostMaintenanceRequest); + return null; + }); + } + + private Optional<IHostMaintenanceRequest> read(String host) { + return storage.read(store -> store.getHostMaintenanceStore().getHostMaintenanceRequest(host)); + } + + private Set<IHostMaintenanceRequest> readAll() { + return storage.read(store -> store.getHostMaintenanceStore().getHostMaintenanceRequests()); + } + + private void truncate(String host) { + storage.write( + (Storage.MutateWork.NoResult.Quiet) store -> store + .getHostMaintenanceStore() + .removeHostMaintenanceRequest(host)); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/backup/RecoveryTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/backup/RecoveryTest.java b/src/test/java/org/apache/aurora/scheduler/storage/backup/RecoveryTest.java index ba03ff9..afba0b7 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/backup/RecoveryTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/backup/RecoveryTest.java @@ -172,6 +172,7 @@ public class RecoveryTest extends EasyMockTest { return new Snapshot() .setHostAttributes(ImmutableSet.of()) + .setHostMaintenanceRequests(ImmutableSet.of()) .setQuotaConfigurations(ImmutableSet.of()) .setJobUpdateDetails(ImmutableSet.of()) .setCronJobs(ImmutableSet.of()) http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java b/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java index 31f9545..3372cec 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/durability/DataCompatibilityTest.java @@ -49,6 +49,7 @@ import org.apache.aurora.gen.Resource; import org.apache.aurora.gen.ResourceAggregate; import org.apache.aurora.gen.storage.Op; import org.apache.aurora.gen.storage.PruneJobUpdateHistory; +import org.apache.aurora.gen.storage.RemoveHostMaintenanceRequest; import org.apache.aurora.gen.storage.RemoveJob; import org.apache.aurora.gen.storage.RemoveJobUpdates; import org.apache.aurora.gen.storage.RemoveLock; @@ -57,6 +58,7 @@ import org.apache.aurora.gen.storage.RemoveTasks; import org.apache.aurora.gen.storage.SaveCronJob; import org.apache.aurora.gen.storage.SaveFrameworkId; import org.apache.aurora.gen.storage.SaveHostAttributes; +import org.apache.aurora.gen.storage.SaveHostMaintenanceRequest; import org.apache.aurora.gen.storage.SaveJobInstanceUpdateEvent; import org.apache.aurora.gen.storage.SaveJobUpdate; import org.apache.aurora.gen.storage.SaveJobUpdateEvent; @@ -111,6 +113,7 @@ public class DataCompatibilityTest { */ private static final List<Op> READ_COMPATIBILITY_OPS = ImmutableList.of( Op.pruneJobUpdateHistory(newStruct(PruneJobUpdateHistory.class)), + Op.removeHostMaintenanceRequest(newStruct(RemoveHostMaintenanceRequest.class)), Op.removeJob(newStruct(RemoveJob.class)), Op.removeJobUpdate(newStruct(RemoveJobUpdates.class)), Op.removeLock(newStruct(RemoveLock.class)), @@ -119,6 +122,7 @@ public class DataCompatibilityTest { Op.saveCronJob(newStruct(SaveCronJob.class)), Op.saveFrameworkId(newStruct(SaveFrameworkId.class)), Op.saveHostAttributes(newStruct(SaveHostAttributes.class)), + Op.saveHostMaintenanceRequest(newStruct(SaveHostMaintenanceRequest.class)), Op.saveJobUpdate(newStruct(SaveJobUpdate.class)), Op.saveJobInstanceUpdateEvent(newStruct(SaveJobInstanceUpdateEvent.class)), Op.saveJobUpdateEvent(newStruct(SaveJobUpdateEvent.class)), http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java index 3dd9ce4..fcca9a5 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/durability/DurableStorageTest.java @@ -29,6 +29,7 @@ import org.apache.aurora.common.testing.easymock.EasyMockTest; import org.apache.aurora.gen.AssignedTask; import org.apache.aurora.gen.Attribute; import org.apache.aurora.gen.HostAttributes; +import org.apache.aurora.gen.HostMaintenanceRequest; import org.apache.aurora.gen.InstanceTaskConfig; import org.apache.aurora.gen.JobConfiguration; import org.apache.aurora.gen.JobInstanceUpdateEvent; @@ -41,13 +42,16 @@ import org.apache.aurora.gen.JobUpdateSettings; import org.apache.aurora.gen.JobUpdateStatus; import org.apache.aurora.gen.JobUpdateSummary; import org.apache.aurora.gen.MaintenanceMode; +import org.apache.aurora.gen.PercentageSlaPolicy; import org.apache.aurora.gen.Range; import org.apache.aurora.gen.ResourceAggregate; import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.gen.ScheduledTask; +import org.apache.aurora.gen.SlaPolicy; import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.gen.storage.Op; import org.apache.aurora.gen.storage.PruneJobUpdateHistory; +import org.apache.aurora.gen.storage.RemoveHostMaintenanceRequest; import org.apache.aurora.gen.storage.RemoveJob; import org.apache.aurora.gen.storage.RemoveJobUpdates; import org.apache.aurora.gen.storage.RemoveLock; @@ -56,6 +60,7 @@ import org.apache.aurora.gen.storage.RemoveTasks; import org.apache.aurora.gen.storage.SaveCronJob; import org.apache.aurora.gen.storage.SaveFrameworkId; import org.apache.aurora.gen.storage.SaveHostAttributes; +import org.apache.aurora.gen.storage.SaveHostMaintenanceRequest; import org.apache.aurora.gen.storage.SaveJobInstanceUpdateEvent; import org.apache.aurora.gen.storage.SaveJobUpdate; import org.apache.aurora.gen.storage.SaveJobUpdateEvent; @@ -75,6 +80,7 @@ import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult.Quiet; import org.apache.aurora.scheduler.storage.durability.Persistence.Edit; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent; import org.apache.aurora.scheduler.storage.entities.IJobKey; @@ -129,6 +135,7 @@ public class DurableStorageTest extends EasyMockTest { storageUtil.quotaStore, storageUtil.attributeStore, storageUtil.jobUpdateStore, + storageUtil.hostMaintenanceStore, eventSink, new ReentrantLock(), TaskTestUtil.THRIFT_BACKFILL); @@ -207,6 +214,7 @@ public class DurableStorageTest extends EasyMockTest { storageUtil.quotaStore.deleteQuotas(); storageUtil.attributeStore.deleteHostAttributes(); storageUtil.jobUpdateStore.deleteAllUpdates(); + storageUtil.hostMaintenanceStore.deleteHostMaintenanceRequests(); RemoveTasks removeTasks = new RemoveTasks(ImmutableSet.of("taskId1")); builder.add(Edit.op(Op.removeTasks(removeTasks))); @@ -784,6 +792,55 @@ public class DurableStorageTest extends EasyMockTest { }.run(); } + @Test + public void testSaveHostMaintenanceRequest() throws Exception { + String host = "hostname"; + IHostMaintenanceRequest hostMaintenanceRequest = IHostMaintenanceRequest.build( + new HostMaintenanceRequest() + .setHost(host) + .setDefaultSlaPolicy(SlaPolicy.percentageSlaPolicy( + new PercentageSlaPolicy() + .setPercentage(95) + .setDurationSecs(1800))) + .setTimeoutSecs(1800) + ); + + new AbstractMutationFixture() { + @Override + protected void setupExpectations() { + storageUtil.expectWrite(); + storageUtil.hostMaintenanceStore.saveHostMaintenanceRequest(hostMaintenanceRequest); + expectPersist(Op.saveHostMaintenanceRequest( + new SaveHostMaintenanceRequest(hostMaintenanceRequest.newBuilder()))); + } + + @Override + protected void performMutations(MutableStoreProvider storeProvider) { + storeProvider.getHostMaintenanceStore().saveHostMaintenanceRequest( + hostMaintenanceRequest); + } + }.run(); + } + + @Test + public void testRemoveHostMaintenanceRequest() throws Exception { + String host = "hostname"; + + new AbstractMutationFixture() { + @Override + protected void setupExpectations() { + storageUtil.expectWrite(); + storageUtil.hostMaintenanceStore.removeHostMaintenanceRequest(host); + expectPersist(Op.removeHostMaintenanceRequest(new RemoveHostMaintenanceRequest(host))); + } + + @Override + protected void performMutations(MutableStoreProvider storeProvider) { + storeProvider.getHostMaintenanceStore().removeHostMaintenanceRequest(host); + } + }.run(); + } + private static IScheduledTask task(String id, ScheduleStatus status) { return IScheduledTask.build(new ScheduledTask() .setStatus(status) http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/durability/WriteRecorderTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/durability/WriteRecorderTest.java b/src/test/java/org/apache/aurora/scheduler/storage/durability/WriteRecorderTest.java index 27c8c82..c247ff1 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/durability/WriteRecorderTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/durability/WriteRecorderTest.java @@ -32,6 +32,7 @@ import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; import org.apache.aurora.scheduler.storage.AttributeStore; import org.apache.aurora.scheduler.storage.CronJobStore; +import org.apache.aurora.scheduler.storage.HostMaintenanceStore; import org.apache.aurora.scheduler.storage.JobUpdateStore; import org.apache.aurora.scheduler.storage.QuotaStore; import org.apache.aurora.scheduler.storage.SchedulerStore; @@ -64,6 +65,8 @@ public class WriteRecorderTest extends EasyMockTest { taskStore = createMock(TaskStore.Mutable.class); attributeStore = createMock(AttributeStore.Mutable.class); jobUpdateStore = createMock(JobUpdateStore.Mutable.class); + HostMaintenanceStore.Mutable hostMaintenanceStore = + createMock(HostMaintenanceStore.Mutable.class); eventSink = createMock(EventSink.class); storage = new WriteRecorder( @@ -74,6 +77,7 @@ public class WriteRecorderTest extends EasyMockTest { createMock(QuotaStore.Mutable.class), attributeStore, jobUpdateStore, + hostMaintenanceStore, LoggerFactory.getLogger(WriteRecorderTest.class), eventSink); } http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotterImplIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotterImplIT.java b/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotterImplIT.java index be07361..4c1918f 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotterImplIT.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/SnapshotterImplIT.java @@ -25,6 +25,7 @@ import org.apache.aurora.common.util.testing.FakeClock; import org.apache.aurora.gen.Attribute; import org.apache.aurora.gen.CronCollisionPolicy; import org.apache.aurora.gen.HostAttributes; +import org.apache.aurora.gen.HostMaintenanceRequest; import org.apache.aurora.gen.Identity; import org.apache.aurora.gen.InstanceTaskConfig; import org.apache.aurora.gen.JobConfiguration; @@ -41,7 +42,9 @@ import org.apache.aurora.gen.JobUpdateState; import org.apache.aurora.gen.JobUpdateStatus; import org.apache.aurora.gen.JobUpdateSummary; import org.apache.aurora.gen.MaintenanceMode; +import org.apache.aurora.gen.PercentageSlaPolicy; import org.apache.aurora.gen.Range; +import org.apache.aurora.gen.SlaPolicy; import org.apache.aurora.gen.storage.QuotaConfiguration; import org.apache.aurora.gen.storage.SchedulerMetadata; import org.apache.aurora.gen.storage.Snapshot; @@ -56,6 +59,7 @@ import org.apache.aurora.scheduler.storage.durability.Loader; import org.apache.aurora.scheduler.storage.durability.Persistence.Edit; import org.apache.aurora.scheduler.storage.durability.ThriftBackfill; import org.apache.aurora.scheduler.storage.entities.IHostAttributes; +import org.apache.aurora.scheduler.storage.entities.IHostMaintenanceRequest; import org.apache.aurora.scheduler.storage.entities.IJobConfiguration; import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails; @@ -156,6 +160,15 @@ public class SnapshotterImplIT { .setStatus(JobUpdateStatus.ERROR))) .setInstanceEvents(ImmutableList.of(new JobInstanceUpdateEvent() .setAction(JobUpdateAction.INSTANCE_UPDATED)))); + private static final IHostMaintenanceRequest HOST_MAINTENANCE_REQUEST = + IHostMaintenanceRequest.build( + new HostMaintenanceRequest() + .setHost("host") + .setDefaultSlaPolicy(SlaPolicy.percentageSlaPolicy( + new PercentageSlaPolicy() + .setPercentage(95) + .setDurationSecs(1800))) + .setTimeoutSecs(1800)); private Snapshot expected() { return new Snapshot() @@ -166,7 +179,8 @@ public class SnapshotterImplIT { .setCronJobs(ImmutableSet.of(new StoredCronJob(CRON_JOB.newBuilder()))) .setSchedulerMetadata(new SchedulerMetadata(FRAMEWORK_ID, METADATA)) .setJobUpdateDetails(ImmutableSet.of( - new StoredJobUpdateDetails().setDetails(UPDATE.newBuilder()))); + new StoredJobUpdateDetails().setDetails(UPDATE.newBuilder()))) + .setHostMaintenanceRequests(ImmutableSet.of(HOST_MAINTENANCE_REQUEST.newBuilder())); } private Snapshot makeNonBackfilled() { http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStoreTest.java new file mode 100644 index 0000000..ce1a9d6 --- /dev/null +++ b/src/test/java/org/apache/aurora/scheduler/storage/mem/MemHostMaintenanceStoreTest.java @@ -0,0 +1,25 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.storage.mem; + +import org.apache.aurora.scheduler.storage.AbstractHostMaintenanceStoreTest; +import org.apache.aurora.scheduler.storage.Storage; + +public class MemHostMaintenanceStoreTest extends AbstractHostMaintenanceStoreTest { + @Override + protected Storage createStorage() { + return MemStorageModule.newEmptyStorage(); + } +} + http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java b/src/test/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java index d59118b..04a60a5 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/testing/StorageTestUtil.java @@ -21,6 +21,7 @@ import org.apache.aurora.common.testing.easymock.EasyMockTest; import org.apache.aurora.scheduler.base.Query; import org.apache.aurora.scheduler.storage.AttributeStore; import org.apache.aurora.scheduler.storage.CronJobStore; +import org.apache.aurora.scheduler.storage.HostMaintenanceStore; import org.apache.aurora.scheduler.storage.JobUpdateStore; import org.apache.aurora.scheduler.storage.QuotaStore; import org.apache.aurora.scheduler.storage.SchedulerStore; @@ -52,6 +53,7 @@ public class StorageTestUtil { public final CronJobStore.Mutable jobStore; public final SchedulerStore.Mutable schedulerStore; public final JobUpdateStore.Mutable jobUpdateStore; + public final HostMaintenanceStore.Mutable hostMaintenanceStore; public final NonVolatileStorage storage; /** @@ -68,6 +70,7 @@ public class StorageTestUtil { this.jobStore = easyMock.createMock(CronJobStore.Mutable.class); this.schedulerStore = easyMock.createMock(SchedulerStore.Mutable.class); this.jobUpdateStore = easyMock.createMock(JobUpdateStore.Mutable.class); + this.hostMaintenanceStore = easyMock.createMock(HostMaintenanceStore.Mutable.class); this.storage = easyMock.createMock(NonVolatileStorage.class); } @@ -93,6 +96,7 @@ public class StorageTestUtil { expect(storeProvider.getCronJobStore()).andReturn(jobStore).anyTimes(); expect(storeProvider.getSchedulerStore()).andReturn(schedulerStore).anyTimes(); expect(storeProvider.getJobUpdateStore()).andReturn(jobUpdateStore).anyTimes(); + expect(storeProvider.getHostMaintenanceStore()).andReturn(hostMaintenanceStore).anyTimes(); expect(mutableStoreProvider.getTaskStore()).andReturn(taskStore).anyTimes(); expect(mutableStoreProvider.getUnsafeTaskStore()).andReturn(taskStore).anyTimes(); expect(mutableStoreProvider.getQuotaStore()).andReturn(quotaStore).anyTimes(); @@ -100,6 +104,8 @@ public class StorageTestUtil { expect(mutableStoreProvider.getCronJobStore()).andReturn(jobStore).anyTimes(); expect(mutableStoreProvider.getSchedulerStore()).andReturn(schedulerStore).anyTimes(); expect(mutableStoreProvider.getJobUpdateStore()).andReturn(jobUpdateStore).anyTimes(); + expect( + mutableStoreProvider.getHostMaintenanceStore()).andReturn(hostMaintenanceStore).anyTimes(); } /** http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/python/apache/aurora/client/cli/test_inspect.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/client/cli/test_inspect.py b/src/test/python/apache/aurora/client/cli/test_inspect.py index e4f43d0..2baba2a 100644 --- a/src/test/python/apache/aurora/client/cli/test_inspect.py +++ b/src/test/python/apache/aurora/client/cli/test_inspect.py @@ -22,10 +22,10 @@ from apache.aurora.config import AuroraConfig from apache.aurora.config.schema.base import Job from apache.thermos.config.schema_base import MB, Process, Resources, Task -from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME - from .util import AuroraClientCommandTest +from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME + class TestInspectCommand(AuroraClientCommandTest): def get_job_config(self): http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/python/apache/aurora/config/test_thrift.py ---------------------------------------------------------------------- diff --git a/src/test/python/apache/aurora/config/test_thrift.py b/src/test/python/apache/aurora/config/test_thrift.py index 8e1d0e1..8872cb5 100644 --- a/src/test/python/apache/aurora/config/test_thrift.py +++ b/src/test/python/apache/aurora/config/test_thrift.py @@ -12,14 +12,17 @@ # limitations under the License. # -import json import getpass +import json import re import pytest from apache.aurora.config import AuroraConfig +from apache.aurora.config.schema.base import CoordinatorSlaPolicy as PystachioCoordinatorSlaPolicy +from apache.aurora.config.schema.base import CountSlaPolicy as PystachioCountSlaPolicy from apache.aurora.config.schema.base import PartitionPolicy as PystachioPartitionPolicy +from apache.aurora.config.schema.base import PercentageSlaPolicy as PystachioPercentageSlaPolicy from apache.aurora.config.schema.base import ( AppcImage, Container, @@ -42,10 +45,13 @@ from apache.thermos.config.schema import Process, Resources, Task from gen.apache.aurora.api.constants import AURORA_EXECUTOR_NAME, GOOD_IDENTIFIER_PATTERN_PYTHON from gen.apache.aurora.api.ttypes import Mode as ThriftMode from gen.apache.aurora.api.ttypes import ( + CoordinatorSlaPolicy, + CountSlaPolicy, CronCollisionPolicy, Identity, JobKey, PartitionPolicy, + PercentageSlaPolicy, Resource ) from gen.apache.aurora.test.constants import INVALID_IDENTIFIERS, VALID_IDENTIFIERS @@ -259,6 +265,72 @@ def test_disable_partition_policy(): assert job.taskConfig.partitionPolicy == PartitionPolicy(False, 0) +def test_no_sla_policy(): + hwc = HELLO_WORLD() + + job = convert_pystachio_to_thrift(hwc) + + assert job.taskConfig.slaPolicy is None + + +def test_percentage_sla_policy(): + hwc = HELLO_WORLD( + sla_policy=PystachioPercentageSlaPolicy(percentage=95.0, duration_secs=1800) + ) + + job = convert_pystachio_to_thrift(hwc) + + assert job.taskConfig.slaPolicy.percentageSlaPolicy == PercentageSlaPolicy( + percentage=95.0, + durationSecs=1800) + assert job.taskConfig.slaPolicy.countSlaPolicy is None + assert job.taskConfig.slaPolicy.coordinatorSlaPolicy is None + + +def test_count_sla_policy(): + hwc = HELLO_WORLD( + sla_policy=PystachioCountSlaPolicy(count=10, duration_secs=1800) + ) + + job = convert_pystachio_to_thrift(hwc) + + assert job.taskConfig.slaPolicy.percentageSlaPolicy is None + assert job.taskConfig.slaPolicy.coordinatorSlaPolicy is None + assert job.taskConfig.slaPolicy.countSlaPolicy == CountSlaPolicy( + count=10, + durationSecs=1800) + + +def test_coordinator_sla_policy_defaults(): + hwc = HELLO_WORLD( + sla_policy=PystachioCoordinatorSlaPolicy(coordinator_url='some-url') + ) + + job = convert_pystachio_to_thrift(hwc) + + assert job.taskConfig.slaPolicy.percentageSlaPolicy is None + assert job.taskConfig.slaPolicy.countSlaPolicy is None + assert job.taskConfig.slaPolicy.coordinatorSlaPolicy == CoordinatorSlaPolicy( + coordinatorUrl='some-url', + statusKey='drain' + ) + + +def test_coordinator_sla_policy_status_key(): + hwc = HELLO_WORLD( + sla_policy=PystachioCoordinatorSlaPolicy(coordinator_url='some-url', status_key='key') + ) + + job = convert_pystachio_to_thrift(hwc) + + assert job.taskConfig.slaPolicy.percentageSlaPolicy is None + assert job.taskConfig.slaPolicy.countSlaPolicy is None + assert job.taskConfig.slaPolicy.coordinatorSlaPolicy == CoordinatorSlaPolicy( + coordinatorUrl='some-url', + statusKey='key' + ) + + def test_config_with_ports(): hwc = HELLO_WORLD( task=HELLO_WORLD.task()( http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/removeHostMaintenanceRequest ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/removeHostMaintenanceRequest b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/removeHostMaintenanceRequest new file mode 100644 index 0000000..93c982e --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/removeHostMaintenanceRequest @@ -0,0 +1,9 @@ +{ + "20": { + "rec": { + "1": { + "str": "string-value" + } + } + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveCronJob ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveCronJob b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveCronJob index 88e1c36..e912b48 100644 --- a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveCronJob +++ b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveCronJob @@ -167,6 +167,20 @@ "i64": 4 } } + }, + "35": { + "rec": { + "3": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "str": "string-value" + } + } + } + } } } }, http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveHostMaintenanceRequest ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveHostMaintenanceRequest b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveHostMaintenanceRequest new file mode 100644 index 0000000..e0e2ac1 --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveHostMaintenanceRequest @@ -0,0 +1,33 @@ +{ + "19": { + "rec": { + "1": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "rec": { + "3": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "str": "string-value" + } + } + } + } + }, + "3": { + "i64": 4 + }, + "4": { + "i64": 4 + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveJobUpdate ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveJobUpdate b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveJobUpdate index 32fdcda..08dfa5b 100644 --- a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveJobUpdate +++ b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveJobUpdate @@ -222,6 +222,20 @@ "i64": 4 } } + }, + "35": { + "rec": { + "3": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "str": "string-value" + } + } + } + } } } }, @@ -402,6 +416,20 @@ "i64": 4 } } + }, + "35": { + "rec": { + "3": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "str": "string-value" + } + } + } + } } } }, http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveTasks ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveTasks b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveTasks index 4323031..df3bc4b 100644 --- a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveTasks +++ b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/current/saveTasks @@ -175,6 +175,20 @@ "i64": 4 } } + }, + "35": { + "rec": { + "3": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "str": "string-value" + } + } + } + } } } }, http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/16-saveHostMaintenanceRequest ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/16-saveHostMaintenanceRequest b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/16-saveHostMaintenanceRequest new file mode 100644 index 0000000..da16d62 --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/16-saveHostMaintenanceRequest @@ -0,0 +1,30 @@ +{ + "19": { + "rec": { + "1": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "rec": { + "3": { + "rec": { + "1": { + "str": "string-value" + }, + "2": { + "i64": 4 + } + } + } + } + }, + "3": { + "i64": 4 + } + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/34be6315/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/17-removeHostMaintenanceRequest ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/17-removeHostMaintenanceRequest b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/17-removeHostMaintenanceRequest new file mode 100644 index 0000000..93c982e --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/storage/durability/goldens/read-compatible/17-removeHostMaintenanceRequest @@ -0,0 +1,9 @@ +{ + "20": { + "rec": { + "1": { + "str": "string-value" + } + } + } +}
