Repository: aurora Updated Branches: refs/heads/master 297ebc157 -> 6b4045397
Remove several scheduler command line arguments. Reviewed at https://reviews.apache.org/r/41786/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/6b404539 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/6b404539 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/6b404539 Branch: refs/heads/master Commit: 6b4045397305040616affdf1989c73d0f78b667a Parents: 297ebc1 Author: Bill Farner <[email protected]> Authored: Sun Jan 3 09:39:44 2016 -0800 Committer: Bill Farner <[email protected]> Committed: Sun Jan 3 09:39:44 2016 -0800 ---------------------------------------------------------------------- NEWS | 5 + docs/scheduler-storage.md | 47 -------- .../aurora/scheduler/http/api/ApiModule.java | 7 +- .../aurora/scheduler/storage/log/Entries.java | 4 +- .../scheduler/storage/log/LogManager.java | 16 --- .../scheduler/storage/log/LogStorageModule.java | 12 -- .../storage/log/StreamManagerImpl.java | 23 +--- .../aurora/scheduler/thrift/aop/AopModule.java | 91 +--------------- .../thrift/aop/FeatureToggleInterceptor.java | 46 -------- .../scheduler/storage/log/LogManagerTest.java | 19 ++-- .../scheduler/storage/log/LogStorageTest.java | 23 ++-- .../storage/log/testing/LogOpMatcher.java | 14 ++- .../scheduler/thrift/aop/AopModuleTest.java | 109 ------------------- .../aop/FeatureToggleInterceptorTest.java | 84 -------------- 14 files changed, 43 insertions(+), 457 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/NEWS ---------------------------------------------------------------------- diff --git a/NEWS b/NEWS index 394b31c..965fbf4 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,11 @@ - Upgraded Mesos to 0.25.0. - Env variables can be passed through to task processes by passing `--preserve_env` to thermos. +- Removed scheduler command line arguments: + - `-enable_cors_support`. Enabling CORS is now implicit by setting the argument + `-enable_cors_for`. + - `-deduplicate_snapshots` and `-deflate_snapshots`. These features are good to always enable. + - `-enable_job_updates` and `-enable_job_creation` 0.11.0 ------ http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/docs/scheduler-storage.md ---------------------------------------------------------------------- diff --git a/docs/scheduler-storage.md b/docs/scheduler-storage.md deleted file mode 100644 index 1cd02f8..0000000 --- a/docs/scheduler-storage.md +++ /dev/null @@ -1,47 +0,0 @@ -# Snapshot Performance - -Periodically the scheduler writes a full snapshot of its state to the replicated log. To do this -it needs to hold a global storage write lock while it writes out this data. In large clusters -this has been observed to take up to 40 seconds. Long pauses can cause issues in the system, -including delays in scheduling new tasks. - -The scheduler has two optimizations to reduce the size of snapshots and thus improve snapshot -performance: compression and deduplication. Most users will want to enable both compression -and deduplication. - -## Compression - -To reduce the size of the snapshot the DEFLATE algorithm can be applied to the serialized bytes -of the snapshot as they are written to the stream. This reduces the total number of bytes that -need to be written to the replicated log at the cost of CPU and generally reduces the amount -of time a snapshot takes. Most users will want to enable both compression and deduplication. - -### Enabling Compression - -Snapshot compression is enabled via the `-deflate_snapshots` flag. This is the default since -Aurora 0.5.0. All released versions of Aurora can read both compressed and uncompressed snapshots, -so there are no backwards compatibility concerns associated with changing this flag. - -### Disabling compression - -Disable compression by passing `-deflate_snapshots=false`. - -## Deduplication - -In Aurora 0.6.0 a new snapshot format was introduced. Rather than write one configuration blob -per Mesos task this format stores each configuration blob once, and each Mesos task with a -pointer to its blob. This format is not backwards compatible with earlier versions of Aurora. - -### Enabling Deduplication - -After upgrading Aurora to 0.6.0, enable deduplication with the `-deduplicate_snapshots` flag. -After the first snapshot the cluster will be using the deduplicated format to write to the -replicated log. Snapshots are created periodically by the scheduler (according to -the `-dlog_snapshot_interval` flag). An administrator can also force a snapshot operation with -`aurora_admin snapshot`. - -### Disabling Deduplication - -To disable deduplication, for example to rollback to Aurora, restart all of the cluster's -schedulers with `-deduplicate_snapshots=false` and either wait for a snapshot or force one -using `aurora_admin snapshot`. http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java index da6894e..cd5adf9 100644 --- a/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java +++ b/src/main/java/org/apache/aurora/scheduler/http/api/ApiModule.java @@ -35,16 +35,13 @@ import org.eclipse.jetty.util.resource.Resource; public class ApiModule extends ServletModule { public static final String API_PATH = "/api"; - @CmdLine(name = "enable_cors_support", help = "Enable CORS support for thrift end points.") - private static final Arg<Boolean> ENABLE_CORS_SUPPORT = Arg.create(false); - /** * Set the {@code Access-Control-Allow-Origin} header for API requests. See * http://www.w3.org/TR/cors/ */ @CmdLine(name = "enable_cors_for", help = "List of domains for which CORS support should be enabled.") - private static final Arg<String> ENABLE_CORS_FOR = Arg.create("*"); + private static final Arg<String> ENABLE_CORS_FOR = Arg.create(null); private static final String API_CLIENT_ROOT = Resource .newClassPathResource("org/apache/aurora/scheduler/gen/client") @@ -52,7 +49,7 @@ public class ApiModule extends ServletModule { @Override protected void configureServlets() { - if (ENABLE_CORS_SUPPORT.get()) { + if (ENABLE_CORS_FOR.get() != null) { filter(API_PATH).through(new CorsFilter(ENABLE_CORS_FOR.get())); } serve(API_PATH).with(TServlet.class); http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java index 548b5e7..413d59a 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/Entries.java @@ -25,7 +25,7 @@ import org.apache.aurora.gen.storage.LogEntry._Fields; /** * Utility class for working with log entries. */ -final class Entries { +public final class Entries { private static final Logger LOG = Logger.getLogger(Entries.class.getName()); @@ -46,7 +46,7 @@ final class Entries { * of the original entry. * @throws CodingException If the value could not be encoded or deflated. */ - static LogEntry deflate(LogEntry entry) throws CodingException { + public static LogEntry deflate(LogEntry entry) throws CodingException { return LogEntry.deflatedEntry(ThriftBinaryCodec.deflateNonNull(entry)); } http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java index 4099503..cfa9c56 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogManager.java @@ -39,22 +39,6 @@ public class LogManager { public @interface MaxEntrySize { } /** - * When true, enable snapshot deflation. - */ - @Retention(RetentionPolicy.RUNTIME) - @Target({ElementType.METHOD, ElementType.PARAMETER}) - @Qualifier - public @interface DeflateSnapshots { } - - /** - * When true, enable snapshot deduplication. - */ - @Retention(RetentionPolicy.RUNTIME) - @Target({ElementType.METHOD, ElementType.PARAMETER}) - @Qualifier - public @interface DeduplicateSnapshots { } - - /** * Hash function used to verify log entries. */ @Retention(RetentionPolicy.RUNTIME) http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java index 561c70b..ed63a74 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/LogStorageModule.java @@ -31,12 +31,10 @@ import org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage; import org.apache.aurora.scheduler.storage.DistributedSnapshotStore; import org.apache.aurora.scheduler.storage.Storage; import org.apache.aurora.scheduler.storage.Storage.NonVolatileStorage; -import org.apache.aurora.scheduler.storage.log.LogManager.DeduplicateSnapshots; import org.apache.aurora.scheduler.storage.log.LogManager.MaxEntrySize; import org.apache.aurora.scheduler.storage.log.LogStorage.Settings; import static org.apache.aurora.scheduler.storage.log.EntrySerializer.EntrySerializerImpl; -import static org.apache.aurora.scheduler.storage.log.LogManager.DeflateSnapshots; import static org.apache.aurora.scheduler.storage.log.LogManager.LogEntryHashFunction; import static org.apache.aurora.scheduler.storage.log.SnapshotDeduplicator.SnapshotDeduplicatorImpl; @@ -64,14 +62,6 @@ public class LogStorageModule extends PrivateModule { public static final Arg<Amount<Integer, Data>> MAX_LOG_ENTRY_SIZE = Arg.create(Amount.of(512, Data.KB)); - @CmdLine(name = "deduplicate_snapshots", - help = "Write snapshots in deduplicated format. For details and backwards compatibility " - + "concerns see docs/scheduler-storage.md.") - private static final Arg<Boolean> DEDUPLICATE_SNAPSHOTS = Arg.create(false); - - @CmdLine(name = "deflate_snapshots", help = "Whether snapshots should be deflate-compressed.") - private static final Arg<Boolean> DEFLATE_SNAPSHOTS = Arg.create(true); - @Override protected void configure() { bind(Settings.class) @@ -80,8 +70,6 @@ public class LogStorageModule extends PrivateModule { bind(new TypeLiteral<Amount<Integer, Data>>() { }).annotatedWith(MaxEntrySize.class) .toInstance(MAX_LOG_ENTRY_SIZE.get()); bind(LogManager.class).in(Singleton.class); - bindConstant().annotatedWith(DeduplicateSnapshots.class).to(DEDUPLICATE_SNAPSHOTS.get()); - bindConstant().annotatedWith(DeflateSnapshots.class).to(DEFLATE_SNAPSHOTS.get()); bind(LogStorage.class).in(Singleton.class); install(CallOrderEnforcingStorage.wrappingModule(LogStorage.class)); http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java index b252468..7a63515 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/log/StreamManagerImpl.java @@ -55,8 +55,6 @@ import static org.apache.aurora.codec.ThriftBinaryCodec.CodingException; import static org.apache.aurora.common.inject.TimedInterceptor.Timed; import static org.apache.aurora.scheduler.log.Log.Stream.InvalidPositionException; import static org.apache.aurora.scheduler.log.Log.Stream.StreamAccessException; -import static org.apache.aurora.scheduler.storage.log.LogManager.DeduplicateSnapshots; -import static org.apache.aurora.scheduler.storage.log.LogManager.DeflateSnapshots; import static org.apache.aurora.scheduler.storage.log.LogManager.LogEntryHashFunction; class StreamManagerImpl implements StreamManager { @@ -79,26 +77,20 @@ class StreamManagerImpl implements StreamManager { private final Object writeMutex = new Object(); private final Log.Stream stream; private final EntrySerializer entrySerializer; - private final boolean deflateSnapshots; private final HashFunction hashFunction; private final SnapshotDeduplicator snapshotDeduplicator; - private final boolean deduplicateSnapshots; @Inject StreamManagerImpl( @Assisted Stream stream, EntrySerializer entrySerializer, - @DeflateSnapshots boolean deflateSnapshots, @LogEntryHashFunction HashFunction hashFunction, - SnapshotDeduplicator snapshotDeduplicator, - @DeduplicateSnapshots boolean deduplicateSnapshots) { + SnapshotDeduplicator snapshotDeduplicator) { this.stream = requireNonNull(stream); this.entrySerializer = requireNonNull(entrySerializer); - this.deflateSnapshots = deflateSnapshots; this.hashFunction = requireNonNull(hashFunction); this.snapshotDeduplicator = requireNonNull(snapshotDeduplicator); - this.deduplicateSnapshots = deduplicateSnapshots; } @Override @@ -203,17 +195,8 @@ class StreamManagerImpl implements StreamManager { public void snapshot(Snapshot snapshot) throws CodingException, InvalidPositionException, StreamAccessException { - LogEntry entry; - if (deduplicateSnapshots) { - entry = LogEntry.deduplicatedSnapshot(snapshotDeduplicator.deduplicate(snapshot)); - } else { - entry = LogEntry.snapshot(snapshot); - } - - if (deflateSnapshots) { - entry = deflate(entry); - } - + LogEntry entry = + deflate(LogEntry.deduplicatedSnapshot(snapshotDeduplicator.deduplicate(snapshot))); Log.Position position = appendAndGetPosition(entry); vars.snapshots.incrementAndGet(); vars.unSnapshottedTransactions.set(0); http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java index 7f29b79..f59ee1a 100644 --- a/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java +++ b/src/main/java/org/apache/aurora/scheduler/thrift/aop/AopModule.java @@ -13,33 +13,13 @@ */ package org.apache.aurora.scheduler.thrift.aop; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Map; - -import javax.inject.Inject; -import javax.inject.Singleton; - import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.inject.AbstractModule; import com.google.inject.Binder; -import com.google.inject.Key; -import com.google.inject.PrivateModule; -import com.google.inject.TypeLiteral; import com.google.inject.matcher.Matcher; import com.google.inject.matcher.Matchers; import org.aopalliance.intercept.MethodInterceptor; -import org.apache.aurora.common.args.Arg; -import org.apache.aurora.common.args.CmdLine; -import org.apache.aurora.gen.AuroraSchedulerManager; import org.apache.aurora.gen.Response; import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift; @@ -48,66 +28,20 @@ import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift; */ public class AopModule extends AbstractModule { - @CmdLine(name = "enable_job_updates", help = "Whether new job updates should be accepted.") - private static final Arg<Boolean> ENABLE_UPDATES = Arg.create(true); - - @CmdLine(name = "enable_job_creation", - help = "Allow new jobs to be created, if false all job creation requests will be denied.") - private static final Arg<Boolean> ENABLE_JOB_CREATION = Arg.create(true); - private static final Matcher<? super Class<?>> THRIFT_IFACE_MATCHER = Matchers.subclassesOf(AnnotatedAuroraAdmin.class) .and(Matchers.annotatedWith(DecoratedThrift.class)); - private final Map<String, Boolean> toggledMethods; - - public AopModule() { - this(ImmutableMap.of( - "createJob", ENABLE_JOB_CREATION.get(), - "acquireLock", ENABLE_UPDATES.get())); - } - - @VisibleForTesting - AopModule(Map<String, Boolean> toggledMethods) { - this.toggledMethods = ImmutableMap.copyOf(toggledMethods); - } - - private static final Function<Method, String> GET_NAME = Method::getName; - @Override protected void configure() { // Layer ordering: - // APIVersion -> Log -> FeatureToggle -> StatsExporter -> SchedulerThriftInterface + // APIVersion -> Log -> StatsExporter -> SchedulerThriftInterface // It's important for this interceptor to be registered first to ensure it's at the 'top' of // the stack and the standard message is always applied. bindThriftDecorator(new ServerInfoInterceptor()); bindThriftDecorator(new LoggingInterceptor()); - - install(new PrivateModule() { - @Override - protected void configure() { - // Ensure that the provided methods exist on the decorated interface. - List<Method> methods = - ImmutableList.copyOf(AuroraSchedulerManager.Iface.class.getMethods()); - for (String toggledMethod : toggledMethods.keySet()) { - Preconditions.checkArgument( - Iterables.any(methods, - Predicates.compose(Predicates.equalTo(toggledMethod), GET_NAME)), - String.format("Method %s was not found in class %s", - toggledMethod, - AuroraSchedulerManager.Iface.class)); - } - - bind(new TypeLiteral<Map<String, Boolean>>() { }).toInstance(toggledMethods); - bind(IsFeatureEnabled.class).in(Singleton.class); - Key<Predicate<Method>> predicateKey = Key.get(new TypeLiteral<Predicate<Method>>() { }); - bind(predicateKey).to(IsFeatureEnabled.class); - expose(predicateKey); - } - }); - bindThriftDecorator(new FeatureToggleInterceptor()); bindThriftDecorator(new ThriftStatsExporterInterceptor()); } @@ -127,27 +61,4 @@ public class AopModule extends AbstractModule { interceptor); binder.requestInjection(interceptor); } - - private static class IsFeatureEnabled implements Predicate<Method> { - private final Predicate<String> methodEnabled; - - @Inject - IsFeatureEnabled(Map<String, Boolean> toggleMethods) { - Predicate<String> builder = Predicates.alwaysTrue(); - for (Map.Entry<String, Boolean> toggleMethod : toggleMethods.entrySet()) { - Predicate<String> enableMethod = Predicates.or( - toggleMethod.getValue() - ? Predicates.alwaysTrue() - : Predicates.alwaysFalse(), - Predicates.not(Predicates.equalTo(toggleMethod.getKey()))); - builder = Predicates.and(builder, enableMethod); - } - methodEnabled = builder; - } - - @Override - public boolean apply(Method method) { - return methodEnabled.apply(method.getName()); - } - } } http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java b/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java deleted file mode 100644 index 2492796..0000000 --- a/src/main/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptor.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.thrift.aop; - -import java.lang.reflect.Method; - -import javax.inject.Inject; - -import com.google.common.base.Predicate; - -import org.aopalliance.intercept.MethodInterceptor; -import org.aopalliance.intercept.MethodInvocation; -import org.apache.aurora.gen.ResponseCode; -import org.apache.aurora.scheduler.thrift.Responses; - -/** - * A method interceptor that blocks access to features based on a supplied predicate. - */ -public class FeatureToggleInterceptor implements MethodInterceptor { - - @Inject private Predicate<Method> allowMethod; - - @Override - public Object invoke(MethodInvocation invocation) throws Throwable { - Method method = invocation.getMethod(); - if (allowMethod.apply(method)) { - return invocation.proceed(); - } else { - return Responses.addMessage( - Responses.empty(), - ResponseCode.ERROR, - "The " + method.getName() + " feature is currently disabled on this scheduler."); - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java index 692ace0..0443bb3 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogManagerTest.java @@ -42,6 +42,7 @@ import org.apache.aurora.gen.HostAttributes; import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.gen.ScheduledTask; import org.apache.aurora.gen.TaskConfig; +import org.apache.aurora.gen.storage.DeduplicatedSnapshot; import org.apache.aurora.gen.storage.Frame; import org.apache.aurora.gen.storage.FrameChunk; import org.apache.aurora.gen.storage.FrameHeader; @@ -103,10 +104,8 @@ public class LogManagerTest extends EasyMockTest { return new StreamManagerImpl( stream, new EntrySerializer.EntrySerializerImpl(maxEntrySize, Hashing.md5()), - false, Hashing.md5(), - new SnapshotDeduplicatorImpl(), - false); + new SnapshotDeduplicatorImpl()); } @Test @@ -261,7 +260,8 @@ public class LogManagerTest extends EasyMockTest { @Test public void testTransactionSnapshot() throws CodingException { Snapshot snapshot = createSnapshot(); - expectAppend(position1, LogEntry.snapshot(snapshot)); + DeduplicatedSnapshot deduplicated = new SnapshotDeduplicatorImpl().deduplicate(snapshot); + expectAppend(position1, Entries.deflate(LogEntry.deduplicatedSnapshot(deduplicated))); stream.truncateBefore(position1); control.replay(); @@ -393,10 +393,8 @@ public class LogManagerTest extends EasyMockTest { final StreamManagerImpl streamManager = new StreamManagerImpl( mockStream, new EntrySerializer.EntrySerializerImpl(message1.chunkSize, Hashing.md5()), - false, Hashing.md5(), - new SnapshotDeduplicatorImpl(), - false); + new SnapshotDeduplicatorImpl()); StreamTransaction tr1 = streamManager.startTransaction(); tr1.add(op1); @@ -485,7 +483,8 @@ public class LogManagerTest extends EasyMockTest { public void testWriteAndReadDeflatedEntry() throws Exception { Snapshot snapshot = createSnapshot(); LogEntry snapshotLogEntry = LogEntry.snapshot(snapshot); - LogEntry deflatedSnapshotEntry = Entries.deflate(snapshotLogEntry); + LogEntry deflatedSnapshotEntry = Entries.deflate( + LogEntry.deduplicatedSnapshot(new SnapshotDeduplicatorImpl().deduplicate(snapshot))); Entry snapshotEntry = createMock(Entry.class); expect(stream.append(entryEq(deflatedSnapshotEntry))).andReturn(position1); @@ -504,10 +503,8 @@ public class LogManagerTest extends EasyMockTest { StreamManagerImpl streamManager = new StreamManagerImpl( stream, new EntrySerializer.EntrySerializerImpl(NO_FRAMES_EVER_SIZE, md5), - true, md5, - new SnapshotDeduplicatorImpl(), - false); + new SnapshotDeduplicatorImpl()); streamManager.snapshot(snapshot); streamManager.readFromBeginning(reader); } http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java index 216f92f..4305270 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/LogStorageTest.java @@ -56,6 +56,7 @@ import org.apache.aurora.gen.ResourceAggregate; import org.apache.aurora.gen.ScheduleStatus; import org.apache.aurora.gen.ScheduledTask; import org.apache.aurora.gen.TaskConfig; +import org.apache.aurora.gen.storage.DeduplicatedSnapshot; import org.apache.aurora.gen.storage.LogEntry; import org.apache.aurora.gen.storage.Op; import org.apache.aurora.gen.storage.PruneJobUpdateHistory; @@ -78,6 +79,7 @@ import org.apache.aurora.gen.storage.Transaction; import org.apache.aurora.gen.storage.storageConstants; import org.apache.aurora.scheduler.base.JobKeys; import org.apache.aurora.scheduler.base.Query; +import org.apache.aurora.scheduler.base.TaskTestUtil; import org.apache.aurora.scheduler.base.Tasks; import org.apache.aurora.scheduler.events.EventSink; import org.apache.aurora.scheduler.events.PubsubEvent; @@ -104,6 +106,7 @@ import org.apache.aurora.scheduler.storage.entities.IResourceAggregate; import org.apache.aurora.scheduler.storage.entities.IScheduledTask; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import org.apache.aurora.scheduler.storage.log.LogStorage.SchedulingService; +import org.apache.aurora.scheduler.storage.log.SnapshotDeduplicator.SnapshotDeduplicatorImpl; import org.apache.aurora.scheduler.storage.log.testing.LogOpMatcher; import org.apache.aurora.scheduler.storage.log.testing.LogOpMatcher.StreamMatcher; import org.apache.aurora.scheduler.storage.testing.StorageTestUtil; @@ -131,6 +134,7 @@ public class LogStorageTest extends EasyMockTest { private LogStorage logStorage; private Log log; + private SnapshotDeduplicator deduplicator; private Stream stream; private Position position; private StreamMatcher streamMatcher; @@ -142,19 +146,15 @@ public class LogStorageTest extends EasyMockTest { @Before public void setUp() { log = createMock(Log.class); - SnapshotDeduplicator snapshotDeduplicator = createMock(SnapshotDeduplicator.class); + deduplicator = createMock(SnapshotDeduplicator.class); StreamManagerFactory streamManagerFactory = logStream -> { HashFunction md5 = Hashing.md5(); return new StreamManagerImpl( logStream, - new EntrySerializer.EntrySerializerImpl( - Amount.of(1, Data.GB), - md5), - false, + new EntrySerializer.EntrySerializerImpl(Amount.of(1, Data.GB), md5), md5, - snapshotDeduplicator, - false); + deduplicator); }; LogManager logManager = new LogManager(log, streamManagerFactory); @@ -227,11 +227,12 @@ public class LogStorageTest extends EasyMockTest { Snapshot snapshotContents = new Snapshot() .setTimestamp(NOW) .setTasks(ImmutableSet.of( - new ScheduledTask() - .setStatus(ScheduleStatus.RUNNING) - .setAssignedTask(new AssignedTask().setTaskId("task_id")))); + TaskTestUtil.makeTask("task_id", TaskTestUtil.JOB).newBuilder())); expect(snapshotStore.createSnapshot()).andReturn(snapshotContents); - streamMatcher.expectSnapshot(snapshotContents).andReturn(position); + DeduplicatedSnapshot deduplicated = + new SnapshotDeduplicatorImpl().deduplicate(snapshotContents); + expect(deduplicator.deduplicate(snapshotContents)).andReturn(deduplicated); + streamMatcher.expectSnapshot(deduplicated).andReturn(position); stream.truncateBefore(position); Capture<MutateWork<Void, RuntimeException>> snapshotWork = createCapture(); expect(storageUtil.storage.write(capture(snapshotWork))).andAnswer( http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java b/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java index 9abdbbb..5a2524d 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/log/testing/LogOpMatcher.java @@ -15,17 +15,19 @@ package org.apache.aurora.scheduler.storage.log.testing; import java.util.Objects; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import org.apache.aurora.codec.ThriftBinaryCodec; import org.apache.aurora.codec.ThriftBinaryCodec.CodingException; +import org.apache.aurora.gen.storage.DeduplicatedSnapshot; import org.apache.aurora.gen.storage.LogEntry; import org.apache.aurora.gen.storage.Op; -import org.apache.aurora.gen.storage.Snapshot; import org.apache.aurora.gen.storage.Transaction; import org.apache.aurora.gen.storage.storageConstants; import org.apache.aurora.scheduler.log.Log.Position; import org.apache.aurora.scheduler.log.Log.Stream; +import org.apache.aurora.scheduler.storage.log.Entries; import org.easymock.EasyMock; import org.easymock.IArgumentMatcher; import org.easymock.IExpectationSetters; @@ -92,9 +94,13 @@ public class LogOpMatcher implements IArgumentMatcher { * @param snapshot Expected snapshot. * @return An expectation setter. */ - public IExpectationSetters<Position> expectSnapshot(Snapshot snapshot) { - LogEntry entry = LogEntry.snapshot(snapshot); - return expect(stream.append(sameEntry(entry))); + public IExpectationSetters<Position> expectSnapshot(DeduplicatedSnapshot snapshot) { + try { + LogEntry entry = Entries.deflate(LogEntry.deduplicatedSnapshot(snapshot)); + return expect(stream.append(sameEntry(entry))); + } catch (CodingException e) { + throw Throwables.propagate(e); + } } } http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java deleted file mode 100644 index 19b7415..0000000 --- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/AopModuleTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.thrift.aop; - -import java.util.Map; - -import com.google.common.collect.ImmutableMap; -import com.google.inject.AbstractModule; -import com.google.inject.CreationException; -import com.google.inject.Guice; -import com.google.inject.Injector; - -import org.apache.aurora.common.testing.easymock.EasyMockTest; -import org.apache.aurora.gen.AuroraAdmin.Iface; -import org.apache.aurora.gen.JobConfiguration; -import org.apache.aurora.gen.Response; -import org.apache.aurora.gen.ResponseCode; -import org.apache.aurora.gen.ServerInfo; -import org.apache.aurora.scheduler.storage.entities.IServerInfo; -import org.junit.Before; -import org.junit.Test; - -import static org.easymock.EasyMock.expect; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertSame; - -public class AopModuleTest extends EasyMockTest { - private AnnotatedAuroraAdmin mockThrift; - - @Before - public void setUp() throws Exception { - mockThrift = createMock(AnnotatedAuroraAdmin.class); - } - - private Iface getIface(Map<String, Boolean> toggledMethods) { - Injector injector = Guice.createInjector( - new AbstractModule() { - @Override - protected void configure() { - bind(IServerInfo.class).toInstance(IServerInfo.build(new ServerInfo())); - MockDecoratedThrift.bindForwardedMock(binder(), mockThrift); - } - }, - new AopModule(toggledMethods)); - return injector.getInstance(AnnotatedAuroraAdmin.class); - } - - @Test - public void testNonFlaggedMethod() throws Exception { - assertCreateAllowed(ImmutableMap.of("acquireLock", false)); - } - - @Test - public void testNoFlaggedMethods() throws Exception { - assertCreateAllowed(ImmutableMap.of()); - } - - @Test - public void testFlaggedMethodEnabled() throws Exception { - assertCreateAllowed(ImmutableMap.of("createJob", true)); - } - - @Test - public void testFlaggedMethodDisabled() throws Exception { - JobConfiguration job = new JobConfiguration(); - - control.replay(); - - Iface thrift = getIface(ImmutableMap.of("createJob", false)); - assertEquals(ResponseCode.ERROR, thrift.createJob(job, null).getResponseCode()); - } - - @Test(expected = CreationException.class) - public void testMissingMethod() { - control.replay(); - getIface(ImmutableMap.of("notamethod", true)); - } - - private void assertCreateAllowed(Map<String, Boolean> toggledMethods) throws Exception { - JobConfiguration job = new JobConfiguration(); - Response response = new Response(); - expect(mockThrift.createJob(job, null)).andReturn(response); - - control.replay(); - - Iface thrift = getIface(toggledMethods); - assertSame(response, thrift.createJob(job, null)); - } - - @Test - public void assertToStringNotIntercepted() { - control.replay(); - - Iface thrift = getIface(ImmutableMap.of()); - assertNotNull(thrift.toString()); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/6b404539/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java b/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java deleted file mode 100644 index 0edb315..0000000 --- a/src/test/java/org/apache/aurora/scheduler/thrift/aop/FeatureToggleInterceptorTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.thrift.aop; - -import java.lang.reflect.Method; - -import com.google.common.base.Predicate; -import com.google.inject.AbstractModule; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.TypeLiteral; -import com.google.inject.matcher.Matchers; - -import org.apache.aurora.common.testing.easymock.EasyMockTest; -import org.apache.aurora.gen.Response; -import org.apache.aurora.gen.ResponseCode; -import org.apache.aurora.gen.TaskQuery; -import org.apache.aurora.scheduler.thrift.auth.DecoratedThrift; -import org.easymock.EasyMock; -import org.junit.Before; -import org.junit.Test; - -import static org.easymock.EasyMock.expect; -import static org.junit.Assert.assertSame; - -public class FeatureToggleInterceptorTest extends EasyMockTest { - - private AnnotatedAuroraAdmin realThrift; - private AnnotatedAuroraAdmin decoratedThrift; - private Predicate<Method> predicate; - - @Before - public void setUp() { - realThrift = createMock(AnnotatedAuroraAdmin.class); - predicate = createMock(new Clazz<Predicate<Method>>() { }); - Injector injector = Guice.createInjector(new AbstractModule() { - @Override - protected void configure() { - MockDecoratedThrift.bindForwardedMock(binder(), realThrift); - bind(new TypeLiteral<Predicate<Method>>() { }).toInstance(predicate); - AopModule.bindThriftDecorator( - binder(), - Matchers.annotatedWith(DecoratedThrift.class), - new FeatureToggleInterceptor()); - } - }); - decoratedThrift = injector.getInstance(AnnotatedAuroraAdmin.class); - } - - @Test - public void testPredicatePass() throws Exception { - TaskQuery query = new TaskQuery(); - Response response = new Response() - .setResponseCode(ResponseCode.OK); - - expect(predicate.apply(EasyMock.anyObject())).andReturn(true); - expect(realThrift.getTasksStatus(query)).andReturn(response); - - control.replay(); - - assertSame(response, decoratedThrift.getTasksStatus(query)); - } - - @Test - public void testPredicateDeny() throws Exception { - TaskQuery query = new TaskQuery(); - expect(predicate.apply(EasyMock.anyObject())).andReturn(false); - - control.replay(); - - assertSame(ResponseCode.ERROR, decoratedThrift.getTasksStatus(query).getResponseCode()); - } -}
