Repository: aurora Updated Branches: refs/heads/master 2df2db951 -> 68c46205e
Removing GcExecutorLauncher code. Bugs closed: AURORA-1334 Reviewed at https://reviews.apache.org/r/35760/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/68c46205 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/68c46205 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/68c46205 Branch: refs/heads/master Commit: 68c46205e5ba0afb4eed3f709abfc393457204fb Parents: 2df2db9 Author: Maxim Khutornenko <[email protected]> Authored: Tue Jun 23 11:08:55 2015 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Tue Jun 23 11:08:55 2015 -0700 ---------------------------------------------------------------------- config/legacy_untested_classes.txt | 1 - debian/aurora-scheduler.default | 3 - debian/aurora-scheduler.init | 1 - debian/aurora-scheduler.upstart | 1 - docs/storage-config.md | 6 +- examples/scheduler/scheduler-local.sh | 1 - .../upstart/aurora-scheduler-kerberos.conf | 1 - examples/vagrant/upstart/aurora-scheduler.conf | 1 - .../aurora/scheduler/SchedulerModule.java | 9 +- .../aurora/scheduler/async/AsyncModule.java | 28 +- .../scheduler/async/GcExecutorLauncher.java | 296 ------------------- .../java/org/apache/aurora/ProtobufsTest.java | 32 ++ .../scheduler/async/GcExecutorLauncherTest.java | 262 ---------------- .../aurora/scheduler/base/CommandUtilTest.java | 42 +-- 14 files changed, 63 insertions(+), 621 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/config/legacy_untested_classes.txt ---------------------------------------------------------------------- diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt index d2f3ca5..4bae43a 100644 --- a/config/legacy_untested_classes.txt +++ b/config/legacy_untested_classes.txt @@ -3,7 +3,6 @@ org/apache/aurora/auth/UnsecureAuthModule$UnsecureCapabilityValidator$1 org/apache/aurora/auth/UnsecureAuthModule$UnsecureCapabilityValidator$2 org/apache/aurora/scheduler/app/SchedulerMain$2 org/apache/aurora/scheduler/app/SchedulerMain$3 -org/apache/aurora/scheduler/async/GcExecutorLauncher$1 org/apache/aurora/scheduler/async/OfferQueue$OfferQueueImpl$2 org/apache/aurora/scheduler/base/Conversions$1 org/apache/aurora/scheduler/base/Conversions$2 http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/debian/aurora-scheduler.default ---------------------------------------------------------------------- diff --git a/debian/aurora-scheduler.default b/debian/aurora-scheduler.default index 47fea5b..bc30627 100644 --- a/debian/aurora-scheduler.default +++ b/debian/aurora-scheduler.default @@ -70,9 +70,6 @@ THERMOS_EXECUTOR_FLAGS="" # Container types that are allowed to be used by jobs. ALLOWED_CONTAINER_TYPES="MESOS,DOCKER" -# Path (on the slave nodes) or URL to garbage collection executor -GC_EXECUTOR_PATH="/usr/share/aurora/bin/gc_executor.pex" - # Scheduler log verbosity LOG_LEVEL="INFO" http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/debian/aurora-scheduler.init ---------------------------------------------------------------------- diff --git a/debian/aurora-scheduler.init b/debian/aurora-scheduler.init index 2ed0159..59b200f 100755 --- a/debian/aurora-scheduler.init +++ b/debian/aurora-scheduler.init @@ -62,7 +62,6 @@ ARGS="-cluster_name=$CLUSTER_NAME -thermos_executor_resources=$THERMOS_EXECUTOR_RESOURCES -thermos_executor_flags=$THERMOS_EXECUTOR_FLAGS -allowed_container_types=$ALLOWED_CONTAINER_TYPES - -gc_executor_path=$GC_EXECUTOR_PATH -vlog=$LOG_LEVEL $EXTRA_SCHEDULER_ARGS" http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/debian/aurora-scheduler.upstart ---------------------------------------------------------------------- diff --git a/debian/aurora-scheduler.upstart b/debian/aurora-scheduler.upstart index ed7fe09..1ceb704 100644 --- a/debian/aurora-scheduler.upstart +++ b/debian/aurora-scheduler.upstart @@ -34,7 +34,6 @@ script -thermos_executor_resources="$THERMOS_EXECUTOR_RESOURCES" \ -thermos_executor_flags="$THERMOS_EXECUTOR_FLAGS" \ -allowed_container_types="$ALLOWED_CONTAINER_TYPES" \ - -gc_executor_path="$GC_EXECUTOR_PATH" \ -vlog="$LOG_LEVEL" \ $EXTRA_SCHEDULER_ARGS end script http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/docs/storage-config.md ---------------------------------------------------------------------- diff --git a/docs/storage-config.md b/docs/storage-config.md index 971bc16..4ec33a1 100644 --- a/docs/storage-config.md +++ b/docs/storage-config.md @@ -100,9 +100,9 @@ accomplished by updating the following scheduler configuration options: registering with Mesos. E.g.: `-mesos_master_address=zk://localhost:2181` * `-max_registration_delay` - set to sufficiently long interval to prevent registration timeout and as a result scheduler suicide. E.g: `-max_registration_delay=360min` - * Make sure `-gc_executor_path` option is not set to prevent accidental task GC. This is - important as scheduler will attempt to reconcile the cluster state and will kill all tasks when - restarted with an empty Mesos replicated log. + * Make sure `-reconciliation_initial_delay` option is set high enough (e.g.: `365days`) to + prevent accidental task GC. This is important as scheduler will attempt to reconcile the cluster + state and will kill all tasks when restarted with an empty Mesos replicated log. * Restart all schedulers http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/examples/scheduler/scheduler-local.sh ---------------------------------------------------------------------- diff --git a/examples/scheduler/scheduler-local.sh b/examples/scheduler/scheduler-local.sh index 6253d50..5b5d2fc 100755 --- a/examples/scheduler/scheduler-local.sh +++ b/examples/scheduler/scheduler-local.sh @@ -51,7 +51,6 @@ AURORA_FLAGS=( # TODO(Kevin Sweeney): Point these to real URLs. -thermos_executor_path=/dev/null - -gc_executor_path=/dev/null -vlog=INFO -logtostderr http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/examples/vagrant/upstart/aurora-scheduler-kerberos.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf b/examples/vagrant/upstart/aurora-scheduler-kerberos.conf index 414539b..3c9e13b 100644 --- a/examples/vagrant/upstart/aurora-scheduler-kerberos.conf +++ b/examples/vagrant/upstart/aurora-scheduler-kerberos.conf @@ -45,7 +45,6 @@ exec bin/aurora-scheduler \ -backup_dir=/var/lib/aurora/backups \ -thermos_executor_path=$DIST_DIR/thermos_executor.pex \ -thermos_executor_flags="--announcer-enable --announcer-ensemble localhost:2181" \ - -gc_executor_path=$DIST_DIR/gc_executor.pex \ -vlog=INFO \ -logtostderr \ -allowed_container_types=MESOS,DOCKER \ http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/examples/vagrant/upstart/aurora-scheduler.conf ---------------------------------------------------------------------- diff --git a/examples/vagrant/upstart/aurora-scheduler.conf b/examples/vagrant/upstart/aurora-scheduler.conf index f4b867c..1c2390f 100644 --- a/examples/vagrant/upstart/aurora-scheduler.conf +++ b/examples/vagrant/upstart/aurora-scheduler.conf @@ -37,7 +37,6 @@ exec bin/aurora-scheduler \ -backup_dir=/var/lib/aurora/backups \ -thermos_executor_path=$DIST_DIR/thermos_executor.pex \ -thermos_executor_flags="--announcer-enable --announcer-ensemble localhost:2181" \ - -gc_executor_path=$DIST_DIR/gc_executor.pex \ -vlog=INFO \ -logtostderr \ -allowed_container_types=MESOS,DOCKER \ http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java index 6edec22..ae31bdb 100644 --- a/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java +++ b/src/main/java/org/apache/aurora/scheduler/SchedulerModule.java @@ -35,7 +35,6 @@ import com.twitter.common.quantity.Time; import org.apache.aurora.scheduler.SchedulerLifecycle.LeadingOptions; import org.apache.aurora.scheduler.TaskIdGenerator.TaskIdGeneratorImpl; -import org.apache.aurora.scheduler.async.GcExecutorLauncher; import org.apache.aurora.scheduler.base.AsyncUtil; import org.apache.aurora.scheduler.events.PubsubEventModule; import org.apache.mesos.Protos; @@ -99,11 +98,7 @@ public class SchedulerModule extends AbstractModule { @Provides @Singleton - List<TaskLauncher> provideTaskLaunchers( - GcExecutorLauncher gcLauncher, - UserTaskLauncher userTaskLauncher) { - - return ImmutableList.of(gcLauncher, userTaskLauncher); + List<TaskLauncher> provideTaskLaunchers(UserTaskLauncher userTaskLauncher) { + return ImmutableList.of(userTaskLauncher); } - } http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java b/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java index 8bcac6c..17b3585 100644 --- a/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java +++ b/src/main/java/org/apache/aurora/scheduler/async/AsyncModule.java @@ -15,7 +15,6 @@ package org.apache.aurora.scheduler.async; import java.lang.annotation.Retention; import java.lang.annotation.Target; -import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.logging.Logger; @@ -25,7 +24,6 @@ import javax.inject.Qualifier; import javax.inject.Singleton; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.util.concurrent.AbstractIdleService; import com.google.common.util.concurrent.RateLimiter; @@ -44,8 +42,6 @@ import com.twitter.common.util.Random; import com.twitter.common.util.TruncatedBinaryBackoff; import org.apache.aurora.scheduler.SchedulerServicesModule; -import org.apache.aurora.scheduler.async.GcExecutorLauncher.GcExecutorSettings; -import org.apache.aurora.scheduler.async.GcExecutorLauncher.RandomGcExecutorSettings; import org.apache.aurora.scheduler.async.OfferManager.OfferManagerImpl; import org.apache.aurora.scheduler.async.OfferManager.OfferReturnDelay; import org.apache.aurora.scheduler.async.RescheduleCalculator.RescheduleCalculatorImpl; @@ -174,19 +170,10 @@ public class AsyncModule extends AbstractModule { private static final Arg<Amount<Long, Time>> RESERVATION_DURATION = Arg.create(Amount.of(3L, Time.MINUTES)); - @CmdLine(name = "executor_gc_interval", - help = "Max interval on which to run the GC executor on a host to clean up dead tasks.") - private static final Arg<Amount<Long, Time>> EXECUTOR_GC_INTERVAL = - Arg.create(Amount.of(1L, Time.HOURS)); - - @CmdLine(name = "gc_executor_path", help = "Path to the gc executor launch script.") - private static final Arg<String> GC_EXECUTOR_PATH = Arg.create(null); - - // TODO(maxim): Disabled by default until AURORA-715 is complete. @CmdLine(name = "reconciliation_initial_delay", help = "Initial amount of time to delay task reconciliation after scheduler start up.") private static final Arg<Amount<Long, Time>> RECONCILIATION_INITIAL_DELAY = - Arg.create(Amount.of((long) Integer.MAX_VALUE, Time.MINUTES)); + Arg.create(Amount.of(0L, Time.MINUTES)); @Positive @CmdLine(name = "reconciliation_explicit_interval", @@ -326,19 +313,6 @@ public class AsyncModule extends AbstractModule { install(new PrivateModule() { @Override protected void configure() { - bind(GcExecutorSettings.class).toInstance(new RandomGcExecutorSettings( - EXECUTOR_GC_INTERVAL.get(), - Optional.fromNullable(GC_EXECUTOR_PATH.get()))); - bind(Executor.class).toInstance(executor); - - bind(GcExecutorLauncher.class).in(Singleton.class); - expose(GcExecutorLauncher.class); - } - }); - - install(new PrivateModule() { - @Override - protected void configure() { bind(TaskReconcilerSettings.class).toInstance(new TaskReconcilerSettings( RECONCILIATION_INITIAL_DELAY.get(), RECONCILIATION_EXPLICIT_INTERVAL.get(), http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/src/main/java/org/apache/aurora/scheduler/async/GcExecutorLauncher.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/async/GcExecutorLauncher.java b/src/main/java/org/apache/aurora/scheduler/async/GcExecutorLauncher.java deleted file mode 100644 index f2ef70d..0000000 --- a/src/main/java/org/apache/aurora/scheduler/async/GcExecutorLauncher.java +++ /dev/null @@ -1,296 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.async; - -import java.util.Collections; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Logger; - -import javax.inject.Inject; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Optional; -import com.google.common.base.Supplier; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Maps; -import com.google.protobuf.ByteString; -import com.twitter.common.quantity.Amount; -import com.twitter.common.quantity.Data; -import com.twitter.common.quantity.Time; -import com.twitter.common.stats.Stats; -import com.twitter.common.stats.StatsProvider; -import com.twitter.common.util.Clock; -import com.twitter.common.util.Random; - -import org.apache.aurora.Protobufs; -import org.apache.aurora.codec.ThriftBinaryCodec; -import org.apache.aurora.codec.ThriftBinaryCodec.CodingException; -import org.apache.aurora.gen.comm.AdjustRetainedTasks; -import org.apache.aurora.scheduler.HostOffer; -import org.apache.aurora.scheduler.TaskLauncher; -import org.apache.aurora.scheduler.base.CommandUtil; -import org.apache.aurora.scheduler.base.Query; -import org.apache.aurora.scheduler.base.Tasks; -import org.apache.aurora.scheduler.configuration.Resources; -import org.apache.aurora.scheduler.mesos.Driver; -import org.apache.aurora.scheduler.storage.Storage; -import org.apache.aurora.scheduler.storage.entities.IScheduledTask; -import org.apache.mesos.Protos; -import org.apache.mesos.Protos.ExecutorID; -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.OfferID; -import org.apache.mesos.Protos.SlaveID; -import org.apache.mesos.Protos.TaskID; -import org.apache.mesos.Protos.TaskInfo; -import org.apache.mesos.Protos.TaskStatus; - -import static java.util.Objects.requireNonNull; - -/** - * A task launcher that periodically initiates garbage collection on a host, re-using a single - * garbage collection executor. - */ -public class GcExecutorLauncher implements TaskLauncher { - private static final Logger LOG = Logger.getLogger(GcExecutorLauncher.class.getName()); - - private final AtomicLong tasksCreated = Stats.exportLong("scheduler_gc_tasks_created"); - private final AtomicLong offersConsumed = Stats.exportLong("scheduler_gc_offers_consumed"); - - @VisibleForTesting - static final Resources TOTAL_GC_EXECUTOR_RESOURCES = - new Resources(0.2, Amount.of(128L, Data.MB), Amount.of(16L, Data.MB), 0); - - // An epsilon is used because we are required to supply executor and task resources. - @VisibleForTesting - static final Resources EPSILON = - new Resources(0.01, Amount.of(32L, Data.MB), Amount.of(1L, Data.MB), 0); - - private static final Resources GC_EXECUTOR_TASK_RESOURCES = - Resources.subtract(TOTAL_GC_EXECUTOR_RESOURCES, EPSILON); - - @VisibleForTesting - static final String SYSTEM_TASK_PREFIX = "system-gc-"; - @VisibleForTesting - static final String LOST_TASKS_STAT_NAME = "gc_executor_tasks_lost"; - @VisibleForTesting - static final String INSUFFICIENT_OFFERS_STAT_NAME = "scheduler_gc_insufficient_offers"; - private static final String EXECUTOR_NAME = "aurora.gc"; - - private final GcExecutorSettings settings; - private final Storage storage; - private final Clock clock; - private final Executor executor; - private final Driver driver; - private final Supplier<String> uuidGenerator; - private final Map<String, Long> pulses; - private final AtomicLong lostTasks; - private final AtomicLong insufficientOffers; - - @Inject - GcExecutorLauncher( - GcExecutorSettings settings, - Storage storage, - Clock clock, - Executor executor, - Driver driver, - StatsProvider statsProvider) { - - this( - settings, - storage, - clock, - executor, - driver, - statsProvider, - new Supplier<String>() { - @Override - public String get() { - return UUID.randomUUID().toString(); - } - }); - } - - @VisibleForTesting - GcExecutorLauncher( - GcExecutorLauncher.GcExecutorSettings settings, - Storage storage, - Clock clock, - Executor executor, - Driver driver, - StatsProvider statsProvider, - Supplier<String> uuidGenerator) { - - this.settings = requireNonNull(settings); - this.storage = requireNonNull(storage); - this.clock = requireNonNull(clock); - this.executor = requireNonNull(executor); - this.driver = requireNonNull(driver); - this.uuidGenerator = requireNonNull(uuidGenerator); - this.pulses = Collections.synchronizedMap(Maps.<String, Long>newHashMap()); - this.lostTasks = statsProvider.makeCounter(LOST_TASKS_STAT_NAME); - this.insufficientOffers = statsProvider.makeCounter(INSUFFICIENT_OFFERS_STAT_NAME); - } - - @VisibleForTesting - static TaskInfo makeGcTask( - String sourceName, - SlaveID slaveId, - String gcExecutorPath, - String uuid, - AdjustRetainedTasks message) { - - ExecutorInfo.Builder executorInfo = ExecutorInfo.newBuilder() - .setExecutorId(ExecutorID.newBuilder().setValue(EXECUTOR_NAME)) - .setName(EXECUTOR_NAME) - .setSource(sourceName) - .addAllResources(GC_EXECUTOR_TASK_RESOURCES.toResourceList()) - .setCommand(CommandUtil.create(gcExecutorPath, ImmutableList.<String>of())); - - byte[] data; - try { - data = ThriftBinaryCodec.encode(message); - } catch (CodingException e) { - LOG.severe("Failed to encode retained tasks message: " + message); - throw Throwables.propagate(e); - } - - return TaskInfo.newBuilder().setName("system-gc") - .setTaskId(TaskID.newBuilder().setValue(SYSTEM_TASK_PREFIX + uuid)) - .setSlaveId(slaveId) - .setData(ByteString.copyFrom(data)) - .setExecutor(executorInfo) - .addAllResources(EPSILON.toResourceList()) - .build(); - } - - private TaskInfo makeGcTask(String hostName, SlaveID slaveId) { - Iterable<IScheduledTask> tasksOnHost = - Storage.Util.fetchTasks(storage, Query.slaveScoped(hostName)); - tasksCreated.incrementAndGet(); - return makeGcTask( - hostName, - slaveId, - settings.getGcExecutorPath().get(), - uuidGenerator.get(), - new AdjustRetainedTasks().setRetainedTasks( - Maps.transformValues(Tasks.mapById(tasksOnHost), Tasks.GET_STATUS))); - } - - private boolean sufficientResources(HostOffer offer) { - boolean sufficient = - Resources.from(offer.getOffer()).greaterThanOrEqual(TOTAL_GC_EXECUTOR_RESOURCES); - if (!sufficient) { - LOG.fine("Offer for host " + offer.getOffer().getHostname() - + " is too small for a GC executor"); - insufficientOffers.incrementAndGet(); - } - return sufficient; - } - - @Override - public boolean willUse(final HostOffer offer) { - if (!settings.getGcExecutorPath().isPresent() - || !sufficientResources(offer) - || !isTimeToCollect(offer.getOffer().getHostname())) { - - return false; - } - - executor.execute(new Runnable() { - @Override - public void run() { - driver.launchTask( - offer.getOffer().getId(), - makeGcTask(offer.getOffer().getHostname(), offer.getOffer().getSlaveId())); - } - }); - offersConsumed.incrementAndGet(); - return true; - } - - @Override - public boolean statusUpdate(TaskStatus status) { - if (status.getTaskId().getValue().startsWith(SYSTEM_TASK_PREFIX)) { - LOG.info("Received status update for GC task: " + Protobufs.toString(status)); - if (status.getState() == Protos.TaskState.TASK_LOST) { - lostTasks.incrementAndGet(); - } - driver.acknowledgeStatusUpdate(status); - return true; - } else { - return false; - } - } - - @Override - public void cancelOffer(OfferID offer) { - // No-op. - } - - private boolean isTimeToCollect(String hostname) { - boolean result = false; - Optional<Long> timestamp = Optional.fromNullable(pulses.get(hostname)); - if (timestamp.isPresent()) { - if (clock.nowMillis() >= timestamp.get()) { - pulses.put(hostname, clock.nowMillis() + settings.getDelayMs()); - result = true; - } - } else { - pulses.put(hostname, clock.nowMillis() + settings.getDelayMs()); - } - - return result; - } - - public static class GcExecutorSettings { - protected final Amount<Long, Time> gcInterval; - private final Optional<String> gcExecutorPath; - - @VisibleForTesting - GcExecutorSettings(Amount<Long, Time> gcInterval, Optional<String> gcExecutorPath) { - this.gcInterval = requireNonNull(gcInterval); - this.gcExecutorPath = requireNonNull(gcExecutorPath); - } - - @VisibleForTesting - int getDelayMs() { - return gcInterval.as(Time.MILLISECONDS).intValue(); - } - - @VisibleForTesting - Optional<String> getGcExecutorPath() { - return gcExecutorPath; - } - } - - /** - * Wraps configuration values for the {@code GcExecutorLauncher}. - */ - static class RandomGcExecutorSettings extends GcExecutorSettings { - private final Random rand = new Random.SystemRandom(new java.util.Random()); - - RandomGcExecutorSettings(Amount<Long, Time> gcInterval, Optional<String> gcExecutorPath) { - super(gcInterval, gcExecutorPath); - } - - @Override - int getDelayMs() { - return rand.nextInt(gcInterval.as(Time.MILLISECONDS).intValue()); - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/src/test/java/org/apache/aurora/ProtobufsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/ProtobufsTest.java b/src/test/java/org/apache/aurora/ProtobufsTest.java new file mode 100644 index 0000000..6c38885 --- /dev/null +++ b/src/test/java/org/apache/aurora/ProtobufsTest.java @@ -0,0 +1,32 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.aurora; + +import com.google.protobuf.Message; + +import org.apache.mesos.Protos; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ProtobufsTest { + + @Test + public void messageToString() { + String id = "test_id"; + Message message = Protos.OfferID.newBuilder().setValue(id).build(); + assertEquals(String.format("value: \"%s\"", id), Protobufs.toString(message)); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/src/test/java/org/apache/aurora/scheduler/async/GcExecutorLauncherTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/async/GcExecutorLauncherTest.java b/src/test/java/org/apache/aurora/scheduler/async/GcExecutorLauncherTest.java deleted file mode 100644 index d2ec944..0000000 --- a/src/test/java/org/apache/aurora/scheduler/async/GcExecutorLauncherTest.java +++ /dev/null @@ -1,262 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.async; - -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import com.google.common.base.Optional; -import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.util.concurrent.MoreExecutors; -import com.twitter.common.quantity.Amount; -import com.twitter.common.quantity.Time; -import com.twitter.common.stats.StatsProvider; -import com.twitter.common.testing.easymock.EasyMockTest; -import com.twitter.common.util.testing.FakeClock; - -import org.apache.aurora.gen.AssignedTask; -import org.apache.aurora.gen.ExecutorConfig; -import org.apache.aurora.gen.HostAttributes; -import org.apache.aurora.gen.Identity; -import org.apache.aurora.gen.ScheduleStatus; -import org.apache.aurora.gen.ScheduledTask; -import org.apache.aurora.gen.TaskConfig; -import org.apache.aurora.gen.comm.AdjustRetainedTasks; -import org.apache.aurora.scheduler.HostOffer; -import org.apache.aurora.scheduler.async.GcExecutorLauncher.GcExecutorSettings; -import org.apache.aurora.scheduler.base.Query; -import org.apache.aurora.scheduler.base.Tasks; -import org.apache.aurora.scheduler.configuration.Resources; -import org.apache.aurora.scheduler.mesos.Driver; -import org.apache.aurora.scheduler.storage.entities.IHostAttributes; -import org.apache.aurora.scheduler.storage.entities.IScheduledTask; -import org.apache.aurora.scheduler.storage.testing.StorageTestUtil; -import org.apache.mesos.Protos.FrameworkID; -import org.apache.mesos.Protos.OfferID; -import org.apache.mesos.Protos.Resource; -import org.apache.mesos.Protos.SlaveID; -import org.apache.mesos.Protos.TaskID; -import org.apache.mesos.Protos.TaskInfo; -import org.apache.mesos.Protos.TaskState; -import org.apache.mesos.Protos.TaskStatus; -import org.junit.Before; -import org.junit.Test; - -import static org.apache.aurora.gen.ScheduleStatus.FAILED; -import static org.apache.aurora.scheduler.async.GcExecutorLauncher.INSUFFICIENT_OFFERS_STAT_NAME; -import static org.apache.aurora.scheduler.async.GcExecutorLauncher.LOST_TASKS_STAT_NAME; -import static org.apache.aurora.scheduler.async.GcExecutorLauncher.SYSTEM_TASK_PREFIX; -import static org.apache.mesos.Protos.Offer; -import static org.easymock.EasyMock.expect; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class GcExecutorLauncherTest extends EasyMockTest { - - private static final String HOST = "slave-host"; - - private static final Offer MESOS_OFFER = Offer.newBuilder() - .setSlaveId(SlaveID.newBuilder().setValue("slave-id")) - .setHostname(HOST) - .setFrameworkId(FrameworkID.newBuilder().setValue("framework-id").build()) - .setId(OfferID.newBuilder().setValue("offer-id")) - .addAllResources(GcExecutorLauncher.TOTAL_GC_EXECUTOR_RESOURCES.toResourceList()) - .build(); - private static final HostOffer OFFER = - new HostOffer(MESOS_OFFER, IHostAttributes.build(new HostAttributes())); - - private static final String JOB_A = "jobA"; - private static final String TASK_UUID = "gc"; - - private static final GcExecutorSettings SETTINGS = - new GcExecutorSettings(Amount.of(1L, Time.HOURS), Optional.of("nonempty")); - - private final AtomicInteger taskIdCounter = new AtomicInteger(); - - private FakeClock clock; - private StorageTestUtil storageUtil; - private Driver driver; - private StatsProvider statsProvider; - private GcExecutorLauncher gcExecutorLauncher; - private AtomicLong lostTasks; - private AtomicLong insufficientOffers; - - @Before - public void setUp() { - storageUtil = new StorageTestUtil(this); - clock = new FakeClock(); - storageUtil.expectOperations(); - driver = createMock(Driver.class); - statsProvider = createMock(StatsProvider.class); - lostTasks = new AtomicLong(); - insufficientOffers = new AtomicLong(); - } - - private void replayAndConstruct() { - expect(statsProvider.makeCounter(LOST_TASKS_STAT_NAME)).andReturn(lostTasks); - expect(statsProvider.makeCounter(INSUFFICIENT_OFFERS_STAT_NAME)).andReturn(insufficientOffers); - control.replay(); - gcExecutorLauncher = new GcExecutorLauncher( - SETTINGS, - storageUtil.storage, - clock, - MoreExecutors.sameThreadExecutor(), - driver, - statsProvider, - Suppliers.ofInstance(TASK_UUID)); - } - - @Test - public void testPruning() throws Exception { - IScheduledTask a = makeTask(JOB_A, FAILED); - IScheduledTask b = makeTask(JOB_A, FAILED); - IScheduledTask c = makeTask(JOB_A, FAILED); - - // Third call - no tasks to be collected. - expectGetTasksByHost(HOST, a, b, c); - expectAdjustRetainedTasks(a, b, c); - - // Fourth call - two tasks collected. - expectGetTasksByHost(HOST, a); - expectAdjustRetainedTasks(a); - - // Fifth call - the last task collected. - expectGetTasksByHost(HOST); - expectAdjustRetainedTasks(); - - replayAndConstruct(); - - // First call - no items in the cache, no tasks collected. - assertFalse(gcExecutorLauncher.willUse(OFFER)); - - // Second call - host item alive, no tasks collected. - clock.advance(Amount.of((long) SETTINGS.getDelayMs() - 1, Time.MILLISECONDS)); - assertFalse(gcExecutorLauncher.willUse(OFFER)); - - // Third call - host item expires (initial delay), no tasks collected - clock.advance(Amount.of(1L, Time.HOURS)); - assertTrue(gcExecutorLauncher.willUse(OFFER)); - - // Fourth call - host item expires (regular delay), two tasks collected - clock.advance(Amount.of(1L, Time.HOURS)); - assertTrue(gcExecutorLauncher.willUse(OFFER)); - - // Fifth call - host item expires (regular delay), one task collected - clock.advance(Amount.of(1L, Time.HOURS)); - assertTrue(gcExecutorLauncher.willUse(OFFER)); - - assertEquals(0, insufficientOffers.get()); - } - - @Test - public void testNoAcceptingSmallOffers() { - replayAndConstruct(); - - Iterable<Resource> resources = - Resources.subtract( - GcExecutorLauncher.TOTAL_GC_EXECUTOR_RESOURCES, - GcExecutorLauncher.EPSILON).toResourceList(); - Offer smallOffer = MESOS_OFFER.toBuilder() - .clearResources() - .addAllResources(resources) - .build(); - assertEquals(0, insufficientOffers.get()); - assertFalse(gcExecutorLauncher.willUse( - new HostOffer(smallOffer, IHostAttributes.build(new HostAttributes())))); - assertEquals(1, insufficientOffers.get()); - } - - private static TaskStatus makeStatus(String taskId) { - return TaskStatus.newBuilder() - .setSlaveId(OFFER.getOffer().getSlaveId()) - .setState(TaskState.TASK_RUNNING) - .setTaskId(TaskID.newBuilder().setValue(taskId)) - .build(); - } - - @Test - public void testStatusUpdate() { - TaskStatus gcStatus1 = makeStatus(SYSTEM_TASK_PREFIX); - TaskStatus gcStatus2 = makeStatus(SYSTEM_TASK_PREFIX + "1"); - TaskStatus lost = makeStatus(SYSTEM_TASK_PREFIX).toBuilder() - .setState(TaskState.TASK_LOST).build(); - - driver.acknowledgeStatusUpdate(gcStatus1); - driver.acknowledgeStatusUpdate(gcStatus2); - driver.acknowledgeStatusUpdate(lost); - - replayAndConstruct(); - - assertTrue(gcExecutorLauncher.statusUpdate(gcStatus1)); - assertTrue(gcExecutorLauncher.statusUpdate(gcStatus2)); - - assertFalse(gcExecutorLauncher.statusUpdate(makeStatus("1" + SYSTEM_TASK_PREFIX))); - assertFalse(gcExecutorLauncher.statusUpdate(makeStatus("asdf"))); - - assertEquals(0, lostTasks.get()); - assertTrue(gcExecutorLauncher.statusUpdate(lost)); - assertEquals(1, lostTasks.get()); - } - - @Test - public void testGcExecutorDisabled() { - expect(statsProvider.makeCounter(LOST_TASKS_STAT_NAME)).andReturn(lostTasks); - expect(statsProvider.makeCounter(INSUFFICIENT_OFFERS_STAT_NAME)).andReturn(insufficientOffers); - control.replay(); - - gcExecutorLauncher = new GcExecutorLauncher( - new GcExecutorSettings(Amount.of(1L, Time.HOURS), Optional.<String>absent()), - storageUtil.storage, - clock, - MoreExecutors.sameThreadExecutor(), - driver, - statsProvider, - Suppliers.ofInstance("gc")); - assertFalse(gcExecutorLauncher.willUse(OFFER)); - assertEquals(0, insufficientOffers.get()); - } - - private void expectAdjustRetainedTasks(IScheduledTask... tasks) { - Map<String, ScheduleStatus> statuses = - Maps.transformValues(Tasks.mapById(ImmutableSet.copyOf(tasks)), Tasks.GET_STATUS); - AdjustRetainedTasks message = new AdjustRetainedTasks().setRetainedTasks(statuses); - TaskInfo task = GcExecutorLauncher.makeGcTask( - HOST, - OFFER.getOffer().getSlaveId(), - SETTINGS.getGcExecutorPath().get(), - TASK_UUID, - message); - driver.launchTask(OFFER.getOffer().getId(), task); - } - - private IScheduledTask makeTask(String jobName, ScheduleStatus status) { - return IScheduledTask.build(new ScheduledTask() - .setStatus(status) - .setAssignedTask(new AssignedTask() - .setTaskId("task-" + taskIdCounter.incrementAndGet()) - .setSlaveHost(HOST) - .setTask(new TaskConfig() - .setJobName(jobName) - .setOwner(new Identity().setRole("role").setUser("user")) - .setExecutorConfig(new ExecutorConfig("aurora", "config"))))); - } - - private void expectGetTasksByHost(String host, IScheduledTask... tasks) { - storageUtil.expectTaskFetch(Query.slaveScoped(host), tasks); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/68c46205/src/test/java/org/apache/aurora/scheduler/base/CommandUtilTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/base/CommandUtilTest.java b/src/test/java/org/apache/aurora/scheduler/base/CommandUtilTest.java index ec43a44..cd02957 100644 --- a/src/test/java/org/apache/aurora/scheduler/base/CommandUtilTest.java +++ b/src/test/java/org/apache/aurora/scheduler/base/CommandUtilTest.java @@ -13,10 +13,8 @@ */ package org.apache.aurora.scheduler.base; -import java.util.Map; - +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.protobuf.TextFormat; import org.apache.mesos.Protos.CommandInfo; @@ -26,16 +24,22 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; public class CommandUtilTest { + + private static final Optional<String> NO_EXTRA_ARGS = Optional.absent(); + private static final ImmutableList<String> NO_RESOURCES = ImmutableList.of(); + private static final String PATH = "./"; + @Test public void testUriBasename() { - test("c", "c", ImmutableMap.<String, String>of()); - test("c", "/a/b/c", ImmutableMap.of("FOO", "1")); - test("foo.zip", "hdfs://twitter.com/path/foo.zip", ImmutableMap.of("PATH", "/bin:/usr/bin")); + test("c", "c"); + test("c", "/a/b/c"); + test("foo.zip", "hdfs://twitter.com/path/foo.zip"); } @Test public void testExecutorOnlyCommand() { - CommandInfo cmd = CommandUtil.create("test/executor", ImmutableList.<String>of()); + CommandInfo cmd = + CommandUtil.create("test/executor", NO_RESOURCES, PATH, NO_EXTRA_ARGS).build(); assertEquals("./executor", cmd.getValue()); assertEquals("test/executor", cmd.getUris(0).getValue()); } @@ -44,7 +48,9 @@ public class CommandUtilTest { public void testWrapperAndExecutorCommand() { CommandInfo cmd = CommandUtil.create( "test/wrapper", - ImmutableList.of("test/executor")); + ImmutableList.of("test/executor"), + PATH, + NO_EXTRA_ARGS).build(); assertEquals("./wrapper", cmd.getValue()); assertEquals("test/executor", cmd.getUris(0).getValue()); assertEquals("test/wrapper", cmd.getUris(1).getValue()); @@ -52,17 +58,17 @@ public class CommandUtilTest { @Test(expected = NullPointerException.class) public void testBadParameters() { - CommandUtil.create(null, ImmutableList.<String>of()); + CommandUtil.create(null, NO_RESOURCES, PATH, NO_EXTRA_ARGS); } @Test(expected = IllegalArgumentException.class) public void testBadUri() { - CommandUtil.create("a/b/c/", ImmutableList.<String>of()); + CommandUtil.create("a/b/c/", NO_RESOURCES, PATH, NO_EXTRA_ARGS); } @Test(expected = IllegalArgumentException.class) public void testEmptyUri() { - CommandUtil.create("", ImmutableList.<String>of()); + CommandUtil.create("", NO_RESOURCES, PATH, NO_EXTRA_ARGS); } @Test @@ -72,19 +78,21 @@ public class CommandUtilTest { // with the same id but different executorInfo. See MESOS-2309 for more details. This is // required because Aurora's GC executor has a constant id. - String uri = "/usr/local/bin/gc_executor"; - String expectedValue = "uris { value: \"/usr/local/bin/gc_executor\" " - + "executable: true } value: \"./gc_executor\""; - CommandInfo actual = CommandUtil.create(uri, ImmutableList.<String>of()); + String uri = "/usr/local/bin/test_executor"; + String expectedValue = "uris { value: \"/usr/local/bin/test_executor\" " + + "executable: true } value: \"./test_executor\""; + CommandInfo actual = CommandUtil.create(uri, NO_RESOURCES, PATH, NO_EXTRA_ARGS).build(); assertEquals(expectedValue, TextFormat.shortDebugString(actual)); } - private void test(String basename, String uri, Map<String, String> env) { + private void test(String basename, String uri) { CommandInfo expectedCommand = CommandInfo.newBuilder() .addUris(URI.newBuilder().setValue(uri).setExecutable(true)) .setValue("./" + basename) .build(); - assertEquals(expectedCommand, CommandUtil.create(uri, ImmutableList.<String>of())); + assertEquals( + expectedCommand, + CommandUtil.create(uri, NO_RESOURCES, PATH, NO_EXTRA_ARGS).build()); } }
