http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/DbQuotaStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbQuotaStore.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbQuotaStore.java deleted file mode 100644 index 0ea21b5..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbQuotaStore.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.Map; -import java.util.stream.Collectors; - -import com.google.common.base.Optional; -import com.google.inject.Inject; - -import org.apache.aurora.scheduler.storage.QuotaStore; -import org.apache.aurora.scheduler.storage.db.views.DBResourceAggregate; -import org.apache.aurora.scheduler.storage.db.views.DBSaveQuota; -import org.apache.aurora.scheduler.storage.db.views.Pairs; -import org.apache.aurora.scheduler.storage.entities.IResourceAggregate; - -import static java.util.Objects.requireNonNull; - -import static org.apache.aurora.common.inject.TimedInterceptor.Timed; - -/** - * Quota store backed by a relational database. - */ -class DbQuotaStore implements QuotaStore.Mutable { - - private final QuotaMapper mapper; - - @Inject - DbQuotaStore(QuotaMapper mapper) { - this.mapper = requireNonNull(mapper); - } - - @Timed("quota_store_fetch_quota") - @Override - public Optional<IResourceAggregate> fetchQuota(String role) { - return Optional.fromNullable(mapper.select(role)) - .transform(DBResourceAggregate::toImmutable); - } - - @Timed("quota_store_fetch_quotas") - @Override - public Map<String, IResourceAggregate> fetchQuotas() { - return Pairs.toMap(mapper.selectAll().stream() - .map(DBSaveQuota::toImmutable) - .collect(Collectors.toList())); - } - - @Timed("quota_store_delete_quotas") - @Override - public void deleteQuotas() { - mapper.truncate(); - } - - @Timed("quota_store_remove_quota") - @Override - public void removeQuota(String role) { - mapper.delete(role); - } - - @Timed("quota_store_save_quota") - @Override - public void saveQuota(String role, IResourceAggregate quota) { - mapper.delete(role); - InsertResult quotaInsert = new InsertResult(); - mapper.insert(role, quota.newBuilder(), quotaInsert); - mapper.insertResources( - quotaInsert.getId(), - Pairs.toMap(DBResourceAggregate.pairsFromResources(quota.getResources()))); - - } -}
http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/DbSchedulerStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbSchedulerStore.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbSchedulerStore.java deleted file mode 100644 index 026dfc9..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbSchedulerStore.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.Objects; - -import com.google.common.base.Optional; -import com.google.inject.Inject; - -import org.apache.aurora.scheduler.storage.SchedulerStore; - -import static org.apache.aurora.common.inject.TimedInterceptor.Timed; - -/** - * A relational database-backed scheduler store. - */ -class DbSchedulerStore implements SchedulerStore.Mutable { - - private final FrameworkIdMapper mapper; - - @Inject - DbSchedulerStore(FrameworkIdMapper mapper) { - this.mapper = Objects.requireNonNull(mapper); - } - - @Timed("scheduler_store_save_framework_id") - @Override - public void saveFrameworkId(String frameworkId) { - mapper.insert(Objects.requireNonNull(frameworkId)); - } - - @Timed("scheduler_store_fetch_framework_id") - @Override - public Optional<String> fetchFrameworkId() { - return Optional.fromNullable(mapper.select()); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/DbStorage.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbStorage.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbStorage.java deleted file mode 100644 index aa7c03b..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbStorage.java +++ /dev/null @@ -1,242 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; - -import javax.sql.DataSource; - -import com.google.common.base.Supplier; -import com.google.common.io.CharStreams; -import com.google.common.util.concurrent.AbstractIdleService; -import com.google.inject.Inject; - -import org.apache.aurora.common.inject.TimedInterceptor.Timed; -import org.apache.aurora.common.stats.StatsProvider; -import org.apache.aurora.scheduler.async.AsyncModule.AsyncExecutor; -import org.apache.aurora.scheduler.async.GatedWorkQueue; -import org.apache.aurora.scheduler.async.GatedWorkQueue.GatedOperation; -import org.apache.aurora.scheduler.storage.AttributeStore; -import org.apache.aurora.scheduler.storage.CronJobStore; -import org.apache.aurora.scheduler.storage.JobUpdateStore; -import org.apache.aurora.scheduler.storage.LockStore; -import org.apache.aurora.scheduler.storage.QuotaStore; -import org.apache.aurora.scheduler.storage.SchedulerStore; -import org.apache.aurora.scheduler.storage.Storage; -import org.apache.aurora.scheduler.storage.TaskStore; -import org.apache.ibatis.builder.StaticSqlSource; -import org.apache.ibatis.datasource.pooled.PoolState; -import org.apache.ibatis.datasource.pooled.PooledDataSource; -import org.apache.ibatis.exceptions.PersistenceException; -import org.apache.ibatis.mapping.MappedStatement.Builder; -import org.apache.ibatis.session.Configuration; -import org.apache.ibatis.session.ExecutorType; -import org.apache.ibatis.session.SqlSession; -import org.apache.ibatis.session.SqlSessionFactory; -import org.mybatis.guice.transactional.Transactional; - -import static java.util.Objects.requireNonNull; - -import static org.apache.ibatis.mapping.SqlCommandType.UPDATE; - -/** - * A storage implementation backed by a relational database. - * <p> - * Delegates read and write concurrency semantics to the underlying database. - */ -public class DbStorage extends AbstractIdleService implements Storage { - - private final SqlSessionFactory sessionFactory; - private final EnumBackfill enumBackfill; - private final MutableStoreProvider storeProvider; - private final GatedWorkQueue gatedWorkQueue; - private final StatsProvider statsProvider; - - @Inject - DbStorage( - SqlSessionFactory sessionFactory, - EnumBackfill enumBackfill, - @AsyncExecutor GatedWorkQueue gatedWorkQueue, - final CronJobStore.Mutable cronJobStore, - final TaskStore.Mutable taskStore, - final SchedulerStore.Mutable schedulerStore, - final AttributeStore.Mutable attributeStore, - final LockStore.Mutable lockStore, - final QuotaStore.Mutable quotaStore, - final JobUpdateStore.Mutable jobUpdateStore, - StatsProvider statsProvider) { - - this.sessionFactory = requireNonNull(sessionFactory); - this.enumBackfill = requireNonNull(enumBackfill); - this.gatedWorkQueue = requireNonNull(gatedWorkQueue); - requireNonNull(cronJobStore); - requireNonNull(taskStore); - requireNonNull(schedulerStore); - requireNonNull(attributeStore); - requireNonNull(lockStore); - requireNonNull(quotaStore); - requireNonNull(jobUpdateStore); - storeProvider = new MutableStoreProvider() { - @Override - public SchedulerStore.Mutable getSchedulerStore() { - return schedulerStore; - } - - @Override - public CronJobStore.Mutable getCronJobStore() { - return cronJobStore; - } - - @Override - public TaskStore getTaskStore() { - return taskStore; - } - - @Override - public TaskStore.Mutable getUnsafeTaskStore() { - return taskStore; - } - - @Override - public LockStore.Mutable getLockStore() { - return lockStore; - } - - @Override - public QuotaStore.Mutable getQuotaStore() { - return quotaStore; - } - - @Override - public AttributeStore.Mutable getAttributeStore() { - return attributeStore; - } - - @Override - public JobUpdateStore.Mutable getJobUpdateStore() { - return jobUpdateStore; - } - }; - this.statsProvider = requireNonNull(statsProvider); - } - - @SuppressWarnings("unchecked") - public <T> T getUnsafeStoreAccess() { - return (T) sessionFactory.getConfiguration().getEnvironment().getDataSource(); - } - - @Timed("db_storage_read_operation") - @Override - @Transactional - public <T, E extends Exception> T read(Work<T, E> work) throws StorageException, E { - try { - return work.apply(storeProvider); - } catch (PersistenceException e) { - throw new StorageException(e.getMessage(), e); - } - } - - @Transactional - <T, E extends Exception> T transactionedWrite(MutateWork<T, E> work) throws E { - return work.apply(storeProvider); - } - - @Timed("db_storage_write_operation") - @Override - public <T, E extends Exception> T write(MutateWork<T, E> work) throws StorageException, E { - // NOTE: Async work is intentionally executed regardless of whether the transaction succeeded. - // Doing otherwise runs the risk of cross-talk between transactions and losing async tasks - // due to failure of an unrelated transaction. This matches behavior prior to the - // introduction of DbStorage, but should be revisited. - // TODO(wfarner): Consider revisiting to execute async work only when the transaction is - // successful. - return gatedWorkQueue.closeDuring((GatedOperation<T, E>) () -> { - try { - return transactionedWrite(work); - } catch (PersistenceException e) { - throw new StorageException(e.getMessage(), e); - } - }); - } - - @Override - public void prepare() { - startAsync().awaitRunning(); - } - - private static void addMappedStatement(Configuration configuration, String name, String sql) { - configuration.addMappedStatement( - new Builder(configuration, name, new StaticSqlSource(configuration, sql), UPDATE).build()); - } - - /** - * Creates the SQL schema during service start-up. - * Note: This design assumes a volatile database engine. - */ - @Override - @Transactional - protected void startUp() throws IOException { - Configuration configuration = sessionFactory.getConfiguration(); - String createStatementName = "create_tables"; - configuration.setMapUnderscoreToCamelCase(true); - - // The ReuseExecutor will cache jdbc Statements with equivalent SQL, improving performance - // slightly when redundant queries are made. - configuration.setDefaultExecutorType(ExecutorType.REUSE); - - addMappedStatement( - configuration, - createStatementName, - CharStreams.toString(new InputStreamReader( - DbStorage.class.getResourceAsStream("schema.sql"), - StandardCharsets.UTF_8))); - - try (SqlSession session = sessionFactory.openSession()) { - session.update(createStatementName); - } - - enumBackfill.backfill(); - - createPoolMetrics(); - } - - @Override - protected void shutDown() { - // noop - } - - private void createPoolMetrics() { - DataSource dataSource = sessionFactory.getConfiguration().getEnvironment().getDataSource(); - // Should not fail because we specify a PoolDataSource in DbModule - PoolState poolState = ((PooledDataSource) dataSource).getPoolState(); - - createPoolGauge("requests", poolState::getRequestCount); - createPoolGauge("average_request_time_ms", poolState::getAverageRequestTime); - createPoolGauge("average_wait_time_ms", poolState::getAverageWaitTime); - createPoolGauge("connections_had_to_wait", poolState::getHadToWaitCount); - createPoolGauge("bad_connections", poolState::getBadConnectionCount); - createPoolGauge("claimed_overdue_connections", poolState::getClaimedOverdueConnectionCount); - createPoolGauge("average_overdue_checkout_time_ms", poolState::getAverageOverdueCheckoutTime); - createPoolGauge("average_checkout_time_ms", poolState::getAverageCheckoutTime); - createPoolGauge("idle_connections", poolState::getIdleConnectionCount); - createPoolGauge("active_connections", poolState::getActiveConnectionCount); - } - - private void createPoolGauge(String name, Supplier<? extends Number> gauge) { - String prefix = "db_storage_mybatis_connection_pool_"; - statsProvider.makeGauge(prefix + name, gauge); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java deleted file mode 100644 index 573d76d..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbTaskStore.java +++ /dev/null @@ -1,207 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.Set; - -import com.google.common.base.Function; -import com.google.common.base.Optional; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.inject.Inject; - -import org.apache.aurora.common.inject.TimedInterceptor.Timed; -import org.apache.aurora.common.quantity.Amount; -import org.apache.aurora.common.quantity.Time; -import org.apache.aurora.common.util.Clock; -import org.apache.aurora.scheduler.base.Query; -import org.apache.aurora.scheduler.base.Query.Builder; -import org.apache.aurora.scheduler.base.Tasks; -import org.apache.aurora.scheduler.storage.TaskStore; -import org.apache.aurora.scheduler.storage.db.views.DbScheduledTask; -import org.apache.aurora.scheduler.storage.entities.IJobKey; -import org.apache.aurora.scheduler.storage.entities.IScheduledTask; -import org.apache.aurora.scheduler.storage.entities.ITaskConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Objects.requireNonNull; - -/** - * A task store implementation based on a relational database. - * <p> - * TODO(wfarner): Consider modifying code generator to support directly producing ITaskConfig, etc - * from myBatis (it will set private final fields just fine). This would reduce memory and time - * spent translating and copying objects. - */ -class DbTaskStore implements TaskStore.Mutable { - - private static final Logger LOG = LoggerFactory.getLogger(DbTaskStore.class); - - private final TaskMapper taskMapper; - private final TaskConfigManager configManager; - private final Clock clock; - private final long slowQueryThresholdNanos; - - @Inject - DbTaskStore( - TaskMapper taskMapper, - TaskConfigManager configManager, - Clock clock, - Amount<Long, Time> slowQueryThreshold) { - - LOG.warn("DbTaskStore is experimental, and should not be used in production clusters!"); - this.taskMapper = requireNonNull(taskMapper); - this.configManager = requireNonNull(configManager); - this.clock = requireNonNull(clock); - this.slowQueryThresholdNanos = slowQueryThreshold.as(Time.NANOSECONDS); - } - - @Timed("db_storage_fetch_task") - @Override - public Optional<IScheduledTask> fetchTask(String taskId) { - requireNonNull(taskId); - return Optional.fromNullable(taskMapper.selectById(taskId)) - .transform(DbScheduledTask::toImmutable); - } - - @Timed("db_storage_fetch_tasks") - @Override - public Iterable<IScheduledTask> fetchTasks(Builder query) { - requireNonNull(query); - - // TODO(wfarner): Consider making slow query logging more reusable, or pushing it down into the - // database. - long start = clock.nowNanos(); - Iterable<IScheduledTask> result = matches(query); - long durationNanos = clock.nowNanos() - start; - boolean infoLevel = durationNanos >= slowQueryThresholdNanos; - long time = Amount.of(durationNanos, Time.NANOSECONDS).as(Time.MILLISECONDS); - String message = "Query took {} ms: {}"; - if (infoLevel) { - LOG.info(message, time, query.get()); - } else if (LOG.isDebugEnabled()) { - LOG.debug(message, time, query.get()); - } - - return result; - } - - @Timed("db_storage_get_job_keys") - @Override - public ImmutableSet<IJobKey> getJobKeys() { - return IJobKey.setFromBuilders(taskMapper.selectJobKeys()); - } - - @Timed("db_storage_save_tasks") - @Override - public void saveTasks(Set<IScheduledTask> tasks) { - if (tasks.isEmpty()) { - return; - } - - // TODO(wfarner): Restrict the TaskStore.Mutable methods to more specific mutations. It would - // simplify this code if we did not have to handle full object tree mutations. - - deleteTasks(Tasks.ids(tasks)); - - // Maintain a cache of all task configs that exist for a job key so that identical entities - LoadingCache<ITaskConfig, Long> configCache = CacheBuilder.newBuilder() - .build(new CacheLoader<ITaskConfig, Long>() { - @Override - public Long load(ITaskConfig config) { - return configManager.insert(config); - } - }); - - for (IScheduledTask task : tasks) { - InsertResult result = new InsertResult(); - taskMapper.insertScheduledTask( - task, - configCache.getUnchecked(task.getAssignedTask().getTask()), - result); - - if (!task.getTaskEvents().isEmpty()) { - taskMapper.insertTaskEvents(result.getId(), task.getTaskEvents()); - } - if (!task.getAssignedTask().getAssignedPorts().isEmpty()) { - taskMapper.insertPorts(result.getId(), task.getAssignedTask().getAssignedPorts()); - } - } - } - - @Timed("db_storage_delete_all_tasks") - @Override - public void deleteAllTasks() { - taskMapper.truncate(); - } - - @Timed("db_storage_delete_tasks") - @Override - public void deleteTasks(Set<String> taskIds) { - if (!taskIds.isEmpty()) { - taskMapper.deleteTasks(taskIds); - } - } - - @Timed("db_storage_mutate_task") - @Override - public Optional<IScheduledTask> mutateTask( - String taskId, - Function<IScheduledTask, IScheduledTask> mutator) { - - requireNonNull(taskId); - requireNonNull(mutator); - - return fetchTask(taskId).transform(original -> { - IScheduledTask maybeMutated = mutator.apply(original); - requireNonNull(maybeMutated); - if (!original.equals(maybeMutated)) { - Preconditions.checkState( - Tasks.id(original).equals(Tasks.id(maybeMutated)), - "A task's ID may not be mutated."); - saveTasks(ImmutableSet.of(maybeMutated)); - } - return maybeMutated; - }); - } - - private FluentIterable<IScheduledTask> matches(Query.Builder query) { - Iterable<DbScheduledTask> results; - Predicate<IScheduledTask> filter; - if (query.get().getTaskIds().size() == 1) { - // Optimize queries that are scoped to a single task, as the dynamic SQL used for arbitrary - // queries comes with a performance penalty. - results = Optional.fromNullable( - taskMapper.selectById(Iterables.getOnlyElement(query.get().getTaskIds()))) - .asSet(); - filter = Util.queryFilter(query); - } else { - results = taskMapper.select(query.get()); - // Additional filtering is not necessary in this case, since the query does it for us. - filter = Predicates.alwaysTrue(); - } - - return FluentIterable.from(results) - .transform(DbScheduledTask::toImmutable) - .filter(filter); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java deleted file mode 100644 index 443b473..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbUtil.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import com.google.inject.AbstractModule; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; - -import org.apache.aurora.common.stats.StatsProvider; -import org.apache.aurora.common.util.Clock; -import org.apache.aurora.common.util.testing.FakeClock; -import org.apache.aurora.scheduler.storage.Storage; -import org.apache.aurora.scheduler.testing.FakeStatsProvider; - -import static org.apache.aurora.scheduler.storage.db.DbModule.testModuleWithWorkQueue; - -/** - * Utility class for creating ad-hoc storage instances. - */ -public final class DbUtil { - - private DbUtil() { - // Utility class. - } - - /** - * Creates a new, empty storage system with bindings for the new storage. - * - * @param dbModule {@link DbModule} to install. - * @return An injector with bindings necessary for a storage system. - */ - public static Injector createStorageInjector(Module dbModule) { - return createStorageInjector(dbModule, new DbModule.MigrationManagerModule()); - } - - public static Injector createStorageInjector(Module dbModule, Module migrationModule) { - Injector injector = Guice.createInjector( - migrationModule, - dbModule, - new AbstractModule() { - @Override - protected void configure() { - FakeStatsProvider stats = new FakeStatsProvider(); - bind(StatsProvider.class).toInstance(stats); - bind(FakeStatsProvider.class).toInstance(stats); - bind(Clock.class).toInstance(new FakeClock()); - } - }); - Storage storage = injector.getInstance(Storage.class); - storage.prepare(); - return injector; - } - - /** - * Creates a new, empty test storage system. - * <p> - * TODO(wfarner): Rename this to createTestStorage() to avoid misuse. - * - * @return A new storage instance. - */ - public static Storage createStorage() { - return createStorageInjector(testModuleWithWorkQueue()).getInstance(Storage.class); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/EnumBackfill.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/EnumBackfill.java b/src/main/java/org/apache/aurora/scheduler/storage/db/EnumBackfill.java deleted file mode 100644 index b4731c9..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/EnumBackfill.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import javax.inject.Inject; - -import org.apache.aurora.gen.CronCollisionPolicy; -import org.apache.aurora.gen.JobUpdateAction; -import org.apache.aurora.gen.JobUpdateStatus; -import org.apache.aurora.gen.MaintenanceMode; -import org.apache.aurora.gen.Mode; -import org.apache.aurora.gen.ScheduleStatus; - -import org.apache.aurora.scheduler.resources.ResourceType; - -import static java.util.Objects.requireNonNull; - -public interface EnumBackfill { - /** - * Hydrates all of the enum tables in the database. - */ - void backfill(); - - class EnumBackfillImpl implements EnumBackfill { - - private final EnumValueMapper enumValueMapper; - - @Inject - public EnumBackfillImpl(EnumValueMapper mapper) { - this.enumValueMapper = requireNonNull(mapper); - } - - @Override - public void backfill() { - for (CronCollisionPolicy policy : CronCollisionPolicy.values()) { - enumValueMapper.addEnumValue("cron_policies", policy.getValue(), policy.name()); - } - - for (MaintenanceMode mode : MaintenanceMode.values()) { - enumValueMapper.addEnumValue("maintenance_modes", mode.getValue(), mode.name()); - } - - for (JobUpdateStatus status : JobUpdateStatus.values()) { - enumValueMapper.addEnumValue("job_update_statuses", status.getValue(), status.name()); - } - - for (JobUpdateAction jua : JobUpdateAction.values()) { - enumValueMapper.addEnumValue("job_instance_update_actions", jua.getValue(), jua.name()); - } - - for (ScheduleStatus status : ScheduleStatus.values()) { - enumValueMapper.addEnumValue("task_states", status.getValue(), status.name()); - } - - for (ResourceType type : ResourceType.values()) { - enumValueMapper.addEnumValue("resource_types", type.getValue(), type.name()); - } - - for (Mode mode : Mode.values()) { - enumValueMapper.addEnumValue("volume_modes", mode.getValue(), mode.name()); - } - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/EnumValueMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/EnumValueMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/EnumValueMapper.java deleted file mode 100644 index 33948a7..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/EnumValueMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import org.apache.ibatis.annotations.Param; - -/** - * Mapper used to populate static enum value tables. - */ -interface EnumValueMapper { - - /** - * Inserts an enum value. - * - * @param table Name of the enum value table. - * @param id Unique enum ID. - * @param name Human-readable enum name. - */ - void addEnumValue(@Param("table") String table, @Param("id") int id, @Param("name") String name); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/FrameworkIdMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/FrameworkIdMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/FrameworkIdMapper.java deleted file mode 100644 index 240767c..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/FrameworkIdMapper.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import javax.annotation.Nullable; - -/** - * MyBatis mapper class for FrameworkIdMapper.xml. - */ -interface FrameworkIdMapper { - @Nullable - String select(); - - void insert(String id); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/GarbageCollectedTableMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/GarbageCollectedTableMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/GarbageCollectedTableMapper.java deleted file mode 100644 index 83e42a9..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/GarbageCollectedTableMapper.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.List; - -import org.apache.ibatis.annotations.Param; - -/** - * Base interface for a table mapper whose rows should be garbage-collected when unreferenced. - */ -interface GarbageCollectedTableMapper { - /** - * Selects the IDs of all rows in the table. - */ - List<Long> selectAllRowIds(); - - /** - * Attempts to delete a row from the table. - */ - void deleteRow(@Param("rowId") long rowId); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/InsertResult.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/InsertResult.java b/src/main/java/org/apache/aurora/scheduler/storage/db/InsertResult.java deleted file mode 100644 index 8a81579..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/InsertResult.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -/** - * MyBatis returns auto-generated IDs through mutable fields in parameters. - * This class can be used as an additional {@link org.apache.ibatis.annotations.Param Param} to - * retrieve the ID when the inserted object is not self-identifying. - */ -public class InsertResult { - private long id = Long.MIN_VALUE; - private boolean isSet; - - public long getId() { - if (!isSet) { - throw new IllegalStateException("Missing ID value."); - } - return id; - } - - void setId(long value) { - id = value; - isSet = true; - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/InstrumentingInterceptor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/InstrumentingInterceptor.java b/src/main/java/org/apache/aurora/scheduler/storage/db/InstrumentingInterceptor.java deleted file mode 100644 index caf1d64..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/InstrumentingInterceptor.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.aurora.scheduler.storage.db; - -import java.util.Properties; -import java.util.function.Function; -import javax.annotation.Nonnull; -import javax.inject.Inject; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; - -import org.apache.aurora.common.stats.SlidingStats; -import org.apache.aurora.common.util.Clock; -import org.apache.ibatis.cache.CacheKey; -import org.apache.ibatis.executor.Executor; -import org.apache.ibatis.mapping.BoundSql; -import org.apache.ibatis.mapping.MappedStatement; -import org.apache.ibatis.plugin.Interceptor; -import org.apache.ibatis.plugin.Intercepts; -import org.apache.ibatis.plugin.Invocation; -import org.apache.ibatis.plugin.Plugin; -import org.apache.ibatis.plugin.Signature; -import org.apache.ibatis.session.ResultHandler; -import org.apache.ibatis.session.RowBounds; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Objects.requireNonNull; - -/** - * A Mybatis Executor invocation interceptor that exports timing information for update and query - * mapped statements. - * - * Currently intercepting the following invocations: - * 1. update(MappedStatement ms, Object parameter) - * 2. query(MappedStatement ms, Object parameter, RowBounds rowBounds, - * ResultHandler resultHandler, CacheKey cacheKey, BoundSql boundSql) - * 3. query(MappedStatement ms, Object parameter, RowBounds rowBounds, - * ResultHandler resultHandler) - * 4. queryCursor(MappedStatement ms, Object parameter, RowBounds rowBounds) - * - * more signatures can be added from: org.apache.ibatis.executors - */ -@Intercepts({ - @Signature( - type = Executor.class, - method = "update", - args = {MappedStatement.class, Object.class}), - @Signature(type = Executor.class, - method = "query", - args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class, - CacheKey.class, BoundSql.class}), - @Signature(type = Executor.class, - method = "query", - args = {MappedStatement.class, Object.class, RowBounds.class, ResultHandler.class}), - @Signature(type = Executor.class, - method = "queryCursor", - args = {MappedStatement.class, Object.class, RowBounds.class}) - }) -public class InstrumentingInterceptor implements Interceptor { - private static final String INVALID_INVOCATION_METRIC_NAME = "invalid_invocations"; - private static final String STATS_NAME_PREFIX = "mybatis."; - private static final Logger LOG = LoggerFactory.getLogger(InstrumentingInterceptor.class); - private final Clock clock; - private final LoadingCache<String, SlidingStats> stats; - - @Inject - public InstrumentingInterceptor(Clock clock) { - this(clock, (String name) -> new SlidingStats(name, "nanos")); - } - - @VisibleForTesting - public InstrumentingInterceptor(Clock clock, Function<String, SlidingStats> statsFactory) { - this.clock = requireNonNull(clock); - - this.stats = CacheBuilder.newBuilder().build(new CacheLoader<String, SlidingStats>() { - @Override public SlidingStats load(String statsName) { - return statsFactory.apply(STATS_NAME_PREFIX + statsName); - } - }); - } - - private String generateStatsName(Invocation invocation) { - if (firstArgumentIsMappedStatement(invocation)) { - MappedStatement statement = (MappedStatement) invocation.getArgs()[0]; - return statement.getId(); - } - - LOG.warn("Received invocation for unknown or invalid target. Invocation target: {}. " - + "Invocation method: {}. Using metric name '{}' instead.", - invocation.getTarget(), - invocation.getMethod(), - INVALID_INVOCATION_METRIC_NAME); - return INVALID_INVOCATION_METRIC_NAME; - } - - private boolean firstArgumentIsMappedStatement(Invocation invocation) { - return invocation != null - && invocation.getArgs() != null - && invocation.getArgs()[0] instanceof MappedStatement; - } - - @Override - public Object intercept(@Nonnull Invocation invocation) throws Throwable { - long start = clock.nowNanos(); - try { - return invocation.proceed(); - } finally { - String statsName = generateStatsName(invocation); - SlidingStats stat = stats.get(statsName); - stat.accumulate(clock.nowNanos() - start); - } - } - - @Override - public Object plugin(Object target) { - return Plugin.wrap(target, this); - } - - @Override - public void setProperties(Properties properties) { - // intentionally left empty as instructed in http://www.mybatis.org/mybatis-3/configuration.html - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/JobInstanceUpdateEventMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/JobInstanceUpdateEventMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/JobInstanceUpdateEventMapper.java deleted file mode 100644 index 591b781..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/JobInstanceUpdateEventMapper.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import org.apache.aurora.gen.JobInstanceUpdateEvent; -import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey; -import org.apache.ibatis.annotations.Param; - -/** - * MyBatis mapper class for JobInstanceUpdateEventMapper.xml - * - * See http://mybatis.github.io/mybatis-3/sqlmap-xml.html for more details. - */ -interface JobInstanceUpdateEventMapper { - - /** - * Inserts a new job instance update event into the database. - * - * @param key Update key of the event. - * @param event Event to insert. - */ - void insert(@Param("key") IJobUpdateKey key, @Param("event") JobInstanceUpdateEvent event); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/JobKeyMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/JobKeyMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/JobKeyMapper.java deleted file mode 100644 index ac0a998..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/JobKeyMapper.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.List; - -import org.apache.aurora.gen.JobKey; -import org.apache.aurora.scheduler.storage.entities.IJobKey; - -/** - * MyBatis mapper class for JobKeyMapper.xml - * - * See http://mybatis.github.io/mybatis-3/sqlmap-xml.html for more details. - */ -interface JobKeyMapper extends GarbageCollectedTableMapper { - /** - * Saves the job key, updating the existing value if it exists. - */ - void merge(IJobKey key); - - /** - * Selects all job keys from the database. - */ - List<JobKey> selectAll(); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.java deleted file mode 100644 index 91053ef..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateDetailsMapper.java +++ /dev/null @@ -1,210 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.List; -import java.util.Set; - -import javax.annotation.Nullable; - -import org.apache.aurora.gen.JobInstanceUpdateEvent; -import org.apache.aurora.gen.JobUpdate; -import org.apache.aurora.gen.JobUpdateQuery; -import org.apache.aurora.gen.JobUpdateSummary; -import org.apache.aurora.gen.Metadata; -import org.apache.aurora.gen.Range; -import org.apache.aurora.scheduler.storage.db.views.DbJobUpdate; -import org.apache.aurora.scheduler.storage.db.views.DbJobUpdateInstructions; -import org.apache.aurora.scheduler.storage.db.views.DbStoredJobUpdateDetails; -import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey; -import org.apache.ibatis.annotations.Param; - -/** - * MyBatis mapper class for JobUpdateDetailsMapper.xml - * - * See http://mybatis.github.io/mybatis-3/sqlmap-xml.html for more details. - */ -interface JobUpdateDetailsMapper { - - /** - * Inserts new job update. - * - * @param jobUpdate Job update to insert. - */ - void insert(JobUpdate jobUpdate); - - /** - * Inserts an association between an update and a lock. - * - * @param key Unique update identifier. - * @param lockToken Unique lock identifier, resulting from - * {@link org.apache.aurora.scheduler.storage.entities.ILock#getToken()}. - */ - void insertLockToken(@Param("key") IJobUpdateKey key, @Param("lockToken") String lockToken); - - /** - * Inserts a task configuration entry for an update. - * - * @param key Update to insert task configs for. - * @param taskConfigRow task configuration row. - * @param isNew Flag to identify if the task config is existing {@code false} or - * desired {@code true}. - * @param result Container for auto-generated ID of the inserted job update row. - */ - void insertTaskConfig( - @Param("key") IJobUpdateKey key, - @Param("taskConfigRow") long taskConfigRow, - @Param("isNew") boolean isNew, - @Param("result") InsertResult result); - - /** - * Inserts the job update metadata entries for an update. - * - * @param key Update to insert task configs for. - * @param metadata Set of metadata (key, value) pairs. - */ - void insertJobUpdateMetadata( - @Param("key") IJobUpdateKey key, - @Param("metadata") Set<Metadata> metadata); - - /** - * Maps inserted task config with a set of associated instance ranges. - * - * @param configId ID of the task config stored. - * @param ranges Set of instance ID ranges. - */ - void insertTaskConfigInstances( - @Param("configId") long configId, - @Param("ranges") Set<Range> ranges); - - /** - * Maps update with an optional set of - * {@link org.apache.aurora.gen.JobUpdateSettings#updateOnlyTheseInstances}. - * - * @param key Update to store overrides for. - * @param ranges Instance ID ranges to associate with an update. - */ - void insertInstanceOverrides(@Param("key") IJobUpdateKey key, @Param("ranges") Set<Range> ranges); - - /** - * Maps update with a set of instance IDs in - * {@link org.apache.aurora.gen.JobUpdateInstructions#desiredState}. - * - * @param key Update to store desired instances for. - * @param ranges Desired instance ID ranges to associate with an update. - */ - void insertDesiredInstances(@Param("key") IJobUpdateKey key, @Param("ranges") Set<Range> ranges); - - /** - * Deletes all updates and events from the database. - */ - void truncate(); - - /** - * Deletes all updates and events with update ID in {@code updates}. - * - * @param rowIds Row IDs of updates to delete. - */ - void deleteCompletedUpdates(@Param("rowIds") Set<Long> rowIds); - - /** - * Selects all distinct job key IDs associated with at least {@code perJobRetainCount} completed - * updates or updates completed before {@code historyPruneThresholdMs}. - * - * @param perJobRetainCount Number of updates to keep per job. - * @param historyPruneThresholdMs History pruning timestamp threshold. - * @return Job key database row IDs. - */ - Set<Long> selectJobKeysForPruning( - @Param("retainCount") int perJobRetainCount, - @Param("pruneThresholdMs") long historyPruneThresholdMs); - - /** - * Groups all updates without a job lock in reverse chronological order of their created times - * and deletes anything in excess of {@code perJobRetainCount} or older than - * {@code historyPruneThresholdMs}. - * - * @param jobKeyId Job key ID to select pruning victims for. - * @param perJobRetainCount Number of updates to keep per job. - * @param historyPruneThresholdMs History pruning timestamp threshold. - * @return Victims to prune. - */ - Set<PruneVictim> selectPruneVictims( - @Param("keyId") long jobKeyId, - @Param("retainCount") int perJobRetainCount, - @Param("pruneThresholdMs") long historyPruneThresholdMs); - - /** - * Gets all job update summaries matching the provided {@code query}. - * All {@code query} fields are ANDed together. - * - * @param query Query to filter results by. - * @return Job update summaries matching the query. - */ - List<JobUpdateSummary> selectSummaries(JobUpdateQuery query); - - /** - * Gets details for the provided {@code key}. - * - * @param key Update to get. - * @return Job update details for the provided update ID, if it exists. - */ - @Nullable - DbStoredJobUpdateDetails selectDetails(@Param("key") IJobUpdateKey key); - - /** - * Gets all job update details matching the provided {@code query}. - * All {@code query} fields are ANDed together. - * - * @param query Query to filter results by. - * @return Job update details matching the query. - */ - List<DbStoredJobUpdateDetails> selectDetailsList(JobUpdateQuery query); - - /** - * Gets job update for the provided {@code update}. - * - * @param key Update to select by. - * @return Job update for the provided update ID, if it exists. - */ - @Nullable - DbJobUpdate selectUpdate(@Param("key") IJobUpdateKey key); - - /** - * Gets job update instructions for the provided {@code update}. - * - * @param key Update to select by. - * @return Job update instructions for the provided update ID, if it exists. - */ - @Nullable - DbJobUpdateInstructions selectInstructions(@Param("key") IJobUpdateKey key); - - /** - * Gets all stored job update details. - * - * @return All stored job update details. - */ - Set<DbStoredJobUpdateDetails> selectAllDetails(); - - /** - * Gets job instance update events for a specific instance within an update. - * - * @param key Update identifier. - * @param instanceId Instance to fetch events for. - * @return Instance events affecting {@code instanceId} within {@code key}. - */ - List<JobInstanceUpdateEvent> selectInstanceUpdateEvents( - @Param("key") IJobUpdateKey key, - @Param("instanceId") int instanceId); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateEventMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateEventMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateEventMapper.java deleted file mode 100644 index d1a3c3f..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/JobUpdateEventMapper.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import org.apache.aurora.gen.JobUpdateEvent; -import org.apache.aurora.scheduler.storage.entities.IJobUpdateKey; -import org.apache.ibatis.annotations.Param; - -/** - * MyBatis mapper class for JobUpdateEventMapper.xml - * - * See http://mybatis.github.io/mybatis-3/sqlmap-xml.html for more details. - */ -interface JobUpdateEventMapper { - - /** - * Inserts a new job update event into the database. - * - * @param key ID of the update associated with the event. - * @param event Event to insert. - */ - void insert(@Param("key") IJobUpdateKey key, @Param("event") JobUpdateEvent event); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/LockKeyMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/LockKeyMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/LockKeyMapper.java deleted file mode 100644 index 9760674..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/LockKeyMapper.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import com.google.inject.Inject; - -import org.apache.aurora.scheduler.storage.entities.ILockKey; - -import static java.util.Objects.requireNonNull; - -/** - * Mapper for LockKeys. Not a MyBatis mapper, this just encapsulates the logic for writing - * union types so it does not leak into the related object's implementation. - * - * TODO(davmclau): - * Consider creating these classes with code generation since something like this will be needed - * for all union field relationships. Might not be possible unless the code generator also defines - * a mapper interface for every field in the union as well as the associated XML mapper config - * with the SQL to satisfy the interface. - * - */ -class LockKeyMapper { - - private final JobKeyMapper jobKeyMapper; - - @Inject - LockKeyMapper(JobKeyMapper jobKeyMapper) { - this.jobKeyMapper = requireNonNull(jobKeyMapper); - } - - public void insert(ILockKey key) { - if (key.isSetJob()) { - jobKeyMapper.merge(requireNonNull(key.getJob())); - } else { - throw new IllegalArgumentException("Unsupported lock type on LockKey."); - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/LockMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/LockMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/LockMapper.java deleted file mode 100644 index b9f338c..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/LockMapper.java +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.List; - -import org.apache.aurora.gen.Lock; -import org.apache.aurora.gen.LockKey; -import org.apache.aurora.scheduler.storage.db.views.LockRow; - -/** - * MyBatis mapper class for LockMapper.xml - * - * See http://mybatis.github.io/mybatis-3/sqlmap-xml.html for more details. - */ -interface LockMapper { - - /** - * Inserts a lock into the database. - */ - void insert(Lock lock); - - /** - * Deletes all locks from the database with the given lockKey. - */ - void delete(LockKey lockKey); - - /** - * Deletes all locks from the database. - */ - void truncate(); - - /** - * Selects all locks from the database as {@link LockRow} instances. - */ - List<LockRow> selectAll(); - - /** - * Fetches the lock with the given lock key. - */ - LockRow select(LockKey lockKey); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManager.java b/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManager.java deleted file mode 100644 index fa986da..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManager.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.sql.SQLException; - -/** - * Manage schema migrations. - */ -public interface MigrationManager { - /** - * Perform a migration, upgrading the schema if there are unapplied changes or downgrading it if - * there are applies changes which do not exist in the current version. - * - * @throws SQLException In the event of a problem performing the migration. - */ - void migrate() throws SQLException; -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManagerImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManagerImpl.java b/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManagerImpl.java deleted file mode 100644 index f9b8c42..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationManagerImpl.java +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; -import javax.inject.Inject; - -import com.google.common.base.Charsets; -import com.google.common.io.CharStreams; - -import org.apache.aurora.scheduler.storage.db.views.MigrationChangelogEntry; -import org.apache.ibatis.migration.Change; -import org.apache.ibatis.migration.DataSourceConnectionProvider; -import org.apache.ibatis.migration.MigrationLoader; -import org.apache.ibatis.migration.operations.UpOperation; -import org.apache.ibatis.migration.options.DatabaseOperationOption; -import org.apache.ibatis.session.SqlSession; -import org.apache.ibatis.session.SqlSessionFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Objects.requireNonNull; - -public class MigrationManagerImpl implements MigrationManager { - private static final Logger LOG = LoggerFactory.getLogger(MigrationManagerImpl.class); - - private final SqlSessionFactory sqlSessionFactory; - private final MigrationLoader migrationLoader; - - @Inject - MigrationManagerImpl(SqlSessionFactory sqlSessionFactory, MigrationLoader migrationLoader) { - this.sqlSessionFactory = requireNonNull(sqlSessionFactory); - this.migrationLoader = requireNonNull(migrationLoader); - } - - @Override - public void migrate() throws SQLException { - LOG.info("Running db migrations."); - - try (SqlSession sqlSession = sqlSessionFactory.openSession(true /* auto commit */)) { - MigrationMapper mapper = sqlSession.getMapper(MigrationMapper.class); - - LOG.info("Bootstrapping changelog table (if necessary)."); - mapper.bootstrapChangelog(); - - if (!checkRollback(mapper, migrationLoader.getMigrations())) { - DatabaseOperationOption options = new DatabaseOperationOption(); - options.setAutoCommit(true); - - new UpOperation().operate( - new DataSourceConnectionProvider( - sqlSessionFactory.getConfiguration().getEnvironment().getDataSource()), - migrationLoader, - options, - null); - - saveDowngradeScript(mapper); - } - } - } - - /** - * Iterates applied changes to ensure they all exist on the classpath. For any changes that do not - * exist on the classpath, their downgrade script is run. - * - * @param mapper A {@link MigrationMapper} instance used to modify the changelog. - * @param changes The list of {@link Change}s found on the classpath. - * @return true if a rollback was detected, false otherwise. - * @throws SQLException in the event a SQL failure. - */ - private boolean checkRollback(MigrationMapper mapper, List<Change> changes) throws SQLException { - boolean rollback = false; - - List<MigrationChangelogEntry> appliedChanges = mapper.selectAll(); - - for (MigrationChangelogEntry change : appliedChanges) { - // We cannot directly call changes.contains(...) since contains relies on Change#equals - // which includes class in its equality check rather than checking instanceof. Instead we just - // find the first element in changes whose id matches our applied change. If it does not exist - // then this must be a rollback. - if (changes.stream().anyMatch(c -> c.getId().equals(change.getId()))) { - LOG.info("Change " + change.getId() + " has been applied, no other downgrades are " - + "necessary"); - break; - } - - LOG.info("No migration corresponding to change id " + change.getId() + " found. Assuming " - + "this is a rollback."); - LOG.info("Downgrade SQL for " + change.getId() + " is: " + change.getDowngradeScript()); - - try (SqlSession session = sqlSessionFactory.openSession(true)) { - try (Connection c = session.getConnection()) { - try (PreparedStatement downgrade = c.prepareStatement(change.getDowngradeScript())) { - downgrade.execute(); - rollback = true; - } - } - } - - LOG.info("Deleting applied change: " + change.getId()); - mapper.delete(change.getId()); - } - - return rollback; - } - - private void saveDowngradeScript(MigrationMapper mapper) { - for (Change c : migrationLoader.getMigrations()) { - try { - String downgradeScript = CharStreams.toString(migrationLoader.getScriptReader(c, true)); - LOG.info("Saving downgrade script for change id " + c.getId() + ": " + downgradeScript); - - mapper.saveDowngradeScript(c.getId(), downgradeScript.getBytes(Charsets.UTF_8)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationMapper.java deleted file mode 100644 index 0c17aa7..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/MigrationMapper.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.math.BigDecimal; -import java.util.List; - -import org.apache.aurora.scheduler.storage.db.views.MigrationChangelogEntry; -import org.apache.ibatis.annotations.Param; - -interface MigrationMapper { - /** - * Creates the changelog table if it does not already exist. - */ - void bootstrapChangelog(); - - /** - * Saves the downgrade script for the supplied change id into the changelog. - * - * @param changeId The id of the change. - * @param downgradeScript The script to be run when a change is rolled back. - */ - void saveDowngradeScript( - @Param("changeId") BigDecimal changeId, - @Param("downgradeScript") byte[] downgradeScript); - - /** - * Select all applied changes from the changelog. - * - * @return A list of changelog entries mapping only their ids and downgrade scripts. - */ - List<MigrationChangelogEntry> selectAll(); - - /** - * Deletes the specified change from the changelog. - * - * @param changeId The id of the change to delete. - */ - void delete(@Param("changeId") BigDecimal changeId); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/MyBatisCacheImpl.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/MyBatisCacheImpl.java b/src/main/java/org/apache/aurora/scheduler/storage/db/MyBatisCacheImpl.java deleted file mode 100644 index d9d406f..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/MyBatisCacheImpl.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReadWriteLock; - -import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; -import com.google.common.cache.CacheBuilder; -import com.google.common.primitives.Ints; - -import org.apache.aurora.common.stats.StatImpl; -import org.apache.aurora.common.stats.Stats; -import org.apache.ibatis.cache.Cache; - -import static java.util.Objects.requireNonNull; - -public class MyBatisCacheImpl implements Cache { - private com.google.common.cache.Cache<Object, Object> delegate; - private final String id; - private Integer size; - private final AtomicInteger clearCount = new AtomicInteger(0); - - public MyBatisCacheImpl(String id) { - this.id = requireNonNull(id); - } - - public void setSize(Integer size) { - this.size = requireNonNull(size); - initCache(); - } - - private void initCache() { - Preconditions.checkState(delegate == null); - requireNonNull(size); - - delegate = CacheBuilder.newBuilder() - .maximumSize(size) - .recordStats() - .softValues() - .build(); - initStats(); - } - - private void initStats() { - makeStat("request_count", () -> delegate.stats().requestCount()); - makeStat("hit_count", () -> delegate.stats().hitCount()); - makeStat("hit_rate", () -> delegate.stats().hitRate()); - makeStat("miss_count", () -> delegate.stats().missCount()); - makeStat("miss_rate", () -> delegate.stats().missRate()); - makeStat("eviction_count", () -> delegate.stats().evictionCount()); - makeStat("size", () -> delegate.size()); - makeStat("clear_count", clearCount::get); - } - - private <T extends Number> void makeStat(String name, Supplier<T> supplier) { - String prefix = "db_storage_mybatis_cache_" + id + "_"; - Stats.export(new StatImpl<Number>(prefix + name) { - @Override - public Number read() { - return supplier.get(); - } - }); - } - - @Override - public String getId() { - return id; - } - - @Override - public void putObject(Object key, Object value) { - if (key == null || value == null) { - return; - } - delegate.put(key, value); - } - - @Override - public Object getObject(Object key) { - return delegate.getIfPresent(key); - } - - @Override - public Object removeObject(Object key) { - delegate.invalidate(key); - // MyBatis says the return value is not used. - return null; - } - - @Override - public void clear() { - delegate.invalidateAll(); - clearCount.incrementAndGet(); - } - - @Override - public int getSize() { - return Ints.saturatedCast(delegate.size()); - } - - @Override - public ReadWriteLock getReadWriteLock() { - // MyBatis says this value is no longer used. - return null; - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/PruneVictim.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/PruneVictim.java b/src/main/java/org/apache/aurora/scheduler/storage/db/PruneVictim.java deleted file mode 100644 index 144f5a3..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/PruneVictim.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import org.apache.aurora.gen.JobUpdateKey; - -/** - * A job update that should be pruned. - */ -public class PruneVictim { - private long rowId; - private JobUpdateKey update; - - public long getRowId() { - return rowId; - } - - public JobUpdateKey getUpdate() { - return update; - } - - public void setRowId(long rowId) { - this.rowId = rowId; - } - - public void setUpdate(JobUpdateKey update) { - this.update = update; - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/QuotaMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/QuotaMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/QuotaMapper.java deleted file mode 100644 index 7ebbb43..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/QuotaMapper.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -import org.apache.aurora.gen.ResourceAggregate; -import org.apache.aurora.scheduler.storage.db.views.DBResourceAggregate; -import org.apache.aurora.scheduler.storage.db.views.DBSaveQuota; -import org.apache.ibatis.annotations.Param; - -/** - * MyBatis mapper class for QuotaMapper.xml. - */ -interface QuotaMapper { - /** - * Inserts the quota for the given {@code role}. - * - * @param role Role to insert quota for. - * @param quota Quota value to store. - * @param result Container for auto-generated ID of the inserted row. - */ - void insert( - @Param("role") String role, - @Param("quota") ResourceAggregate quota, - @Param("result") InsertResult result); - - /** - * Insert quota resources. - * - * @param quotaId Quota ID to merge resources for. - * @param values Resources to merge. - */ - void insertResources( - @Param("quotaId") long quotaId, - @Param("values") Map<Integer, String> values); - - /** - * Gets the quota assigned to a role. - * - * @param role Role to select quota for. - * @return The previously-saved quota for the role, if it exists. - */ - @Nullable - DBResourceAggregate select(String role); - - /** - * Gets all saved quotas. - * - * @return All quotas stored in the database. - */ - List<DBSaveQuota> selectAll(); - - /** - * Removes the quota stored for a role. - * - * @param role Role to delete the quota entry for, if one exists. - */ - void delete(String role); - - /** - * Removes all stored quota records. - */ - void truncate(); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollector.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollector.java b/src/main/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollector.java deleted file mode 100644 index d29bcf3..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/RowGarbageCollector.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -import javax.inject.Inject; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.AbstractScheduledService; - -import org.apache.aurora.common.inject.TimedInterceptor.Timed; -import org.apache.aurora.common.stats.StatsProvider; -import org.apache.aurora.scheduler.storage.Storage; -import org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult; -import org.apache.ibatis.exceptions.PersistenceException; -import org.apache.ibatis.session.SqlSession; -import org.apache.ibatis.session.SqlSessionFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Objects.requireNonNull; - -/** - * A periodic cleanup routine for unreferenced database relations. - */ -class RowGarbageCollector extends AbstractScheduledService { - - private static final Logger LOG = LoggerFactory.getLogger(RowGarbageCollector.class); - - // Note: these are deliberately ordered to remove 'parent' references first, but since - // this is an iterative process, it is not strictly necessary. - private static final List<Class<? extends GarbageCollectedTableMapper>> TABLES = - ImmutableList.of(TaskConfigMapper.class, JobKeyMapper.class); - - private final AtomicLong deletedCount; - private final Scheduler iterationScheduler; - private final SqlSessionFactory sessionFactory; - - // Note: Storage is only used to acquire the same application-level lock used by other storage - // mutations. This sidesteps the issue of DB deadlocks (e.g. AURORA-1401). - private final Storage storage; - - @Inject - RowGarbageCollector( - Scheduler iterationScheduler, - SqlSessionFactory sessionFactory, - Storage storage, - StatsProvider statsProvider) { - - this.iterationScheduler = requireNonNull(iterationScheduler); - this.sessionFactory = requireNonNull(sessionFactory); - this.storage = requireNonNull(storage); - this.deletedCount = statsProvider.makeCounter("row_garbage_collector_deleted"); - } - - @Override - protected Scheduler scheduler() { - return iterationScheduler; - } - - @Timed("row_garbage_collector_run") - @VisibleForTesting - @Override - public void runOneIteration() { - LOG.info("Scanning database tables for unreferenced rows."); - - long startCount = deletedCount.get(); - for (Class<? extends GarbageCollectedTableMapper> tableClass : TABLES) { - storage.write((NoResult.Quiet) storeProvider -> { - try (SqlSession session = sessionFactory.openSession(true)) { - GarbageCollectedTableMapper table = session.getMapper(tableClass); - for (long rowId : table.selectAllRowIds()) { - try { - table.deleteRow(rowId); - deletedCount.incrementAndGet(); - } catch (PersistenceException e) { - // Expected for rows that are still referenced. - } - } - } - }); - } - LOG.info("Deleted {} unreferenced rows.", Long.valueOf(deletedCount.get() - startCount)); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/94276046/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java deleted file mode 100644 index d2eb6aa..0000000 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.scheduler.storage.db; - -import java.util.Map; -import java.util.Optional; - -import javax.inject.Inject; - -import com.google.common.collect.Maps; - -import org.apache.aurora.scheduler.storage.db.views.DBResourceAggregate; -import org.apache.aurora.scheduler.storage.db.views.DbTaskConfig; -import org.apache.aurora.scheduler.storage.entities.IAppcImage; -import org.apache.aurora.scheduler.storage.entities.IConstraint; -import org.apache.aurora.scheduler.storage.entities.IDockerContainer; -import org.apache.aurora.scheduler.storage.entities.IDockerImage; -import org.apache.aurora.scheduler.storage.entities.IImage; -import org.apache.aurora.scheduler.storage.entities.IMesosContainer; -import org.apache.aurora.scheduler.storage.entities.ITaskConfig; -import org.apache.aurora.scheduler.storage.entities.IValueConstraint; - -import static java.util.Objects.requireNonNull; - -class TaskConfigManager { - private final TaskConfigMapper configMapper; - private final JobKeyMapper jobKeyMapper; - - @Inject - TaskConfigManager(TaskConfigMapper configMapper, JobKeyMapper jobKeyMapper) { - this.configMapper = requireNonNull(configMapper); - this.jobKeyMapper = requireNonNull(jobKeyMapper); - } - - private Optional<Long> getConfigRow(ITaskConfig config) { - // NOTE: The 'config' object passed in MUST have all version-relevant fields populated in order - // to correctly compare with objects loaded from DB. This may not hold true if a 'config' is - // passed from storage recovery routine during version downgrade and fields are not properly - // backfilled. See AURORA-1603 for more details. - - // We could optimize this slightly by first comparing the un-hydrated row and breaking early. - Map<ITaskConfig, DbTaskConfig> rowsByConfig = - Maps.uniqueIndex( - configMapper.selectConfigsByJob(config.getJob()), - DbTaskConfig::toImmutable); - - return Optional.ofNullable(rowsByConfig.get(config)).map(DbTaskConfig::getRowId); - } - - long insert(ITaskConfig config) { - InsertResult configInsert = new InsertResult(); - - // Determine whether this config is already stored. - Optional<Long> existingRow = getConfigRow(config); - if (existingRow.isPresent()) { - return existingRow.get(); - } - - jobKeyMapper.merge(config.getJob()); - configMapper.insert(config, configInsert); - for (IConstraint constraint : config.getConstraints()) { - InsertResult constraintResult = new InsertResult(); - configMapper.insertConstraint(configInsert.getId(), constraint, constraintResult); - switch (constraint.getConstraint().getSetField()) { - case VALUE: - IValueConstraint valueConstraint = constraint.getConstraint().getValue(); - InsertResult valueResult = new InsertResult(); - configMapper.insertValueConstraint( - constraintResult.getId(), - valueConstraint, - valueResult); - configMapper.insertValueConstraintValues( - valueResult.getId(), - valueConstraint.getValues()); - break; - - case LIMIT: - configMapper.insertLimitConstraint( - constraintResult.getId(), - constraint.getConstraint().getLimit()); - break; - - default: - throw new IllegalStateException( - "Unhandled constraint type " + constraint.getConstraint().getSetField()); - } - } - - if (!config.getResources().isEmpty()) { - configMapper.insertResources( - configInsert.getId(), - DBResourceAggregate.pairsFromResources(config.getResources())); - } - - if (!config.getRequestedPorts().isEmpty()) { - configMapper.insertRequestedPorts(configInsert.getId(), config.getRequestedPorts()); - } - - if (!config.getTaskLinks().isEmpty()) { - configMapper.insertTaskLinks(configInsert.getId(), config.getTaskLinks()); - } - - if (!config.getMetadata().isEmpty()) { - configMapper.insertMetadata(configInsert.getId(), config.getMetadata()); - } - - if (!config.getMesosFetcherUris().isEmpty()) { - configMapper.insertMesosFetcherUris(configInsert.getId(), config.getMesosFetcherUris()); - } - - if (config.getContainer().isSetDocker()) { - IDockerContainer container = config.getContainer().getDocker(); - InsertResult containerInsert = new InsertResult(); - configMapper.insertContainer(configInsert.getId(), container, containerInsert); - if (!container.getParameters().isEmpty()) { - configMapper.insertDockerParameters(containerInsert.getId(), container.getParameters()); - } - } else if (config.getContainer().isSetMesos() - && config.getContainer().getMesos().isSetImage()) { - - IMesosContainer container = config.getContainer().getMesos(); - IImage image = container.getImage(); - - switch (image.getSetField()) { - case DOCKER: - IDockerImage dockerImage = image.getDocker(); - configMapper.insertDockerImage( - configInsert.getId(), - dockerImage.getName(), - dockerImage.getTag()); - break; - case APPC: - IAppcImage appcImage = image.getAppc(); - configMapper.insertAppcImage( - configInsert.getId(), - appcImage.getName(), - appcImage.getImageId()); - break; - default: - throw new IllegalStateException("Unexpected image type: " + image.getSetField()); - } - - if (!container.getVolumes().isEmpty()) { - configMapper.insertVolumes(configInsert.getId(), container.getVolumes()); - } - } - - return configInsert.getId(); - } -}
