Repository: aurora Updated Branches: refs/heads/master 102f5f0a9 -> 4e28b9c8b
Add support for Mesos Fetcher Reviewed at https://reviews.apache.org/r/49218/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/4e28b9c8 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/4e28b9c8 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/4e28b9c8 Branch: refs/heads/master Commit: 4e28b9c8b29b66f2f10b0a6cafdec1f8e2c1bd7b Parents: 102f5f0 Author: Renan DelValle <[email protected]> Authored: Wed Jun 29 08:59:19 2016 -0700 Committer: Maxim Khutornenko <[email protected]> Committed: Wed Jun 29 08:59:19 2016 -0700 ---------------------------------------------------------------------- RELEASE-NOTES.md | 12 +++++ .../thrift/org/apache/aurora/gen/api.thrift | 12 +++++ docs/features/mesos-fetcher.md | 46 ++++++++++++++++++++ .../apache/aurora/scheduler/app/AppModule.java | 8 +++- .../aurora/scheduler/base/TaskTestUtil.java | 5 +++ .../configuration/ConfigurationManager.java | 12 ++++- .../scheduler/mesos/MesosTaskFactory.java | 13 ++++++ .../scheduler/storage/db/TaskConfigManager.java | 4 ++ .../scheduler/storage/db/TaskConfigMapper.java | 13 +++++- .../V007_CreateMesosFetcherURIsTable.java | 46 ++++++++++++++++++++ .../storage/db/views/DbTaskConfig.java | 3 ++ .../scheduler/storage/db/TaskConfigMapper.xml | 26 +++++++++++ .../aurora/scheduler/storage/db/schema.sql | 8 ++++ .../configuration/ConfigurationManagerTest.java | 25 +++++++++++ .../mesos/MesosTaskFactoryImplTest.java | 13 ++++++ .../storage/AbstractTaskStoreTest.java | 13 ++++++ .../aurora/scheduler/thrift/Fixtures.java | 1 + .../aurora/scheduler/thrift/ThriftIT.java | 1 + 18 files changed, 258 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/RELEASE-NOTES.md ---------------------------------------------------------------------- diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index af2061c..5beda5d 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,15 @@ +0.15.0 +====== + +### New/updated: + +- New scheduler commandline argument -enable_mesos_fetcher to allow job submissions +to contain URIs which will be passed to the Mesos Fetcher and subsequently downloaded into +the sandbox. Please note that enabling job submissions to download resources from arbitrary +URIs may have security implications. + +### Deprecations and removals: + 0.14.0 ====== http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/api/src/main/thrift/org/apache/aurora/gen/api.thrift ---------------------------------------------------------------------- diff --git a/api/src/main/thrift/org/apache/aurora/gen/api.thrift b/api/src/main/thrift/org/apache/aurora/gen/api.thrift index 3e6daf4..1d66208 100644 --- a/api/src/main/thrift/org/apache/aurora/gen/api.thrift +++ b/api/src/main/thrift/org/apache/aurora/gen/api.thrift @@ -142,6 +142,16 @@ struct InstanceKey { 2: i32 instanceId } +/** URI which mirrors CommandInfo.URI in the Mesos Protobuf */ +struct MesosFetcherURI { + /** Where to get the resource from */ + 1: string value + /** Extract compressed archive after downloading */ + 2: optional bool extract + /** Cache value using Mesos Fetcher caching mechanism **/ + 3: optional bool cache +} + struct ExecutorConfig { /** Name identifying the Executor. */ 1: string name @@ -253,6 +263,8 @@ struct TaskConfig { 20: set<Constraint> constraints /** a list of named ports this task requests */ 21: set<string> requestedPorts + /** Resources to retrieve with Mesos Fetcher */ + 33: optional set<MesosFetcherURI> mesosFetcherUris /** * Custom links to include when displaying this task on the scheduler dashboard. Keys are anchor * text, values are URLs. Wildcards are supported for dynamic link crafting based on host, ports, http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/docs/features/mesos-fetcher.md ---------------------------------------------------------------------- diff --git a/docs/features/mesos-fetcher.md b/docs/features/mesos-fetcher.md new file mode 100644 index 0000000..3df64c9 --- /dev/null +++ b/docs/features/mesos-fetcher.md @@ -0,0 +1,46 @@ +Mesos Fetcher +============= + +Mesos has support for downloading resources into the sandbox through the +use of the [Mesos Fetcher](http://mesos.apache.org/documentation/latest/fetcher/) + +Aurora supports passing URIs to the Mesos Fetcher dynamically by including +a list of URIs in job submissions. + +How to use +---------- +The scheduler flag `-enable_mesos_fetcher` must be set to true. + +Currently only the scheduler side of this feature has been implemented +so a modification to the existing client, or a custom Thrift client are required +to make use of this feature. + +If using a custom Thrift client, the list of URIs must be included in TaskConfig +as the `mesosFetcherUris` field. + +Each Mesos Fetcher URI has the following data members: + +|Property | Description| +|---------|------| +|value (required) |Path to the resource needed in the sandbox.| +|extract (optional)|Extract files from packed or compressed archives into the sandbox.| +|cache (optional) | Use caching mechanism provided by Mesos for resources.| + +Note that this structure is very similar to the one provided for downloading +resources needed for a [custom executor](../operations/configuration.md). + +This is because both features use the Mesos fetcher to retrieve resources into +the sandbox. However, one, the custom executor feature, has a static set of URIs +set in the server side, and the other, the Mesos Fetcher feature, is a dynamic set +of URIs set at the time of job submission. + +Security Implications +--------------------- +There are security implications that must be taken into account when enabling this feature. +**Enabling this feature may potentially enable any job submitting user to perform a privilege escalation.** + +Until a more through solution is created, one step that has been taken to mitigate this issue +is to statically mark every user submitted URI as non-executable. This is in contrast to the set of URIs +set in the custom executor feature which may mark any URI as executable. + +If the need arises to mark a downloaded URI as executable, please consider using the custom executor feature. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/app/AppModule.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java index 6c7c75a..e5bafa4 100644 --- a/src/main/java/org/apache/aurora/scheduler/app/AppModule.java +++ b/src/main/java/org/apache/aurora/scheduler/app/AppModule.java @@ -97,6 +97,11 @@ public class AppModule extends AbstractModule { @CmdLine(name = "allow_gpu_resource", help = "Allow jobs to request Mesos GPU resource.") private static final Arg<Boolean> ALLOW_GPU_RESOURCE = Arg.create(false); + @CmdLine(name = "enable_mesos_fetcher", help = "Allow jobs to pass URIs " + + "to the Mesos Fetcher. Note that enabling this feature could pose " + + "a privilege escalation threat.") + private static final Arg<Boolean> ENABLE_MESOS_FETCHER = Arg.create(false); + private final ConfigurationManagerSettings configurationManagerSettings; @VisibleForTesting @@ -110,7 +115,8 @@ public class AppModule extends AbstractModule { ENABLE_DOCKER_PARAMETERS.get(), DEFAULT_DOCKER_PARAMETERS.get(), REQUIRE_DOCKER_USE_EXECUTOR.get(), - ALLOW_GPU_RESOURCE.get())); + ALLOW_GPU_RESOURCE.get(), + ENABLE_MESOS_FETCHER.get())); } @Override http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java index 8103163..c43e04a 100644 --- a/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java +++ b/src/main/java/org/apache/aurora/scheduler/base/TaskTestUtil.java @@ -30,6 +30,7 @@ import org.apache.aurora.gen.DockerParameter; import org.apache.aurora.gen.ExecutorConfig; import org.apache.aurora.gen.Identity; import org.apache.aurora.gen.LimitConstraint; +import org.apache.aurora.gen.MesosFetcherURI; import org.apache.aurora.gen.Metadata; import org.apache.aurora.gen.Resource; import org.apache.aurora.gen.ScheduleStatus; @@ -78,6 +79,7 @@ public final class TaskTestUtil { false, ImmutableMultimap.of(), true, + true, true); public static final ConfigurationManager CONFIGURATION_MANAGER = new ConfigurationManager(CONFIGURATION_MANAGER_SETTINGS, TIER_MANAGER, THRIFT_BACKFILL); @@ -110,6 +112,9 @@ public final class TaskTestUtil { .setTaskLinks(ImmutableMap.of("http", "link", "admin", "otherLink")) .setContactEmail("[email protected]") .setMetadata(ImmutableSet.of(new Metadata("key", "value"))) + .setMesosFetcherUris(ImmutableSet.of( + new MesosFetcherURI("pathA").setExtract(true).setCache(true), + new MesosFetcherURI("pathB").setExtract(true).setCache(true))) .setExecutorConfig(new ExecutorConfig("name", "config")) .setContainer(Container.docker( new DockerContainer("imagename") http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java index fe18c0f..93ed360 100644 --- a/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java +++ b/src/main/java/org/apache/aurora/scheduler/configuration/ConfigurationManager.java @@ -116,19 +116,22 @@ public class ConfigurationManager { private final Multimap<String, String> defaultDockerParameters; private final boolean requireDockerUseExecutor; private final boolean allowGpuResource; + private final boolean enableMesosFetcher; public ConfigurationManagerSettings( ImmutableSet<Container._Fields> allowedContainerTypes, boolean allowDockerParameters, Multimap<String, String> defaultDockerParameters, boolean requireDockerUseExecutor, - boolean allowGpuResource) { + boolean allowGpuResource, + boolean enableMesosFetcher) { this.allowedContainerTypes = requireNonNull(allowedContainerTypes); this.allowDockerParameters = allowDockerParameters; this.defaultDockerParameters = requireNonNull(defaultDockerParameters); this.requireDockerUseExecutor = requireDockerUseExecutor; this.allowGpuResource = allowGpuResource; + this.enableMesosFetcher = enableMesosFetcher; } } @@ -217,6 +220,9 @@ public class ConfigurationManager { static final String EXECUTOR_REQUIRED_WITH_DOCKER = "This scheduler is configured to require an executor for Docker-based tasks."; + @VisibleForTesting + static final String MESOS_FETCHER_DISABLED = + "Mesos Fetcher for individual jobs is disabled in this cluster."; /** * Check validity of and populates defaults in a task configuration. This will return a deep copy * of the provided task configuration with default configuration values applied, and configuration @@ -339,6 +345,10 @@ public class ConfigurationManager { throw new TaskDescriptionException("GPU resource support is disabled in this cluster."); } + if (!settings.enableMesosFetcher && !config.getMesosFetcherUris().isEmpty()) { + throw new TaskDescriptionException(MESOS_FETCHER_DISABLED); + } + maybeFillLinks(builder); return ITaskConfig.build(builder); http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java index 3b01801..cbbd6be 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/MesosTaskFactory.java @@ -13,8 +13,10 @@ */ package org.apache.aurora.scheduler.mesos; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -261,6 +263,17 @@ public interface MesosTaskFactory { ExecutorInfo.Builder builder = executorSettings.getExecutorConfig().getExecutor().toBuilder() .setExecutorId(getExecutorId(task.getTaskId())) .setSource(getInstanceSourceName(task.getTask(), task.getInstanceId())); + + //TODO: (rdelvalle) add output_file when Aurora's Mesos dep is updated (MESOS-4735) + List<CommandInfo.URI> mesosFetcherUris = task.getTask().getMesosFetcherUris().stream() + .map(u -> Protos.CommandInfo.URI.newBuilder().setValue(u.getValue()) + .setExecutable(false) + .setExtract(u.isExtract()) + .setCache(u.isCache()).build()) + .collect(Collectors.toList()); + + builder.setCommand(builder.getCommand().toBuilder().addAllUris(mesosFetcherUris)); + Iterable<Resource> executorResources = acceptedOffer.getExecutorResources(); LOG.debug( "Setting executor resources to {}", http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java index c761642..e137e57 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigManager.java @@ -114,6 +114,10 @@ class TaskConfigManager { configMapper.insertMetadata(configInsert.getId(), config.getMetadata()); } + if (!config.getMesosFetcherUris().isEmpty()) { + configMapper.insertMesosFetcherUris(configInsert.getId(), config.getMesosFetcherUris()); + } + if (config.getContainer().isSetDocker()) { IDockerContainer container = config.getContainer().getDocker(); InsertResult containerInsert = new InsertResult(); http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java index 5712010..151306a 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.java @@ -24,6 +24,7 @@ import org.apache.aurora.scheduler.storage.entities.IDockerContainer; import org.apache.aurora.scheduler.storage.entities.IDockerParameter; import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.ILimitConstraint; +import org.apache.aurora.scheduler.storage.entities.IMesosFetcherURI; import org.apache.aurora.scheduler.storage.entities.IMetadata; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import org.apache.aurora.scheduler.storage.entities.IValueConstraint; @@ -126,7 +127,7 @@ interface TaskConfigMapper extends GarbageCollectedTableMapper { @Param("result") InsertResult result); /** - * Inserts docker parameters in associationw ith an {@link IDockerContainer}. + * Inserts docker parameters in association with an {@link IDockerContainer}. * * @param containerId Docker container row ID. * @param parameters Parameters to insert. @@ -146,6 +147,16 @@ interface TaskConfigMapper extends GarbageCollectedTableMapper { @Param("metadata") Set<IMetadata> metadata); /** + * Inserts the Mesos Fetcher URIs in association with an {@link ITaskConfig}. + * + * @param configId Task config ID. + * @param uris Resources Mesos Fetcher should place in sandbox. + */ + void insertMesosFetcherUris( + @Param("configId") long configId, + @Param("uris") Set<IMesosFetcherURI> uris); + + /** * Deletes task configs. * * @param configIds Configs to delete. http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V007_CreateMesosFetcherURIsTable.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V007_CreateMesosFetcherURIsTable.java b/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V007_CreateMesosFetcherURIsTable.java new file mode 100644 index 0000000..d474e17 --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/migration/V007_CreateMesosFetcherURIsTable.java @@ -0,0 +1,46 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.storage.db.migration; + +import java.math.BigDecimal; + +import org.apache.ibatis.migration.MigrationScript; + +public class V007_CreateMesosFetcherURIsTable implements MigrationScript { + @Override + public BigDecimal getId() { + return BigDecimal.valueOf(7L); + } + + @Override + public String getDescription() { + return "Create the task_config_mesos_fetcher_uris table."; + } + + @Override + public String getUpScript() { + return "CREATE TABLE IF NOT EXISTS task_config_mesos_fetcher_uris(" + + "id IDENTITY," + + "task_config_id BIGINT NOT NULL REFERENCES task_configs(id) ON DELETE CASCADE," + + "value VARCHAR NOT NULL," + + "extract BOOLEAN NOT NULL," + + "cache BOOLEAN NOT NULL" + + ");"; + } + + @Override + public String getDownScript() { + return "DROP TABLE IF EXISTS task_config_mesos_fetcher_uris;"; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java index a90cb00..cf65a28 100644 --- a/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java +++ b/src/main/java/org/apache/aurora/scheduler/storage/db/views/DbTaskConfig.java @@ -23,6 +23,7 @@ import org.apache.aurora.gen.ExecutorConfig; import org.apache.aurora.gen.Identity; import org.apache.aurora.gen.JobKey; import org.apache.aurora.gen.MesosContainer; +import org.apache.aurora.gen.MesosFetcherURI; import org.apache.aurora.gen.Metadata; import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; @@ -46,6 +47,7 @@ public final class DbTaskConfig { private String contactEmail; private ExecutorConfig executorConfig; private List<Metadata> metadata; + private List<MesosFetcherURI> mesosFetcherUris; private DbContainer container; private String tier; private List<DBResource> resources; @@ -77,6 +79,7 @@ public final class DbTaskConfig { .setContactEmail(contactEmail) .setExecutorConfig(executorConfig) .setMetadata(ImmutableSet.copyOf(metadata)) + .setMesosFetcherUris(ImmutableSet.copyOf(mesosFetcherUris)) .setContainer( container == null ? Container.mesos(new MesosContainer()) : container.toThrift()) .setResources(resources.stream().map(DBResource::toThrift).collect(toImmutableSet())); http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml index 2c8af8b..3fce25f 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/TaskConfigMapper.xml @@ -126,6 +126,10 @@ <id column="id" /> </resultMap> + <resultMap id="mesosFetcherUrisMap" type="org.apache.aurora.gen.MesosFetcherURI"> + <id column="id" /> + </resultMap> + <resultMap id="resourceMap" type="org.apache.aurora.scheduler.storage.db.views.DBResource"> <id column="id" /> <result property="type" @@ -151,6 +155,7 @@ <result column="port_name" /> </collection> <collection property="metadata" resultMap="metadataMap" columnPrefix="m_"/> + <collection property="mesosFetcherUris" resultMap="mesosFetcherUrisMap" columnPrefix="u_"/> <collection property="taskLinks" select="selectTaskLinks" @@ -188,6 +193,10 @@ m.id AS m_id, m.key AS m_key, m.value AS m_value, + u.id AS u_id, + u.value AS u_value, + u.extract AS u_extract, + u.cache AS u_cache, di.name as c_image_docker_name, di.tag as c_image_docker_tag, ai.name as c_image_appc_name, @@ -204,6 +213,7 @@ LEFT OUTER JOIN task_config_requested_ports AS p ON p.task_config_id = c.id LEFT OUTER JOIN task_config_docker_containers AS d ON d.task_config_id = c.id LEFT OUTER JOIN task_config_metadata AS m ON m.task_config_id = c.id + LEFT OUTER JOIN task_config_mesos_fetcher_uris AS u ON u.task_config_id = c.id LEFT OUTER JOIN task_config_docker_images AS di ON di.task_config_id = c.id LEFT OUTER JOIN task_config_appc_images AS ai ON ai.task_config_id = c.id LEFT OUTER JOIN task_constraints AS tc ON tc.task_config_id = c.id @@ -393,6 +403,22 @@ ) </insert> + <insert id="insertMesosFetcherUris"> + INSERT INTO task_config_mesos_fetcher_uris ( + task_config_id, + value, + extract, + cache + ) VALUES ( + <foreach item="entry" collection="uris" separator="),("> + #{configId}, + #{entry.value}, + #{entry.extract}, + #{entry.cache} + </foreach> + ) + </insert> + <select id="selectAllRowIds" resultType="long"> SELECT id FROM task_configs </select> http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql ---------------------------------------------------------------------- diff --git a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql index 5069bed..a40830f 100644 --- a/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql +++ b/src/main/resources/org/apache/aurora/scheduler/storage/db/schema.sql @@ -181,6 +181,14 @@ CREATE TABLE task_config_metadata( value VARCHAR NOT NULL ); +CREATE TABLE task_config_mesos_fetcher_uris( + id IDENTITY, + task_config_id BIGINT NOT NULL REFERENCES task_configs(id) ON DELETE CASCADE, + value VARCHAR NOT NULL, + extract BOOLEAN NOT NULL, + cache BOOLEAN NOT NULL +); + CREATE TABLE task_config_docker_containers( id IDENTITY, task_config_id BIGINT NOT NULL REFERENCES task_configs(id) ON DELETE CASCADE, http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java index 2dff80b..1473270 100644 --- a/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java +++ b/src/test/java/org/apache/aurora/scheduler/configuration/ConfigurationManagerTest.java @@ -29,6 +29,7 @@ import org.apache.aurora.gen.Identity; import org.apache.aurora.gen.JobConfiguration; import org.apache.aurora.gen.JobKey; import org.apache.aurora.gen.LimitConstraint; +import org.apache.aurora.gen.MesosFetcherURI; import org.apache.aurora.gen.TaskConfig; import org.apache.aurora.gen.TaskConstraint; import org.apache.aurora.gen.ValueConstraint; @@ -118,6 +119,7 @@ public class ConfigurationManagerTest { false, ImmutableMultimap.of(), true, + false, false), TaskTestUtil.TIER_MANAGER, TaskTestUtil.THRIFT_BACKFILL); @@ -127,6 +129,7 @@ public class ConfigurationManagerTest { true, ImmutableMultimap.of("foo", "bar"), false, + true, true), TaskTestUtil.TIER_MANAGER, TaskTestUtil.THRIFT_BACKFILL); @@ -288,12 +291,34 @@ public class ConfigurationManagerTest { true, ImmutableMultimap.of("foo", "bar"), false, + false, false), TaskTestUtil.TIER_MANAGER, TaskTestUtil.THRIFT_BACKFILL).validateAndPopulate(ITaskConfig.build(builder)); } @Test + public void testMesosFetcherDisabled() throws Exception { + TaskConfig builder = CONFIG_WITH_CONTAINER.newBuilder(); + builder.setMesosFetcherUris( + ImmutableSet.of( + new MesosFetcherURI("pathA").setExtract(true).setCache(true), + new MesosFetcherURI("pathB").setExtract(true).setCache(true))); + + expectTaskDescriptionException(ConfigurationManager.MESOS_FETCHER_DISABLED); + new ConfigurationManager( + new ConfigurationManagerSettings( + ALL_CONTAINER_TYPES, + true, + ImmutableMultimap.of("foo", "bar"), + false, + false, + false), + TaskTestUtil.TIER_MANAGER, + TaskTestUtil.THRIFT_BACKFILL).validateAndPopulate(ITaskConfig.build(builder)); + } + + @Test public void testTaskLinks() throws Exception { TaskConfig builder = CONFIG_WITH_CONTAINER.newBuilder(); builder.addToResources(namedPort("health")); http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java index 58785bf..500fd43 100644 --- a/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java +++ b/src/test/java/org/apache/aurora/scheduler/mesos/MesosTaskFactoryImplTest.java @@ -45,6 +45,7 @@ import org.apache.aurora.scheduler.storage.entities.IJobKey; import org.apache.aurora.scheduler.storage.entities.IServerInfo; import org.apache.aurora.scheduler.storage.entities.ITaskConfig; import org.apache.mesos.Protos; +import org.apache.mesos.Protos.CommandInfo.URI; import org.apache.mesos.Protos.ContainerInfo; import org.apache.mesos.Protos.ContainerInfo.DockerInfo; import org.apache.mesos.Protos.ContainerInfo.MesosInfo; @@ -152,6 +153,18 @@ public class MesosTaskFactoryImplTest extends EasyMockTest { .setExecutorId(MesosTaskFactoryImpl.getExecutorId(task.getTaskId())) .setSource( MesosTaskFactoryImpl.getInstanceSourceName(task.getTask(), task.getInstanceId())) + .setCommand(executor.getCommand().toBuilder().addAllUris( + ImmutableSet.of( + URI.newBuilder() + .setValue("pathA") + .setExecutable(false) + .setExtract(true) + .setCache(true).build(), + URI.newBuilder() + .setValue("pathB") + .setExecutable(false) + .setExtract(true) + .setCache(true).build()))) .build(); } http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java b/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java index b1593f6..3d07292 100644 --- a/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java +++ b/src/test/java/org/apache/aurora/scheduler/storage/AbstractTaskStoreTest.java @@ -49,6 +49,7 @@ import org.apache.aurora.gen.HostAttributes; import org.apache.aurora.gen.Image; import org.apache.aurora.gen.MaintenanceMode; import org.apache.aurora.gen.MesosContainer; +import org.apache.aurora.gen.MesosFetcherURI; import org.apache.aurora.gen.Metadata; import org.apache.aurora.gen.ScheduledTask; import org.apache.aurora.gen.TaskQuery; @@ -179,6 +180,18 @@ public abstract class AbstractTaskStoreTest extends TearDownTestCase { } @Test + public void testSaveWithMesosFetcherUris() { + ScheduledTask builder = TASK_A.newBuilder(); + builder.getAssignedTask().getTask().setMesosFetcherUris( + ImmutableSet.of( + new MesosFetcherURI("pathA").setExtract(true).setCache(true), + new MesosFetcherURI("pathB").setExtract(true).setCache(true))); + IScheduledTask task = IScheduledTask.build(builder); + saveTasks(task); + assertStoreContents(task); + } + + @Test public void testSaveWithDockerImage() { ScheduledTask builder = TASK_A.newBuilder(); http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java b/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java index a883b0e..a79b0f4 100644 --- a/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java +++ b/src/test/java/org/apache/aurora/scheduler/thrift/Fixtures.java @@ -113,6 +113,7 @@ final class Fixtures { .setMaxTaskFailures(1) .setConstraints(ImmutableSet.of()) .setMetadata(ImmutableSet.of()) + .setMesosFetcherUris(ImmutableSet.of()) .setContainer(Container.mesos(new MesosContainer())) .setResources(ImmutableSet.of( Resource.numCpus(1), http://git-wip-us.apache.org/repos/asf/aurora/blob/4e28b9c8/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java b/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java index ed69996..3caad32 100644 --- a/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java +++ b/src/test/java/org/apache/aurora/scheduler/thrift/ThriftIT.java @@ -155,6 +155,7 @@ public class ThriftIT extends EasyMockTest { true, ImmutableMultimap.of(), false, + true, true); createThrift(configurationManagerSettings);
