Repository: aurora Updated Branches: refs/heads/master 7b1e8484a -> 6ba06ae84
Introduce a utility class to read executor configurations in JSON. Reviewed at https://reviews.apache.org/r/40391/ Project: http://git-wip-us.apache.org/repos/asf/aurora/repo Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/6ba06ae8 Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/6ba06ae8 Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/6ba06ae8 Branch: refs/heads/master Commit: 6ba06ae84440021b5821e666ac5a628ca094af61 Parents: 7b1e848 Author: Bill Farner <[email protected]> Authored: Wed Nov 18 20:09:46 2015 -0800 Committer: Bill Farner <[email protected]> Committed: Wed Nov 18 20:09:46 2015 -0800 ---------------------------------------------------------------------- build.gradle | 4 + .../executor/ExecutorSettingsLoader.java | 164 +++++++++++++++++++ .../scheduler/mesos/TestExecutorSettings.java | 1 + .../executor/ExecutorSettingsLoaderTest.java | 71 ++++++++ .../executor/test-missing-field.json | 4 + .../executor/test-thermos-executor.json | 49 ++++++ 6 files changed, 293 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 17dbd25..32295e6 100644 --- a/build.gradle +++ b/build.gradle @@ -110,6 +110,8 @@ For more details, please see https://issues.apache.org/jira/browse/AURORA-1169 force "org.hamcrest:hamcrest-core:1.3" force "org.apache.thrift:libthrift:${thriftRev}" force "org.slf4j:slf4j-jdk14:${slf4jRev}" + force 'com.fasterxml.jackson.core:jackson-annotations:2.5.1' + force 'com.fasterxml.jackson.core:jackson-core:2.5.1' } } } @@ -347,6 +349,8 @@ dependencies { compile "com.google.inject.extensions:guice-assistedinject:${guiceRev}" compile 'com.google.protobuf:protobuf-java:2.5.0' compile 'com.h2database:h2:1.4.187' + compile 'com.hubspot.jackson:jackson-datatype-protobuf:0.9.2' + compile 'com.fasterxml.jackson.core:jackson-core:2.5.1' compile "com.sun.jersey:jersey-core:${jerseyRev}" compile "com.sun.jersey:jersey-json:${jerseyRev}" compile "com.sun.jersey:jersey-server:${jerseyRev}" http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java new file mode 100644 index 0000000..fafd90d --- /dev/null +++ b/src/main/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoader.java @@ -0,0 +1,164 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.aurora.scheduler.configuration.executor; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.MoreObjects; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.io.CharStreams; +import com.google.protobuf.UninitializedMessageException; +import com.hubspot.jackson.datatype.protobuf.ProtobufModule; + +import org.apache.mesos.Protos.ExecutorID; +import org.apache.mesos.Protos.ExecutorInfo; +import org.apache.mesos.Protos.Volume; + +import static java.util.Objects.requireNonNull; + +import static com.fasterxml.jackson.databind.PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES; + +/** + * A utility class to read JSON-formatted executor configurations. + */ +public final class ExecutorSettingsLoader { + public static final ExecutorID PLACEHOLDER_EXECUTOR_ID = ExecutorID.newBuilder() + .setValue("PLACEHOLDER") + .build(); + + private ExecutorSettingsLoader() { + // Utility class + } + + /** + * Thrown when an executor configuration could not be read. + */ + public static class ExecutorConfigException extends Exception { + public ExecutorConfigException(Throwable cause) { + super(cause); + } + } + + /** + * Reads an executor configuration from a JSON-encoded source. + * + * @param input The configuration data source. + * @return An executor configuration. + * @throws ExecutorConfigException If the input cannot be read or is not properly formatted. + */ + public static ExecutorConfig read(Readable input) throws ExecutorConfigException { + String configContents; + try { + configContents = CharStreams.toString(input); + } catch (IOException e) { + throw new ExecutorConfigException(e); + } + + ObjectMapper mapper = new ObjectMapper() + .registerModule(new ProtobufModule()) + .setPropertyNamingStrategy(CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); + Schema parsed; + try { + parsed = mapper.readValue(configContents, Schema.class); + } catch (IOException e) { + throw new ExecutorConfigException(e); + } + + ExecutorInfo executorInfo; + try { + // We apply a placeholder value for the executor ID so that we can construct and validate + // the protobuf schema. This allows us to catch many validation errors here rather than + // later on when launching tasks. + executorInfo = parsed.executor.setExecutorId(PLACEHOLDER_EXECUTOR_ID).build(); + } catch (UninitializedMessageException e) { + throw new ExecutorConfigException(e); + } + + return new ExecutorConfig( + executorInfo, + Optional.fromNullable(parsed.volumeMounts).or(ImmutableList.of())); + } + + /** + * The JSON schema. This is separated from the public {@link ExecutorConfig} so we can read + * objects that do not have all fields required by the protobuf set in the JSON config. + */ + private static class Schema { + private ExecutorInfo.Builder executor; + private List<Volume> volumeMounts; + + ExecutorInfo.Builder getExecutor() { + return executor; + } + + void setExecutor(ExecutorInfo.Builder executor) { + this.executor = executor; + } + + List<Volume> getVolumeMounts() { + return volumeMounts; + } + + void setVolumeMounts(List<Volume> volumeMounts) { + this.volumeMounts = volumeMounts; + } + } + + public static class ExecutorConfig { + private final ExecutorInfo executor; + private final List<Volume> volumeMounts; + + public ExecutorConfig(ExecutorInfo executor, List<Volume> volumeMounts) { + this.executor = requireNonNull(executor); + this.volumeMounts = requireNonNull(volumeMounts); + } + + public ExecutorInfo getExecutor() { + return executor; + } + + public List<Volume> getVolumeMounts() { + return volumeMounts; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("executor", executor) + .add("volumeMounts", volumeMounts) + .toString(); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ExecutorConfig)) { + return false; + } + + ExecutorConfig other = (ExecutorConfig) obj; + return Objects.equals(executor, other.executor) + && Objects.equals(volumeMounts, other.volumeMounts); + } + + @Override + public int hashCode() { + return Objects.hash(executor, volumeMounts); + } + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java b/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java index 556a3f8..6214882 100644 --- a/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java +++ b/src/main/java/org/apache/aurora/scheduler/mesos/TestExecutorSettings.java @@ -37,6 +37,7 @@ public final class TestExecutorSettings { } public static final ExecutorInfo THERMOS_EXECUTOR_INFO = ExecutorInfo.newBuilder() + .setName("thermos") .setExecutorId(Executors.PLACEHOLDER_EXECUTOR_ID) .setCommand(CommandInfo.newBuilder().setValue("thermos_executor.pex") .addAllArguments(ImmutableList.of( http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java b/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java new file mode 100644 index 0000000..90986a5 --- /dev/null +++ b/src/test/java/org/apache/aurora/scheduler/configuration/executor/ExecutorSettingsLoaderTest.java @@ -0,0 +1,71 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.scheduler.configuration.executor; + +import java.io.InputStreamReader; +import java.io.StringReader; + +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableList; + +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettingsLoader.ExecutorConfig; +import org.apache.aurora.scheduler.configuration.executor.ExecutorSettingsLoader.ExecutorConfigException; +import org.apache.aurora.scheduler.mesos.TestExecutorSettings; +import org.apache.mesos.Protos.Volume; +import org.apache.mesos.Protos.Volume.Mode; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ExecutorSettingsLoaderTest { + private static final ExecutorConfig THERMOS_CONFIG = new ExecutorConfig( + TestExecutorSettings.THERMOS_CONFIG.getExecutor(), + ImmutableList.of( + Volume.newBuilder() + .setHostPath("/path/to/observer_root") + .setContainerPath("/path/to/observer_root") + .setMode(Mode.RO) + .build(), + Volume.newBuilder() + .setHostPath("/host") + .setContainerPath("/container") + .setMode(Mode.RW) + .build())); + + private ExecutorConfig loadResource(String name) throws ExecutorConfigException { + return ExecutorSettingsLoader.read( + new InputStreamReader(getClass().getResourceAsStream(name), Charsets.UTF_8)); + } + + private void assertParsedResult(ExecutorConfig expected, String file) + throws ExecutorConfigException { + + assertEquals(expected, loadResource(file)); + } + + @Test + public void testParse() throws Exception { + assertParsedResult(THERMOS_CONFIG, "test-thermos-executor.json"); + } + + @Test(expected = ExecutorConfigException.class) + public void testInvalidJson() throws Exception { + ExecutorSettingsLoader.read(new StringReader("this is not json")); + } + + @Test(expected = ExecutorConfigException.class) + public void testMissingField() throws Exception { + loadResource("test-missing-field.json"); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json new file mode 100644 index 0000000..4646170 --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-missing-field.json @@ -0,0 +1,4 @@ +{ + "executor": { + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/6ba06ae8/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json ---------------------------------------------------------------------- diff --git a/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json new file mode 100644 index 0000000..8d4d90b --- /dev/null +++ b/src/test/resources/org/apache/aurora/scheduler/configuration/executor/test-thermos-executor.json @@ -0,0 +1,49 @@ +{ + "executor": { + "name": "thermos", + "command": { + "value": "thermos_executor.pex", + "arguments": [ + "--announcer-enable", + "--announcer-ensemble", + "localhost:2181" + ], + "uris": [ + { + "value": "/home/vagrant/aurora/dist/thermos_executor.pex", + "executable": true, + "extract": false, + "cache": false + } + ] + }, + "resources": [ + { + "name": "cpus", + "type": "SCALAR", + "scalar": { + "value": 0.25 + } + }, + { + "name": "mem", + "type": "SCALAR", + "scalar": { + "value": 128 + } + } + ] + }, + "volume_mounts": [ + { + "mode": "RO", + "container_path": "/path/to/observer_root", + "host_path": "/path/to/observer_root" + }, + { + "mode": "RW", + "container_path": "/container", + "host_path": "/host" + } + ] +}
