Upgrade mesos driver to Mesos 1.5 with protobuf 2.5
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/b5b468b9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/b5b468b9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/b5b468b9 Branch: refs/heads/upgrademesos1.5 Commit: b5b468b96f3cab34e8f8b87d87e22f31130ec72b Parents: 6af9d37 Author: jpgilaberte <jpgilabe...@stratio.com> Authored: Thu Sep 6 15:42:31 2018 +0200 Committer: jpgilaberte <jpgilabe...@stratio.com> Committed: Thu Sep 6 15:42:31 2018 +0200 ---------------------------------------------------------------------- build.gradle | 6 +- gradle/quality.gradle | 16 + myriad-commons/build.gradle | 22 + myriad-commons/proto/mesos/executor.proto | 212 + myriad-commons/proto/mesos/mesos.proto | 3190 + myriad-commons/proto/mesos/scheduler.proto | 433 + myriad-commons/proto/mesos/v1/executor.proto | 214 + myriad-commons/proto/mesos/v1/mesos.proto | 3173 + myriad-commons/proto/mesos/v1/scheduler.proto | 420 + .../main/java/org/apache/mesos/Executor.java | 149 + .../java/org/apache/mesos/ExecutorDriver.java | 112 + .../src/main/java/org/apache/mesos/Log.java | 468 + .../org/apache/mesos/MesosExecutorDriver.java | 97 + .../org/apache/mesos/MesosNativeLibrary.java | 245 + .../org/apache/mesos/MesosSchedulerDriver.java | 307 + .../src/main/java/org/apache/mesos/Protos.java | 162923 ++++++++++++++++ .../main/java/org/apache/mesos/Scheduler.java | 211 + .../java/org/apache/mesos/SchedulerDriver.java | 323 + .../java/org/apache/mesos/executor/Protos.java | 10759 + .../java/org/apache/mesos/scheduler/Protos.java | 24780 +++ .../org/apache/mesos/state/AbstractState.java | 403 + .../org/apache/mesos/state/InMemoryState.java | 143 + .../org/apache/mesos/state/LevelDBState.java | 36 + .../java/org/apache/mesos/state/LogState.java | 79 + .../main/java/org/apache/mesos/state/State.java | 95 + .../java/org/apache/mesos/state/Variable.java | 48 + .../org/apache/mesos/state/ZooKeeperState.java | 75 + .../main/java/org/apache/mesos/v1/Protos.java | 162577 +++++++++++++++ .../org/apache/mesos/v1/executor/Protos.java | 10785 + .../org/apache/mesos/v1/scheduler/Mesos.java | 58 + .../org/apache/mesos/v1/scheduler/Protos.java | 24570 +++ .../apache/mesos/v1/scheduler/Scheduler.java | 53 + .../org/apache/mesos/v1/scheduler/V0Mesos.java | 84 + .../org/apache/mesos/v1/scheduler/V1Mesos.java | 84 + 34 files changed, 407148 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 075b768..575f1b6 100644 --- a/build.gradle +++ b/build.gradle @@ -52,6 +52,7 @@ idea { } } + subprojects { apply plugin: 'java' @@ -69,7 +70,7 @@ subprojects { version = "0.2.0" ext { - mesosVer = "0.28.1" + //mesosVer = "1.5.0" //Allows passing -PhadoopVer=2.6.0 from command line if(!project.hasProperty('hadoopVer')) { hadoopVer = "2.7.0" @@ -102,7 +103,7 @@ subprojects { capsule "co.paralleluniverse:capsule:0.7.1" myriadExecutorConf 'org.slf4j:slf4j-log4j12:1.7.10' - compile "org.apache.mesos:mesos:${mesosVer}" + //compile "org.apache.mesos:mesos:${mesosVer}" compile 'com.google.code.gson:gson:2.3.1' // marshalling between the scheduler and executor testCompile 'org.apache.zookeeper:zookeeper:3.4.6' // to resolve temporary mavenlocal issue @@ -115,4 +116,5 @@ subprojects { run { systemProperty "myriad.config", "build/resources/main/myriad-config-default.yml" } + } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/gradle/quality.gradle ---------------------------------------------------------------------- diff --git a/gradle/quality.gradle b/gradle/quality.gradle index c013d7d..e58d9ef 100644 --- a/gradle/quality.gradle +++ b/gradle/quality.gradle @@ -25,6 +25,22 @@ tasks.withType(FindBugs) { excludeFilter = file("$rootProject.projectDir/config/findbugs/excludeFilter.xml") } +tasks.withType(FindBugs) { + exclude '**/org/apache/mesos/*' + classes = classes.filter { + !it.path.contains(new File("org/apache/mesos/").path) + } +} + +tasks.withType(Pmd) { + exclude 'org/apache/mesos/**' +} + +tasks.withType(Checkstyle) { + exclude 'org/apache/mesos/**' +} + + pmd { ruleSets = [ // 'java-basic', // need brandon's http config to add in http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/myriad-commons/build.gradle ---------------------------------------------------------------------- diff --git a/myriad-commons/build.gradle b/myriad-commons/build.gradle new file mode 100644 index 0000000..c66e8f7 --- /dev/null +++ b/myriad-commons/build.gradle @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +dependencies { + compile group: 'com.google.protobuf', name: 'protobuf-java', version: '2.5.0' +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5b468b9/myriad-commons/proto/mesos/executor.proto ---------------------------------------------------------------------- diff --git a/myriad-commons/proto/mesos/executor.proto b/myriad-commons/proto/mesos/executor.proto new file mode 100644 index 0000000..1b5fa5d --- /dev/null +++ b/myriad-commons/proto/mesos/executor.proto @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +import "mesos/mesos.proto"; + +package mesos.executor; + +option java_package = "org.apache.mesos.executor"; +option java_outer_classname = "Protos"; + + +/** + * Executor event API. + * + * An event is described using the standard protocol buffer "union" + * trick, see https://developers.google.com/protocol-buffers/docs/techniques#union. + */ +message Event { + // Possible event types, followed by message definitions if + // applicable. + enum Type { + // This must be the first enum value in this list, to + // ensure that if 'type' is not set, the default value + // is UNKNOWN. This enables enum values to be added + // in a backwards-compatible way. See: MESOS-4997. + UNKNOWN = 0; + + SUBSCRIBED = 1; // See 'Subscribed' below. + LAUNCH = 2; // See 'Launch' below. + LAUNCH_GROUP = 8; // See 'LaunchGroup' below. + KILL = 3; // See 'Kill' below. + ACKNOWLEDGED = 4; // See 'Acknowledged' below. + MESSAGE = 5; // See 'Message' below. + ERROR = 6; // See 'Error' below. + + // Received when the agent asks the executor to shutdown/kill itself. + // The executor is then required to kill all its active tasks, send + // `TASK_KILLED` status updates and gracefully exit. The executor + // should terminate within a `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD` + // (an environment variable set by the agent upon executor startup); + // it can be configured via `ExecutorInfo.shutdown_grace_period`. If + // the executor fails to do so, the agent will forcefully destroy the + // container where the executor is running. The agent would then send + // `TASK_LOST` updates for any remaining active tasks of this executor. + // + // NOTE: The executor must not assume that it will always be allotted + // the full grace period, as the agent may decide to allot a shorter + // period and failures / forcible terminations may occur. + // + // TODO(alexr): Consider adding a duration field into the `Shutdown` + // message so that the agent can communicate when a shorter period + // has been allotted. + SHUTDOWN = 7; + } + + // First event received when the executor subscribes. + // The 'id' field in the 'framework_info' will be set. + message Subscribed { + required ExecutorInfo executor_info = 1; + required FrameworkInfo framework_info = 2; + required SlaveInfo slave_info = 3; + + // Uniquely identifies the container of an executor run. + optional ContainerID container_id = 4; + } + + // Received when the framework attempts to launch a task. Once + // the task is successfuly launched, the executor must respond with + // a TASK_RUNNING update (See TaskState in mesos.proto). + message Launch { + required TaskInfo task = 1; + } + + // Received when the framework attempts to launch a group of tasks atomically. + // Similar to `Launch` above the executor must send TASK_RUNNING updates for + // tasks that are successfully launched. + message LaunchGroup { + required TaskGroupInfo task_group = 1; + } + + // Received when the scheduler wants to kill a specific task. Once + // the task is terminated, the executor should send a TASK_KILLED + // (or TASK_FAILED) update. The terminal update is necessary so + // Mesos can release the resources associated with the task. + message Kill { + required TaskID task_id = 1; + + // If set, overrides any previously specified kill policy for this task. + // This includes 'TaskInfo.kill_policy' and 'Executor.kill.kill_policy'. + optional KillPolicy kill_policy = 2; + } + + // Received when the slave acknowledges the receipt of status + // update. Schedulers are responsible for explicitly acknowledging + // the receipt of status updates that have 'update.status().uuid()' + // field set. Unacknowledged updates can be retried by the executor. + // They should also be sent by the executor whenever it + // re-subscribes. + message Acknowledged { + required TaskID task_id = 1; + required bytes uuid = 2; + } + + // Received when a custom message generated by the scheduler is + // forwarded by the slave. Note that this message is not + // interpreted by Mesos and is only forwarded (without reliability + // guarantees) to the executor. It is up to the scheduler to retry + // if the message is dropped for any reason. + message Message { + required bytes data = 1; + } + + // Received in case the executor sends invalid calls (e.g., + // required values not set). + // TODO(arojas): Remove this once the old executor driver is no + // longer supported. With HTTP API all errors will be signaled via + // HTTP response codes. + message Error { + required string message = 1; + } + + // Type of the event, indicates which optional field below should be + // present if that type has a nested message definition. + // Enum fields should be optional, see: MESOS-4997. + optional Type type = 1; + + optional Subscribed subscribed = 2; + optional Acknowledged acknowledged = 3; + optional Launch launch = 4; + optional LaunchGroup launch_group = 8; + optional Kill kill = 5; + optional Message message = 6; + optional Error error = 7; +} + + +/** + * Executor call API. + * + * Like Event, a Call is described using the standard protocol buffer + * "union" trick (see above). + */ +message Call { + // Possible call types, followed by message definitions if + // applicable. + enum Type { + // See comments above on `Event::Type` for more details on this enum value. + UNKNOWN = 0; + + SUBSCRIBE = 1; // See 'Subscribe' below. + UPDATE = 2; // See 'Update' below. + MESSAGE = 3; // See 'Message' below. + } + + // Request to subscribe with the slave. If subscribing after a disconnection, + // it must include a list of all the tasks and updates which haven't been + // acknowledged by the scheduler. + message Subscribe { + repeated TaskInfo unacknowledged_tasks = 1; + repeated Update unacknowledged_updates = 2; + } + + // Notifies the scheduler that a task has transitioned from one + // state to another. Status updates should be used by executors + // to reliably communicate the status of the tasks that they + // manage. It is crucial that a terminal update (see TaskState + // in mesos.proto) is sent to the scheduler as soon as the task + // terminates, in order for Mesos to release the resources allocated + // to the task. It is the responsibility of the scheduler to + // explicitly acknowledge the receipt of a status update. See + // 'Acknowledged' in the 'Events' section above for the semantics. + message Update { + required TaskStatus status = 1; + } + + // Sends arbitrary binary data to the scheduler. Note that Mesos + // neither interprets this data nor makes any guarantees about the + // delivery of this message to the scheduler. + // See 'Message' in the 'Events' section. + message Message { + required bytes data = 2; + } + + // Identifies the executor which generated this call. + required ExecutorID executor_id = 1; + required FrameworkID framework_id = 2; + + // Type of the call, indicates which optional field below should be + // present if that type has a nested message definition. + // In case type is SUBSCRIBED, no message needs to be set. + // See comments on `Event::Type` above on the reasoning behind this field being optional. + optional Type type = 3; + + optional Subscribe subscribe = 4; + optional Update update = 5; + optional Message message = 6; +}
