[2/3] incubator-myriad git commit: Implementation of MYRIAD-229, MYRIAD-237, MYRIAD-238, MYRIAD-225 JIRA: [MYRIAD-225] https://issues.apache.org/jira/browse/MYRIAD-225 [MYRIAD-229] https://issues.apac
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/577c30b1/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsTest.java
--
diff --git
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsTest.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsTest.java
new file mode 100644
index 000..cf47b8c
--- /dev/null
+++
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/SchedulerUtilsTest.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.myriad.scheduler;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.mesos.Protos.Offer;
+import org.apache.mesos.Protos.TaskID;
+import org.apache.myriad.BaseConfigurableTest;
+import org.apache.myriad.TestObjectFactory;
+import org.apache.myriad.state.NodeTask;
+import org.apache.myriad.state.SchedulerState;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Unit tests for SchedulerUtils
+ */
+public class SchedulerUtilsTest extends BaseConfigurableTest {
+ private SchedulerState sState;
+ private TaskID idOne, idTwo, idThree;
+ private NodeTask taskOne, taskTwo, taskThree;
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+super.setUp();
+sState = TestObjectFactory.getSchedulerState(this.cfg,
"tmp/scheduler-utils-test");
+idOne = TaskID.newBuilder().setValue("Task1").build();
+idTwo = TaskID.newBuilder().setValue("Task2").build();
+idThree = TaskID.newBuilder().setValue("Task3").build();
+taskOne = TestObjectFactory.getNodeTask("zero", "server1", 0.0, 0.0,
Long.valueOf(1), Long.valueOf(2));
+taskTwo = TestObjectFactory.getNodeTask("low", "localhost", 0.2, 1024.0,
Long.valueOf(1), Long.valueOf(2));
+taskThree = TestObjectFactory.getNodeTask("medium", "localhost", 0.4,
2048.0, Long.valueOf(1), Long.valueOf(2));
+
+sState.addTask(idOne, taskOne);
+sState.addTask(idTwo, taskTwo);
+sState.addTask(idThree, taskThree);
+
+this.baseStateStoreDirectory = "/tmp/scheduler-utils-test";
+ }
+
+ @Test
+ public void testIsUniqueFilenameTrue() throws Exception {
+List tasks = Lists.newArrayList(taskOne, taskTwo, taskThree);
+NodeTask newTask = TestObjectFactory.getNodeTask("medium", "server1", 0.4,
2048.0, Long.valueOf(1), Long.valueOf(2));
+Offer offer = TestObjectFactory.getOffer("server2", "slave1",
"mock-framework", "offer1", 0.0, 0.0);
+assertTrue(SchedulerUtils.isUniqueHostname(offer, newTask, tasks));
+ }
+
+ @Test
+ public void testIsUniqueFilenameFalse() throws Exception {
+List tasks = Lists.newArrayList(taskOne, taskTwo, taskThree);
+NodeTask newTask = TestObjectFactory.getNodeTask("medium", "localhost",
0.4, 2048.0, Long.valueOf(1), Long.valueOf(2));
+Offer offer = TestObjectFactory.getOffer("localhost", "slave1",
"mock-framework", "offer1", 0.2, 512.0);
+assertFalse(SchedulerUtils.isUniqueHostname(offer, newTask, tasks));
+ }
+
+ @Test
+ public void testIsEligibleForFineGrainedSchedulingFalse() throws Exception {
+assertFalse(SchedulerUtils.isEligibleForFineGrainedScaling("localhost",
sState));
+ }
+
+ @Test
+ public void testIsEligibleForFineGrainedSchedulingTrue() throws Exception {
+assertFalse(SchedulerUtils.isEligibleForFineGrainedScaling("server1",
sState));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/577c30b1/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
--
diff --git
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
index 21b5ad0..c4b2b45 100644
---
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
+++
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestTaskUtils.java
@@ -127,5 +127,4 @@ public class TestTaskUtils
[3/3] incubator-myriad git commit: Implementation of MYRIAD-229, MYRIAD-237, MYRIAD-238, MYRIAD-225 JIRA: [MYRIAD-225] https://issues.apache.org/jira/browse/MYRIAD-225 [MYRIAD-229] https://issues.apac
Implementation of MYRIAD-229, MYRIAD-237, MYRIAD-238, MYRIAD-225
JIRA:
[MYRIAD-225] https://issues.apache.org/jira/browse/MYRIAD-225
[MYRIAD-229] https://issues.apache.org/jira/browse/MYRIAD-239
[MYRIAD-237] https://issues.apache.org/jira/browse/MYRIAD-237
[MYRIAD-237] https://issues.apache.org/jira/browse/MYRIAD-237
Pull Request:
Closes #91
Author:hokiegeek2 <hokiege...@gmail.com>
Date: Wed Aug 17 15:21:56 2016 -0400
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/577c30b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/577c30b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/577c30b1
Branch: refs/heads/master
Commit: 577c30b1abdaa30313251c887c014aebac3fd93c
Parents: 7aea259
Author: hokiegeek2 <hokiege...@gmail.com>
Authored: Wed Aug 17 15:21:56 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Tue Aug 30 13:08:36 2016 -0400
--
build.gradle| 1 -
gradle/spock.gradle | 46 ---
.../src/main/java/org/apache/myriad/Main.java | 18 +-
.../java/org/apache/myriad/MyriadModule.java| 2 +-
.../myriad/api/SchedulerStateResource.java | 2 +-
.../configuration/NodeManagerConfiguration.java | 18 +-
.../myriad/scheduler/MyriadOperations.java | 9 +-
.../org/apache/myriad/scheduler/Rebalancer.java | 4 +-
.../apache/myriad/scheduler/SchedulerUtils.java | 2 +-
.../apache/myriad/scheduler/TaskFactory.java| 2 +-
.../apache/myriad/scheduler/TaskTerminator.java | 4 +-
.../org/apache/myriad/scheduler/TaskUtils.java | 8 +-
.../handlers/StatusUpdateEventHandler.java | 2 +-
.../scheduler/fgs/NMHeartBeatHandler.java | 67 +++--
.../scheduler/fgs/OfferLifecycleManager.java| 67 +++--
.../scheduler/fgs/YarnNodeCapacityManager.java | 10 +-
.../scheduler/yarn/MyriadFairScheduler.java | 5 +
.../org/apache/myriad/state/SchedulerState.java | 14 +-
.../MyriadFileSystemRMStateStoreTest.java | 22 +-
.../org/apache/myriad/BaseConfigurableTest.java | 77 -
.../org/apache/myriad/MyriadTestModule.java | 32 +-
.../org/apache/myriad/TestObjectFactory.java| 290 +--
.../myriad/api/ArtifactsResourceTest.java | 17 ++
.../myriad/api/SchedulerStateResourceTest.java | 17 ++
.../configuration/MyriadConfigurationTest.java | 5 +-
.../myriad/health/HealthCheckUtilsTest.java | 17 ++
.../health/MesosDriverHealthCheckTest.java | 17 ++
.../myriad/scheduler/MyriadOperationsTest.java | 97 ---
.../myriad/scheduler/SchedulerUtilsSpec.groovy | 90 --
.../myriad/scheduler/SchedulerUtilsTest.java| 89 ++
.../apache/myriad/scheduler/TestTaskUtils.java | 3 +-
.../constraints/LikeConstraintSpec.groovy | 93 --
.../constraints/LikeConstraintTest.java | 86 ++
.../myriad/scheduler/fgs/FGSTestBaseSpec.groovy | 175 ---
.../scheduler/fgs/NMHeartBeatHandlerSpec.groovy | 114
.../scheduler/fgs/NMHeartBeatHandlerTest.java | 240 +++
.../myriad/scheduler/fgs/NodeStoreTest.java | 23 +-
.../apache/myriad/scheduler/fgs/NodeTest.java | 89 ++
.../fgs/OfferLifeCycleManagerTest.java | 22 +-
.../fgs/YarnNodeCapacityManagerSpec.groovy | 143 -
.../fgs/YarnNodeCapacityManagerTest.java| 149 ++
.../org/apache/myriad/state/ClusterTest.java| 17 ++
.../org/apache/myriad/state/MockDispatcher.java | 28 +-
.../org/apache/myriad/state/MockFuture.java | 17 ++
.../java/org/apache/myriad/state/MockRMApp.java | 17 ++
.../org/apache/myriad/state/MockRMContext.java | 25 +-
.../org/apache/myriad/state/MockRMNode.java | 17 ++
.../java/org/apache/myriad/state/MockState.java | 17 ++
.../org/apache/myriad/state/MockVariable.java | 17 ++
.../apache/myriad/state/MyriadStateTest.java| 17 ++
.../org/apache/myriad/state/NodeTaskTest.java | 17 ++
.../apache/myriad/state/SchedulerStateTest.java | 159 +++---
.../state/utils/ByteBufferSupportTest.java | 17 ++
.../webapp/HttpConnectorProviderTest.java | 17 ++
.../myriad/webapp/MyriadWebServerTest.java | 17 ++
.../resources/myriad-config-test-default.yml| 39 +--
56 files changed, 1590 insertions(+), 1036 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/577c30b1/build.gradle
--
diff --git a/build.gradle b/build.gradle
index b48cfc9..a551ba0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -57,7 +57,6 @@ subprojects {
apply plugin: 'java'
apply plugin: 'application'
-apply from: "$rootDir/gradle/spock.
[1/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc
Repository: incubator-myriad
Updated Branches:
refs/heads/master df4cbc0e9 -> 7aea259cf
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
--
diff --git
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
index 87aee56..b6806a2 100644
---
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
+++
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
@@ -18,53 +18,106 @@
*/
package org.apache.myriad.scheduler;
+import org.apache.commons.lang.StringUtils;
+import org.apache.mesos.Protos;
import static org.junit.Assert.assertTrue;
import org.apache.mesos.Protos.CommandInfo;
+import org.apache.myriad.configuration.ServiceConfiguration;
import org.apache.myriad.BaseConfigurableTest;
-import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl;
import org.junit.Test;
+import java.util.*;
+
/**
* Class to test CommandLine generation
*/
public class TestServiceCommandLine extends BaseConfigurableTest {
+
+ public static final String KEY_NM_ADDRESS =
"myriad.yarn.nodemanager.address";
+ public static final String KEY_NM_LOCALIZER_ADDRESS =
"myriad.yarn.nodemanager.localizer.address";
+ public static final String KEY_NM_WEBAPP_ADDRESS =
"myriad.yarn.nodemanager.webapp.address";
+ public static final String KEY_NM_SHUFFLE_PORT =
"myriad.mapreduce.shuffle.port";
+
+ public static final String KEY_JHS_WEBAPP_ADDRESS =
"myriad.mapreduce.jobhistory.webapp.address";
+ public static final String KEY_JHS_ADMIN_ADDRESS =
"myriad.mapreduce.jobhistory.admin.address";
+ public static final String KEY_JHS_ADDRESS =
"myriad.mapreduce.jobhistory.address";
+
+ private static final String msgFormat = System.lineSeparator() + "%s" +
System.lineSeparator() + "!="
+ + System.lineSeparator() + "%s";
+ protected static final String CMD_FORMAT = "echo \"%1$s\" && %1$s";
static String toJHSCompare =
- "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf
/usr/local/hadoop/etc/hadoop/yarn-site.xml; " +
- "export TASK_DIR=`basename $PWD`; sudo chmod +wx
/sys/fs/cgroup/cpu/mesos/$TASK_DIR;" +
- "sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\"; sudo tar
-zxpf hadoop-2.5.0.tar.gz && sudo cp" +
- " conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H
$YARN_HOME/bin/mapred historyserver";
+ " sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp yarnConfiguration
/usr/local/hadoop/etc/hadoop/yarn-site.xml && " +
+ "sudo -E -u hduser -H bin/mapred historyserver";
static String toCompare =
- "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp conf
/usr/local/hadoop/etc/hadoop/yarn-site.xml;";
+ " sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo cp yarnConfiguration
/usr/local/hadoop/etc/hadoop/yarn-site.xml && " +
+ "sudo -E -u hduser -H $YARN_HOME/bin/yarn nodemanager";
@Test
public void testJHSCommandLineGeneration() throws Exception {
-ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null);
-String executorCmd = "$YARN_HOME/bin/mapred historyserver";
-ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory",
10.0, 15.0);
+Map portsMap = new TreeMap<>();
+portsMap.put(KEY_JHS_ADDRESS, 0L);
+portsMap.put(KEY_JHS_WEBAPP_ADDRESS, 3L);
+portsMap.put(KEY_JHS_ADMIN_ADDRESS, 0L);
+
+ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory",
10.0, 15.0, portsMap);
+ServiceConfiguration serviceConfiguration =
cfg.getServiceConfiguration("jobhistory").get();
+ServiceCommandLineGenerator serviceCommandLineGenerator = new
ServiceCommandLineGenerator(cfg);
+List ports = new ArrayList<>();
+ports.add(2L);
+ports.add(1L);
+ports.add(3L);
-CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd);
-System.out.println(toJHSCompare);
-System.out.println(cInfo.getValue());
+CommandInfo cInfo =
serviceCommandLineGenerator.generateCommandLine(profile,
+serviceConfiguration,
+ports);
+String testVal = String.format(CMD_FORMAT, toJHSCompare);
+assertTrue(String.format(msgFormat, cInfo.getValue(), testVal),
+cInfo.getValue().equals(testVal));
-assertTrue(cInfo.getValue().startsWith(toCompare));
+List environmentList =
cInfo.getEnvironment().getVariablesList();
+String yarnOpts = "";
+for (Protos.Environment.Variable variable: environmentList) {
+ if
(variable.getName().equals(ServiceCommandLineGenerator.ENV_HADOOP_OPTS)){
+yarnOpts = variable.getValue();
+ }
+}
+assertTrue("Environment contains " +
[2/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
deleted file mode 100644
index ace9928..000
---
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.myriad.scheduler;
-
-/**
- * Generic interface to represent some constraints that task can impose
- * while figuring out whether to accept or reject the offer
- * We may start small and then eventually add more constraints
- */
-public interface TaskConstraints {
-
- /**
- * Required number of ports
- *
- * @return portsNumber
- */
- public int portsCount();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
deleted file mode 100644
index 0665190..000
---
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.myriad.scheduler;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Factory class to keep map of the constraints
- */
-public class TaskConstraintsManager {
-
- /**
- * Since all the additions will happen during init time, there is no need to
make this map Concurrent
- * if/when later on it will change we may need to change HashMap to
Concurrent one
- */
- private Map taskConstraintsMap = new HashMap<>();
-
- public TaskConstraints getConstraints(String taskPrefix) {
-return taskConstraintsMap.get(taskPrefix);
- }
-
- public void addTaskConstraints(final String taskPrefix, final
TaskConstraints taskConstraints) {
-if (taskConstraints != null) {
- taskConstraintsMap.put(taskPrefix, taskConstraints);
-}
- }
-
- public boolean exists(String taskPrefix) {
-return taskConstraintsMap.containsKey(taskPrefix);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
index 6b398a3..7e63e0d 100644
---
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
+++
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
@@ -9,233 +9,151 @@
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or
[3/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc
Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the process). -Refactor TackFactory classes as necessary to work with this. This adds a removes several methods to TaskUtils to get port Resources. -Created a ResourceOfferContainer moving the work of checking offers and creating the resources from them out of the ResourceOfferEventHandler and TaskFactory Classes. -Remove NMPorts and Ports classes. JIRA: [MYRIAD-136] https://issues.apache.org/jira/browse/MYRIAD-136 [MYRIAD-213] https://issues.apache.org/jira/browse/MYRIAD-213 [MYRIAD-214] https://issues.apache.org/jira/browse/MYRIAD-214 Pull Request: Closes #79 Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/7aea259c Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/7aea259c Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/7aea259c Branch: refs/heads/master Commit: 7aea259cf231655bdbd033f61584eec7715fc97f Parents: df4cbc0 Author: DarinJ <dar...@apache.org> Authored: Thu Jun 9 10:08:46 2016 -0400 Committer: darinj <dar...@apache.org> Committed: Fri Aug 12 09:27:14 2016 -0400 -- build.gradle| 4 + .../myriad/executor/MyriadExecutorDefaults.java | 23 -- .../src/main/java/org/apache/myriad/Main.java | 47 ++- .../java/org/apache/myriad/MyriadModule.java| 29 +- .../configuration/MyriadConfiguration.java | 15 +- .../configuration/NodeManagerConfiguration.java | 25 +- .../configuration/ServiceConfiguration.java | 6 +- .../scheduler/DownloadNMExecutorCLGenImpl.java | 90 -- .../scheduler/ExecutorCommandLineGenerator.java | 148 - .../scheduler/ExtendedResourceProfile.java | 27 +- .../myriad/scheduler/MyriadOperations.java | 2 +- .../myriad/scheduler/NMExecutorCLGenImpl.java | 176 --- .../NMExecutorCommandLineGenerator.java | 150 + .../org/apache/myriad/scheduler/NMPorts.java| 78 - .../apache/myriad/scheduler/NMTaskFactory.java | 74 + .../java/org/apache/myriad/scheduler/Ports.java | 26 -- .../scheduler/ServiceCommandLineGenerator.java | 86 +- .../scheduler/ServiceResourceProfile.java | 30 +- .../scheduler/ServiceTaskConstraints.java | 49 --- .../myriad/scheduler/ServiceTaskFactory.java| 76 + .../scheduler/ServiceTaskFactoryImpl.java | 253 --- .../myriad/scheduler/TaskConstraints.java | 35 --- .../scheduler/TaskConstraintsManager.java | 48 --- .../apache/myriad/scheduler/TaskFactory.java| 304 +++ .../org/apache/myriad/scheduler/TaskUtils.java | 86 +- .../handlers/ResourceOffersEventHandler.java| 175 ++- .../scheduler/resource/RangeResource.java | 218 + .../resource/ResourceOfferContainer.java| 207 + .../scheduler/resource/ScalarResource.java | 88 ++ .../scheduler/yarn/MyriadFairScheduler.java | 1 - .../org/apache/myriad/MyriadTestModule.java | 14 +- .../myriad/api/SchedulerStateResourceTest.java | 9 +- .../myriad/scheduler/MockSchedulerDriver.java | 17 ++ .../myriad/scheduler/MyriadDriverTest.java | 17 ++ .../myriad/scheduler/MyriadOperationsTest.java | 22 +- .../myriad/scheduler/NMProfileManagerTest.java | 17 ++ .../myriad/scheduler/SchedulerUtilsSpec.groovy | 6 +- .../scheduler/ServiceResourceProfileTest.java | 24 +- .../myriad/scheduler/TMSTaskFactoryImpl.java| 9 +- .../scheduler/TaskConstraintsManagerTest.java | 32 -- .../myriad/scheduler/TestNMTaskFactory.java | 72 + .../myriad/scheduler/TestRandomPorts.java | 203 - .../scheduler/TestServiceCommandLine.java | 97 -- .../scheduler/TestServiceTaskFactory.java | 77 + .../apache/myriad/scheduler/TestTaskUtils.java | 37 +-- .../fgs/YarnNodeCapacityManagerTest.java| 4 +- .../myriad/scheduler/offer/OfferBuilder.java| 117 +++ .../resource/TestResourceOfferContainer.java| 166 ++ .../org/apache/myriad/state/ClusterTest.java| 9 +- .../org/apache/myriad/state/NodeTaskTest.java | 9 +- .../apache/myriad/state/SchedulerStateTest.java | 5 +- .../state/utils/ByteBufferSupportTest.java | 5 +- ...iad-config-test-default-with-docker-info.yml | 1 + .../resources/myriad-config-test-default.yml| 32 +- 54 files changed, 1928 insertions(+), 1649 deletions(-) -- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/build.gradle -- diff --git a/build.gradle b/build.gradle i
incubator-myriad git commit: MYRIAD-218 Cleaned up TaskUtils. Removed unused methods and methods which are only used in unit tests JIRA: [MYRIAD-218] https://issues.apache.org/jira/browse/MYRIAD-218 P
Repository: incubator-myriad
Updated Branches:
refs/heads/master dcccf2e3b -> df4cbc0e9
MYRIAD-218 Cleaned up TaskUtils. Removed unused methods and methods
which are only used in unit tests
JIRA:
[MYRIAD-218] https://issues.apache.org/jira/browse/MYRIAD-218
Pull Request:
Closes #87
Author:ponnu <ponnulinga...@gmail.com>
Date: Mon Jul 25 21:03:15 2016 -0400
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/df4cbc0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/df4cbc0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/df4cbc0e
Branch: refs/heads/master
Commit: df4cbc0e97024cff4064e58fa792b3cbf3eee4f3
Parents: dcccf2e
Author: ponnu <ponnulinga...@gmail.com>
Authored: Mon Jul 25 21:03:15 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Fri Jul 29 16:33:39 2016 -0400
--
.../org/apache/myriad/scheduler/TaskUtils.java | 162 +--
.../apache/myriad/scheduler/TestTaskUtils.java | 33 +---
2 files changed, 13 insertions(+), 182 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/df4cbc0e/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
index c8e2a21..6ab6ee0 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskUtils.java
@@ -18,60 +18,28 @@
*/
package org.apache.myriad.scheduler;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerConfigurationException;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.xpath.XPath;
-import javax.xml.xpath.XPathConstants;
-import javax.xml.xpath.XPathExpression;
-import javax.xml.xpath.XPathExpressionException;
-import javax.xml.xpath.XPathFactory;
-
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
import org.apache.mesos.Protos;
-import org.apache.myriad.configuration.MyriadBadConfigurationException;
import org.apache.myriad.configuration.MyriadConfiguration;
import org.apache.myriad.configuration.MyriadContainerConfiguration;
import org.apache.myriad.configuration.MyriadDockerConfiguration;
-import org.apache.myriad.configuration.NodeManagerConfiguration;
-import org.apache.myriad.configuration.ServiceConfiguration;
import org.apache.myriad.executor.MyriadExecutorDefaults;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
/**
* utility class for working with tasks and node manager profiles
*/
public class TaskUtils {
private static final Logger LOGGER =
LoggerFactory.getLogger(TaskUtils.class);
- private static final String YARN_NODEMANAGER_RESOURCE_CPU_VCORES =
"yarn.nodemanager.resource.cpu-vcores";
- private static final String YARN_NODEMANAGER_RESOURCE_MEMORY_MB =
"yarn.nodemanager.resource.memory-mb";
private static final String CONTAINER_PATH_KEY = "containerPath";
private static final String HOST_PATH_KEY = "hostPath";
private static final String RW_MODE = "mode";
@@ -79,90 +47,12 @@ public class TaskUtils {
private static final String PARAMETER_VALUE_KEY = "value";
private MyriadConfiguration cfg;
- Random random = new Random();
@Inject
public TaskUtils(MyriadConfiguration cfg) {
this.cfg = cfg;
}
- public static String getRevisedConfig(Double cpu, Double me
[2/2] incubator-myriad git commit: MYRIAD-200
MYRIAD-200
Increase JUnit Test Coverage
JIRA:
[MYRIAD-200] https://issues.apache.org/jira/browse/MYRIAD-200
Pull Request:
Closes #78
Author:hokiegeek2 <hokiege...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/4a6e50c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/4a6e50c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/4a6e50c4
Branch: refs/heads/master
Commit: 4a6e50c41ce3098a393c96ff8f55f9d4eb78b390
Parents: 6354ce6
Author: hokiegeek2 <hokiege...@gmail.com>
Authored: Thu Jun 9 15:03:24 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Tue Jul 12 14:48:32 2016 -0400
--
.../myriad/executor/MyriadExecutorDefaults.java | 6 +-
.../apache/myriad/executor/NMTaskConfig.java| 2 +-
.../recovery/MyriadFileSystemRMStateStore.java | 5 +-
.../src/main/java/org/apache/myriad/Main.java | 30 +-
.../java/org/apache/myriad/MyriadModule.java| 22 +-
.../apache/myriad/health/HealthCheckUtils.java | 39 ++-
.../scheduler/ExtendedResourceProfile.java | 47 ++-
.../org/apache/myriad/scheduler/NMProfile.java | 47 ++-
.../myriad/scheduler/NMProfileManager.java | 25 +-
.../scheduler/ServiceResourceProfile.java | 96 +-
.../scheduler/event/OfferRescindedEvent.java| 6 +-
.../handlers/OfferRescindedEventHandler.java| 5 +-
.../event/handlers/SlaveLostEventHandler.java | 1 +
.../scheduler/fgs/OfferLifecycleManager.java| 9 +
.../java/org/apache/myriad/state/Cluster.java | 4 +
.../java/org/apache/myriad/state/NodeTask.java | 15 +-
.../myriad/state/utils/ByteBufferSupport.java | 27 +-
.../apache/myriad/webapp/MyriadWebServer.java | 21 +-
.../MyriadFileSystemRMStateStoreTest.java | 69
.../org/apache/myriad/BaseConfigurableTest.java | 35 ++
.../org/apache/myriad/MultiBindingsTest.java| 33 +-
.../org/apache/myriad/TestObjectFactory.java| 90 +
.../myriad/api/ArtifactsResourceTest.java | 51 +++
.../myriad/api/SchedulerStateResourceTest.java | 58
.../configuration/MyriadConfigurationTest.java | 95 +++---
.../myriad/health/HealthCheckUtilsTest.java | 27 ++
.../health/MesosDriverHealthCheckTest.java | 69
.../myriad/scheduler/MockSchedulerDriver.java | 124 +++
.../myriad/scheduler/MyriadDriverTest.java | 43 +++
.../myriad/scheduler/MyriadOperationsTest.java | 112 ++
.../myriad/scheduler/NMProfileManagerTest.java | 63
.../scheduler/ServiceResourceProfileTest.java | 31 ++
.../scheduler/TaskConstraintsManagerTest.java | 32 ++
.../scheduler/TestServiceCommandLine.java | 27 +-
.../apache/myriad/scheduler/TestTaskUtils.java | 41 +--
.../fgs/OfferLifeCycleManagerTest.java | 54 +++
.../myriad/scheduler/fgs/OfferUtilsTest.java| 74
.../org/apache/myriad/state/ClusterTest.java| 62
.../org/apache/myriad/state/MockDispatcher.java | 32 ++
.../org/apache/myriad/state/MockFuture.java | 44 +++
.../java/org/apache/myriad/state/MockRMApp.java | 202 +++
.../org/apache/myriad/state/MockRMContext.java | 339 +++
.../java/org/apache/myriad/state/MockState.java | 45 +++
.../org/apache/myriad/state/MockVariable.java | 35 ++
.../apache/myriad/state/MyriadStateTest.java| 21 ++
.../org/apache/myriad/state/NodeTaskTest.java | 40 +++
.../apache/myriad/state/SchedulerStateTest.java | 65
.../state/utils/ByteBufferSupportTest.java | 184 ++
.../webapp/HttpConnectorProviderTest.java | 22 ++
.../myriad/webapp/MyriadWebServerTest.java | 29 ++
.../resources/myriad-config-test-default.yml| 7 +-
51 files changed, 2455 insertions(+), 207 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java
--
diff --git
a/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java
b/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java
index bda7ff0..c7e4515 100644
---
a/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java
+++
b/myriad-commons/src/main/java/org/apache/myriad/executor/MyriadExecutorDefaults.java
@@ -69,9 +69,7 @@ public class MyriadExecutorDefaults {
public static final double DEFAULT_JVM_MAX_MEMORY_MB = 256;
/**
- * Default cpus for executor JVM.
+ * Default CPU cores for executor JVM.
*/
public static final double DEFAULT_CPUS = 0.2;
-
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4a6e50c4/myria
svn commit: r1750722 - in /incubator/myriad/site: feed.xml news/index.html
Author: darinj Date: Wed Jun 29 20:57:46 2016 New Revision: 1750722 URL: http://svn.apache.org/viewvc?rev=1750722=rev Log: Update Myriad website for 0.2.0 release, update news Modified: incubator/myriad/site/feed.xml incubator/myriad/site/news/index.html Modified: incubator/myriad/site/feed.xml URL: http://svn.apache.org/viewvc/incubator/myriad/site/feed.xml?rev=1750722=1750721=1750722=diff == --- incubator/myriad/site/feed.xml (original) +++ incubator/myriad/site/feed.xml Wed Jun 29 20:57:46 2016 @@ -24,15 +24,15 @@ under the License. http://myriad.incubator.apache.org/ http://myriad.incubator.apache.org/feed.xml; rel="self" type="application/rss+xml"/> -Wed, 29 Jun 2016 16:45:19 -0400 -Wed, 29 Jun 2016 16:45:19 -0400 +Wed, 29 Jun 2016 16:57:22 -0400 +Wed, 29 Jun 2016 16:57:22 -0400 Jekyll v3.1.6 Apache Myriad 0.2.0 (Incubating) Released pThe Apache Myriad community is excited to make its second Apache release. Thanks to a diverse community of developers, mentors, Apache members and users!/p -pa href=/blogs/2016/06/29/myriad-0-2-0-release-announcementBlog/a/p +pa href=/blogs/2016/06/29/myriad-0.2.0-release-announcement.htmlBlog/a/p pa href=/downloads/#latestDownload/a/p @@ -113,7 +113,7 @@ To try out the latest and greatest code, Apache Myriad 0.1.0 (Incubating) Released pThe Apache Myriad community is excited to make the first Apache release of the project. Thanks to a diverse community of developers, mentors, Apache members and users!/p -pa href=/blogs/2015/12/09/myriad-0-1-0-release-announcementBlog/a/p +pa href=/blogs/2015/12/09/myriad-0-1-0-release-announcement.htmlBlog/a/p pa href=/downloads/#latestDownload/a/p Modified: incubator/myriad/site/news/index.html URL: http://svn.apache.org/viewvc/incubator/myriad/site/news/index.html?rev=1750722=1750721=1750722=diff == --- incubator/myriad/site/news/index.html (original) +++ incubator/myriad/site/news/index.html Wed Jun 29 20:57:46 2016 @@ -80,7 +80,7 @@ Jun 29, 2016 The Apache Myriad community is excited to make its second Apache release. Thanks to a diverse community of developers, mentors, Apache members and users! -Blog +Blog Download @@ -102,7 +102,7 @@ Dec 9, 2015 The Apache Myriad community is excited to make the first Apache release of the project. Thanks to a diverse community of developers, mentors, Apache members and users! -Blog +Blog Download
svn commit: r1750721 - /incubator/myriad/site/news/2016/06/29/myriad-0.2.0-release-announcement.html
Author: darinj Date: Wed Jun 29 20:54:59 2016 New Revision: 1750721 URL: http://svn.apache.org/viewvc?rev=1750721=rev Log: Update Myriad website for 0.2.0 release, update news Modified: incubator/myriad/site/news/2016/06/29/myriad-0.2.0-release-announcement.html Modified: incubator/myriad/site/news/2016/06/29/myriad-0.2.0-release-announcement.html URL: http://svn.apache.org/viewvc/incubator/myriad/site/news/2016/06/29/myriad-0.2.0-release-announcement.html?rev=1750721=1750720=1750721=diff == --- incubator/myriad/site/news/2016/06/29/myriad-0.2.0-release-announcement.html (original) +++ incubator/myriad/site/news/2016/06/29/myriad-0.2.0-release-announcement.html Wed Jun 29 20:54:59 2016 @@ -70,7 +70,7 @@ The Apache Myriad community is excited to make its second Apache release. Thanks to a diverse community of developers, mentors, Apache members and users! -Blog +Blog Download
svn commit: r1750720 - in /incubator/myriad/site: blogs/2016/ blogs/2016/06/ blogs/2016/06/28/ blogs/2016/06/29/ news/2016/ news/2016/06/ news/2016/06/28/ news/2016/06/29/
Author: darinj Date: Wed Jun 29 20:50:15 2016 New Revision: 1750720 URL: http://svn.apache.org/viewvc?rev=1750720=rev Log: Update Myriad website for 0.2.0 release, with correct time Added: incubator/myriad/site/blogs/2016/ incubator/myriad/site/blogs/2016/06/ incubator/myriad/site/blogs/2016/06/28/ incubator/myriad/site/blogs/2016/06/28/myriad-0.2.0-release-announcement.html incubator/myriad/site/blogs/2016/06/29/ incubator/myriad/site/blogs/2016/06/29/myriad-0.2.0-release-announcement.html incubator/myriad/site/news/2016/ incubator/myriad/site/news/2016/06/ incubator/myriad/site/news/2016/06/28/ incubator/myriad/site/news/2016/06/28/myriad-0.2.0-release-announcement.html incubator/myriad/site/news/2016/06/29/ incubator/myriad/site/news/2016/06/29/myriad-0.2.0-release-announcement.html Added: incubator/myriad/site/blogs/2016/06/28/myriad-0.2.0-release-announcement.html URL: http://svn.apache.org/viewvc/incubator/myriad/site/blogs/2016/06/28/myriad-0.2.0-release-announcement.html?rev=1750720=auto == --- incubator/myriad/site/blogs/2016/06/28/myriad-0.2.0-release-announcement.html (added) +++ incubator/myriad/site/blogs/2016/06/28/myriad-0.2.0-release-announcement.html Wed Jun 29 20:50:15 2016 @@ -0,0 +1,219 @@ + + + + + + + + + + + Apache Myriad 0.2.0 (Incubating) Released + + + + https://fonts.googleapis.com/icon?family=Material+Icons;> + + + + + + + + + + + + + + + + +Apache Myriad 0.2.0 (Incubating) Released + + + more_vert + + + http://www.apache.org/;>Apache Homepage + http://www.apache.org/licenses/;>License + http://www.apache.org/foundation/sponsorship.html;>Sponsorship + http://www.apache.org/foundation/thanks.html;>Thanks + http://www.apache.org/security/;>Security + + + + + + + + +Home +News +Documentation +Downloads +Blogs +Community + + + + + + + + + + + + + + + Apache Myriad 0.2.0 (Incubating) Released + +On behalf of the entire Myriad Team, Iâm happy to announce the release of Apache Myriad 0.2.0 (incubating). This is the +second release of Myriad in the Apache Incubator. Among the features of the 0.2.0 release are increased reliability of +Fine Grained Scaling, support for running the Node Manager and Auxiliary Services (JobHistory Server, etc) in Docker +Containers, and better distribution management of the Hadoop configuration and binaries. This release brings Myriad much +closer to being production quality. + +In addition to new features, we also welcome our newest committer Yuliya Feldman (MapR) and a new contributor Zachary Jaffee +(Case Western Reserve University). + +Many thanks to the following individuals who contributed code/documentation to the Myriad project. This release wouldnât +have been possible without their effort. + + + Aashreya Ravi Shankar + Adam B + Brandon Gulla + Darin Johnson + Jim Klucar + Kannan Rajah + Ken Sipe + Kris Chi + Larry Edelstein + Mohit Soni + Naveen Manivannan + Paul Read + Ruth Harris + Santosh Marella + Sarjeet Singh + Shingo Omura + Suneel Marthi + Swapnil Daingade + Yuliya Feldman + Zachary Jaffee + Zhongyue Luo + + +This is the second of many Apache Myriad releases. Weâve already started merging new pull requests for the next release, +where we plan to add support for YARN Security and advanced Mesos features like reservations/volumes and oversubscription. + +If youâd like to get started we encourage you to start with our https://cwiki.apache.org/confluence/display/MYRIAD/;>documentation wiki. +For more details on the 0.2.0 release, please refer to our https://cwiki.apache.org/confluence/display/MYRIAD/Release+Notes#ReleaseNotes-myriad-0.2.0-incubating;>release notes. +The release sources can be downloaded from http://www.apache.org/dist/incubator/myriad/myriad-0.2.0-incubating/;>here. +To try out the latest and greatest code, please checkout Myriadâs https://github.com/apache/incubator-myriad.git;>git repository. + +If you have any questions, please feel free to reach us out at . + +Enjoy! + +https://github.com/darinj;>Darin Johnson (Apache Myriad committer, 0.2.0 Release Manager) | 28 Jun 2016 + + + + + + + + + + + +https://twitter.com/apachemyriad;> + + + + +https://github.com/apache/incubator-myriad;> + + + + +http://stackoverflow.com/search?q=apache+myriad;> + + + + + + https://freenode.net/;>Join us on Freenode IRC #myriad + + + + + Disclaimer:Apache Myriad is an effort undergoing incubation at The Apache Software +Foundation (ASF), sponsored by the Apache Incubator.
[incubator-myriad] Git Push Summary
Repository: incubator-myriad Updated Tags: refs/tags/myriad-0.2.0-incubating-rc3 [deleted] 700dd44ac
incubator-myriad git commit: Bulk commit of MYRIAD-198 changes
Repository: incubator-myriad
Updated Branches:
refs/heads/master 63b58698e -> cbdcbbe03
Bulk commit of MYRIAD-198 changes
We overuse Optionals in the config and then use an or method in various
factories later. In many cases having the configuration return a default when
the parameter was specified would create cleaner code.
Pull Request:
Closes #76
Author: hokiegeek2 <hokiege...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/cbdcbbe0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/cbdcbbe0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/cbdcbbe0
Branch: refs/heads/master
Commit: cbdcbbe0349f7a10a3fba98ccd120e228982a193
Parents: 63b5869
Author: hokiegeek2 <hokiege...@gmail.com>
Authored: Fri May 20 12:11:32 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Wed Jun 8 13:10:44 2016 -0400
--
.../src/main/java/org/apache/myriad/Main.java | 21 ++---
.../java/org/apache/myriad/MesosModule.java | 23 +++--
.../java/org/apache/myriad/MyriadModule.java| 25 ++---
.../configuration/MyriadConfiguration.java | 61 ++--
.../MyriadExecutorConfiguration.java| 12 ++-
.../configuration/NodeManagerConfiguration.java | 40 +---
.../configuration/ServiceConfiguration.java | 51 ++
.../myriad/scheduler/MyriadOperations.java | 40
.../myriad/scheduler/NMExecutorCLGenImpl.java | 18 ++--
.../scheduler/ServiceTaskConstraints.java | 9 +-
.../scheduler/ServiceTaskFactoryImpl.java | 39 +---
.../apache/myriad/scheduler/TaskFactory.java| 32 +--
.../org/apache/myriad/scheduler/TaskUtils.java | 63 +++--
.../handlers/ResourceOffersEventHandler.java| 2 +-
.../org/apache/myriad/state/SchedulerState.java | 9 +-
.../main/resources/myriad-config-default.yml| 4 +-
.../configuration/MyriadConfigurationTest.java | 97 +---
.../apache/myriad/scheduler/TestTaskUtils.java | 5 +-
.../fgs/YarnNodeCapacityManagerSpec.groovy | 6 +-
...iad-config-test-default-with-docker-info.yml | 2 +-
...-config-test-default-with-framework-role.yml | 2 +-
.../resources/myriad-config-test-default.yml| 14 +--
22 files changed, 363 insertions(+), 212 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/cbdcbbe0/myriad-scheduler/src/main/java/org/apache/myriad/Main.java
--
diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/Main.java
b/myriad-scheduler/src/main/java/org/apache/myriad/Main.java
index 0f0703e..e825256 100644
--- a/myriad-scheduler/src/main/java/org/apache/myriad/Main.java
+++ b/myriad-scheduler/src/main/java/org/apache/myriad/Main.java
@@ -240,16 +240,15 @@ public class Main {
TaskConstraintsManager taskConstraintsManager =
injector.getInstance(TaskConstraintsManager.class);
Map<String, ServiceConfiguration> servicesConfigs =
injector.getInstance(MyriadConfiguration.class).getServiceConfigurations();
-if (servicesConfigs != null) {
- for (Map.Entry<String, ServiceConfiguration> entry :
servicesConfigs.entrySet()) {
-final String taskPrefix = entry.getKey();
-ServiceConfiguration config = entry.getValue();
-final Double cpu =
config.getCpus().or(ServiceConfiguration.DEFAULT_CPU);
-final Double mem =
config.getJvmMaxMemoryMB().or(ServiceConfiguration.DEFAULT_MEMORY);
-
-profileManager.add(new ServiceResourceProfile(taskPrefix, cpu, mem));
-taskConstraintsManager.addTaskConstraints(taskPrefix, new
ServiceTaskConstraints(cfg, taskPrefix));
- }
+
+for (Map.Entry<String, ServiceConfiguration> entry :
servicesConfigs.entrySet()) {
+ final String taskPrefix = entry.getKey();
+ ServiceConfiguration config = entry.getValue();
+ final Double cpu = config.getCpus();
+ final Double mem = config.getJvmMaxMemoryMB();
+
+ profileManager.add(new ServiceResourceProfile(taskPrefix, cpu, mem));
+ taskConstraintsManager.addTaskConstraints(taskPrefix, new
ServiceTaskConstraints(cfg, taskPrefix));
}
}
@@ -262,7 +261,7 @@ public class Main {
}
private void initRebalancerService(MyriadConfiguration cfg, Injector
injector) {
-if (cfg.isRebalancer()) {
+if (cfg.isRebalancerEnabled()) {
LOGGER.info("Initializing Rebalancer");
rebalancerService = Executors.newScheduledThreadPool(1);
final int initialDelay = 100;
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/cbdcbbe0/myriad-scheduler/src/main/java/org/apache/myriad/MesosModule.java
-
[incubator-myriad] Git Push Summary
Repository: incubator-myriad Updated Tags: refs/tags/myriad-0.2.0-incubating-rc4 [created] 63b58698e
incubator-myriad git commit: Update cgroups documentation - was missing a necessary yarn-site configuration. Fix minor typo hierachy->hierarchy. Pull Request: Closes #65 Author: Darin Johnson <dar...@
Repository: incubator-myriad
Updated Branches:
refs/heads/0.2.x 700dd44ac -> 63b58698e
Update cgroups documentation - was missing a necessary yarn-site configuration.
Fix minor typo hierachy->hierarchy.
Pull Request:
Closes #65
Author: Darin Johnson <dar...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/63b58698
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/63b58698
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/63b58698
Branch: refs/heads/0.2.x
Commit: 63b58698e668729f9f5fb65d5ebc426bf6a8680e
Parents: 700dd44
Author: darinj <dar...@apache.org>
Authored: Sat May 21 14:37:28 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Sun May 22 22:44:17 2016 -0400
--
docs/cgroups.md | 4
.../java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java| 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/63b58698/docs/cgroups.md
--
diff --git a/docs/cgroups.md b/docs/cgroups.md
index d28eeab..b90c781 100644
--- a/docs/cgroups.md
+++ b/docs/cgroups.md
@@ -82,6 +82,10 @@ Modify the `$YARN_HOME/etc/hadoop/yarn-site.xml` file by
adding the following co
yarn.nodemanager.linux-container-executor.path
${yarn.home}/bin/container-executor
+
+yarn.nodemanager.linux-container-executor.cgroups.hierarchy
+${yarn.nodemanager.linux-container-executor.cgroups.hierarchy}
+
svn commit: r13758 - in /dev/incubator/myriad/myriad-0.2.0-incubating-rc3: ./ myriad-0.2.0-incubating-rc3.tar.gz myriad-0.2.0-incubating-rc3.tar.gz.asc myriad-0.2.0-incubating-rc3.tar.gz.md5 myriad-0.
Author: darinj Date: Tue May 24 04:10:03 2016 New Revision: 13758 Log: Adding myriad-0.2.0-incubating-rc3. Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc3/ dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz (with props) dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.asc dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.md5 dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.sha512 Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz == Binary file - no diff available. Propchange: dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz -- svn:mime-type = application/octet-stream Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.asc == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.asc (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.asc Tue May 24 04:10:03 2016 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v2 + +iQIcBAABCAAGBQJXQ9QVAAoJEApMeUgqrp4/M54QAKy83W2TR8/OOk8lnCeGjFDR +sZ5bEnR/dLElth1CaiHomfcdnUakqgE9Wuvi45ObHR/6LwZzY1wENRJOWbCOhHmY +UV/SkVqZzes3Ljyr0yRBRwOYVs7uiweXOm8FH8K0COBTXat1pKfZCTdwe+1iwUXn +PdnbtZBgLYuij4TD91T4diydDgErg+VDwxPJwbTBm6BUwERhil29HRHqSpdNISKN +UDdOTIv4N2zS1PSH5Pacn7SBAeeNHt7hZemInLJwrww/a32ZvRgTDmc7x72conul +gmmnGeabJtOq9LBuj1vJPG0gkikKCq8F/RAlZyUlAp03hyLz2nxRMbFHnsvaoel9 +nrcgBDLsiV/44/93Z3yPmtko2QsAGO/dPeycLv38kYAh+GT1uLSTh9Jb4d9ybFjR +S2iRxpEqL+XXIam5AF2R+CJ0VkHjfzMlPVewMFZY2NPcUTvvPpjUp/mYeKaF2o4i +FsXhm2R+6Nu+RfFgnQYbI0YIuoBBbI3xkHVgKpcsB5e18mDJsTOXODlOjXwj0XRY +lzROivG58kGY/0uG3jvwjOStCCYdHzmyYBYG0zpyE19vlH9J2apSH89Ko6efz4rJ +u5LmQbU73Vu3uZ8waHwx8N/OHXyTE2UFiL9IqeZ5C5ukgzl3VsIGXEmn4+C1kprN +BTcA1bMlf+cArOWNjvg5 +=Ump6 +-END PGP SIGNATURE- Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.md5 == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.md5 (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.md5 Tue May 24 04:10:03 2016 @@ -0,0 +1,2 @@ +myriad-0.2.0-incubating-rc3.tar.gz: 76 73 7A 88 AA FE C6 F5 A9 2F F3 64 63 7B +FD 3C Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.sha512 == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.sha512 (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc3/myriad-0.2.0-incubating-rc3.tar.gz.sha512 Tue May 24 04:10:03 2016 @@ -0,0 +1,4 @@ +myriad-0.2.0-incubating-rc3.tar.gz: FB7884BD 424DDFF0 834B3EE2 8B453513 1A0792C9 +4725B5F1 C439D1D1 D23ACD45 64BBFDB9 71DBA3F9 +50E3DD84 B47E52B0 8913C650 7D042C65 87A4A99B +603FDE76
[incubator-myriad] Git Push Summary
Repository: incubator-myriad Updated Tags: refs/tags/myriad-0.2.0-incubating-rc3 [created] 700dd44ac
incubator-myriad git commit: Update cgroups documentation - was missing a necessary yarn-site configuration. Fix minor typo hierachy->hierarchy. Pull Request: Closes #65 Author: Darin Johnson <dar...@
Repository: incubator-myriad
Updated Branches:
refs/heads/master 700dd44ac -> 63b58698e
Update cgroups documentation - was missing a necessary yarn-site configuration.
Fix minor typo hierachy->hierarchy.
Pull Request:
Closes #65
Author: Darin Johnson <dar...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/63b58698
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/63b58698
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/63b58698
Branch: refs/heads/master
Commit: 63b58698e668729f9f5fb65d5ebc426bf6a8680e
Parents: 700dd44
Author: darinj <dar...@apache.org>
Authored: Sat May 21 14:37:28 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Sun May 22 22:44:17 2016 -0400
--
docs/cgroups.md | 4
.../java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java| 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/63b58698/docs/cgroups.md
--
diff --git a/docs/cgroups.md b/docs/cgroups.md
index d28eeab..b90c781 100644
--- a/docs/cgroups.md
+++ b/docs/cgroups.md
@@ -82,6 +82,10 @@ Modify the `$YARN_HOME/etc/hadoop/yarn-site.xml` file by
adding the following co
yarn.nodemanager.linux-container-executor.path
${yarn.home}/bin/container-executor
+
+yarn.nodemanager.linux-container-executor.cgroups.hierarchy
+${yarn.nodemanager.linux-container-executor.cgroups.hierarchy}
+
svn commit: r13669 - in /dev/incubator/myriad/myriad-0.2.0-incubating-rc2: ./ myriad-0.2.0-incubating-rc2.tar.gz myriad-0.2.0-incubating-rc2.tar.gz.asc myriad-0.2.0-incubating-rc2.tar.gz.md5 myriad-0.
Author: darinj Date: Tue May 17 21:21:37 2016 New Revision: 13669 Log: Adding myriad-0.2.0-incubating-rc2. Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc2/ dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz (with props) dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.asc dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.md5 dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.sha512 Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz == Binary file - no diff available. Propchange: dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz -- svn:mime-type = application/octet-stream Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.asc == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.asc (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.asc Tue May 17 21:21:37 2016 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v2 + +iQIcBAABCAAGBQJXO4tYAAoJEApMeUgqrp4/UD0P/28FLKYrhj/xHuT+CQAxwrQp +kEor5y6Bg+Wuq71I3sJfuTt9WrEFX0JREsG+CvKVKYZH0uCtfION4chYFIqrDs4G +aanJzD70QQBcIIjb61A1NcaN997g10r+FyiZ0K+Iw/mEfHX+gNnIrHGtKeKwHZII +CokpkvZZY5LsDYCszSnyldlxv6mgEGwpUX3aBJP6Z3CYqNJ+ZptnFugiDqZJiuwx +6KiEFUVUBJeWDWXGtJ00qhcrGT7PLuJw7GCCI4qewLWu5TuhwTNUG0DkSm9B/uwV +PgRX4QcjLhjp9xCIsRT+vucJt6Va4IlcZiaxD0P3Il/fgJ0RapWEEyBn+RQH6T8B +vbk77f8HMxErwWtzlAV30/D0729qqqvZMHJVIlJsoPYtmj+bKTCDgCpGwmfa20wH +Rwl54RMQJb3rD4lz/cOM8EiYybHGmAFH181MMfGNoBRDOe2E2JZROARFiP5Wbsvp +FcDxNnLfkN2nlWtbSMP0GiqXoJDBa+h7B9EPbj9kVEvtp+X+duNX1334GoTGpY9f +TWGEb90kHzbcYw3pakME8ZDtHH+cFjavdVXFOnL84uiG8twlS3HcM+0ULFL5BAjr +E2RUer+yzNLt+vLwk+BIzPhN8Pu9aJ/uIbnghxiUR0jVD33olPVk01QM4surq2bt +JKPF8cYWFHnZguS5k9GZ +=N2sD +-END PGP SIGNATURE- Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.md5 == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.md5 (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.md5 Tue May 17 21:21:37 2016 @@ -0,0 +1,2 @@ +myriad-0.2.0-incubating-rc2.tar.gz: 10 E2 6F D1 D0 2B EB F5 CC DC 1F 4F 02 53 +BD 57 Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.sha512 == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.sha512 (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc2/myriad-0.2.0-incubating-rc2.tar.gz.sha512 Tue May 17 21:21:37 2016 @@ -0,0 +1,4 @@ +myriad-0.2.0-incubating-rc2.tar.gz: B7E94CAC 67F647A1 FFA24DE8 BB7FF22C 82B86E6D +9E6E1A10 1A72117F BDA6A596 E80CBA5E CF3144E7 +E5DE8FC3 4D9CA287 51538867 73732E3A 677CD876 +A38F681A
[incubator-myriad] Git Push Summary
Repository: incubator-myriad Updated Tags: refs/tags/myriad-0.2.0-incubating-rc2 [created] 700dd44ac
[2/2] incubator-myriad git commit: Update NOTICE file to 2016
Update NOTICE file to 2016 Author: Darin Johnson <dar...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/700dd44a Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/700dd44a Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/700dd44a Branch: refs/heads/0.2.x Commit: 700dd44ac1b30ba2359b3e2d793f1387044c1672 Parents: 5531103 Author: darinj <dar...@apache.org> Authored: Tue May 17 17:03:01 2016 -0400 Committer: darinj <dar...@apache.org> Committed: Tue May 17 17:03:01 2016 -0400 -- NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/700dd44a/NOTICE -- diff --git a/NOTICE b/NOTICE index 004b1e8..55dbd59 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Myriad -Copyright 2015, The Apache Software Foundation +Copyright 2016, The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
[1/2] incubator-myriad git commit: Couple of minor bug fixes for 0.2.0 RC: 1. UI frameworkShutdown is broken because of PR#58. 2. Additional 'sudo' from cgroup cmdline. 3. Adding profile info to flexd
Repository: incubator-myriad
Updated Branches:
refs/heads/0.2.x 6c11c7338 -> 700dd44ac
Couple of minor bug fixes for 0.2.0 RC:
1. UI frameworkShutdown is broken because of PR#58.
2. Additional 'sudo' from cgroup cmdline.
3. Adding profile info to flexdown UI (keep same with flexup).
Pull Request:
Closes #74
Author: sarjeet2013
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/55311033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/55311033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/55311033
Branch: refs/heads/0.2.x
Commit: 553110336d05111e0dc8be3952881abdf81ba8b8
Parents: 6c11c73
Author: Sarjeet Singh <sarjeetsi...@maprtech.com>
Authored: Mon May 16 01:41:16 2016 -0700
Committer: darinj <dar...@apache.org>
Committed: Tue May 17 17:00:07 2016 -0400
--
.../main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java | 1 -
.../src/main/resources/webapp/js/components/FlexDownComponent.js | 2 +-
.../resources/webapp/js/components/ShutdownFrameworkComponent.js | 2 +-
3 files changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/55311033/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
index 2700488..8fca9e2 100644
---
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
+++
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
@@ -110,7 +110,6 @@ public class NMExecutorCLGenImpl implements
ExecutorCommandLineGenerator {
protected void appendCgroupsCmds(StringBuilder cmdLine) {
if (cfg.getFrameworkSuperUser().isPresent()) {
cmdLine.append(" export TASK_DIR=`basename $PWD`&&");
- appendSudo(cmdLine);
//The container executor script expects mount-path to exist and owned by
the yarn user
//See:
https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html
//If YARN ever moves to cgroup/mem it will be necessary to add a mem
version.
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/55311033/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
--
diff --git
a/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
b/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
index 0a44195..1ea5218 100644
---
a/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
+++
b/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
@@ -114,7 +114,7 @@ var FlexDownComponent = React.createClass({
}
for( var ii = 0; ii < keys.length; ii++) {
var key = keys[ii];
- var txt = key;
+ var txt = key + '\t' + JSON.stringify(this.props.profiles[key]);
options.push( {txt} );
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/55311033/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
--
diff --git
a/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
b/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
index a26f070..689b4e4 100644
---
a/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
+++
b/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
@@ -81,7 +81,7 @@ var ShutdownFrameworkComponent = React.createClass({
},
onRequestShutdown: function() {
console.log( "shutting down Myriad ");
- request.get('/api/framework/shutdown/framework')
+ request.post('/api/framework/shutdown/framework')
.set('Content-Type', 'application/json')
.end(function(err, res){
console.log("Result from
/api/framework/shutdown/framework");
[1/2] incubator-myriad git commit: Couple of minor bug fixes for 0.2.0 RC: 1. UI frameworkShutdown is broken because of PR#58. 2. Additional 'sudo' from cgroup cmdline. 3. Adding profile info to flexd
Repository: incubator-myriad
Updated Branches:
refs/heads/master 6c11c7338 -> 700dd44ac
Couple of minor bug fixes for 0.2.0 RC:
1. UI frameworkShutdown is broken because of PR#58.
2. Additional 'sudo' from cgroup cmdline.
3. Adding profile info to flexdown UI (keep same with flexup).
Pull Request:
Closes #74
Author: sarjeet2013
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/55311033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/55311033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/55311033
Branch: refs/heads/master
Commit: 553110336d05111e0dc8be3952881abdf81ba8b8
Parents: 6c11c73
Author: Sarjeet Singh <sarjeetsi...@maprtech.com>
Authored: Mon May 16 01:41:16 2016 -0700
Committer: darinj <dar...@apache.org>
Committed: Tue May 17 17:00:07 2016 -0400
--
.../main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java | 1 -
.../src/main/resources/webapp/js/components/FlexDownComponent.js | 2 +-
.../resources/webapp/js/components/ShutdownFrameworkComponent.js | 2 +-
3 files changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/55311033/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
index 2700488..8fca9e2 100644
---
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
+++
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/NMExecutorCLGenImpl.java
@@ -110,7 +110,6 @@ public class NMExecutorCLGenImpl implements
ExecutorCommandLineGenerator {
protected void appendCgroupsCmds(StringBuilder cmdLine) {
if (cfg.getFrameworkSuperUser().isPresent()) {
cmdLine.append(" export TASK_DIR=`basename $PWD`&&");
- appendSudo(cmdLine);
//The container executor script expects mount-path to exist and owned by
the yarn user
//See:
https://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/NodeManagerCgroups.html
//If YARN ever moves to cgroup/mem it will be necessary to add a mem
version.
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/55311033/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
--
diff --git
a/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
b/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
index 0a44195..1ea5218 100644
---
a/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
+++
b/myriad-scheduler/src/main/resources/webapp/js/components/FlexDownComponent.js
@@ -114,7 +114,7 @@ var FlexDownComponent = React.createClass({
}
for( var ii = 0; ii < keys.length; ii++) {
var key = keys[ii];
- var txt = key;
+ var txt = key + '\t' + JSON.stringify(this.props.profiles[key]);
options.push( {txt} );
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/55311033/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
--
diff --git
a/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
b/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
index a26f070..689b4e4 100644
---
a/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
+++
b/myriad-scheduler/src/main/resources/webapp/js/components/ShutdownFrameworkComponent.js
@@ -81,7 +81,7 @@ var ShutdownFrameworkComponent = React.createClass({
},
onRequestShutdown: function() {
console.log( "shutting down Myriad ");
- request.get('/api/framework/shutdown/framework')
+ request.post('/api/framework/shutdown/framework')
.set('Content-Type', 'application/json')
.end(function(err, res){
console.log("Result from
/api/framework/shutdown/framework");
svn commit: r13620 - in /dev/incubator/myriad/myriad-0.2.0-incubating-rc1: ./ myriad-0.2.0-incubating-rc1.tar.gz myriad-0.2.0-incubating-rc1.tar.gz.asc myriad-0.2.0-incubating-rc1.tar.gz.md5 myriad-0.
Author: darinj Date: Fri May 13 03:00:32 2016 New Revision: 13620 Log: Adding myriad-0.2.0-incubating-rc1. Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc1/ dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz (with props) dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.asc dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.md5 dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.sha512 Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz == Binary file - no diff available. Propchange: dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz -- svn:mime-type = application/octet-stream Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.asc == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.asc (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.asc Fri May 13 03:00:32 2016 @@ -0,0 +1,17 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v2 + +iQIcBAABCAAGBQJXNUNJAAoJEApMeUgqrp4/CoEP/AqrQJWGbjXxGtTPhwq9herq +7I7hxTfwF8O28iQRdGYiKOSVt22RJ1m6WfGsJodl8ySDQhfg1e7Nb/cQcN2IU2Rj +erp6i4DUUMfV/AQFiW8w1wKa96lCYrUJpZ0m08Rn7OId+ky1nR2wy/eDyaHcoVzP +atTvRHkdz2vxxg+8qq2vpHdLQTS7KnrJYJ1T5d5EJFyXUHoyHLfcw0VYtFnVOcS7 +gvSoy+SVFTR3bfNimgRAcC0RZFHNNwD7H3lSxdjrxpf8OLZNwOJyT/sNbEHGPS/t +E1ymeLtiOjrH7KXQRiTX+ZQleUH95Mkq9gXUOx7hAZ9loiJMiDF27Fzs2QYC4M4l +OSFQWjp8d8wKptiV3Hl2GZIjevdmFGfGrh6z4J0hWimFk/+wLGxt82ZzQDsuEi28 +Bqtl+HZP5brY49DsV+NKelUMNXhVVsIB4FnZE1ifgSTcNNEFcfPOsyN+G9t34arz +Q36cCEZ9t2jx5sIPlDKqwR5xtstdYyjHCrVcCz32Iv00DevPQyJnobkBkKqp+seW +Bhzrfy41SKnjhxXyDVjZAigmWHtenO2garfttM7apJnHweYSTL82hCaRmJB9b8Qn +Oj7+VcV4tRf0cqjl7nks5zP4riGhhaZMpQ8LHGK0pb9xH86I4kvpKXSLZH5uscdj +4kBI0YdJi/4fUnJs7ARx +=TgaU +-END PGP SIGNATURE- Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.md5 == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.md5 (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.md5 Fri May 13 03:00:32 2016 @@ -0,0 +1,2 @@ +myriad-0.2.0-incubating-rc1.tar.gz: DE DA A0 6A 05 9E 6E ED 75 0B 27 E0 EB 2A +22 DD Added: dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.sha512 == --- dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.sha512 (added) +++ dev/incubator/myriad/myriad-0.2.0-incubating-rc1/myriad-0.2.0-incubating-rc1.tar.gz.sha512 Fri May 13 03:00:32 2016 @@ -0,0 +1,4 @@ +myriad-0.2.0-incubating-rc1.tar.gz: 335D54AA 9E60553D C7565638 37C4E803 FA320FC0 +ADB0BC00 A3C6B423 26297576 B3E05E39 1BB5B209 +0BEE9393 47974454 2EA84C14 15E27076 AC9C32C0 +62DDB0CE
incubator-myriad git commit: docker/build.gradle was missing header file Pull Request: Closes #73 Author: DarinJ <dar...@apache.org>
Repository: incubator-myriad
Updated Branches:
refs/heads/0.2.x 048727634 -> 6c11c7338
docker/build.gradle was missing header file
Pull Request:
Closes #73
Author: DarinJ <dar...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/6c11c733
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/6c11c733
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/6c11c733
Branch: refs/heads/0.2.x
Commit: 6c11c73387c67b6b06db5fcf8d3923065f89d6e1
Parents: 0487276
Author: darinj <darinj.w...@gmail.com>
Authored: Thu May 12 21:56:25 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Thu May 12 22:00:57 2016 -0400
--
docker/build.gradle | 18 ++
1 file changed, 18 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/6c11c733/docker/build.gradle
--
diff --git a/docker/build.gradle b/docker/build.gradle
index 38f61e6..5c5e23d 100644
--- a/docker/build.gradle
+++ b/docker/build.gradle
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
task wrapper(type: Wrapper) {
gradleVersion = '2.9'
}
[incubator-myriad] Git Push Summary
Repository: incubator-myriad Updated Branches: refs/heads/0.2.0 [created] 048727634
incubator-myriad git commit: Release prep: -Version bump 0.2.0 -Update Docs -Update Sample Configuration -Minor changes to docker scripts Pull Request: Closes: #72
Repository: incubator-myriad
Updated Branches:
refs/heads/master 12a679c2b -> 048727634
Release prep:
-Version bump 0.2.0
-Update Docs
-Update Sample Configuration
-Minor changes to docker scripts
Pull Request:
Closes: #72
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/04872763
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/04872763
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/04872763
Branch: refs/heads/master
Commit: 04872763495a2c21ae2649b4c8134933cb59d932
Parents: 12a679c
Author: darinj <darinj.w...@gmail.com>
Authored: Thu May 12 14:56:49 2016 -0400
Committer: darinj <dar...@apache.org>
Committed: Thu May 12 16:50:56 2016 -0400
--
README.md | 3 +-
build.gradle| 2 +-
docker/Dockerfile | 11 +++-
docker/README.md| 60 +---
docker/myriad-bin/configure-yarn.sh | 32 +--
docker/myriad-bin/install-yarn.sh | 4 +-
.../myriad-config-default.yml.template | 43 +++---
docker/myriad-etc/yarn-site.xml.template| 49 +---
docs/cgroups.md | 2 +-
docs/myriad-docker.md | 0
docs/myriad-multi-tenancy.md| 10
.../myriad-remote-distribution-configuration.md | 24 ++--
.../main/resources/myriad-config-default.yml| 6 +-
13 files changed, 115 insertions(+), 131 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/04872763/README.md
--
diff --git a/README.md b/README.md
index 02925ae..bffcf79 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,8 @@ Please keep checking this section for updates.
- [x] Framework checkpointing
- [x] Launch Job History Server
- [x] Constraints based Node Manager placement
-- [ ] Support multi-tenancy for Node Managers
+- [x] Docker support
+- [x] Support multi-tenancy for Node Managers
- [ ] Configuration store for storing rules and policies for clusters managed
by Myriad
## Mailing Lists
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/04872763/build.gradle
--
diff --git a/build.gradle b/build.gradle
index ac416ec..55e8d85 100644
--- a/build.gradle
+++ b/build.gradle
@@ -64,7 +64,7 @@ subprojects {
mainClassName = "org.apache.myriad.Main"
group = "org.apache.myriad"
-version = "0.1.0"
+version = "0.2.0"
ext {
mesosVer = "0.28.1"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/04872763/docker/Dockerfile
--
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4f344a6..8626e13 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -22,12 +22,17 @@
FROM debian
MAINTAINER Apache Myriad d...@myriad.incubator.apache.org
-ENV HADOOP_USER="yarn"
+ENV YARN_USER="yarn"
+ENV YARN_UID="108"
+ENV HADOOP_GROUP="hadoop"
+ENV HADOOP_GID="112"
+ENV YARN_GROUP="yarn"
+ENV YARN_GID="113"
ENV HADOOP_VER="2.7.0"
ENV HADOOP_HOME=/usr/local/hadoop
ENV JAVA_HOME=/usr
ENV MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so
-
+ENV MESOS_VER=0.28.1
# Setup mesosphere repositories
RUN apt-get -y update
@@ -37,7 +42,7 @@ RUN DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
CODENAME=$(lsb_releas
RUN apt-get -y update
# Install Mesos
-RUN apt-get install -y mesos curl tar
+RUN apt-get install -y mesos=`apt-cache madison mesos | grep $MESOS_VER | head
-1 | awk {'print $3'}` curl tar
# Run local YARN install
ADD myriad-bin/install-yarn.sh /install-yarn.sh
RUN sh /install-yarn.sh
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/04872763/docker/README.md
--
diff --git a/docker/README.md b/docker/README.md
index 1650691..f8fdb62 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,41 +1,63 @@
# Using Myriad with Docker #
Docker is the easiest way to from 0 to Yarn on Mesos within minutes.
-## Building the Resource Manager Docker
+# Building the Resource Manager Docker
-#Configuration Guide#
+YARN runs as a user and group, and expects consistent uid and gids in HDFS and
accross the cluster. Hence it is necessary
+to edit docker/Dockerfile and modify the following:
+```
+ENV YARN_USER="yarn"
+ENV YARN_UID="107"
+ENV HADOOP_GROUP="hadoop"
+ENV
incubator-myriad git commit: [Myriad 171] - updateFile had parameter change between Hadoop 2.7.0->2.7.1 [Myriad 190] - Tests fail on Hadoop 2.6.2+
Repository: incubator-myriad
Updated Branches:
refs/heads/master e1db4d8b8 -> 1daad6324
[Myriad 171] - updateFile had parameter change between Hadoop 2.7.0->2.7.1
[Myriad 190] - Tests fail on Hadoop 2.6.2+
Corrects behavior for hadoop 2.7.1+ and hadoop 2.6.2+.
Added support to specify hadoopVer from command line as follows:
```
./gradlew -PhadoopVer=X.Y.Z build
```
where `X.Y.Z` is the hadoop version.
JIRA:
[Myriad-171] https://issues.apache.org/jira/browse/MYRIAD-171
[Myriad-190] https://issues.apache.org/jira/browse/MYRIAD-190
Pull Request:
Closes #63
Author:
darinj dar...@apache.org
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/1daad632
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/1daad632
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/1daad632
Branch: refs/heads/master
Commit: 1daad6324ce8b74df0f144600785c1db3d7348d9
Parents: e1db4d8
Author: darinj <darinj.w...@gmail.com>
Authored: Mon Mar 14 02:04:26 2016 -0400
Committer: darinj <darinj.w...@gmail.com>
Committed: Sat Apr 2 06:59:57 2016 -0400
--
build.gradle| 5 ++-
gradle.properties | 1 -
.../recovery/MyriadFileSystemRMStateStore.java | 37 ++--
.../myriad/scheduler/fgs/FGSTestBaseSpec.groovy | 2 ++
4 files changed, 41 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/build.gradle
--
diff --git a/build.gradle b/build.gradle
index 918f84c..067a52d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -68,7 +68,10 @@ subprojects {
ext {
mesosVer = "0.24.1"
-hadoopVer = "2.7.0"
+//Allows passing -PhadoopVer=2.6.0 from command line
+if(!project.hasProperty('hadoopVer')) {
+hadoopVer = "2.7.0"
+}
metricsVer = "3.1.0"
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/gradle.properties
--
diff --git a/gradle.properties b/gradle.properties
index 9667034..bca1f77 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -18,6 +18,5 @@
org.gradle.jvmargs=-Xmx1024m -XX:MaxPermSize=256m
-XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
org.gradle.parallel=true
org.gradle.daemon=true
-
# faster builds: gradle build -x findBugsM
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
index dda9b28..923e29d 100644
---
a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
+++
b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
@@ -20,6 +20,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -43,11 +46,28 @@ public class MyriadFileSystemRMStateStore extends
FileSystemRMStateStore impleme
private Path myriadPathRoot = null;
private byte[] myriadStateBytes = null;
+ private Method updateFileMethod = null; //This is a cache Method so we do
fewer reflection calls
+
@Override
public synchronized void initInternal(Configuration conf) throws Exception {
super.initInternal(conf);
Path rootPath = new Path(fsWorkingPath, ROOT_NAME);
myriadPathRoot = new Path(rootPath, RM_MYRIAD_ROOT);
+updateFileMethod = getUpdateFileMethod();
+if (updateFileMethod == null) {
+ //something is broken
+ throw new RuntimeException("Could not find valid updateFile Method");
+}
+ }
+
+ private Method getUpdateFileMethod() {
+Method[] methods = super.getClass().getSuperclass().getDeclaredMethods();
+for (Method m : methods) {
+ if (m.getName().equals("updateFile")) {
+return m;
+ }
+}
+return null;
}
@Override
@@ -59,7 +79,6 @@ public class MyriadFileSystemRMStateStore extends
FileSystemRMSta
incubator-myriad git commit: Docker Gradle Integration
Repository: incubator-myriad
Updated Branches:
refs/heads/master 4bce035ec -> e1db4d8b8
Docker Gradle Integration
Docker RM now is built via Gradle.
Pull Request:
Closes #55
Author:
Brandon Gulla bran...@brandongulla.com
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/e1db4d8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/e1db4d8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/e1db4d8b
Branch: refs/heads/master
Commit: e1db4d8b8cdb4fd1994be9097ab6e439e8f28cc0
Parents: 4bce035
Author: Brandon Gulla <bran...@brandongulla.com>
Authored: Wed Nov 18 12:51:49 2015 -0500
Committer: darinj <darinj.w...@gmail.com>
Committed: Wed Mar 30 23:51:13 2016 -0400
--
build.gradle | 15 ++
docker/Dockerfile | 2 +-
docker/README.md | 20 +
docker/ResourceManager.dockerfile | 55 --
docker/build.gradle | 50 +++
docker/myriad-bin/create-user.sh | 2 +-
settings.gradle | 1 +
7 files changed, 75 insertions(+), 70 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/build.gradle
--
diff --git a/build.gradle b/build.gradle
index 7e57403..918f84c 100644
--- a/build.gradle
+++ b/build.gradle
@@ -21,6 +21,21 @@ allprojects {
apply plugin: 'eclipse'
}
+buildscript {
+repositories {
+jcenter()
+mavenCentral()
+}
+
+dependencies {
+classpath 'com.bmuschko:gradle-docker-plugin:2.6.1'
+}
+}
+
+repositories {
+mavenCentral()
+}
+
idea {
project {
languageLevel = '1.7'
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/Dockerfile
--
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 18d4084..52ad7f3 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -43,7 +43,7 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh
RUN sh /install-yarn.sh
# Copy over myriad libraries
-ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
+ADD ./libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
# Initialize hadoop confs with env vars
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/README.md
--
diff --git a/docker/README.md b/docker/README.md
index 8f00d64..7ce77ac 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,12 +1,14 @@
# Using Myriad with Docker #
+Docker is the easiest way to from 0 to Yarn on Mesos within minutes.
## Building the Resource Manager Docker
-
-`./build-myriad.sh` will run the gradle scripts from the root myriad folder
and compile all necessary libraries.
+Run the following command:
+```./gradlew buildRMDocker```
+This will build the ResourceManager from src and save the image as
*mesos/myriad*.
#Configuration Guide#
-In order for the ResourceManager to operate correctly, you will need to
provide 2 configuration files:
+In order for the ResourceManager to operate correctly, you will need to
provide 2 configuration files. These files will need to mounted from a
directory into */myriad-conf* within the docker container.
*
[myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml)
* modified
[yarn-site.xml](https://github.com/mesos/myriad/blob/phase1/docs/myriad-dev.md)
@@ -17,19 +19,11 @@ In order for the ResourceManager to operate correctly, you
will need to provide
```bash
docker run --net=host --name='myriad-resourcemanager' -t \
-v /path/to/configs:/myriad-conf \
- -e HADOOP_NAMENODE="10.100.3.237:9000" \
- mesos/myriad-resourcemanager
+ mesos/myriad
```
#Environment Variables#
-
-* *HADOOP_NAMENODE* : *Required*
-* *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop
distribution used by Myriad.
-
-
-If you already had a working Vagrant instance, you will need to run `vagrant
reload` in order to allow zookeeper and hdfs port-forwarding.
-
-Still having problems connecting to Zookeeper? Be sure that your zkServer and
MesosMaster values are correct in the
[myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml)
file.
+* *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop
distribution used by Myriad. This will download the *.tar.gz file to be used as
the hadoop distribution of choice for Myriad.
---
http://git-wip-us.apache.org/repos/asf/incub
incubator-myriad git commit: [Myriad 188] - NodeManager switch to UNHEALTHY causes NPE on ResourceManager.
Repository: incubator-myriad
Updated Branches:
refs/heads/master 79ba4a5f0 -> 4bce035ec
[Myriad 188] - NodeManager switch to UNHEALTHY causes NPE on ResourceManager.
JIRA:
[Myriad-188] https://issues.apache.org/jira/browse/MYRIAD-188
[Myriad-156] https://issues.apache.org/jira/browse/MYRIAD-156
Pull Request:
Closes #62
Author:
darinj dar...@apache.org
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/4bce035e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/4bce035e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/4bce035e
Branch: refs/heads/master
Commit: 4bce035ec8feff66f4804cc121c3051f5ecff4fa
Parents: 79ba4a5
Author: darinj <darinj.w...@gmail.com>
Authored: Tue Mar 8 05:51:51 2016 -0500
Committer: darinj <darinj.w...@gmail.com>
Committed: Sun Mar 13 17:03:54 2016 -0400
--
.../myriad/scheduler/fgs/YarnNodeCapacityManager.java | 12 +---
.../apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy | 7 +--
.../scheduler/fgs/YarnNodeCapacityManagerSpec.groovy| 5 +++--
3 files changed, 17 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4bce035e/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManager.java
--
diff --git
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManager.java
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManager.java
index 1a5d185..1dee5fa 100644
---
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManager.java
+++
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/fgs/YarnNodeCapacityManager.java
@@ -70,6 +70,7 @@ import org.slf4j.LoggerFactory;
*/
public class YarnNodeCapacityManager extends BaseInterceptor {
private static final Logger LOGGER =
LoggerFactory.getLogger(YarnNodeCapacityManager.class);
+
private final AbstractYarnScheduler yarnScheduler;
private final RMContext rmContext;
private final MyriadDriver myriadDriver;
@@ -247,9 +248,14 @@ public class YarnNodeCapacityManager extends
BaseInterceptor {
rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
LOGGER.debug("Setting capacity for node {} to {}", rmNode.getHostName(),
newCapacity);
// updates the scheduler with the new capacity for the NM.
- // the event is handled by the scheduler asynchronously
- rmContext.getDispatcher().getEventHandler().handle(new
NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(
- rmNode.getTotalCapability(),
RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
+ synchronized (yarnScheduler) {
+if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) != null) {
+ yarnScheduler.updateNodeResource(rmNode,
+ ResourceOption.newInstance(rmNode.getTotalCapability(),
RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
+} else {
+ LOGGER.info("Yarn Scheduler doesn't have node {}, probably
UNHEALTHY", rmNode.getNodeID());
+}
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4bce035e/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
--
diff --git
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
index 5b15e59..c76 100644
---
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
+++
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
@@ -33,6 +33,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSSchedulerNode
import org.apache.hadoop.yarn.util.resource.Resources
import org.apache.mesos.Protos
import org.apache.mesos.SchedulerDriver
@@ -65,6 +66,7 @@ class FGSTestBaseSpec extends Specification {
def rmNodes = new ConcurrentHashMap<NodeId, RMNode>()
+
RMNode getRMNode(int cpu, int mem, String host, Protos.SlaveID slaveId) {
RMNode rmNode = MockNodes.newNodeInfo(0, Resources.createResource(mem,
cpu), 0, host)
if (rmNodes[rmNode.getNodeID()]) {
@@ -80,
incubator-myriad git commit: MYRIAD-186 Clean up the build [Forced Update!]
Repository: incubator-myriad
Updated Branches:
refs/heads/master 4246a2a56 -> b56565046 (forced update)
MYRIAD-186 Clean up the build
JIRA:
[MYRIAD-186] https://issues.apache.org/jira/browse/MYRIAD-186
Pull Request:
Closes #60
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/b5656504
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/b5656504
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/b5656504
Branch: refs/heads/master
Commit: b56565046c60382d99c40bd31d1e7fc8e6fdd635
Parents: 9c03f6a
Author: Zachary Jaffee <zjaf...@salesforce.com>
Authored: Fri Feb 26 15:31:08 2016 -0800
Committer: darinj <darinj.w...@gmail.com>
Committed: Wed Mar 9 23:56:35 2016 -0500
--
docker/Dockerfile | 1 -
docker/ResourceManager.dockerfile | 1 -
docker/build-myriad.sh| 1 -
myriad-executor/build.gradle | 23 ---
4 files changed, 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5656504/docker/Dockerfile
--
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 47cc2d2..18d4084 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -43,7 +43,6 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh
RUN sh /install-yarn.sh
# Copy over myriad libraries
-ADD /libs/myriad-executor-runnable-0.1.0.jar /usr/local/libexec/mesos/
ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5656504/docker/ResourceManager.dockerfile
--
diff --git a/docker/ResourceManager.dockerfile
b/docker/ResourceManager.dockerfile
index 7c3dbef..c4448d7 100644
--- a/docker/ResourceManager.dockerfile
+++ b/docker/ResourceManager.dockerfile
@@ -44,7 +44,6 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh
RUN sh /install-yarn.sh
# Copy over myriad libraries
-ADD /libs/myriad-executor-runnable-0.1.0.jar /usr/local/libexec/mesos/
ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
# Initialize hadoop confs with env vars
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5656504/docker/build-myriad.sh
--
diff --git a/docker/build-myriad.sh b/docker/build-myriad.sh
index 38d6b95..d3b0e74 100644
--- a/docker/build-myriad.sh
+++ b/docker/build-myriad.sh
@@ -37,6 +37,5 @@ fi
# Copy over the Java Libraries
cp -rf ../myriad-scheduler/build/libs/* libs/
-cp -rf ../myriad-executor/build/libs/myriad-executor-runnable-0.1.0.jar libs/
echo "Init complete! " #Modify config/myriad-default-config.yml to your liking
before building the docker image"
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/b5656504/myriad-executor/build.gradle
--
diff --git a/myriad-executor/build.gradle b/myriad-executor/build.gradle
index 15529c9..05594d7 100644
--- a/myriad-executor/build.gradle
+++ b/myriad-executor/build.gradle
@@ -24,26 +24,3 @@ dependencies {
compile "org.apache.hadoop:hadoop-common:${hadoopVer}"
}
-
-task capsule(type: Jar, dependsOn: jar) {
-archiveName = "myriad-executor-runnable-${version}.jar"
-
-from jar
-from { configurations.runtime }
-from { configurations.myriadExecutorConf }
-from { sourceSets.main.resources }
-
-from(configurations.capsule.collect { zipTree(it) }) { include
'Capsule.class' }
-
-manifest {
-attributes(
-'Main-Class': 'Capsule',
-'Application-Class':
'org.apache.myriad.executor.MyriadExecutor',
-'Min-Java-Version': '1.7.0',
-'JVM-Args': run.jvmArgs.join(' '),
-'System-Properties': (run.systemProperties).collect { k, v ->
"$k=$v" }.join(' ')
-)
-}
-}
-
-build.dependsOn capsule
