date:20160812

[1/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc

2016-08-12 Thread darinj

Repository: incubator-myriad
Updated Branches:
  refs/heads/master df4cbc0e9 -> 7aea259cf


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
--
diff --git 
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
 
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
index 87aee56..b6806a2 100644
--- 
a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
+++ 
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java
@@ -18,53 +18,106 @@
  */
 package org.apache.myriad.scheduler;
 
+import org.apache.commons.lang.StringUtils;
+import org.apache.mesos.Protos;
 import static org.junit.Assert.assertTrue;
 
 import org.apache.mesos.Protos.CommandInfo;
+import org.apache.myriad.configuration.ServiceConfiguration;
 import org.apache.myriad.BaseConfigurableTest;
-import org.apache.myriad.scheduler.TaskFactory.NMTaskFactoryImpl;
 import org.junit.Test;
 
+import java.util.*;
+
 /**
  * Class to test CommandLine generation
  */
 public class TestServiceCommandLine extends BaseConfigurableTest {
+
+  public static final String KEY_NM_ADDRESS = 
"myriad.yarn.nodemanager.address";
+  public static final String KEY_NM_LOCALIZER_ADDRESS = 
"myriad.yarn.nodemanager.localizer.address";
+  public static final String KEY_NM_WEBAPP_ADDRESS = 
"myriad.yarn.nodemanager.webapp.address";
+  public static final String KEY_NM_SHUFFLE_PORT = 
"myriad.mapreduce.shuffle.port";
+
+  public static final String KEY_JHS_WEBAPP_ADDRESS = 
"myriad.mapreduce.jobhistory.webapp.address";
+  public static final String KEY_JHS_ADMIN_ADDRESS = 
"myriad.mapreduce.jobhistory.admin.address";
+  public static final String KEY_JHS_ADDRESS = 
"myriad.mapreduce.jobhistory.address";
+
+  private static final String msgFormat = System.lineSeparator() + "%s" + 
System.lineSeparator() + "!="
+  + System.lineSeparator() + "%s";
+  protected static final String CMD_FORMAT = "echo \"%1$s\" && %1$s";
   static String toJHSCompare =
-  "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp conf 
/usr/local/hadoop/etc/hadoop/yarn-site.xml; " +
-  "export TASK_DIR=`basename $PWD`; sudo  chmod +wx 
/sys/fs/cgroup/cpu/mesos/$TASK_DIR;" +
-  "sudo -E -u hduser -H  $YARN_HOME/bin/mapred historyserver\"; sudo tar 
-zxpf hadoop-2.5.0.tar.gz &&  sudo  cp" +
-  " conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H 
$YARN_HOME/bin/mapred historyserver";
+  " sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp yarnConfiguration 
/usr/local/hadoop/etc/hadoop/yarn-site.xml &&  " +
+  "sudo -E -u hduser -H  bin/mapred historyserver";
   static String toCompare =
-  "echo \" sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp conf 
/usr/local/hadoop/etc/hadoop/yarn-site.xml;";
+  " sudo tar -zxpf hadoop-2.7.0.tar.gz &&  sudo  cp yarnConfiguration 
/usr/local/hadoop/etc/hadoop/yarn-site.xml &&  " +
+  "sudo -E -u hduser -H  $YARN_HOME/bin/yarn nodemanager";
 
   @Test
   public void testJHSCommandLineGeneration() throws Exception {
-ServiceTaskFactoryImpl jhs = new ServiceTaskFactoryImpl(cfg, null);
-String executorCmd = "$YARN_HOME/bin/mapred historyserver";
-ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 
10.0, 15.0);
+Map portsMap = new TreeMap<>();
+portsMap.put(KEY_JHS_ADDRESS, 0L);
+portsMap.put(KEY_JHS_WEBAPP_ADDRESS, 3L);
+portsMap.put(KEY_JHS_ADMIN_ADDRESS, 0L);
+
+ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 
10.0, 15.0, portsMap);
+ServiceConfiguration serviceConfiguration = 
cfg.getServiceConfiguration("jobhistory").get();
+ServiceCommandLineGenerator serviceCommandLineGenerator = new 
ServiceCommandLineGenerator(cfg);
+List ports = new ArrayList<>();
+ports.add(2L);
+ports.add(1L);
+ports.add(3L);
 
-CommandInfo cInfo = jhs.createCommandInfo(profile, executorCmd);
-System.out.println(toJHSCompare);
-System.out.println(cInfo.getValue());
+CommandInfo cInfo = 
serviceCommandLineGenerator.generateCommandLine(profile,
+serviceConfiguration,
+ports);
+String testVal =  String.format(CMD_FORMAT, toJHSCompare);
+assertTrue(String.format(msgFormat, cInfo.getValue(), testVal),
+cInfo.getValue().equals(testVal));
 
-assertTrue(cInfo.getValue().startsWith(toCompare));
+List environmentList = 
cInfo.getEnvironment().getVariablesList();
+String yarnOpts = "";
+for (Protos.Environment.Variable variable: environmentList) {
+  if 
(variable.getName().equals(ServiceCommandLineGenerator.ENV_HADOOP_OPTS)){
+yarnOpts = variable.getValue();
+  }
+}
+assertTrue("Environment contains " +

[2/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc

2016-08-12 Thread darinj

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
--
diff --git 
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
 
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
deleted file mode 100644
index ace9928..000
--- 
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraints.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.myriad.scheduler;
-
-/**
- * Generic interface to represent some constraints that task can impose
- * while figuring out whether to accept or reject the offer
- * We may start small and then eventually add more constraints
- */
-public interface TaskConstraints {
-
-  /**
-   * Required number of ports
-   *
-   * @return portsNumber
-   */
-  public int portsCount();
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
--
diff --git 
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
 
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
deleted file mode 100644
index 0665190..000
--- 
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskConstraintsManager.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.myriad.scheduler;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Factory class to keep map of the constraints
- */
-public class TaskConstraintsManager {
-
-  /**
-   * Since all the additions will happen during init time, there is no need to 
make this map Concurrent
-   * if/when later on it will change we may need to change HashMap to 
Concurrent one
-   */
-  private Map taskConstraintsMap = new HashMap<>();
-
-  public TaskConstraints getConstraints(String taskPrefix) {
-return taskConstraintsMap.get(taskPrefix);
-  }
-
-  public void addTaskConstraints(final String taskPrefix, final 
TaskConstraints taskConstraints) {
-if (taskConstraints != null) {
-  taskConstraintsMap.put(taskPrefix, taskConstraints);
-}
-  }
-
-  public boolean exists(String taskPrefix) {
-return taskConstraintsMap.containsKey(taskPrefix);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
--
diff --git 
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java 
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
index 6b398a3..7e63e0d 100644
--- 
a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
+++ 
b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java
@@ -9,233 +9,151 @@
  * 
  * http://www.apache.org/licenses/LICENSE-2.0
  * 
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or

[3/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc

2016-08-12 Thread darinj

Refactor which addresses Myriad 213, 214, and 136 in the process.
-Refactored ExecutorCommandLineGenerator classes to use this class (resolves 
Myriad-214 in the process).
-Refactor TackFactory classes as necessary to work with this. This adds a 
removes several methods to TaskUtils to get port Resources.
-Created a ResourceOfferContainer moving the work of checking offers and 
creating the resources from them out of the ResourceOfferEventHandler and 
TaskFactory Classes.
-Remove NMPorts and Ports classes.
JIRA:
[MYRIAD-136] https://issues.apache.org/jira/browse/MYRIAD-136
[MYRIAD-213] https://issues.apache.org/jira/browse/MYRIAD-213
[MYRIAD-214] https://issues.apache.org/jira/browse/MYRIAD-214
Pull Request:
  Closes #79


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/7aea259c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/7aea259c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/7aea259c

Branch: refs/heads/master
Commit: 7aea259cf231655bdbd033f61584eec7715fc97f
Parents: df4cbc0
Author: DarinJ 
Authored: Thu Jun 9 10:08:46 2016 -0400
Committer: darinj 
Committed: Fri Aug 12 09:27:14 2016 -0400

--
 build.gradle|   4 +
 .../myriad/executor/MyriadExecutorDefaults.java |  23 --
 .../src/main/java/org/apache/myriad/Main.java   |  47 ++-
 .../java/org/apache/myriad/MyriadModule.java|  29 +-
 .../configuration/MyriadConfiguration.java  |  15 +-
 .../configuration/NodeManagerConfiguration.java |  25 +-
 .../configuration/ServiceConfiguration.java |   6 +-
 .../scheduler/DownloadNMExecutorCLGenImpl.java  |  90 --
 .../scheduler/ExecutorCommandLineGenerator.java | 148 -
 .../scheduler/ExtendedResourceProfile.java  |  27 +-
 .../myriad/scheduler/MyriadOperations.java  |   2 +-
 .../myriad/scheduler/NMExecutorCLGenImpl.java   | 176 ---
 .../NMExecutorCommandLineGenerator.java | 150 +
 .../org/apache/myriad/scheduler/NMPorts.java|  78 -
 .../apache/myriad/scheduler/NMTaskFactory.java  |  74 +
 .../java/org/apache/myriad/scheduler/Ports.java |  26 --
 .../scheduler/ServiceCommandLineGenerator.java  |  86 +-
 .../scheduler/ServiceResourceProfile.java   |  30 +-
 .../scheduler/ServiceTaskConstraints.java   |  49 ---
 .../myriad/scheduler/ServiceTaskFactory.java|  76 +
 .../scheduler/ServiceTaskFactoryImpl.java   | 253 ---
 .../myriad/scheduler/TaskConstraints.java   |  35 ---
 .../scheduler/TaskConstraintsManager.java   |  48 ---
 .../apache/myriad/scheduler/TaskFactory.java| 304 +++
 .../org/apache/myriad/scheduler/TaskUtils.java  |  86 +-
 .../handlers/ResourceOffersEventHandler.java| 175 ++-
 .../scheduler/resource/RangeResource.java   | 218 +
 .../resource/ResourceOfferContainer.java| 207 +
 .../scheduler/resource/ScalarResource.java  |  88 ++
 .../scheduler/yarn/MyriadFairScheduler.java |   1 -
 .../org/apache/myriad/MyriadTestModule.java |  14 +-
 .../myriad/api/SchedulerStateResourceTest.java  |   9 +-
 .../myriad/scheduler/MockSchedulerDriver.java   |  17 ++
 .../myriad/scheduler/MyriadDriverTest.java  |  17 ++
 .../myriad/scheduler/MyriadOperationsTest.java  |  22 +-
 .../myriad/scheduler/NMProfileManagerTest.java  |  17 ++
 .../myriad/scheduler/SchedulerUtilsSpec.groovy  |   6 +-
 .../scheduler/ServiceResourceProfileTest.java   |  24 +-
 .../myriad/scheduler/TMSTaskFactoryImpl.java|   9 +-
 .../scheduler/TaskConstraintsManagerTest.java   |  32 --
 .../myriad/scheduler/TestNMTaskFactory.java |  72 +
 .../myriad/scheduler/TestRandomPorts.java   | 203 -
 .../scheduler/TestServiceCommandLine.java   |  97 --
 .../scheduler/TestServiceTaskFactory.java   |  77 +
 .../apache/myriad/scheduler/TestTaskUtils.java  |  37 +--
 .../fgs/YarnNodeCapacityManagerTest.java|   4 +-
 .../myriad/scheduler/offer/OfferBuilder.java| 117 +++
 .../resource/TestResourceOfferContainer.java| 166 ++
 .../org/apache/myriad/state/ClusterTest.java|   9 +-
 .../org/apache/myriad/state/NodeTaskTest.java   |   9 +-
 .../apache/myriad/state/SchedulerStateTest.java |   5 +-
 .../state/utils/ByteBufferSupportTest.java  |   5 +-
 ...iad-config-test-default-with-docker-info.yml |   1 +
 .../resources/myriad-config-test-default.yml|  32 +-
 54 files changed, 1928 insertions(+), 1649 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/7aea259c/build.gradle
--
diff --git a/build.gradle b/build.gradle
index

[1/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc

[2/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc

[3/3] incubator-myriad git commit: Refactor which addresses Myriad 213, 214, and 136 in the process. -Refactored ExecutorCommandLineGenerator classes to use this class (resolves Myriad-214 in the proc

3 matches

Site Navigation

Mail list logo

Footer information