Repository: incubator-myriad Updated Branches: refs/heads/master e1db4d8b8 -> 1daad6324
[Myriad 171] - updateFile had parameter change between Hadoop 2.7.0->2.7.1 [Myriad 190] - Tests fail on Hadoop 2.6.2+ Corrects behavior for hadoop 2.7.1+ and hadoop 2.6.2+. Added support to specify hadoopVer from command line as follows: ``` ./gradlew -PhadoopVer=X.Y.Z build ``` where `X.Y.Z` is the hadoop version. JIRA: [Myriad-171] https://issues.apache.org/jira/browse/MYRIAD-171 [Myriad-190] https://issues.apache.org/jira/browse/MYRIAD-190 Pull Request: Closes #63 Author: darinj dar...@apache.org Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/1daad632 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/1daad632 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/1daad632 Branch: refs/heads/master Commit: 1daad6324ce8b74df0f144600785c1db3d7348d9 Parents: e1db4d8 Author: darinj <darinj.w...@gmail.com> Authored: Mon Mar 14 02:04:26 2016 -0400 Committer: darinj <darinj.w...@gmail.com> Committed: Sat Apr 2 06:59:57 2016 -0400 ---------------------------------------------------------------------- build.gradle | 5 ++- gradle.properties | 1 - .../recovery/MyriadFileSystemRMStateStore.java | 37 ++++++++++++++++++-- .../myriad/scheduler/fgs/FGSTestBaseSpec.groovy | 2 ++ 4 files changed, 41 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 918f84c..067a52d 100644 --- a/build.gradle +++ b/build.gradle @@ -68,7 +68,10 @@ subprojects { ext { mesosVer = "0.24.1" - hadoopVer = "2.7.0" + //Allows passing -PhadoopVer=2.6.0 from command line + if(!project.hasProperty('hadoopVer')) { + hadoopVer = "2.7.0" + } metricsVer = "3.1.0" } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/gradle.properties ---------------------------------------------------------------------- diff --git a/gradle.properties b/gradle.properties index 9667034..bca1f77 100644 --- a/gradle.properties +++ b/gradle.properties @@ -18,6 +18,5 @@ org.gradle.jvmargs=-Xmx1024m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 org.gradle.parallel=true org.gradle.daemon=true - # faster builds: gradle build -x findBugsM http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java index dda9b28..923e29d 100644 --- a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java +++ b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java @@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.recovery; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; @@ -43,11 +46,28 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme private Path myriadPathRoot = null; private byte[] myriadStateBytes = null; + private Method updateFileMethod = null; //This is a cache Method so we do fewer reflection calls + @Override public synchronized void initInternal(Configuration conf) throws Exception { super.initInternal(conf); Path rootPath = new Path(fsWorkingPath, ROOT_NAME); myriadPathRoot = new Path(rootPath, RM_MYRIAD_ROOT); + updateFileMethod = getUpdateFileMethod(); + if (updateFileMethod == null) { + //something is broken + throw new RuntimeException("Could not find valid updateFile Method"); + } + } + + private Method getUpdateFileMethod() { + Method[] methods = super.getClass().getSuperclass().getDeclaredMethods(); + for (Method m : methods) { + if (m.getName().equals("updateFile")) { + return m; + } + } + return null; } @Override @@ -59,7 +79,6 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme @Override public synchronized RMState loadState() throws Exception { RMState rmState = super.loadState(); - Path myriadStatePath = new Path(myriadPathRoot, MYRIAD_STATE_FILE); LOGGER.info("Loading state information for Myriad from: " + myriadStatePath); @@ -92,12 +111,26 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme LOGGER.debug("Storing state information for Myriad at: " + myriadStatePath); try { - updateFile(myriadStatePath, sc.toSerializedContext().toByteArray()); + reflectedUpdateFile(myriadStatePath, sc.toSerializedContext().toByteArray()); } catch (Exception e) { LOGGER.error("State information for Myriad could not be stored at: " + myriadStatePath, e); } } + + + protected void reflectedUpdateFile(Path outputPath, byte[] data) throws InvocationTargetException, IllegalAccessException { + Class [] parameters = updateFileMethod.getParameterTypes(); + if (parameters.length == 2 && parameters[0].equals(Path.class) && parameters[1].isArray()) { + updateFileMethod.invoke(this, outputPath, data); + } else if (parameters.length == 3 && parameters[0].equals(Path.class) && parameters[1].isArray() && parameters[2].isPrimitive()) { + updateFileMethod.invoke(this, outputPath, data, true); + } else { + //something is broken + throw new RuntimeException("updateFile Method has unexpected parameters"); + } + } + @Override public synchronized void removeMyriadState() throws Exception { if (fs.exists(myriadPathRoot)) { http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy index c769999..b87c1be 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy @@ -21,6 +21,7 @@ package org.apache.myriad.scheduler.fgs import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import org.apache.hadoop.yarn.api.records.* +import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.hadoop.yarn.event.Dispatcher import org.apache.hadoop.yarn.event.EventHandler import org.apache.hadoop.yarn.server.resourcemanager.MockNodes @@ -108,6 +109,7 @@ class FGSTestBaseSpec extends Specification { getRMApplicationHistoryWriter() >> writer getSystemMetricsPublisher() >> publisher getRMNodes() >> rmNodes + getYarnConfiguration() >> new YarnConfiguration() } /******************* Offers Related ****************/
