Repository: incubator-slider Updated Branches: refs/heads/develop 4bebbf711 -> 2954a6f19
SLIDER-1090 Speed up SliderClient and AM by reading metainfo from summary file (shanyu zhao via gourksaha) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2954a6f1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2954a6f1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2954a6f1 Branch: refs/heads/develop Commit: 2954a6f19fa0de65ac3b5f452199df343f24b416 Parents: 4bebbf7 Author: Gour Saha <gourks...@apache.org> Authored: Wed Mar 23 17:15:09 2016 -0700 Committer: Gour Saha <gourks...@apache.org> Committed: Wed Mar 23 17:15:09 2016 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 33 +++++++++ .../slider/providers/agent/AgentUtils.java | 60 +++++++++++++--- .../slider/providers/agent/TestAgentUtils.java | 76 ++++++++++++++++++++ 3 files changed, 160 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2954a6f1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index cf15dfc..2e5c45a 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -25,6 +25,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -179,6 +180,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.io.PrintStream; @@ -1355,6 +1357,36 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } + private void createSummaryMetainfoFile(Path srcFile, Path destFile, + boolean overwrite) throws IOException { + FileSystem srcFs = srcFile.getFileSystem(getConfig()); + try (InputStream inputStreamJson = SliderUtils + .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json"); + InputStream inputStreamXml = SliderUtils + .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) { + InputStream inputStream = null; + Path summaryFileInFs = null; + if (inputStreamJson != null) { + inputStream = inputStreamJson; + summaryFileInFs = new Path(destFile.getParent(), destFile.getName() + + ".metainfo.json"); + log.info("Found JSON metainfo file in package"); + } else if (inputStreamXml != null) { + inputStream = inputStreamXml; + summaryFileInFs = new Path(destFile.getParent(), destFile.getName() + + ".metainfo.xml"); + log.info("Found XML metainfo file in package"); + } + if (inputStream != null) { + try (FSDataOutputStream dataOutputStream = sliderFileSystem + .getFileSystem().create(summaryFileInFs, overwrite)) { + log.info("Creating summary metainfo file"); + IOUtils.copy(inputStream, dataOutputStream); + } + } + } + } + private int actionPackageInstall(ActionPackageArgs actionPackageArgs) throws YarnException, IOException { requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); @@ -1375,6 +1407,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.info("Installing package {} to {} (overwrite set to {})", srcFile, fileInFs, actionPackageArgs.replacePkg); fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs); + createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg); String destPathWithHomeDir = Path .getPathWithoutSchemeAndAuthority(fileInFs).toString(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2954a6f1/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index b38dc6e..cfcfc5d 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -16,6 +16,7 @@ */ package org.apache.slider.providers.agent; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.SliderFileSystem; @@ -40,30 +41,55 @@ import java.io.InputStream; public class AgentUtils { private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); + public static Metainfo getApplicationMetainfoFromSummaryFile( + SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) { + FileSystem fs = fileSystem.getFileSystem(); + Path appPathXML = new Path(metainfoPath + ".metainfo.xml"); + Path appPathJson = new Path(metainfoPath + ".metainfo.json"); + Path appPathUsed = null; + try { + FSDataInputStream appStream = null; + if (fs.exists(appPathXML)) { + appPathUsed = appPathXML; + appStream = fs.open(appPathXML); + return parseMetainfo(appStream, metainfoForAddon, "xml"); + } else if (fs.exists(appPathJson)) { + appPathUsed = appPathJson; + appStream = fs.open(appPathJson); + return parseMetainfo(appStream, metainfoForAddon, "json"); + } + } catch (IOException e) { + log.info("Failed to get metainfo from summary file {} - {}", appPathUsed, + e.getMessage()); + log.debug("Failed to get metainfo", e); + } + return null; + } + public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) throws IOException, BadConfigException { log.info("Reading metainfo at {}", metainfoPath); + Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem, + metainfoPath, metainfoForAddon); + if (metainfo != null) { + log.info("Got metainfo from summary file"); + return metainfo; + } + FileSystem fs = fileSystem.getFileSystem(); Path appPath = new Path(metainfoPath); - Metainfo metainfo = null; - AbstractMetainfoParser metainfoParser = null; - if (metainfoForAddon) { - metainfoParser = new AddonPackageMetainfoParser(); - } else { - metainfoParser = new MetainfoParser(); - } InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream( fs, appPath, "metainfo.json"); if (metainfoJsonStream == null) { InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream( fs, appPath, "metainfo.xml"); if (metainfoXMLStream != null) { - metainfo = metainfoParser.fromXmlStream(metainfoXMLStream); + metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml"); } } else { - metainfo = metainfoParser.fromJsonStream(metainfoJsonStream); + metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json"); } if (metainfo == null) { @@ -74,6 +100,22 @@ public class AgentUtils { return metainfo; } + private static Metainfo parseMetainfo(InputStream stream, + boolean metainfoForAddon, String type) throws IOException { + AbstractMetainfoParser metainfoParser = null; + if (metainfoForAddon) { + metainfoParser = new AddonPackageMetainfoParser(); + } else { + metainfoParser = new MetainfoParser(); + } + if (type.equals("xml")) { + return metainfoParser.fromXmlStream(stream); + } else if (type.equals("json")) { + return metainfoParser.fromJsonStream(stream); + } + return null; + } + static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem, String appDef, String configFileName) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2954a6f1/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java new file mode 100644 index 0000000..db8c138 --- /dev/null +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java @@ -0,0 +1,76 @@ +package org.apache.slider.providers.agent; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.providers.agent.application.metadata.Metainfo; +import org.apache.slider.tools.TestUtility; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestAgentUtils { + protected static final Logger log = + LoggerFactory.getLogger(TestAgentUtils.class); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + private static final String metainfo_str = "<metainfo>\n" + + " <schemaVersion>2.0</schemaVersion>\n" + + " <application>\n" + + " <name>MYTESTAPPLICATION</name>\n" + + " <comment>\n" + + " My Test Application\n" + + " </comment>\n" + + " <version>1.0</version>\n" + + " <type>YARN-APP</type>\n" + + " <components>\n" + + " <component>\n" + + " <name>REST</name>\n" + + " <category>MASTER</category>\n" + + " <commandScript>\n" + + " <script>scripts/rest.py</script>\n" + + " <scriptType>PYTHON</scriptType>\n" + + " <timeout>600</timeout>\n" + + " </commandScript>\n" + + " </component>\n" + + " </components>\n" + + " </application>\n" + + "</metainfo>"; + + @Test + public void testGetApplicationMetainfo() throws Exception { + String zipFileName = TestUtility.createAppPackage( + folder, + "testpkg", + "test.zip", + "target/test-classes/org/apache/slider/common/tools/test"); + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + // Without accompany metainfo file, read metainfo from the zip file + Metainfo metainfo = AgentUtils.getApplicationMetainfo( + sliderFileSystem, zipFileName, false); + Assert.assertNotNull(metainfo.getApplication()); + Assert.assertEquals("STORM", metainfo.getApplication().getName()); + + // With accompany metainfo file, read metainfo from the accompany file + String acompanyFileName = zipFileName + ".metainfo.xml"; + File f = new File(acompanyFileName); + try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) { + writer.write(metainfo_str); + } + metainfo = AgentUtils.getApplicationMetainfo( + sliderFileSystem, zipFileName, false); + Assert.assertNotNull(metainfo.getApplication()); + Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName()); + } +}