SLIDER-663. Make it easy to develop and deploy application packages that are essentially shell commands (part-II)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/210a1c94 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/210a1c94 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/210a1c94 Branch: refs/heads/develop Commit: 210a1c94ca347380f7b22fba807c12dc1e30deff Parents: df35225 Author: Sumit Mohanty <[email protected]> Authored: Wed Mar 18 14:44:07 2015 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Mar 18 14:44:07 2015 -0700 ---------------------------------------------------------------------- .../core/persist/AppDefinitionPersister.java | 64 +++++- .../agent/TestAgentClientProvider2.java | 10 +- .../agent/TestAppDefinitionPersister.java | 227 +++++++++++++++++++ 3 files changed, 288 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/210a1c94/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java index 8f0f7b0..f394542 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java @@ -18,6 +18,7 @@ package org.apache.slider.core.persist; +import com.google.common.annotations.VisibleForTesting; import com.google.common.io.Files; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.Path; @@ -26,7 +27,6 @@ import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.agent.AgentKeys; @@ -107,11 +107,23 @@ public class AppDefinitionPersister { } public void processSuppliedDefinitions(String clustername, - AbstractClusterBuildingActionArgs buildInfo, - ConfTreeOperations appConf) + AbstractClusterBuildingActionArgs buildInfo, + ConfTreeOperations appConf) throws BadConfigException, IOException, BadCommandArgumentsException { // if metainfo is provided add to the app instance if (buildInfo.appMetaInfo != null) { + + if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) { + throw new BadConfigException("--metainfo file cannot be read."); + } + + if (buildInfo.appDef != null) { + throw new BadConfigException("both --metainfo and --appdef may not be specified."); + } + if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { + throw new BadConfigException("application.def must not be set if --metainfo is provided."); + } + File tempDir = Files.createTempDir(); File pkgSrcDir = new File(tempDir, "default"); pkgSrcDir.mkdirs(); @@ -127,6 +139,14 @@ public class AppDefinitionPersister { } if (buildInfo.appDef != null) { + if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { + throw new BadConfigException("application.def must not be set if --appdef is provided."); + } + + if (!buildInfo.appDef.exists()) { + throw new BadConfigException("--appdef is not a valid path."); + } + Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG)); Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); @@ -135,16 +155,30 @@ public class AppDefinitionPersister { } if (buildInfo.addonDelegate.getAddonMap().size() > 0) { + if (SliderUtils.isUnset(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { + throw new BadConfigException("addon package can only be specified if main app package is specified."); + } + List<String> addons = new ArrayList<String>(); Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap(); for (String key : addonMap.keySet()) { + File defPath = new File(addonMap.get(key)); + if (SliderUtils.isUnset(addonMap.get(key))) { + throw new BadConfigException("Invalid path for addon package " + key); + } + + if (!defPath.exists()) { + throw new BadConfigException("addon folder or package path is not valid."); + } + Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key); String addonPkgName = "addon_" + key + ".zip"; - appDefinitions.add(new AppDefinition(addonPath, buildInfo.appDef, addonPkgName)); + appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName)); String addOnKey = AgentKeys.ADDON_PREFIX + key; Path addonPkgPath = new Path(addonPath, addonPkgName); log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath); appConf.getGlobalOptions().set(addOnKey, addonPkgPath); + addons.add(addOnKey); } String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS); @@ -156,19 +190,33 @@ public class AppDefinitionPersister { } + @VisibleForTesting + public List<AppDefinitionPersister.AppDefinition> getAppDefinitions() { + return appDefinitions; + } + // Helper class to hold details for the app and addon packages - class AppDefinition { + public class AppDefinition { // The target folder where the package will be stored - Path targetFolderInFs; + public Path targetFolderInFs; // The on disk location of the app def package or folder - File appDefPkgOrFolder; + public File appDefPkgOrFolder; // Package name - String pkgName; + public String pkgName; public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) { this.targetFolderInFs = targetFolderInFs; this.appDefPkgOrFolder = appDefPkgOrFolder; this.pkgName = pkgName; } + + @Override + public String toString() { + return new StringBuilder().append("targetFolderInFs").append(" : ").append(targetFolderInFs.toString()) + .append(", ") + .append("appDefPkgOrFolder").append(" : ").append(appDefPkgOrFolder.toString()) + .append(", ") + .append("pkgName").append(" : ").append(pkgName).toString(); + } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/210a1c94/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java index 4bd9842..65e0fbd 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java @@ -225,7 +225,7 @@ public class TestAgentClientProvider2 { args.install = true; try { client.actionClient(args); - }catch(BadCommandArgumentsException e) { + } catch (BadCommandArgumentsException e) { log.info(e.getMessage()); Assert.assertTrue(e.getMessage().contains("A valid install location must be provided for the client")); } @@ -235,7 +235,7 @@ public class TestAgentClientProvider2 { args.installLocation = dest; try { client.actionClient(args); - }catch(BadCommandArgumentsException e) { + } catch (BadCommandArgumentsException e) { log.info(e.getMessage()); Assert.assertTrue(e.getMessage().contains("Install path does not exist at")); } @@ -243,7 +243,7 @@ public class TestAgentClientProvider2 { dest.mkdir(); try { client.actionClient(args); - }catch(BadCommandArgumentsException e) { + } catch (BadCommandArgumentsException e) { log.info(e.getMessage()); Assert.assertTrue(e.getMessage().contains("A valid application package location required")); } @@ -253,7 +253,7 @@ public class TestAgentClientProvider2 { args.clientConfig = tmpFile; try { client.actionClient(args); - }catch(SliderException e) { + } catch (SliderException e) { log.info(e.getMessage()); Assert.assertTrue(e.getMessage().contains("Invalid configuration. Must be a valid json file")); } @@ -261,7 +261,7 @@ public class TestAgentClientProvider2 { args.clientConfig = null; try { client.actionClient(args); - }catch(SliderException e) { + } catch (SliderException e) { log.info(e.getMessage()); Assert.assertTrue(e.getMessage().contains("Not a valid app package. Could not read metainfo")); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/210a1c94/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java new file mode 100644 index 0000000..eaf496c --- /dev/null +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.providers.agent; + +import com.google.common.io.Files; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.slider.common.params.ActionCreateArgs; +import org.apache.slider.common.params.AddonArgsDelegate; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.core.conf.ConfTreeOperations; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.persist.AppDefinitionPersister; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class TestAppDefinitionPersister { + protected static final Logger log = + LoggerFactory.getLogger(TestAppDefinitionPersister.class); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + /** + * @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration(); + * BasicConfigurator.configure(); }* + */ + + + @Test + public void testAppDefinitionPersister() throws Exception { + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.getLocal(configuration); + log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + + AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem); + String clustername = "c1"; + ActionCreateArgs buildInfo = new ActionCreateArgs(); + buildInfo.appMetaInfo = null; + buildInfo.appDef = null; + buildInfo.addonDelegate = new AddonArgsDelegate(); + + // nothing to do + adp.processSuppliedDefinitions(clustername, buildInfo, null); + adp.persistPackages(); + List<AppDefinitionPersister.AppDefinition> appDefinitions = adp.getAppDefinitions(); + Assert.assertTrue(appDefinitions.size() == 0); + + ConfTree ct = new ConfTree(); + ConfTreeOperations appConf = new ConfTreeOperations(ct); + final File tempDir = Files.createTempDir(); + final File metainfo = new File(tempDir, "metainfo.json"); + + // unreadable metainfo + buildInfo.appMetaInfo = metainfo; + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("--metainfo file cannot be read")); + } + + try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) { + writer.println("{"); + writer.println("}"); + } + buildInfo.appDef = metainfo; + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("both --metainfo and --appdef may not be specified")); + } + + buildInfo.appDef = null; + + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --metainfo is provided")); + } + + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + appDefinitions = adp.getAppDefinitions(); + Assert.assertTrue(appDefinitions.size() == 1); + Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); + log.info(appDefinitions.get(0).toString()); + Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default")); + Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef")); + Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName); + + buildInfo.appDef = tempDir; + buildInfo.appMetaInfo = null; + + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided")); + } + + adp.getAppDefinitions().clear(); + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + appDefinitions = adp.getAppDefinitions(); + Assert.assertTrue(appDefinitions.size() == 1); + Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); + log.info(appDefinitions.get(0).toString()); + Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString())); + Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef")); + Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName); + + adp.getAppDefinitions().clear(); + buildInfo.appDef = null; + buildInfo.appMetaInfo = null; + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + + ArrayList<String> list = new ArrayList<String>() {{ + add("addon1"); + add(""); + add("addon2"); + add(metainfo.getAbsolutePath()); + }}; + + buildInfo.addonDelegate.addonTuples = list; + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified")); + } + + buildInfo.appMetaInfo = metainfo; + + try { + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + } catch (BadConfigException bce) { + log.info(bce.getMessage()); + Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1")); + } + + appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); + + list = new ArrayList<String>() {{ + add("addon1"); + add(tempDir.getAbsolutePath()); + add("addon2"); + add(metainfo.getAbsolutePath()); + }}; + + buildInfo.addonDelegate.addonTuples = list; + adp.getAppDefinitions().clear(); + + adp.processSuppliedDefinitions(clustername, buildInfo, appConf); + appDefinitions = adp.getAppDefinitions(); + + Assert.assertTrue(appDefinitions.size() == 3); + Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); + Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains( + "addons/addon1/addon_addon1.zip")); + Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains( + "addons/addon2/addon_addon2.zip")); + log.info(appConf.getGlobalOptions().get("application.addons")); + Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains( + "application.addon.addon2,application.addon.addon1") + || appConf.getGlobalOptions().get("application.addons").contains( + "application.addon.addon1,application.addon.addon2")); + int seen = 0; + for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) { + if (adp_ad.pkgName.equals("appPkg.zip")) { + log.info(adp_ad.toString()); + Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default")); + Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef")); + seen++; + } + if (adp_ad.pkgName.equals("addon_addon1.zip")) { + log.info(adp_ad.toString()); + Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString())); + Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1")); + seen++; + } + if (adp_ad.pkgName.equals("addon_addon2.zip")) { + log.info(adp_ad.toString()); + Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json")); + Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2")); + seen++; + } + } + Assert.assertEquals(3, seen); + } +}
