Repository: incubator-slider Updated Branches: refs/heads/develop f5f837cc7 -> 4f92ab9fa
SLIDER-827 Slider package --instances does not list any applications Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4f92ab9f Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4f92ab9f Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4f92ab9f Branch: refs/heads/develop Commit: 4f92ab9faf7bdc134c3c126e1691725921be6021 Parents: f5f837c Author: Gour Saha <[email protected]> Authored: Thu Mar 26 13:14:44 2015 -0700 Committer: Gour Saha <[email protected]> Committed: Thu Mar 26 13:14:44 2015 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 63 +++++-- .../slider/common/params/ActionPackageArgs.java | 4 + .../client/TestPackageCommandOptions.groovy | 172 +++++++++++++++++++ 3 files changed, 226 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4f92ab9f/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index c23177a..658fe7f 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -204,6 +204,7 @@ import static org.apache.slider.common.params.SliderActions.*; public class SliderClient extends AbstractSliderLaunchedService implements RunService, SliderExitCodes, SliderKeys, ErrorStrings, SliderClientAPI { private static final Logger log = LoggerFactory.getLogger(SliderClient.class); + private static PrintStream clientOutputStream = System.out; // value should not be changed without updating string find in slider.py private static final String PASSWORD_PROMPT = "Enter password for"; @@ -1077,26 +1078,49 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override public int actionPackage(ActionPackageArgs actionPackageInfo) throws YarnException, IOException { + initializeOutputStream(actionPackageInfo.out); + int exitCode = -1; if (actionPackageInfo.install) { - return actionPackageInstall(actionPackageInfo); + exitCode = actionPackageInstall(actionPackageInfo); } if (actionPackageInfo.delete) { - return actionPackageDelete(actionPackageInfo); + exitCode = actionPackageDelete(actionPackageInfo); } if (actionPackageInfo.list) { - return actionPackageList(); + exitCode = actionPackageList(); } if (actionPackageInfo.instances) { - return actionPackageInstances(); + exitCode = actionPackageInstances(); + } + finalizeOutputStream(actionPackageInfo.out); + if (exitCode != -1) { + return exitCode; } throw new BadCommandArgumentsException( "Select valid package operation option"); } + private void initializeOutputStream(String outFile) + throws FileNotFoundException { + if (outFile != null) { + clientOutputStream = new PrintStream(new FileOutputStream(outFile)); + } else { + clientOutputStream = System.out; + } + } + + private void finalizeOutputStream(String outFile) { + if (outFile != null && clientOutputStream != null) { + clientOutputStream.flush(); + clientOutputStream.close(); + } + clientOutputStream = System.out; + } + private int actionPackageInstances() throws YarnException, IOException { Map<String, Path> persistentInstances = sliderFileSystem .listPersistentInstances(); - if(persistentInstances.isEmpty()) { + if (persistentInstances.isEmpty()) { log.info("No slider cluster specification available"); return EXIT_SUCCESS; } @@ -1105,26 +1129,39 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe FileSystem fs = sliderFileSystem.getFileSystem(); Iterator<Map.Entry<String, Path>> instanceItr = persistentInstances .entrySet().iterator(); - log.info("List of application with its package name and path"); + log.info("List of applications with its package name and path"); + println("%-25s %15s %s", "Cluster Name", "Package Name", + "Application Location"); while(instanceItr.hasNext()) { Map.Entry<String, Path> entry = instanceItr.next(); String clusterName = entry.getKey(); Path clusterPath = entry.getValue(); AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( clusterName, clusterPath); - Path appDefPath = new Path(instanceDefinition.getAppConfOperations() - .getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF)); + Path appDefPath = null; + try { + appDefPath = new Path(instanceDefinition.getAppConfOperations() + .getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF)); + } catch (BadConfigException e) { + // Invalid cluster state, so move on to next. No need to log anything + // as this is just listing of instances. + continue; + } + if (!appDefPath.isUriPathAbsolute()) { + appDefPath = new Path(fs.getHomeDirectory(), appDefPath); + } + String appDefPathStr = appDefPath.toUri().getPath(); try { - if (appDefPath.toString().contains(pkgPathValue) + if (appDefPathStr.contains(pkgPathValue) && fs.isFile(appDefPath)) { String packageName = appDefPath.getParent().getName(); - println("\t" + clusterName + "\t" + packageName + "\t" - + appDefPath.toString()); + println("%-25s %15s %s", clusterName, packageName, + appDefPathStr); } } catch(IOException e) { if(log.isDebugEnabled()) { log.debug(clusterName + " application definition path " - + appDefPath.toString() + " is not found."); + + appDefPathStr + " is not found."); } } } @@ -3836,7 +3873,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe */ @SuppressWarnings("UseOfSystemOutOrSystemErr") private static void print(CharSequence src) { - System.out.append(src); + clientOutputStream.append(src); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4f92ab9f/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java index e98aba1..0ea7351 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java @@ -58,6 +58,10 @@ public class ActionPackageArgs extends AbstractActionArgs { @Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package") public boolean replacePkg = false; + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, + description = "Output file for package data") + public String out; + /** * Get the min #of params expected * @return the min number of params in the {@link #parameters} field http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4f92ab9f/slider-core/src/test/groovy/org/apache/slider/client/TestPackageCommandOptions.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestPackageCommandOptions.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestPackageCommandOptions.groovy new file mode 100644 index 0000000..80fec96 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/client/TestPackageCommandOptions.groovy @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils +import org.apache.hadoop.fs.FileUtil +import org.apache.hadoop.fs.Path +import org.apache.hadoop.fs.RawLocalFileSystem +import org.apache.hadoop.yarn.api.records.YarnApplicationState +import org.apache.hadoop.yarn.conf.YarnConfiguration + +import org.apache.slider.agent.AgentMiniClusterTestBase +import org.apache.slider.common.params.ActionPackageArgs +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.ClientArgs +import org.apache.slider.common.params.SliderActions +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.core.main.ServiceLauncherBaseTest; +import org.apache.slider.providers.agent.AgentKeys +import org.apache.slider.test.YarnZKMiniClusterTestBase + +import org.junit.Before; +import org.junit.Test; + +/** + * Test the package commands options + */ +class TestPackageCommandOptions extends AgentMiniClusterTestBase { + final shouldFail = new GroovyTestCase().&shouldFail + private static SliderFileSystem testFileSystem + private static String APP_NAME = "HBASE" + + @Before + public void setupFilesystem() { + org.apache.hadoop.fs.FileSystem fileSystem = new RawLocalFileSystem() + YarnConfiguration configuration = SliderUtils.createConfiguration() + fileSystem.setConf(configuration) + testFileSystem = new SliderFileSystem(fileSystem, configuration) + File testFolderDir = new File(testFileSystem.buildPackageDirPath(APP_NAME) + .toUri().path) + testFolderDir.deleteDir() + } + + @Test + public void testPackageInstall() throws Throwable { + // create a mock app package file + File localPackage = + FileUtil.createLocalTempFile(tempLocation, "hbase.zip", false); + String contents = UUID.randomUUID().toString() + FileUtils.write(localPackage, contents); + // install the package + YarnConfiguration conf = SliderUtils.createConfiguration() + ServiceLauncher launcher = launch(TestSliderClient, + conf, + [ + ClientArgs.ACTION_PACKAGE, + ClientArgs.ARG_INSTALL, + ClientArgs.ARG_NAME, + APP_NAME, + ClientArgs.ARG_PACKAGE, + localPackage.absolutePath + ]) + Path installedPath = new Path(testFileSystem.buildPackageDirPath(APP_NAME), + localPackage.getName()) + File installedPackage = new File(installedPath.toUri().path) + // verify file was installed successfully + assert installedPackage.exists() + assert FileUtils.readFileToString(installedPackage).equals( + FileUtils.readFileToString(localPackage)) + } + + @Test + public void testPackageInstances() throws Throwable { + describe("Create mini cluster") + YarnConfiguration yarnConfig = new YarnConfiguration(configuration) + String clustername = createMiniCluster("", yarnConfig, 1, true) + describe("Created cluster - " + clustername) + + // get the default application.def file and install it as a package + String appDefPath = agentDefOptions.getAt(AgentKeys.APP_DEF) + File appDefFile = new File(new URI(appDefPath)) + YarnConfiguration conf = SliderUtils.createConfiguration() + ServiceLauncher<SliderClient> launcher = launch(TestSliderClient, + conf, + [ + ClientArgs.ACTION_PACKAGE, + ClientArgs.ARG_INSTALL, + ClientArgs.ARG_NAME, + APP_NAME, + ClientArgs.ARG_PACKAGE, + appDefFile.absolutePath + ]) + Path installedPath = new Path(testFileSystem.buildPackageDirPath(APP_NAME), + appDefFile.getName()) + File installedPackage = new File(installedPath.toUri().path) + assert installedPackage.exists() + describe("Installed app package to - " + installedPackage.toURI() + .toString()) + // overwrite the application.def property with the new installed path + agentDefOptions.putAt(AgentKeys.APP_DEF, installedPackage.toURI() + .toString()) + // start the app and AM + describe("Starting the app") + launcher = createStandaloneAM(clustername, true, false) + SliderClient sliderClient = launcher.service + addToTeardown(sliderClient); + + describe("Listing all instances of installed packages") + String outFileName = "target/packageInstances.out" + launcher = launchClientAgainstMiniMR( + //config includes RM binding info + yarnConfig, + //varargs list of command line params + [SliderActions.ACTION_PACKAGE, + Arguments.ARG_PKGINSTANCES, + ClientArgs.ARG_OUTPUT, + outFileName + ] + ) + assert launcher.serviceExitCode == 0 + def client = launcher.service + def instances = client.enumSliderInstances(false, null, null) + def enumeratedInstance = instances[clustername] + assert enumeratedInstance != null + assert enumeratedInstance.applicationReport.name == + clustername + File outFile = new File(outFileName) + def outText = outFile.text + assert outText.contains(installedPackage.absolutePath) + assert outText.contains(APP_NAME) + assert outText.contains(clustername) + assert outText.matches("(?s).*" + clustername + " +" + APP_NAME + " +" + + installedPackage.absolutePath + ".*") + } + + private File getTempLocation () { + return new File(System.getProperty("user.dir") + "/target/_"); + } + + static class TestSliderClient extends SliderClient { + public TestSliderClient() { + super() + } + + @Override + protected void initHadoopBinding() throws IOException, SliderException { + sliderFileSystem = testFileSystem + } + } +}
