Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-1107_AM_config_generation 41de54d76 -> d3923d889
SLIDER-1107 address suggestions from gour Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d3923d88 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d3923d88 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d3923d88 Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: d3923d88915997f62b35ec43bbe86b232c9bcccf Parents: 41de54d Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Wed Aug 3 14:16:09 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Wed Aug 3 14:16:09 2016 -0700 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 5 ++- .../slider/common/params/SliderActions.java | 2 +- .../slider/common/tools/CoreFileSystem.java | 31 ++++++++++---- .../slider/core/conf/ConfTreeOperations.java | 14 ------ .../slider/core/launch/AbstractLauncher.java | 10 ++--- .../providers/agent/AgentClientProvider.java | 45 +++++++++++++------- .../providers/agent/AgentProviderService.java | 26 +++++------ .../agent/TestAgentProviderService.java | 4 ++ 8 files changed, 80 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index eb0630d..af134b3 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -1112,7 +1112,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override public int actionResource(ActionResourceArgs resourceInfo) throws YarnException, IOException { - if (resourceInfo.install) { + if (resourceInfo.help) { + actionHelp(ACTION_RESOURCE); + return EXIT_SUCCESS; + } else if (resourceInfo.install) { return actionInstallResource(resourceInfo); } else if (resourceInfo.delete) { return actionDeleteResource(resourceInfo); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index e209cdc..204ad9a 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -107,7 +107,7 @@ public interface SliderActions { " Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'."; String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory"; String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client"; - String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the sub-folder 'resources' of the user's Slider base directory"; + String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index 0b0f1bc..aa5edf1 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -830,23 +830,36 @@ public class CoreFileSystem { } public void touch(Path path, boolean overwrite) throws IOException { - FSDataOutputStream out = fileSystem.create(path, overwrite); - out.close(); + FSDataOutputStream out = null; + try { + out = fileSystem.create(path, overwrite); + } finally { + IOUtils.closeStream(out); + } } public void cat(Path path, boolean overwrite, String data) throws IOException { - FSDataOutputStream out = fileSystem.create(path, overwrite); - byte[] bytes = data.getBytes(Charset.forName("UTF-8")); - out.write(bytes); - out.close(); + FSDataOutputStream out = null; + try { + out = fileSystem.create(path, overwrite); + byte[] bytes = data.getBytes(Charset.forName("UTF-8")); + out.write(bytes); + } finally { + IOUtils.closeStream(out); + } } public String cat(Path path) throws IOException { FileStatus status = fileSystem.getFileStatus(path); byte[] b = new byte[(int) status.getLen()]; - FSDataInputStream in = fileSystem.open(path); - int count = in.read(b); - return new String(b, 0, count, UTF_8); + FSDataInputStream in = null; + try { + in = fileSystem.open(path); + int count = in.read(b); + return new String(b, 0, count, UTF_8); + } finally { + IOUtils.closeStream(in); + } } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java index d376c72..d24a158 100644 --- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java @@ -347,20 +347,6 @@ public class ConfTreeOperations { } /** - * Load from a json String. The inner conf tree is the loaded data -unresolved - * @param json json string - * @return loaded value - * @throws IOException load failure - */ - public static ConfTreeOperations fromString(String json) throws - IOException { - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - ConfTreeOperations ops = new ConfTreeOperations( - confTreeSerDeser.fromJson(json) ); - return ops; - } - - /** * Build from an existing instance -which is cloned via JSON ser/deser * @param instance the source instance * @return loaded value http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index efc58b4..5a3eb3d 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -133,13 +133,13 @@ public abstract class AbstractLauncher extends Configured { return localResources; } - public void addLocalResource(String subpath, LocalResource resource) { - localResources.put(subpath, resource); + public void addLocalResource(String subPath, LocalResource resource) { + localResources.put(subPath, resource); } - public void addLocalResource(String subpath, LocalResource resource, String mountPath) { - localResources.put(subpath, resource); - mountPaths.put(subpath, mountPath); + public void addLocalResource(String subPath, LocalResource resource, String mountPath) { + localResources.put(subPath, resource); + mountPaths.put(subPath, mountPath); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 3eef0b0..4c6a50b 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -27,6 +27,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.slider.api.InternalKeys; @@ -335,9 +336,21 @@ public class AgentClientProvider extends AbstractClientProvider extractFile(zipInputStream, filePath); if ("metainfo.xml".equals(zipEntry.getName())) { - metaInfo = new MetainfoParser().fromXmlStream(new FileInputStream(filePath)); + FileInputStream input = null; + try { + input = new FileInputStream(filePath); + metaInfo = new MetainfoParser().fromXmlStream(input); + } finally { + IOUtils.closeStream(input); + } } else if ("metainfo.json".equals(zipEntry.getName())) { - metaInfo = new MetainfoParser().fromJsonStream(new FileInputStream(filePath)); + FileInputStream input = null; + try { + input = new FileInputStream(filePath); + metaInfo = new MetainfoParser().fromJsonStream(input); + } finally { + IOUtils.closeStream(input); + } } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { try { defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset())); @@ -362,22 +375,22 @@ public class AgentClientProvider extends AbstractClientProvider throw new BadConfigException(E_COULD_NOT_READ_METAINFO); } - String client_script = null; + String clientScript = null; String clientComponent = null; for (Component component : metaInfo.getApplication().getComponents()) { if (component.getCategory().equals("CLIENT")) { clientComponent = component.getName(); if (component.getCommandScript() != null) { - client_script = component.getCommandScript().getScript(); + clientScript = component.getCommandScript().getScript(); } break; } } - if (SliderUtils.isUnset(client_script)) { + if (SliderUtils.isUnset(clientScript)) { log.info("Installing CLIENT without script"); List<Package> packages = metaInfo.getApplication().getPackages(); - if (packages != null && packages.size() > 0) { + if (packages.size() > 0) { // retrieve package resources from HDFS and extract for (Package pkg : packages) { Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); @@ -416,20 +429,22 @@ public class AgentClientProvider extends AbstractClientProvider } File confInstallDir; String clientRoot = null; - if (defaultConfig != null) { + if (config != null) { try { - clientRoot = defaultConfig.getJSONObject("global") + clientRoot = config.getJSONObject("global") .getString(AgentKeys.APP_CLIENT_ROOT); } catch (JSONException e) { - e.printStackTrace(); + log.info("Couldn't read {} from provided client config, falling " + + "back on default", AgentKeys.APP_CLIENT_ROOT); } } - if (config != null) { + if (clientRoot == null && defaultConfig != null) { try { - clientRoot = config.getJSONObject("global") + clientRoot = defaultConfig.getJSONObject("global") .getString(AgentKeys.APP_CLIENT_ROOT); } catch (JSONException e) { - e.printStackTrace(); + log.info("Couldn't read {} from default client config, using {}", + AgentKeys.APP_CLIENT_ROOT, clientInstallPath); } } if (clientRoot == null) { @@ -446,7 +461,7 @@ public class AgentClientProvider extends AbstractClientProvider confInstallDir); } } else { - log.info("Installing CLIENT using script {}", client_script); + log.info("Installing CLIENT using script {}", clientScript); expandAgentTar(agentPkgDir); JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name); @@ -455,13 +470,13 @@ public class AgentClientProvider extends AbstractClientProvider file.write(commandJson.toString()); } catch (IOException e) { - e.printStackTrace(); + log.error("Couldn't write command.json to file"); } finally { file.flush(); file.close(); } - runCommand(appPkgDir, agentPkgDir, cmdDir, client_script); + runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript); } } catch (IOException ioex) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index bc04220..4ffae7c 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; @@ -170,7 +171,6 @@ public class AgentProviderService extends AbstractProviderService implements private AgentClientProvider clientProvider; private AtomicInteger taskId = new AtomicInteger(0); private volatile Metainfo metaInfo = null; - private AggregateConf instanceDefinition = null; private SliderFileSystem fileSystem = null; private Map<String, DefaultConfig> defaultConfigs = null; private ComponentCommandOrder commandOrder = null; @@ -284,7 +284,6 @@ public class AgentProviderService extends AbstractProviderService implements if (metaInfo == null) { synchronized (syncLock) { if (metaInfo == null) { - this.instanceDefinition = instanceDefinition; this.fileSystem = fileSystem; readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); @@ -702,12 +701,15 @@ public class AgentProviderService extends AbstractProviderService implements } Path destPath = new Path(parentDir, resource.getName()); if (!fileSystem.getFileSystem().exists(destPath)) { - FSDataOutputStream os = fileSystem.getFileSystem().create(destPath); - byte[] contents = FileUtils.readFileToByteArray(resource); - os.write(contents, 0, contents.length); - - os.flush(); - os.close(); + FSDataOutputStream os = null; + try { + os = fileSystem.getFileSystem().create(destPath); + byte[] contents = FileUtils.readFileToByteArray(resource); + os.write(contents, 0, contents.length); + os.flush(); + } finally { + IOUtils.closeStream(os); + } log.info("Uploaded {} to localization path {}", resource, destPath); } else { log.info("Resource {} already existed at localization path {}", resource, @@ -1362,7 +1364,7 @@ public class AgentProviderService extends AbstractProviderService implements // identify client component Component client = null; for (Component component : getMetaInfo().getApplication().getComponents()) { - if (component != null && component.getCategory().equals("CLIENT")) { + if (component.getCategory().equals("CLIENT")) { client = component; break; } @@ -1373,7 +1375,7 @@ public class AgentProviderService extends AbstractProviderService implements } // register AM-generated client configs - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); + ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); MapOperations clientOperations = appConf.getOrAddComponent(client.getName()); appConf.resolve(); if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION, @@ -1765,7 +1767,7 @@ public class AgentProviderService extends AbstractProviderService implements Application application = getMetaInfo().getApplication(); if ((!canAnyMasterPublishConfig() || canPublishConfig(componentGroup)) && - !instanceDefinition.getAppConfOperations().getComponentOptBool( + !getAmState().getAppConfSnapshot().getComponentOptBool( componentGroup, AgentKeys.AM_CONFIG_GENERATION, false)) { // If no Master can explicitly publish then publish if its a master // Otherwise, wait till the master that can publish is ready @@ -1890,7 +1892,7 @@ public class AgentProviderService extends AbstractProviderService implements simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue()); } } - if (!instanceDefinition.getAppConfOperations().getComponentOptBool( + if (!getAmState().getAppConfSnapshot().getComponentOptBool( groupName, AgentKeys.AM_CONFIG_GENERATION, false)) { publishApplicationInstanceData(groupName, groupName, simpleEntries.entrySet()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index af1c61f..0d56ff7 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -902,6 +902,8 @@ public class TestAgentProviderService { doReturn(access).when(mockAps).getAmState(); PublishedExportsSet pubExpSet = new PublishedExportsSet(); expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes(); + expect(access.getAppConfSnapshot()).andReturn(new ConfTreeOperations( + new ConfTree())).anyTimes(); replay(access); Map<String, String> ports = new HashMap<String, String>(); @@ -1016,6 +1018,8 @@ public class TestAgentProviderService { doReturn(access).when(mockAps).getAmState(); PublishedExportsSet pubExpSet = new PublishedExportsSet(); expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes(); + expect(access.getAppConfSnapshot()).andReturn(new ConfTreeOperations( + new ConfTree())).anyTimes(); replay(access); mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_MASTER");
