SLIDER-1107 fix client installation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/511891c1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/511891c1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/511891c1 Branch: refs/heads/feature/SLIDER-1107_AM_config_generation Commit: 511891c151746b2bd9e5de9cd781b8dd58ec512d Parents: 1699a71 Author: Billie Rinaldi <billie.rina...@gmail.com> Authored: Thu May 12 18:30:07 2016 -0700 Committer: Billie Rinaldi <billie.rina...@gmail.com> Committed: Thu May 12 18:51:38 2016 -0700 ---------------------------------------------------------------------- app-packages/hbase-nopkg/appConfig-default.json | 4 + .../resources/hbase-env-client.sh.j2 | 49 +++++++++++ app-packages/hbase-nopkg/src/assembly/hbase.xml | 7 ++ .../slider/common/tools/CoreFileSystem.java | 2 - .../providers/agent/AgentClientProvider.java | 93 +++++++++----------- .../TestPublishedConfigurationOutputter.java | 19 ++-- 6 files changed, 113 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/appConfig-default.json b/app-packages/hbase-nopkg/appConfig-default.json index 980538a..1095efd 100644 --- a/app-packages/hbase-nopkg/appConfig-default.json +++ b/app-packages/hbase-nopkg/appConfig-default.json @@ -103,6 +103,10 @@ }, "HBASE_THRIFT2": { "site.global.hbase_component": "thrift2" + }, + "HBASE_CLIENT": { + "site.hbase-env.template.file": "hbase-env-client.sh.j2", + "site.hbase-site.instance": "${CLUSTER_NAME}" } } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 new file mode 100644 index 0000000..2511a29 --- /dev/null +++ b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 @@ -0,0 +1,49 @@ +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME={{java64_home}} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}" +export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}" +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false +export HBASE_OPTS="$HBASE_OPTS {{hbase_opts}}" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/src/assembly/hbase.xml ---------------------------------------------------------------------- diff --git a/app-packages/hbase-nopkg/src/assembly/hbase.xml b/app-packages/hbase-nopkg/src/assembly/hbase.xml index da10fb6..0558319 100644 --- a/app-packages/hbase-nopkg/src/assembly/hbase.xml +++ b/app-packages/hbase-nopkg/src/assembly/hbase.xml @@ -42,6 +42,12 @@ <fileMode>0755</fileMode> </file> <file> + <source>clientInstallConfig-default.json</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> <source>metainfo.xml</source> <outputDirectory>/</outputDirectory> <filtered>true</filtered> @@ -71,6 +77,7 @@ <exclude>target/**</exclude> <exclude>appConfig-default.json</exclude> <exclude>appConfig-secured-default.json</exclude> + <exclude>clientInstallConfig-default.json</exclude> <exclude>metainfo.xml</exclude> <exclude>metainfo-secured.xml</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index df086d1..0b0f1bc 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -774,8 +774,6 @@ public class CoreFileSystem { Path destPath = new Path(destFile.getPath()); fileSystem.copyToLocalFile(hdfsPath, destPath); - // set file permissions of the destPath - fileSystem.setPermission(destPath, new FsPermission("755")); } /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index e5c3bec..18e4702 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -22,6 +22,7 @@ import com.google.common.io.Files; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; @@ -60,7 +61,6 @@ import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -328,51 +328,25 @@ public class AgentClientProvider extends AbstractClientProvider { ZipEntry zipEntry = zipInputStream.getNextEntry(); while (zipEntry != null) { - if ("metainfo.xml".equals(zipEntry.getName())) { - int size = (int) zipEntry.getSize(); - if (size != -1) { - log.info("Reading {} of size {}", zipEntry.getName(), - zipEntry.getSize()); - byte[] content = new byte[size]; - int offset = 0; - while (offset < size) { - offset += zipInputStream.read(content, offset, size - offset); - } - metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content)); - } - } else if ("metainfo.json".equals(zipEntry.getName())) { - int size = (int) zipEntry.getSize(); - if (size != -1) { - log.info("Reading {} of size {}", zipEntry.getName(), - zipEntry.getSize()); - byte[] content = new byte[size]; - int offset = 0; - while (offset < size) { - offset += zipInputStream.read(content, offset, size - offset); - } - metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content)); - } - } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { - int size = (int) zipEntry.getSize(); - if (size != -1) { - log.info("Reading {} of size {}", zipEntry.getName(), - zipEntry.getSize()); - byte[] content = new byte[size]; - int offset = 0; - while (offset < size) { - offset += zipInputStream.read(content, offset, size - offset); - } + log.info("Processing {}", zipEntry.getName()); + String filePath = appPkgDir + File.separator + zipEntry.getName(); + if (!zipEntry.isDirectory()) { + log.info("Extracting file {}", filePath); + extractFile(zipInputStream, filePath); + + if ("metainfo.xml".equals(zipEntry.getName())) { + metaInfo = new MetainfoParser().fromXmlStream(new FileInputStream(filePath)); + } else if ("metainfo.json".equals(zipEntry.getName())) { + metaInfo = new MetainfoParser().fromJsonStream(new FileInputStream(filePath)); + } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { try { - defaultConfig = new JSONObject(new String(content, Charset.defaultCharset())); + defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset())); } catch (JSONException jex) { throw new SliderException("Unable to read default client config.", jex); } } - } - String filePath = appPkgDir + File.separator + zipEntry.getName(); - if (!zipEntry.isDirectory()) { - extractFile(zipInputStream, filePath); } else { + log.info("Creating dir {}", filePath); File dir = new File(filePath); dir.mkdir(); } @@ -393,7 +367,9 @@ public class AgentClientProvider extends AbstractClientProvider for (Component component : metaInfo.getApplication().getComponents()) { if (component.getCategory().equals("CLIENT")) { clientComponent = component; - client_script = component.getCommandScript().getScript(); + if (component.getCommandScript() != null) { + client_script = component.getCommandScript().getScript(); + } break; } } @@ -435,16 +411,26 @@ public class AgentClientProvider extends AbstractClientProvider } if (name == null) { log.warn("Conf files not being generated because no app name was " + - "providied"); + "provided"); return; } File confInstallDir; String clientRoot = null; - try { - clientRoot = defaultConfig.getJSONObject("global") - .getString(AgentKeys.APP_CLIENT_ROOT); - } catch (JSONException e) { - e.printStackTrace(); + if (defaultConfig != null) { + try { + clientRoot = defaultConfig.getJSONObject("global") + .getString(AgentKeys.APP_CLIENT_ROOT); + } catch (JSONException e) { + e.printStackTrace(); + } + } + if (config != null) { + try { + clientRoot = config.getJSONObject("global") + .getString(AgentKeys.APP_CLIENT_ROOT); + } catch (JSONException e) { + e.printStackTrace(); + } } if (clientRoot == null) { confInstallDir = clientInstallPath; @@ -485,8 +471,6 @@ public class AgentClientProvider extends AbstractClientProvider } catch (IOException ioex) { log.warn("Error while executing INSTALL command {}", ioex.getMessage()); throw new SliderException("INSTALL client failed."); - } finally { - tmpDir.delete(); } } @@ -561,6 +545,7 @@ public class AgentClientProvider extends AbstractClientProvider } private void expandTar(File tarFile, File destDir) throws IOException { + log.info("Expanding tar {} to {}", tarFile, destDir); TarArchiveInputStream tarIn = new TarArchiveInputStream( new GzipCompressorInputStream( new BufferedInputStream( @@ -572,10 +557,13 @@ public class AgentClientProvider extends AbstractClientProvider TarArchiveEntry tarEntry = tarIn.getNextTarEntry(); while (tarEntry != null) { File destPath = new File(destDir, tarEntry.getName()); + File parent = destPath.getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } if (tarEntry.isDirectory()) { destPath.mkdirs(); } else { - destPath.createNewFile(); byte[] byteToRead = new byte[1024]; BufferedOutputStream buffOut = new BufferedOutputStream(new FileOutputStream(destPath)); @@ -588,6 +576,9 @@ public class AgentClientProvider extends AbstractClientProvider buffOut.close(); } } + if ((tarEntry.getMode() & 0100) != 0) { + destPath.setExecutable(true); + } tarEntry = tarIn.getNextTarEntry(); } } finally { @@ -598,6 +589,8 @@ public class AgentClientProvider extends AbstractClientProvider private void retrieveConfigFile(RegistryOperations rops, Configuration configuration, ConfigFile configFile, String name, String user, File destDir) throws IOException, SliderException { + log.info("Retrieving config {} to {}", configFile.getDictionaryName(), + destDir); PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops, configuration, configFile.getDictionaryName(), name, user, true); ClientUtils.saveOrReturnConfig(published, configFile.getType(), http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java index eec5c38..63d5961 100644 --- a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java +++ b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java @@ -21,7 +21,6 @@ import com.google.common.base.Charsets; import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter.TemplateOutputter; import org.codehaus.jackson.map.ObjectMapper; import org.junit.Before; import org.junit.Rule; @@ -58,7 +57,7 @@ public class TestPublishedConfigurationOutputter { PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON, new PublishedConfiguration("description", - config.entrySet(), null, null)); + config.entrySet())); String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", ""); @@ -78,7 +77,7 @@ public class TestPublishedConfigurationOutputter { PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML, new PublishedConfiguration("description", - config.entrySet(), null, null)); + config.entrySet())); String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", ""); @@ -99,7 +98,7 @@ public class TestPublishedConfigurationOutputter { PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML, new PublishedConfiguration("description", - config.entrySet(), null, null)); + config.entrySet())); String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", ""); @@ -118,7 +117,7 @@ public class TestPublishedConfigurationOutputter { PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES, new PublishedConfiguration("description", - config.entrySet(), null, null)); + config.entrySet())); String output = configurationOutputter.asString(); assert output.contains("key1=val1"); @@ -145,7 +144,7 @@ public class TestPublishedConfigurationOutputter { PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML, new PublishedConfiguration("description", - config.entrySet(), null, null)); + config.entrySet())); String output = configurationOutputter.asString().replaceAll("(\\r|\\n)", ""); @@ -177,7 +176,7 @@ public class TestPublishedConfigurationOutputter { PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV, new PublishedConfiguration("description", - envConfig.entrySet(), null, null)); + envConfig.entrySet())); String output = configurationOutputter.asString(); assert "content val1 ".equals(output); @@ -192,7 +191,7 @@ public class TestPublishedConfigurationOutputter { @Test public void testTemplate1() throws IOException { HashMap<String, String> templateConfig = new HashMap<>(config); - templateConfig.put(TemplateOutputter.TEMPLATE_FILE, "templateFileName"); + templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName"); SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class); expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes(); @@ -201,10 +200,12 @@ public class TestPublishedConfigurationOutputter { PowerMock.replay(fileSystem); + ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE, + templateConfig, fileSystem, "clusterName", null); PublishedConfigurationOutputter configurationOutputter = PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE, new PublishedConfiguration("description", - templateConfig.entrySet(), fileSystem, "clusterName")); + templateConfig.entrySet())); String output = configurationOutputter.asString(); assert "content val1\n more val1 content".equals(output);