Repository: hive Updated Branches: refs/heads/branch-2.1 606922834 -> 9a52491c3
HIVE-14057. Add an option to llapstatus to generate output to a file, and minor bug fixes. (Siddharth Seth, reviewed by Sergey Shelukhin) (cherry picked from commit f80f65da9bbc03eea55ebc6ed726419dfda3d224) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9a52491c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9a52491c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9a52491c Branch: refs/heads/branch-2.1 Commit: 9a52491c37c7e6a2c72bdafa2dfa3c6e07886fc1 Parents: 6069228 Author: Siddharth Seth <ss...@apache.org> Authored: Wed Jun 22 11:14:41 2016 -0700 Committer: Siddharth Seth <ss...@apache.org> Committed: Wed Jun 22 11:15:54 2016 -0700 ---------------------------------------------------------------------- .../llap/cli/LlapStatusOptionsProcessor.java | 38 +++++++--- .../hive/llap/cli/LlapStatusServiceDriver.java | 73 +++++++++++++++----- .../java/org/apache/hive/http/LlapServlet.java | 3 +- 3 files changed, 86 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9a52491c/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java index cb848c0..306391b 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java @@ -40,12 +40,13 @@ public class LlapStatusOptionsProcessor { enum OptionConstants { - NAME("name", 'n', "LLAP cluster name"), + NAME("name", 'n', "LLAP cluster name", true), FIND_APP_TIMEOUT("findAppTimeout", 'f', "Amount of time(s) that the tool will sleep to wait for the YARN application to start. negative values=wait forever, 0=Do not wait. default=" + - TimeUnit.SECONDS.convert(FIND_YARN_APP_TIMEOUT_MS, TimeUnit.MILLISECONDS) + "s"), + TimeUnit.SECONDS.convert(FIND_YARN_APP_TIMEOUT_MS, TimeUnit.MILLISECONDS) + "s", true), + OUTPUT_FILE("outputFile", 'o', "File to which output should be written (Default stdout)", true), HIVECONF("hiveconf", null, "Use value for given property. Overridden by explicit parameters", "property=value", 2), - HELP("help", 'H', "Print help information"); + HELP("help", 'H', "Print help information", false); private final String longOpt; @@ -54,9 +55,8 @@ public class LlapStatusOptionsProcessor { private final String argName; private final int numArgs; - OptionConstants(String longOpt, char shortOpt, String description) { - this(longOpt, shortOpt, description, longOpt, 1); - + OptionConstants(String longOpt, char shortOpt, String description, boolean hasArgs) { + this(longOpt, shortOpt, description, longOpt, hasArgs ? 1 : 0); } OptionConstants(String longOpt, Character shortOpt, String description, String argName, int numArgs) { @@ -93,11 +93,18 @@ public class LlapStatusOptionsProcessor { private final String name; private final Properties conf; private final long findAppTimeoutMs; + private final String outputFile; - LlapStatusOptions(String name, Properties hiveProperties, long findAppTimeoutMs) { + public LlapStatusOptions(String name, Properties hiveProperties, long findAppTimeoutMs, + String outputFile) { this.name = name; this.conf = hiveProperties; this.findAppTimeoutMs = findAppTimeoutMs; + this.outputFile = outputFile; + } + + public LlapStatusOptions(String name) { + this(name, new Properties(), FIND_YARN_APP_TIMEOUT_MS, null); } public String getName() { @@ -111,6 +118,10 @@ public class LlapStatusOptionsProcessor { public long getFindAppTimeoutMs() { return findAppTimeoutMs; } + + public String getOutputFile() { + return outputFile; + } } private final Options options = new Options(); @@ -154,11 +165,16 @@ public class LlapStatusOptionsProcessor { hiveConf = new Properties(); } - return new LlapStatusOptions(name, hiveConf, findAppTimeoutMs); + String outputFile = null; + if (commandLine.hasOption(OptionConstants.OUTPUT_FILE.getLongOpt())) { + outputFile = commandLine.getOptionValue(OptionConstants.OUTPUT_FILE.getLongOpt()); + } + + return new LlapStatusOptions(name, hiveConf, findAppTimeoutMs, outputFile); } - private void printUsage() { + public static void printUsage() { HelpFormatter hf = new HelpFormatter(); try { int width = hf.getWidth(); @@ -167,7 +183,9 @@ public class LlapStatusOptionsProcessor { hf.setWidth(width); } catch (Throwable t) { // Ignore } - hf.printHelp(LLAPSTATUS_CONSTANT, options); + + LlapStatusOptionsProcessor optionsProcessor = new LlapStatusOptionsProcessor(); + hf.printHelp(LLAPSTATUS_CONSTANT, optionsProcessor.options); } } http://git-wip-us.apache.org/repos/asf/hive/blob/9a52491c/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java index 646c286..5209226 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusServiceDriver.java @@ -19,7 +19,10 @@ package org.apache.hadoop.hive.llap.cli; +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.PrintWriter; import java.util.EnumSet; import java.util.HashMap; @@ -70,19 +73,29 @@ public class LlapStatusServiceDriver { conf = (ss != null) ? ss.getConf() : new HiveConf(SessionState.class); } + /** + * Parse command line options. + * + * @param args + * @return command line options. + */ + public LlapStatusOptions parseOptions(String[] args) throws LlapStatusCliException { + + LlapStatusOptionsProcessor optionsProcessor = new LlapStatusOptionsProcessor(); + LlapStatusOptions options; + try { + options = optionsProcessor.processOptions(args); + return options; + } catch (Exception e) { + LOG.info("Failed to parse arguments", e); + throw new LlapStatusCliException(ExitCode.INCORRECT_USAGE, "Incorrect usage"); + } + } - public int run(String[] args) { + public int run(LlapStatusOptions options) { SliderClient sliderClient = null; try { - LlapStatusOptionsProcessor optionsProcessor = new LlapStatusOptionsProcessor(); - LlapStatusOptions options; - try { - options = optionsProcessor.processOptions(args); - } catch (Exception e) { - LOG.info("Failed to parse arguments", e); - return ExitCode.INCORRECT_USAGE.getInt(); - } for (String f : LlapDaemonConfiguration.DAEMON_CONFIGS) { conf.addResource(f); @@ -111,6 +124,9 @@ public class LlapStatusServiceDriver { LOG.info(message); return ExitCode.INCORRECT_USAGE.getInt(); } + if (LOG.isDebugEnabled()) { + LOG.debug("Using appName: {}", appName); + } try { sliderClient = createSliderClient(); @@ -157,7 +173,7 @@ public class LlapStatusServiceDriver { return ret.getInt(); } else { try { - ret = populateAppStatusFromLlapRegistry(options, appStatusBuilder); + ret = populateAppStatusFromLlapRegistry(appName, appStatusBuilder); } catch (LlapStatusCliException e) { logError(e); return e.getExitCode().getInt(); @@ -382,16 +398,16 @@ public class LlapStatusServiceDriver { /** * - * @param options + * @param appName * @param appStatusBuilder * @return an ExitCode. An ExitCode other than ExitCode.SUCCESS implies future progress not possible * @throws LlapStatusCliException */ - private ExitCode populateAppStatusFromLlapRegistry(LlapStatusOptions options, AppStatusBuilder appStatusBuilder) throws + private ExitCode populateAppStatusFromLlapRegistry(String appName, AppStatusBuilder appStatusBuilder) throws LlapStatusCliException { Configuration llapRegistryConf= new Configuration(conf); llapRegistryConf - .set(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + options.getName()); + .set(HiveConf.ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname, "@" + appName); LlapRegistryService llapRegistry; try { llapRegistry = LlapRegistryService.getClient(llapRegistryConf); @@ -836,12 +852,35 @@ public class LlapStatusServiceDriver { public static void main(String[] args) { LOG.info("LLAP status invoked with arguments = {}", args); - int ret; + int ret = ExitCode.SUCCESS.getInt(); + + LlapStatusServiceDriver statusServiceDriver = null; + LlapStatusOptions options = null; + try { + statusServiceDriver = new LlapStatusServiceDriver(); + options = statusServiceDriver.parseOptions(args); + } catch (Throwable t) { + logError(t); + if (t instanceof LlapStatusCliException) { + LlapStatusCliException ce = (LlapStatusCliException) t; + ret = ce.getExitCode().getInt(); + } else { + ret = ExitCode.INTERNAL_ERROR.getInt(); + } + } finally { + LOG.info("LLAP status finished"); + } + if (ret != 0 || options == null) { // Failure / help + System.exit(ret); + } + try { - LlapStatusServiceDriver statusServiceDriver = new LlapStatusServiceDriver(); - ret = statusServiceDriver.run(args); + ret = statusServiceDriver.run(options); if (ret == ExitCode.SUCCESS.getInt()) { - try (PrintWriter pw = new PrintWriter(System.out)) { + try (OutputStream os = options.getOutputFile() == null ? System.out : + new BufferedOutputStream( + new FileOutputStream(options.getOutputFile())); PrintWriter pw = new PrintWriter( + os)) { statusServiceDriver.outputJson(pw); } } http://git-wip-us.apache.org/repos/asf/hive/blob/9a52491c/service/src/java/org/apache/hive/http/LlapServlet.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/http/LlapServlet.java b/service/src/java/org/apache/hive/http/LlapServlet.java index 65e23fc..993766b 100644 --- a/service/src/java/org/apache/hive/http/LlapServlet.java +++ b/service/src/java/org/apache/hive/http/LlapServlet.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.llap.cli.LlapStatusOptionsProcessor; import org.apache.hadoop.hive.llap.cli.LlapStatusServiceDriver; @SuppressWarnings("serial") @@ -97,7 +98,7 @@ public class LlapServlet extends HttpServlet { LOG.info("Retrieving info for cluster: " + clusterName); LlapStatusServiceDriver driver = new LlapStatusServiceDriver(); - int ret = driver.run(new String[] { "-n", clusterName }); + int ret = driver.run(new LlapStatusOptionsProcessor.LlapStatusOptions(clusterName)); if (ret == LlapStatusServiceDriver.ExitCode.SUCCESS.getInt()) { driver.outputJson(writer); }
