Repository: tez Updated Branches: refs/heads/master c07d6b751 -> 56f7847d2
TEZ-2788. Allow TezAnalyzerBase to parse SimpleHistory logs (bikas) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/56f7847d Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/56f7847d Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/56f7847d Branch: refs/heads/master Commit: 56f7847d26418879cfa32d7033147c8f95c56469 Parents: c07d6b7 Author: Bikas Saha <[email protected]> Authored: Tue Sep 8 18:50:47 2015 -0700 Committer: Bikas Saha <[email protected]> Committed: Tue Sep 8 18:50:47 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../tez/analyzer/plugins/TezAnalyzerBase.java | 63 +++++++++++++++++--- 2 files changed, 55 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/56f7847d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 4717249..77535b1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ Release 0.8.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2788. Allow TezAnalyzerBase to parse SimpleHistory logs TEZ-2782. VertexInfo.getAvgExecutionTimeInterval throws NPE when task does not have any valid attempts info TEZ-2778. Improvements to handle multiple read errors with complex DAGs TEZ-2768. Log a useful error message when the summary stream cannot be closed when shutting http://git-wip-us.apache.org/repos/asf/tez/blob/56f7847d/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java ---------------------------------------------------------------------- diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java index 3eb2f57..fa3bd97 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java @@ -27,9 +27,12 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Tool; import org.apache.tez.analyzer.Analyzer; +import org.apache.tez.history.ATSImportTool; import org.apache.tez.history.parser.ATSFileParser; +import org.apache.tez.history.parser.SimpleHistoryParser; import org.apache.tez.history.parser.datamodel.DagInfo; import com.google.common.base.Preconditions; @@ -37,9 +40,10 @@ import com.google.common.base.Preconditions; public abstract class TezAnalyzerBase extends Configured implements Tool, Analyzer { - private static final String ATS_FILE_NAME = "atsFileName"; + private static final String EVENT_FILE_NAME = "eventFileName"; private static final String OUTPUT_DIR = "outputDir"; private static final String DAG_ID = "dagId"; + private static final String FROM_SIMPLE_HISTORY = "fromSimpleHistory"; private static final String HELP = "help"; private String outputDir; @@ -52,10 +56,16 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz Option outputDirOption = OptionBuilder.withArgName(OUTPUT_DIR).withLongOpt(OUTPUT_DIR) .withDescription("Directory to write outputs to.").hasArg().isRequired(false).create(); - Option inputATSFileNameOption = OptionBuilder.withArgName(ATS_FILE_NAME).withLongOpt - (ATS_FILE_NAME) - .withDescription("File with ATS data for the DAG").hasArg() - .isRequired(true).create(); + Option eventFileNameOption = OptionBuilder.withArgName(EVENT_FILE_NAME).withLongOpt + (EVENT_FILE_NAME) + .withDescription("File with event data for the DAG").hasArg() + .isRequired(false).create(); + + Option fromSimpleHistoryOption = OptionBuilder.withArgName(FROM_SIMPLE_HISTORY).withLongOpt + (FROM_SIMPLE_HISTORY) + .withDescription("Event data from Simple History logging. Must also specify event file") + .isRequired(false).create(); + Option help = OptionBuilder.withArgName(HELP).withLongOpt (HELP) .withDescription("print help") @@ -64,7 +74,8 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz Options opts = new Options(); opts.addOption(dagIdOption); opts.addOption(outputDirOption); - opts.addOption(inputATSFileNameOption); + opts.addOption(eventFileNameOption); + opts.addOption(fromSimpleHistoryOption); opts.addOption(help); return opts; } @@ -103,12 +114,46 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyz if (outputDir == null) { outputDir = System.getProperty("user.dir"); } + + File file = null; + if (cmdLine.hasOption(EVENT_FILE_NAME)) { + file = new File(cmdLine.getOptionValue(EVENT_FILE_NAME)); + } - File file = new File(cmdLine.getOptionValue(ATS_FILE_NAME)); String dagId = cmdLine.getOptionValue(DAG_ID); - ATSFileParser parser = new ATSFileParser(file); - DagInfo dagInfo = parser.getDAGData(dagId); + DagInfo dagInfo = null; + + if (file == null) { + if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) { + System.err.println("Event file name must be specified when using simple history"); + printUsage(); + return -2; + } + // using ATS - try to download directly + String[] importArgs = { "--dagId=" + dagId, "--downloadDir=" + outputDir }; + + int result = ATSImportTool.process(importArgs); + if (result != 0) { + System.err.println("Error downloading data from ATS"); + return -3; + } + + //Parse ATS data and verify results + //Parse downloaded contents + file = new File(outputDir + + Path.SEPARATOR + dagId + + Path.SEPARATOR + dagId + ".zip"); + } + + Preconditions.checkState(file != null); + if (!cmdLine.hasOption(FROM_SIMPLE_HISTORY)) { + ATSFileParser parser = new ATSFileParser(file); + dagInfo = parser.getDAGData(dagId); + } else { + SimpleHistoryParser parser = new SimpleHistoryParser(file); + dagInfo = parser.getDAGData(dagId); + } Preconditions.checkState(dagInfo.getDagId().equals(dagId)); analyze(dagInfo); return 0;
