Repository: hive Updated Branches: refs/heads/branch-1 f0b8f1786 -> 92730ca5b
HIVE-15840: Webhcat test TestPig_5 failing with Pig on Tez at check for percent complete of job (Daniel Dai, reviewed by Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92730ca5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92730ca5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92730ca5 Branch: refs/heads/branch-1 Commit: 92730ca5baeb56b809e25227ccfd6e670bd8bbea Parents: f0b8f17 Author: Daniel Dai <[email protected]> Authored: Tue Feb 7 19:43:34 2017 -0800 Committer: Daniel Dai <[email protected]> Committed: Tue Feb 7 19:43:34 2017 -0800 ---------------------------------------------------------------------- .../hcatalog/templeton/tool/TempletonUtils.java | 29 ++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/92730ca5/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java index a7c6137..78a549e 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java @@ -118,7 +118,17 @@ public class TempletonUtils { * "(Map|Reducer) (\\d+:) ((-/-)|(\\d+(\\(\\+\\d+\\))?/\\d+))" is the complete pattern but we'll drop "-/-" to exclude * groups that don't add information such as "Map 1: -/-" */ - public static final Pattern TEZ_COMPLETE = Pattern.compile("(Map|Reducer) (\\d+:) (\\d+(\\(\\+\\d+\\))?/\\d+)"); + public static final Pattern HIVE_TEZ_COMPLETE = Pattern.compile("(Map|Reducer) (\\d+:) (\\d+(\\(\\+\\d+\\))?/\\d+)"); + /** + * Pig on Tez produces progress report that looks like this + * DAG Status: status=RUNNING, progress=TotalTasks: 3 Succeeded: 0 Running: 0 Failed: 0 Killed: 0 + * + * Use Succeeded/TotalTasks to report progress + * There is a hole as Pig might launch more than one DAGs. If this happens, user might + * see progress rewind since the percentage is for the new DAG. To fix this, We need to fix + * Pig print total number of DAGs on console, and track complete DAGs in WebHCat. + */ + public static final Pattern PIG_TEZ_COMPLETE = Pattern.compile("progress=TotalTasks: (\\d+) Succeeded: (\\d+)"); public static final Pattern TEZ_COUNTERS = Pattern.compile("\\d+"); /** @@ -137,14 +147,14 @@ public class TempletonUtils { if(hive.find()) { return "map " + hive.group(1) + " reduce " + hive.group(2); } - Matcher tez = TEZ_COMPLETE.matcher(line); - if(tez.find()) { + Matcher hiveTez = HIVE_TEZ_COMPLETE.matcher(line); + if(hiveTez.find()) { int totalTasks = 0; int completedTasks = 0; do { //here each group looks something like "Map 2: 2/4" "Reducer 3: 1(+2)/4" //just parse the numbers and ignore one from "Map 2" and from "(+2)" if it's there - Matcher counts = TEZ_COUNTERS.matcher(tez.group()); + Matcher counts = TEZ_COUNTERS.matcher(hiveTez.group()); List<String> items = new ArrayList<String>(4); while(counts.find()) { items.add(counts.group()); @@ -156,12 +166,21 @@ public class TempletonUtils { else { totalTasks += Integer.parseInt(items.get(3)); } - } while(tez.find()); + } while(hiveTez.find()); if(totalTasks == 0) { return "0% complete (0 total tasks)"; } return completedTasks * 100 / totalTasks + "% complete"; } + Matcher pigTez = PIG_TEZ_COMPLETE.matcher(line); + if(pigTez.find()) { + int totalTasks = Integer.parseInt(pigTez.group(1)); + int completedTasks = Integer.parseInt(pigTez.group(2)); + if(totalTasks == 0) { + return "0% complete (0 total tasks)"; + } + return completedTasks * 100 / totalTasks + "% complete"; + } return null; }
