Repository: hive Updated Branches: refs/heads/master 38c757cd7 -> bd9264fb3
HIVE-19567: Fix flakiness in TestTriggers (Prasanth Jayachandran, reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bd9264fb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bd9264fb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bd9264fb Branch: refs/heads/master Commit: bd9264fb30403b6e04cb4fbddb3635d3fde67e5f Parents: 38c757c Author: Prasanth Jayachandran <[email protected]> Authored: Wed May 16 07:14:46 2018 -0700 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Wed May 16 07:14:46 2018 -0700 ---------------------------------------------------------------------- .../jdbc/TestTriggersMoveWorkloadManager.java | 48 ++++++++++---------- .../hooks/PostExecWMEventsSummaryPrinter.java | 1 + 2 files changed, 25 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/bd9264fb/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersMoveWorkloadManager.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersMoveWorkloadManager.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersMoveWorkloadManager.java index 40af04f..e403d28 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersMoveWorkloadManager.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersMoveWorkloadManager.java @@ -101,11 +101,6 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter"); List<String> errCaptureExpect = new ArrayList<>(); errCaptureExpect.add("Workload Manager Events Summary"); - errCaptureExpect.add("Event: GET Pool: BI"); - // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET - errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); - errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); - errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); errCaptureExpect.add("\"eventType\" : \"GET\""); errCaptureExpect.add("\"eventType\" : \"MOVE\""); errCaptureExpect.add("\"eventType\" : \"KILL\""); @@ -117,6 +112,11 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { // violation in ETL queue errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated"); errCaptureExpect.add("\"subscribedCounters\" : [ \"EXECUTION_TIME\" ]"); + errCaptureExpect.add("Event: GET Pool: BI"); + // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET + errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); + errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); + errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect); } @@ -137,10 +137,6 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter"); List<String> errCaptureExpect = new ArrayList<>(); errCaptureExpect.add("Workload Manager Events Summary"); - errCaptureExpect.add("Event: GET Pool: BI"); - // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET - errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); - errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); errCaptureExpect.add("\"eventType\" : \"GET\""); errCaptureExpect.add("\"eventType\" : \"MOVE\""); errCaptureExpect.add("\"eventType\" : \"RETURN\""); @@ -149,6 +145,10 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { // violation in BI queue errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger + " violated"); errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"EXECUTION_TIME\" ]"); + errCaptureExpect.add("Event: GET Pool: BI"); + // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET + errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); + errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); runQueryWithTrigger(query, setCmds, null, errCaptureExpect); } @@ -172,12 +172,6 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter"); List<String> errCaptureExpect = new ArrayList<>(); errCaptureExpect.add("Workload Manager Events Summary"); - errCaptureExpect.add("Event: GET Pool: BI"); - // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET - errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); - errCaptureExpect.add("Event: MOVE Pool: BI Cluster %: 80.00"); - errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); - errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); errCaptureExpect.add("\"eventType\" : \"GET\""); errCaptureExpect.add("\"eventType\" : \"MOVE\""); errCaptureExpect.add("\"eventType\" : \"MOVE\""); @@ -193,6 +187,12 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { // violation in BI queue errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated"); errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"EXECUTION_TIME\", \"SHUFFLE_BYTES\" ]"); + errCaptureExpect.add("Event: GET Pool: BI"); + // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET + errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); + errCaptureExpect.add("Event: MOVE Pool: BI Cluster %: 80.00"); + errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); + errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect); } @@ -215,10 +215,6 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { // setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter"); // List<String> errCaptureExpect = new ArrayList<>(); // errCaptureExpect.add("Workload Manager Events Summary"); -// errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00"); -// errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); -// errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); -// errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); // errCaptureExpect.add("\"eventType\" : \"GET\""); // errCaptureExpect.add("\"eventType\" : \"MOVE\""); // errCaptureExpect.add("\"eventType\" : \"KILL\""); @@ -230,6 +226,10 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { // // violation in ETL queue // errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated"); // errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"HDFS_BYTES_WRITTEN\" ]"); +// errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00"); +// errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00"); +// errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); +// errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); // runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect); // } @@ -250,10 +250,6 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter"); List<String> errCaptureExpect = new ArrayList<>(); errCaptureExpect.add("Workload Manager Events Summary"); - errCaptureExpect.add("Event: GET Pool: BI"); - // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET - errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); - errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); errCaptureExpect.add("\"eventType\" : \"GET\""); errCaptureExpect.add("\"eventType\" : \"KILL\""); errCaptureExpect.add("\"eventType\" : \"RETURN\""); @@ -262,6 +258,10 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { // violation in BI queue errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated"); errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\" ]"); + errCaptureExpect.add("Event: GET Pool: BI"); + // HIVE-19061 introduces UPDATE event which will capture changes to allocation % after GET + errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00"); + errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00"); runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect); } @@ -288,4 +288,4 @@ public class TestTriggersMoveWorkloadManager extends AbstractJdbcTriggersTest { } wm.updateResourcePlanAsync(plan).get(10, TimeUnit.SECONDS); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hive/blob/bd9264fb/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java index 83cca89..94c66c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecWMEventsSummaryPrinter.java @@ -54,6 +54,7 @@ public class PostExecWMEventsSummaryPrinter implements ExecuteWithHookContext { for (TezTask tezTask : rootTasks) { WmContext wmContext = tezTask.getDriverContext().getCtx().getWmContext(); if (wmContext != null) { + wmContext.printJson(console); wmContext.shortPrint(console); } }
