This is an automated email from the ASF dual-hosted git repository.
prasanthj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 079a720 HIVE-21597: WM trigger validation should happen at the time
of create or alter (Prasanth Jayachandran reviewed by Daniel Dai)
079a720 is described below
commit 079a7208363e4798d24a54d997d5b0f1cb7cd657
Author: Prasanth Jayachandran <prasan...@apache.org>
AuthorDate: Fri Apr 12 16:37:13 2019 -0700
HIVE-21597: WM trigger validation should happen at the time of create or
alter (Prasanth Jayachandran reviewed by Daniel Dai)
---
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 11 +++++++
.../hive/ql/exec/tez/TriggerValidatorRunnable.java | 8 ++++-
ql/src/test/queries/clientpositive/resourceplan.q | 10 ++++--
.../results/clientpositive/llap/resourceplan.q.out | 36 ++++++++++++++--------
.../apache/hive/service/server/KillQueryImpl.java | 4 +--
5 files changed, 52 insertions(+), 17 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 7c5a47e..7f0eb40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -190,6 +190,7 @@ import
org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObje
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveRoleGrant;
import
org.apache.hadoop.hive.ql.security.authorization.plugin.HiveV1Authorizer;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.wm.ExecutionTrigger;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
@@ -595,11 +596,21 @@ public class DDLTask extends Task<DDLWork> implements
Serializable {
}
private int createWMTrigger(Hive db, CreateWMTriggerDesc desc) throws
HiveException {
+ validateTrigger(desc.getTrigger());
db.createWMTrigger(desc.getTrigger());
return 0;
}
+ private void validateTrigger(final WMTrigger trigger) throws HiveException {
+ try {
+ ExecutionTrigger.fromWMTrigger(trigger);
+ } catch (IllegalArgumentException e) {
+ throw new HiveException(e);
+ }
+ }
+
private int alterWMTrigger(Hive db, AlterWMTriggerDesc desc) throws
HiveException {
+ validateTrigger(desc.getTrigger());
db.alterWMTrigger(desc.getTrigger());
return 0;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
index 670184b..8fb0695 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TriggerValidatorRunnable.java
@@ -72,10 +72,16 @@ public class TriggerValidatorRunnable implements Runnable {
currentCounterValue);
violatedSessions.put(sessionState, currentTrigger);
LOG.info("KILL trigger replacing MOVE for query {}",
queryId);
- } else {
+ } else if
(existingTrigger.getAction().getType().equals(Action.Type.MOVE_TO_POOL) &&
+
currentTrigger.getAction().getType().equals(Action.Type.MOVE_TO_POOL)){
// if multiple MOVE happens, only first move will be chosen
LOG.warn("Conflicting MOVE triggers ({} and {}). Choosing
the first MOVE trigger: {}",
existingTrigger, currentTrigger,
existingTrigger.getName());
+ } else if
(existingTrigger.getAction().getType().equals(Action.Type.KILL_QUERY) &&
+
currentTrigger.getAction().getType().equals(Action.Type.KILL_QUERY)){
+ // if multiple KILL happens, only first kill will be chosen
+ LOG.warn("Conflicting KILL triggers ({} and {}). Choosing
the first KILL trigger: {}",
+ existingTrigger, currentTrigger,
existingTrigger.getName());
}
} else {
// first violation for the session
diff --git a/ql/src/test/queries/clientpositive/resourceplan.q
b/ql/src/test/queries/clientpositive/resourceplan.q
index 46aae72..93d848b 100644
--- a/ql/src/test/queries/clientpositive/resourceplan.q
+++ b/ql/src/test/queries/clientpositive/resourceplan.q
@@ -179,11 +179,17 @@ CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME >=
'30seconds' DO MOVE TO slow
CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME < '30hour' DO MOVE TO
slow_pool;
CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME <= '30min' DO MOVE TO
slow_pool;
CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME = '0day' DO MOVE TO
slow_pool;
+-- invalid size unit
+CREATE TRIGGER plan_1.trigger_2 WHEN BYTES_READ > '10k' DO KILL;
+-- invalid time unit
+CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME > '10 millis' DO KILL;
+-- invalid long value
+CREATE TRIGGER plan_1.trigger_2 WHEN BYTES_READ > '-1000' DO KILL;
CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME > '30hour' DO MOVE TO
slow_pool;
SELECT * FROM SYS.WM_TRIGGERS;
-ALTER TRIGGER plan_1.trigger_1 WHEN BYTES_READ > '1min' DO KILL;
+ALTER TRIGGER plan_1.trigger_1 WHEN BYTES_READ > '1GB' DO KILL;
SELECT * FROM SYS.WM_TRIGGERS;
DROP TRIGGER plan_1.trigger_1;
@@ -193,7 +199,7 @@ SELECT * FROM SYS.WM_TRIGGERS;
CREATE TRIGGER plan_2.trigger_1 WHEN BYTES_READ > '100mb' DO MOVE TO null_pool;
-- Add trigger with reserved keywords.
-CREATE TRIGGER `table`.`table` WHEN BYTES_WRITTEN > '100KB' DO MOVE TO `table`;
+CREATE TRIGGER `table`.`table` WHEN BYTES_WRITTEN > '100KB' DO MOVE TO
`default`;
CREATE TRIGGER `table`.`trigger` WHEN BYTES_WRITTEN > '100MB' DO MOVE TO
`default`;
CREATE TRIGGER `table`.`database` WHEN BYTES_WRITTEN > "1GB" DO MOVE TO
`default`;
CREATE TRIGGER `table`.`trigger1` WHEN ELAPSED_TIME > 10 DO KILL;
diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out
b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
index 9ae68f4..c0d6ec2 100644
--- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out
+++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
@@ -4059,6 +4059,18 @@ FAILED: ParseException line 2:50 mismatched input '>='
expecting > near 'ELAPSED
FAILED: ParseException line 2:50 mismatched input '<' expecting > near
'ELAPSED_TIME' in comparisionOperator
FAILED: ParseException line 2:50 mismatched input '<=' expecting > near
'ELAPSED_TIME' in comparisionOperator
FAILED: ParseException line 2:50 mismatched input '=' expecting > near
'ELAPSED_TIME' in comparisionOperator
+PREHOOK: query: CREATE TRIGGER plan_1.trigger_2 WHEN BYTES_READ > '10k' DO KILL
+PREHOOK: type: CREATE TRIGGER
+PREHOOK: Output: dummyHostnameForTest
+FAILED: Execution Error, return code 1 from
org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.IllegalArgumentException:
Invalid size unit k
+PREHOOK: query: CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME > '10
millis' DO KILL
+PREHOOK: type: CREATE TRIGGER
+PREHOOK: Output: dummyHostnameForTest
+FAILED: Execution Error, return code 1 from
org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.IllegalArgumentException:
Invalid time unit millis
+PREHOOK: query: CREATE TRIGGER plan_1.trigger_2 WHEN BYTES_READ > '-1000' DO
KILL
+PREHOOK: type: CREATE TRIGGER
+PREHOOK: Output: dummyHostnameForTest
+FAILED: Execution Error, return code 1 from
org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.IllegalArgumentException:
Illegal value for counter limit. Expected a positive long value.
PREHOOK: query: CREATE TRIGGER plan_1.trigger_2 WHEN ELAPSED_TIME > '30hour'
DO MOVE TO slow_pool
PREHOOK: type: CREATE TRIGGER
PREHOOK: Output: dummyHostnameForTest
@@ -4074,10 +4086,10 @@ POSTHOOK: Input: sys@wm_triggers
#### A masked pattern was here ####
plan_1 default trigger_1 BYTES_READ > '10kb' KILL
plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool
-PREHOOK: query: ALTER TRIGGER plan_1.trigger_1 WHEN BYTES_READ > '1min' DO KILL
+PREHOOK: query: ALTER TRIGGER plan_1.trigger_1 WHEN BYTES_READ > '1GB' DO KILL
PREHOOK: type: ALTER TRIGGER
PREHOOK: Output: dummyHostnameForTest
-POSTHOOK: query: ALTER TRIGGER plan_1.trigger_1 WHEN BYTES_READ > '1min' DO
KILL
+POSTHOOK: query: ALTER TRIGGER plan_1.trigger_1 WHEN BYTES_READ > '1GB' DO KILL
POSTHOOK: type: ALTER TRIGGER
PREHOOK: query: SELECT * FROM SYS.WM_TRIGGERS
PREHOOK: type: QUERY
@@ -4087,7 +4099,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_TRIGGERS
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@wm_triggers
#### A masked pattern was here ####
-plan_1 default trigger_1 BYTES_READ > '1min' KILL
+plan_1 default trigger_1 BYTES_READ > '1GB' KILL
plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool
PREHOOK: query: DROP TRIGGER plan_1.trigger_1
PREHOOK: type: DROP TRIGGER
@@ -4107,10 +4119,10 @@ PREHOOK: query: CREATE TRIGGER plan_2.trigger_1 WHEN
BYTES_READ > '100mb' DO MOV
PREHOOK: type: CREATE TRIGGER
PREHOOK: Output: dummyHostnameForTest
FAILED: Execution Error, return code 1 from
org.apache.hadoop.hive.ql.exec.DDLTask.
InvalidOperationException(message:Resource plan must be disabled to edit it.)
-PREHOOK: query: CREATE TRIGGER `table`.`table` WHEN BYTES_WRITTEN > '100KB' DO
MOVE TO `table`
+PREHOOK: query: CREATE TRIGGER `table`.`table` WHEN BYTES_WRITTEN > '100KB' DO
MOVE TO `default`
PREHOOK: type: CREATE TRIGGER
PREHOOK: Output: dummyHostnameForTest
-POSTHOOK: query: CREATE TRIGGER `table`.`table` WHEN BYTES_WRITTEN > '100KB'
DO MOVE TO `table`
+POSTHOOK: query: CREATE TRIGGER `table`.`table` WHEN BYTES_WRITTEN > '100KB'
DO MOVE TO `default`
POSTHOOK: type: CREATE TRIGGER
PREHOOK: query: CREATE TRIGGER `table`.`trigger` WHEN BYTES_WRITTEN > '100MB'
DO MOVE TO `default`
PREHOOK: type: CREATE TRIGGER
@@ -4142,7 +4154,7 @@ POSTHOOK: Input: sys@wm_triggers
#### A masked pattern was here ####
plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool
table default database BYTES_WRITTEN > "1GB" MOVE TO default
-table default table BYTES_WRITTEN > '100KB' MOVE TO table
+table default table BYTES_WRITTEN > '100KB' MOVE TO default
table default trigger BYTES_WRITTEN > '100MB' MOVE TO default
table default trigger1 ELAPSED_TIME > 10 KILL
table default trigger2 ELAPSED_TIME > '1hour' KILL
@@ -4160,7 +4172,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: sys@wm_triggers
#### A masked pattern was here ####
plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool
-table default table BYTES_WRITTEN > '100KB' MOVE TO table
+table default table BYTES_WRITTEN > '100KB' MOVE TO default
table default trigger BYTES_WRITTEN > '100MB' MOVE TO default
table default trigger1 ELAPSED_TIME > 10 KILL
table default trigger2 ELAPSED_TIME > '1hour' KILL
@@ -4232,7 +4244,7 @@ POSTHOOK: Input: sys@wm_triggers
#### A masked pattern was here ####
plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool
plan_2 default trigger_1 BYTES_READ > 0 MOVE TO null_pool
-table default table BYTES_WRITTEN > '100KB' MOVE TO table
+table default table BYTES_WRITTEN > '100KB' MOVE TO default
table default trigger BYTES_WRITTEN > '100MB' MOVE TO default
table default trigger1 ELAPSED_TIME > 10 KILL
table default trigger2 ELAPSED_TIME > '1hour' KILL
@@ -4615,13 +4627,13 @@ POSTHOOK: query: SHOW RESOURCE PLAN `table`
POSTHOOK: type: SHOW RESOURCEPLAN
table[status=DISABLED,parallelism=1,defaultPool=null]
+ table[allocFraction=0.0,schedulingPolicy=fifo,parallelism=1]
- | trigger table: if (BYTES_WRITTEN > '100KB') { MOVE TO table }
+ | trigger table: if (BYTES_WRITTEN > '100KB') { MOVE TO default }
+ pool[allocFraction=0.9,schedulingPolicy=fair,parallelism=3]
+ child2[allocFraction=0.7,schedulingPolicy=fair,parallelism=3]
| trigger trigger1: if (ELAPSED_TIME > 10) { KILL }
| trigger trigger2: if (ELAPSED_TIME > '1hour') { KILL }
+ child1[allocFraction=0.3,schedulingPolicy=fair,parallelism=1]
- | trigger table: if (BYTES_WRITTEN > '100KB') { MOVE TO table }
+ | trigger table: if (BYTES_WRITTEN > '100KB') { MOVE TO default }
| trigger trigger1: if (ELAPSED_TIME > 10) { KILL }
+ <unmanaged queries>
| trigger trigger1: if (ELAPSED_TIME > 10) { KILL }
@@ -4871,7 +4883,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: sys@wm_triggers
#### A masked pattern was here ####
plan_1 default trigger_2 ELAPSED_TIME > '30hour' MOVE TO slow_pool
-table default table BYTES_WRITTEN > '100KB' MOVE TO table
+table default table BYTES_WRITTEN > '100KB' MOVE TO default
table default trigger BYTES_WRITTEN > '100MB' MOVE TO default
table default trigger1 ELAPSED_TIME > 10 KILL
table default trigger2 ELAPSED_TIME > '1hour' KILL
@@ -4977,7 +4989,7 @@ plan_4a default trigger_1 BYTES_READ > '10GB'
KILL
plan_4a default trigger_2 BYTES_READ > '11GB' KILL
plan_4b default trigger_1 BYTES_READ > '10GB' KILL
plan_4b default trigger_2 BYTES_READ > '11GB' KILL
-table default table BYTES_WRITTEN > '100KB' MOVE TO table
+table default table BYTES_WRITTEN > '100KB' MOVE TO default
table default trigger BYTES_WRITTEN > '100MB' MOVE TO default
table default trigger1 ELAPSED_TIME > 10 KILL
table default trigger2 ELAPSED_TIME > '1hour' KILL
diff --git a/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
b/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
index c7f2c91..d9a5033 100644
--- a/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
+++ b/service/src/java/org/apache/hive/service/server/KillQueryImpl.java
@@ -162,7 +162,7 @@ public class KillQueryImpl implements KillQuery {
killChildYarnJobs(conf, queryTag);
} else {
// no privilege to cancel
- throw new HiveSQLException("No privilege");
+ throw new HiveSQLException("No privilege to kill query id");
}
break;
case TAG:
@@ -174,7 +174,7 @@ public class KillQueryImpl implements KillQuery {
}
killChildYarnJobs(conf, queryIdOrTag);
if (numCanceled == 0) {
- throw new HiveSQLException("No privilege");
+ throw new HiveSQLException("No privilege to kill query tag");
}
break;
case UNKNOWN:
