HIVE-18438 : WM RP: it's impossible to unset things (Sergey Shelukhin, reviewed by Harish Jaiprakash, Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/900da829 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/900da829 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/900da829 Branch: refs/heads/master Commit: 900da82915209630ec7c194619028e5fa1b87d28 Parents: 572eacc Author: sergey <ser...@apache.org> Authored: Thu Jan 18 13:28:54 2018 -0800 Committer: sergey <ser...@apache.org> Committed: Thu Jan 18 13:28:54 2018 -0800 ---------------------------------------------------------------------- .../listener/DummyRawStoreFailEvent.java | 7 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 7 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 13 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 41 +- .../hadoop/hive/ql/parse/ResourcePlanParser.g | 17 + .../hive/ql/plan/AlterResourcePlanDesc.java | 9 +- .../hive/ql/plan/CreateOrAlterWMPoolDesc.java | 32 +- .../test/queries/clientpositive/resourceplan.q | 13 + .../clientpositive/llap/resourceplan.q.out | 124 +- .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2326 +++++++++--------- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 1295 ++++++---- .../gen/thrift/gen-cpp/hive_metastore_types.h | 179 +- .../hive/metastore/api/WMAlterPoolRequest.java | 16 +- .../api/WMAlterResourcePlanRequest.java | 16 +- .../hive/metastore/api/WMNullablePool.java | 901 +++++++ .../metastore/api/WMNullableResourcePlan.java | 918 +++++++ .../src/gen/thrift/gen-php/metastore/Types.php | 392 ++- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 275 ++- .../gen/thrift/gen-rb/hive_metastore_types.rb | 62 +- .../hive/metastore/HiveMetaStoreClient.java | 4 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 6 +- .../hadoop/hive/metastore/ObjectStore.java | 50 +- .../apache/hadoop/hive/metastore/RawStore.java | 6 +- .../hive/metastore/cache/CachedStore.java | 9 +- .../src/main/thrift/hive_metastore.thrift | 23 +- .../DummyRawStoreControlledCommit.java | 6 +- .../DummyRawStoreForJdoConnection.java | 6 +- 27 files changed, 5018 insertions(+), 1735 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index bc9ef62..140b566 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -58,6 +58,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.WMResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; @@ -73,6 +74,7 @@ import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.WMMapping; import org.apache.hadoop.hive.metastore.api.WMPool; +import org.apache.hadoop.hive.metastore.api.WMNullablePool; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; import org.apache.thrift.TException; @@ -997,7 +999,8 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public WMFullResourcePlan alterResourcePlan(String name, WMResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) + public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, + boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { return objectStore.alterResourcePlan(name, resourcePlan, canActivateDisabled, canDeactivate, isReplace); } @@ -1050,7 +1053,7 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public void alterPool(WMPool pool, String poolPath) throws AlreadyExistsException, + public void alterPool(WMNullablePool pool, String poolPath) throws AlreadyExistsException, NoSuchObjectException, InvalidOperationException, MetaException { objectStore.alterPool(pool, poolPath); } http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 81e1dd9..aa66a39 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -105,6 +105,7 @@ import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.TxnInfo; import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; import org.apache.hadoop.hive.metastore.api.WMResourcePlan; import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus; import org.apache.hadoop.hive.metastore.api.WMTrigger; @@ -725,7 +726,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable { return 0; } - WMResourcePlan resourcePlan = desc.getResourcePlan(); + WMNullableResourcePlan resourcePlan = desc.getResourcePlan(); final WorkloadManager wm = WorkloadManager.getInstance(); final TezSessionPoolManager pm = TezSessionPoolManager.getInstance(); boolean isActivate = false, isInTest = HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST); @@ -818,9 +819,9 @@ public class DDLTask extends Task<DDLWork> implements Serializable { private int createOrAlterWMPool(Hive db, CreateOrAlterWMPoolDesc desc) throws HiveException { if (desc.isUpdate()) { - db.alterWMPool(desc.getPool(), desc.getPoolPath()); + db.alterWMPool(desc.getAlterPool(), desc.getPoolPath()); } else { - db.createWMPool(desc.getPool()); + db.createWMPool(desc.getCreatePool()); } return 0; } http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index aa9bf9a..16861bc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -132,6 +132,8 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; import org.apache.hadoop.hive.metastore.api.WMMapping; +import org.apache.hadoop.hive.metastore.api.WMNullablePool; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.api.WMResourcePlan; import org.apache.hadoop.hive.metastore.api.WMTrigger; @@ -4857,12 +4859,11 @@ private void constructOneLBLocationMap(FileStatus fSta, } } - public WMFullResourcePlan alterResourcePlan(String rpName, WMResourcePlan resourcePlan, - boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) - throws HiveException { + public WMFullResourcePlan alterResourcePlan(String rpName, WMNullableResourcePlan resourcePlan, + boolean canActivateDisabled, boolean isForceDeactivate, boolean isReplace) throws HiveException { try { - return getMSC().alterResourcePlan( - rpName, resourcePlan, canActivateDisabled, isForceDeactivate, isReplace); + return getMSC().alterResourcePlan(rpName, resourcePlan, canActivateDisabled, + isForceDeactivate, isReplace); } catch (Exception e) { throw new HiveException(e); } @@ -4916,7 +4917,7 @@ private void constructOneLBLocationMap(FileStatus fSta, } } - public void alterWMPool(WMPool pool, String poolPath) throws HiveException { + public void alterWMPool(WMNullablePool pool, String poolPath) throws HiveException { try { getMSC().alterWMPool(pool, poolPath); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index db86320..a731579 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -62,6 +62,8 @@ import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.WMMapping; +import org.apache.hadoop.hive.metastore.api.WMNullablePool; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.api.WMResourcePlan; import org.apache.hadoop.hive.metastore.api.WMResourcePlanStatus; @@ -951,7 +953,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { // This command exists solely to output this message. TODO: can we do it w/o an error? throw new SemanticException("Activate a resource plan to enable workload management"); case HiveParser.TOK_DISABLE: - WMResourcePlan anyRp = new WMResourcePlan(); + WMNullableResourcePlan anyRp = new WMNullableResourcePlan(); anyRp.setStatus(WMResourcePlanStatus.ENABLED); AlterResourcePlanDesc desc = new AlterResourcePlanDesc( anyRp, null, false, false, true, false); @@ -963,7 +965,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException("Invalid syntax for ALTER RESOURCE PLAN statement"); } String rpName = unescapeIdentifier(ast.getChild(0).getText()); - WMResourcePlan resourcePlan = new WMResourcePlan(); + WMNullableResourcePlan resourcePlan = new WMNullableResourcePlan(); boolean isEnableActivate = false, isReplace = false; boolean validate = false; for (int i = 1; i < ast.getChildCount(); ++i) { @@ -1008,18 +1010,32 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { resourcePlan.setStatus(WMResourcePlanStatus.ACTIVE); } break; - case HiveParser.TOK_QUERY_PARALLELISM: + case HiveParser.TOK_QUERY_PARALLELISM: { if (child.getChildCount() != 1) { throw new SemanticException("Expected one argument"); } - resourcePlan.setQueryParallelism(Integer.parseInt(child.getChild(0).getText())); + Tree val = child.getChild(0); + resourcePlan.setIsSetQueryParallelism(true); + if (val.getType() == HiveParser.TOK_NULL) { + resourcePlan.unsetQueryParallelism(); + } else { + resourcePlan.setQueryParallelism(Integer.parseInt(val.getText())); + } break; - case HiveParser.TOK_DEFAULT_POOL: + } + case HiveParser.TOK_DEFAULT_POOL: { if (child.getChildCount() != 1) { throw new SemanticException("Expected one argument"); } - resourcePlan.setDefaultPoolPath(poolPath(child.getChild(0))); + Tree val = child.getChild(0); + resourcePlan.setIsSetDefaultPoolPath(true); + if (val.getType() == HiveParser.TOK_NULL) { + resourcePlan.unsetDefaultPoolPath(); + } else { + resourcePlan.setDefaultPoolPath(poolPath(child.getChild(0))); + } break; + } case HiveParser.TOK_RENAME: if (child.getChildCount() != 1) { throw new SemanticException("Expected one argument"); @@ -1179,7 +1195,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { private void analyzeAlterPool(ASTNode ast) throws SemanticException { if (ast.getChildCount() < 3) { - throw new SemanticException("Invalid syntax for alter pool."); + throw new SemanticException("Invalid syntax for alter pool: " + ast.toStringTree()); } String rpName = unescapeIdentifier(ast.getChild(0).getText()); Tree poolTarget = ast.getChild(1); @@ -1192,9 +1208,11 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { poolPath = poolPath(ast.getChild(1)); } - WMPool poolChanges = null; + WMNullablePool poolChanges = null; for (int i = 2; i < ast.getChildCount(); ++i) { Tree child = ast.getChild(i); + LOG.error("TODO# got2 " + child.toStringTree()); + if (child.getChildCount() != 1) { throw new SemanticException("Invalid syntax in alter pool expected parameter."); } @@ -1211,7 +1229,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException("Cannot alter the unmanaged pool"); } if (poolChanges == null) { - poolChanges = new WMPool(rpName, null); + poolChanges = new WMNullablePool(rpName, null); } switch (child.getType()) { case HiveParser.TOK_ALLOC_FRACTION: @@ -1221,7 +1239,10 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { poolChanges.setQueryParallelism(Integer.parseInt(param.getText())); break; case HiveParser.TOK_SCHEDULING_POLICY: - poolChanges.setSchedulingPolicy(PlanUtils.stripQuotes(param.getText())); + poolChanges.setIsSetSchedulingPolicy(true); + if (param.getType() != HiveParser.TOK_NULL) { + poolChanges.setSchedulingPolicy(PlanUtils.stripQuotes(param.getText())); + } break; case HiveParser.TOK_PATH: poolChanges.setPoolPath(poolPath(param)); http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g index b47f7b3..e3ea1f2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g @@ -56,6 +56,21 @@ rpAssignList : rpAssign (COMMA rpAssign)* -> rpAssign+ ; +rpUnassign +@init { gParent.pushMsg("rpAssign", state); } +@after { gParent.popMsg(state); } + : ( + (KW_QUERY_PARALLELISM) -> ^(TOK_QUERY_PARALLELISM TOK_NULL) + | (KW_DEFAULT KW_POOL) -> ^(TOK_DEFAULT_POOL TOK_NULL) + ) + ; + +rpUnassignList +@init { gParent.pushMsg("rpAssignList", state); } +@after { gParent.popMsg(state); } + : rpUnassign (COMMA rpUnassign)* -> rpUnassign+ + ; + createResourcePlanStatement @init { gParent.pushMsg("create resource plan statement", state); } @after { gParent.popMsg(state); } @@ -80,6 +95,7 @@ alterResourcePlanStatement (KW_VALIDATE -> ^(TOK_ALTER_RP $name TOK_VALIDATE)) | (KW_DISABLE -> ^(TOK_ALTER_RP $name TOK_DISABLE)) | (KW_SET rpAssignList -> ^(TOK_ALTER_RP $name rpAssignList)) + | (KW_UNSET rpUnassignList -> ^(TOK_ALTER_RP $name rpUnassignList)) | (KW_RENAME KW_TO newName=identifier -> ^(TOK_ALTER_RP $name ^(TOK_RENAME $newName))) | ((activate enable? | enable activate?) -> ^(TOK_ALTER_RP $name activate? enable?)) ) @@ -219,6 +235,7 @@ alterPoolStatement @after { gParent.popMsg(state); } : KW_ALTER KW_POOL rpName=identifier DOT poolPath ( (KW_SET poolAssignList -> ^(TOK_ALTER_POOL $rpName poolPath poolAssignList)) + | (KW_UNSET KW_SCHEDULING_POLICY -> ^(TOK_ALTER_POOL $rpName poolPath ^(TOK_SCHEDULING_POLICY TOK_NULL))) | (KW_ADD KW_TRIGGER triggerName=identifier -> ^(TOK_ALTER_POOL $rpName poolPath ^(TOK_ADD_TRIGGER $triggerName))) | (KW_DROP KW_TRIGGER triggerName=identifier http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java index 7339520..dd2eace 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; import org.apache.hadoop.hive.metastore.api.WMResourcePlan; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -28,7 +29,7 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level; public class AlterResourcePlanDesc extends DDLDesc implements Serializable { private static final long serialVersionUID = -3514685833183437279L; - private WMResourcePlan resourcePlan; + private WMNullableResourcePlan resourcePlan; private String rpName; private boolean validate; private boolean isEnableActivate, isForceDeactivate, isReplace; @@ -36,7 +37,7 @@ public class AlterResourcePlanDesc extends DDLDesc implements Serializable { public AlterResourcePlanDesc() {} - public AlterResourcePlanDesc(WMResourcePlan resourcePlan, String rpName, boolean validate, + public AlterResourcePlanDesc(WMNullableResourcePlan resourcePlan, String rpName, boolean validate, boolean isEnableActivate, boolean isForceDeactivate, boolean isReplace) { this.resourcePlan = resourcePlan; this.rpName = rpName; @@ -48,11 +49,11 @@ public class AlterResourcePlanDesc extends DDLDesc implements Serializable { @Explain(displayName="Resource plan changed fields", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public WMResourcePlan getResourcePlan() { + public WMNullableResourcePlan getResourcePlan() { return resourcePlan; } - public void setResourcePlan(WMResourcePlan resourcePlan) { + public void setResourcePlan(WMNullableResourcePlan resourcePlan) { this.resourcePlan = resourcePlan; } http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateOrAlterWMPoolDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateOrAlterWMPoolDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateOrAlterWMPoolDesc.java index 73806c5..6835faa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateOrAlterWMPoolDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateOrAlterWMPoolDesc.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import org.apache.hadoop.hive.metastore.api.WMNullablePool; import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -26,25 +27,44 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level; public class CreateOrAlterWMPoolDesc extends DDLDesc implements Serializable { private static final long serialVersionUID = 4872940135771213510L; - private WMPool pool; + private WMPool createPool; + private WMNullablePool alterPool; private String poolPath; private boolean update; public CreateOrAlterWMPoolDesc() {} public CreateOrAlterWMPoolDesc(WMPool pool, String poolPath, boolean update) { - this.pool = pool; + this.createPool = pool; + this.poolPath = poolPath; + this.update = update; + } + + public CreateOrAlterWMPoolDesc(WMNullablePool pool, String poolPath, boolean update) { + this.alterPool = pool; this.poolPath = poolPath; this.update = update; } @Explain(displayName="pool", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public WMPool getPool() { - return pool; + public Object getPool() { + return createPool == null ? alterPool : createPool; + } + + public WMPool getCreatePool() { + return createPool; + } + + public WMNullablePool getAlterPool() { + return alterPool; + } + + public void setCreatePool(WMPool pool) { + this.createPool = pool; } - public void setPool(WMPool pool) { - this.pool = pool; + public void setAlterPool(WMNullablePool pool) { + this.alterPool = pool; } @Explain(displayName="poolPath", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/test/queries/clientpositive/resourceplan.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/resourceplan.q b/ql/src/test/queries/clientpositive/resourceplan.q index c616c22..d2aec73 100644 --- a/ql/src/test/queries/clientpositive/resourceplan.q +++ b/ql/src/test/queries/clientpositive/resourceplan.q @@ -51,6 +51,11 @@ SELECT * FROM SYS.WM_RESOURCEPLANS; ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 4; SELECT * FROM SYS.WM_RESOURCEPLANS; +-- Change query parallelism, success. +ALTER RESOURCE PLAN plan_3 UNSET QUERY_PARALLELISM; +SELECT * FROM SYS.WM_RESOURCEPLANS; + + -- Will fail for now; there are no pools. ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 30, DEFAULT POOL = default1; SELECT * FROM SYS.WM_RESOURCEPLANS; @@ -214,6 +219,10 @@ ALTER RESOURCE PLAN plan_2 VALIDATE; ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE; ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1; +ALTER POOL plan_2.default.c2 SET SCHEDULING_POLICY='fair'; +SELECT * FROM SYS.WM_POOLS; +ALTER POOL plan_2.default.c2 UNSET SCHEDULING_POLICY; +SELECT * FROM SYS.WM_POOLS; -- Now we can activate. ALTER RESOURCE PLAN plan_2 VALIDATE; @@ -262,6 +271,10 @@ ALTER RESOURCE PLAN `table` SET DEFAULT POOL = `table`.pool; DROP POOL `table`.default; SELECT * FROM SYS.WM_POOLS; +-- Change query parallelism, success. +ALTER RESOURCE PLAN `table` UNSET DEFAULT POOL; +SELECT * FROM SYS.WM_RESOURCEPLANS; + -- -- Pool to trigger mappings. -- http://git-wip-us.apache.org/repos/asf/hive/blob/900da829/ql/src/test/results/clientpositive/llap/resourceplan.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index 22883bc..68508e9 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -3279,6 +3279,20 @@ POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default plan_3 DISABLED 4 default +PREHOOK: query: ALTER RESOURCE PLAN plan_3 UNSET QUERY_PARALLELISM +PREHOOK: type: ALTER RESOURCEPLAN +POSTHOOK: query: ALTER RESOURCE PLAN plan_3 UNSET QUERY_PARALLELISM +POSTHOOK: type: ALTER RESOURCEPLAN +PREHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS +PREHOOK: type: QUERY +PREHOOK: Input: sys@wm_resourceplans +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@wm_resourceplans +#### A masked pattern was here #### +plan_2 DISABLED 4 default +plan_3 DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 30, DEFAULT POOL = default1 PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. NoSuchObjectException(message:Cannot find pool: default1) @@ -3291,7 +3305,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 DISABLED 4 default +plan_3 DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE @@ -3315,7 +3329,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 DISABLED 4 default +plan_3 DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Resource plan plan_3 is disabled and should be enabled before activation (or in the same command)) @@ -3328,7 +3342,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 DISABLED 4 default +plan_3 DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE @@ -3342,7 +3356,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 DISABLED 4 default +plan_3 DISABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE @@ -3356,7 +3370,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 ENABLED 4 default +plan_3 ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE @@ -3370,7 +3384,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 ACTIVE 4 default +plan_3 ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE @@ -3384,7 +3398,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 ACTIVE 4 default +plan_3 ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Resource plan plan_3 is active; activate another plan first, or disable workload management.) @@ -3397,7 +3411,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 ACTIVE 4 default +plan_3 ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Resource plan plan_3 is active; activate another plan first, or disable workload management.) @@ -3410,7 +3424,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 ACTIVE 4 default +plan_3 ACTIVE NULL default PREHOOK: query: DISABLE WORKLOAD MANAGEMENT PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: DISABLE WORKLOAD MANAGEMENT @@ -3424,7 +3438,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 ENABLED 4 default +plan_3 ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE @@ -3442,7 +3456,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 DISABLED 4 default -plan_3 ACTIVE 4 default +plan_3 ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE @@ -3456,7 +3470,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 ENABLED 4 default -plan_3 ACTIVE 4 default +plan_3 ACTIVE NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_2 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_2 ACTIVATE @@ -3470,7 +3484,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 ACTIVE 4 default -plan_3 ENABLED 4 default +plan_3 ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE @@ -3484,7 +3498,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 ACTIVE 4 default -plan_3 ENABLED 4 default +plan_3 ENABLED NULL default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE @@ -3498,7 +3512,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_2 ACTIVE 4 default -plan_3 DISABLED 4 default +plan_3 DISABLED NULL default PREHOOK: query: DROP RESOURCE PLAN plan_2 PREHOOK: type: DROP RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Cannot drop an active resource plan) @@ -3757,6 +3771,40 @@ PREHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PAR PREHOOK: type: ALTER POOL POSTHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1 POSTHOOK: type: ALTER POOL +PREHOOK: query: ALTER POOL plan_2.default.c2 SET SCHEDULING_POLICY='fair' +PREHOOK: type: ALTER POOL +POSTHOOK: query: ALTER POOL plan_2.default.c2 SET SCHEDULING_POLICY='fair' +POSTHOOK: type: ALTER POOL +PREHOOK: query: SELECT * FROM SYS.WM_POOLS +PREHOOK: type: QUERY +PREHOOK: Input: sys@wm_pools +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM SYS.WM_POOLS +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@wm_pools +#### A masked pattern was here #### +plan_1 default 1.0 4 NULL +plan_2 default 1.0 4 NULL +plan_2 default.c1 0.3 3 fair +plan_2 default.c2 0.7 1 fair +table default 1.0 4 NULL +PREHOOK: query: ALTER POOL plan_2.default.c2 UNSET SCHEDULING_POLICY +PREHOOK: type: ALTER POOL +POSTHOOK: query: ALTER POOL plan_2.default.c2 UNSET SCHEDULING_POLICY +POSTHOOK: type: ALTER POOL +PREHOOK: query: SELECT * FROM SYS.WM_POOLS +PREHOOK: type: QUERY +PREHOOK: Input: sys@wm_pools +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM SYS.WM_POOLS +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@wm_pools +#### A masked pattern was here #### +plan_1 default 1.0 4 NULL +plan_2 default 1.0 4 NULL +plan_2 default.c1 0.3 3 fair +plan_2 default.c2 0.7 1 NULL +table default 1.0 4 NULL PREHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE @@ -3788,7 +3836,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.7 1 fair +plan_2 def.c2 0.7 1 NULL table default 1.0 4 NULL PREHOOK: query: DROP POOL plan_2.default PREHOOK: type: DROP POOL @@ -3804,7 +3852,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.7 1 fair +plan_2 def.c2 0.7 1 NULL table default 1.0 4 NULL PREHOOK: query: CREATE POOL plan_2.child1.child2 WITH QUERY_PARALLELISM=2, SCHEDULING_POLICY='fifo', ALLOC_FRACTION=0.8 @@ -3849,7 +3897,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.7 1 fair +plan_2 def.c2 0.7 1 NULL table default 1.0 4 NULL table table 0.0 1 fifo table table.pool1 0.9 3 fair @@ -3870,7 +3918,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.7 1 fair +plan_2 def.c2 0.7 1 NULL table default 1.0 4 NULL table table 0.0 1 fifo table table.pool 0.9 3 fair @@ -3890,7 +3938,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.7 1 fair +plan_2 def.c2 0.7 1 NULL table default 1.0 4 NULL table table 0.0 1 fifo table table.pool 0.9 3 fair @@ -3910,7 +3958,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.7 1 fair +plan_2 def.c2 0.7 1 NULL table default 1.0 4 NULL table table 0.0 1 fifo table table.pool 0.9 3 fair @@ -3946,11 +3994,26 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 fair -plan_2 def.c2 0.7 1 fair +plan_2 def.c2 0.7 1 NULL table table 0.0 1 fifo table table.pool 0.9 3 fair table table.pool.child1 0.3 1 fair table table.pool.child2 0.7 3 fair +PREHOOK: query: ALTER RESOURCE PLAN `table` UNSET DEFAULT POOL +PREHOOK: type: ALTER RESOURCEPLAN +POSTHOOK: query: ALTER RESOURCE PLAN `table` UNSET DEFAULT POOL +POSTHOOK: type: ALTER RESOURCEPLAN +PREHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS +PREHOOK: type: QUERY +PREHOOK: Input: sys@wm_resourceplans +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@wm_resourceplans +#### A masked pattern was here #### +plan_1 ACTIVE NULL default +plan_2 DISABLED 4 def +table DISABLED 1 NULL PREHOOK: query: ALTER POOL plan_2.def.c1 ADD TRIGGER trigger_1 PREHOOK: type: ALTER POOL POSTHOOK: query: ALTER POOL plan_2.def.c1 ADD TRIGGER trigger_1 @@ -4003,11 +4066,10 @@ PREHOOK: query: SHOW RESOURCE PLAN `table` PREHOOK: type: SHOW RESOURCEPLAN POSTHOOK: query: SHOW RESOURCE PLAN `table` POSTHOOK: type: SHOW RESOURCEPLAN -table[status=DISABLED,parallelism=1,defaultPool=table.pool] +table[status=DISABLED,parallelism=1,defaultPool=null] + table[allocFraction=0.0,schedulingPolicy=fifo,parallelism=1] | trigger table: if (BYTES_WRITTEN > 100K) { MOVE TO table } + pool[allocFraction=0.9,schedulingPolicy=fair,parallelism=3] - | mapped for default + child2[allocFraction=0.7,schedulingPolicy=fair,parallelism=3] | trigger trigger2: if (BYTES_READ > 100) { KILL } | trigger trigger1: if (ELAPSED_TIME > 10) { KILL } @@ -4121,7 +4183,7 @@ plan_2[status=DISABLED,parallelism=4,defaultPool=def] + def[allocFraction=1.0,schedulingPolicy=null,parallelism=4] | mapped for users: user2 | mapped for default - + c2[allocFraction=0.7,schedulingPolicy=fair,parallelism=1] + + c2[allocFraction=0.7,schedulingPolicy=null,parallelism=1] | trigger trigger_1: if (BYTES_READ = 0) { MOVE TO null_pool } | mapped for groups: group2 + c1[allocFraction=0.3,schedulingPolicy=fair,parallelism=3] @@ -4188,7 +4250,7 @@ POSTHOOK: type: SHOW RESOURCEPLAN plan_2[status=DISABLED,parallelism=4,defaultPool=def] + def[allocFraction=1.0,schedulingPolicy=null,parallelism=4] | mapped for default - + c2[allocFraction=0.7,schedulingPolicy=fair,parallelism=1] + + c2[allocFraction=0.7,schedulingPolicy=null,parallelism=1] | trigger trigger_1: if (BYTES_READ = 0) { MOVE TO null_pool } + c1[allocFraction=0.3,schedulingPolicy=fair,parallelism=3] | mapped for groups: group1 @@ -4213,7 +4275,7 @@ POSTHOOK: Input: sys@wm_resourceplans plan_1 ENABLED NULL default plan_2 DISABLED NULL default plan_4 ACTIVE NULL default -table DISABLED 1 table.pool +table DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4302,7 +4364,7 @@ plan_2 DISABLED NULL default plan_4 ACTIVE NULL default plan_4a DISABLED NULL default plan_4b DISABLED NULL default -table DISABLED 1 table.pool +table DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4376,7 +4438,7 @@ plan_2 DISABLED NULL default plan_4 ACTIVE NULL default plan_4a DISABLED NULL default plan_4a-old-0 DISABLED NULL default -table DISABLED 1 table.pool +table DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools @@ -4412,7 +4474,7 @@ plan_2 DISABLED NULL default plan_4 ACTIVE NULL default plan_4-old-0 DISABLED NULL default plan_4a-old-0 DISABLED NULL default -table DISABLED 1 table.pool +table DISABLED 1 NULL PREHOOK: query: CREATE RESOURCE PLAN plan_4a LIKE plan_4 PREHOOK: type: CREATE RESOURCEPLAN POSTHOOK: query: CREATE RESOURCE PLAN plan_4a LIKE plan_4 @@ -4439,7 +4501,7 @@ plan_4 ACTIVE NULL default plan_4-old-0 DISABLED NULL default plan_4-old-1 DISABLED NULL default plan_4a-old-0 DISABLED NULL default -table DISABLED 1 table.pool +table DISABLED 1 NULL PREHOOK: query: SELECT * FROM SYS.WM_POOLS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_pools
