HIVE-18179 : Implement validate resource plan (part 1) (Harish Jaiprakash, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d832335 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d832335 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d832335 Branch: refs/heads/master Commit: 0d8323357c9bf944f80823d8607d03f1b1f5fc52 Parents: fb85336 Author: sergey <[email protected]> Authored: Tue Dec 5 15:54:14 2017 -0800 Committer: sergey <[email protected]> Committed: Tue Dec 5 15:54:14 2017 -0800 ---------------------------------------------------------------------- .../listener/DummyRawStoreFailEvent.java | 4 +- .../hive/metastore/HiveMetaStoreClient.java | 4 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 2 +- .../DummyRawStoreControlledCommit.java | 2 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 8 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 2 +- .../formatting/JsonMetaDataFormatter.java | 19 + .../metadata/formatting/MetaDataFormatter.java | 3 + .../formatting/TextMetaDataFormatter.java | 10 + .../hive/ql/parse/DDLSemanticAnalyzer.java | 12 +- .../hive/ql/plan/AlterResourcePlanDesc.java | 14 + .../test/queries/clientpositive/resourceplan.q | 18 +- .../clientpositive/llap/resourceplan.q.out | 131 +- .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2132 ++++++++-------- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 518 ++-- .../gen/thrift/gen-cpp/hive_metastore_types.h | 14 +- .../hive/metastore/api/ThriftHiveMetastore.java | 2324 +++++++++--------- .../WMGetTriggersForResourePlanResponse.java | 36 +- .../api/WMValidateResourcePlanResponse.java | 178 +- .../gen-php/metastore/ThriftHiveMetastore.php | 1308 +++++----- .../src/gen/thrift/gen-php/metastore/Types.php | 68 +- .../hive_metastore/ThriftHiveMetastore.py | 882 +++---- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 42 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 4 +- .../hadoop/hive/metastore/HiveMetaStore.java | 4 +- .../hadoop/hive/metastore/ObjectStore.java | 85 +- .../apache/hadoop/hive/metastore/RawStore.java | 2 +- .../hive/metastore/cache/CachedStore.java | 2 +- .../src/main/thrift/hive_metastore.thrift | 2 +- .../DummyRawStoreForJdoConnection.java | 4 +- 30 files changed, 4052 insertions(+), 3782 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 7965ca3..62c9172 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1008,10 +1008,10 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { @Override public WMFullResourcePlan getActiveResourcePlan() throws MetaException { return objectStore.getActiveResourcePlan(); - } + } @Override - public boolean validateResourcePlan(String name) + public List<String> validateResourcePlan(String name) throws NoSuchObjectException, InvalidObjectException, MetaException { return objectStore.validateResourcePlan(name); } http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 4a32704..4afc03b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -2685,11 +2685,11 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { } @Override - public boolean validateResourcePlan(String resourcePlanName) + public List<String> validateResourcePlan(String resourcePlanName) throws NoSuchObjectException, InvalidObjectException, MetaException, TException { WMValidateResourcePlanRequest request = new WMValidateResourcePlanRequest(); request.setResourcePlanName(resourcePlanName); - return client.validate_resource_plan(request).isIsValid(); + return client.validate_resource_plan(request).getErrors(); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 0020136..40e7930 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -1790,7 +1790,7 @@ public interface IMetaStoreClient { WMFullResourcePlan getActiveResourcePlan() throws MetaException, TException; - boolean validateResourcePlan(String resourcePlanName) + List<String> validateResourcePlan(String resourcePlanName) throws NoSuchObjectException, InvalidObjectException, MetaException, TException; void createWMTrigger(WMTrigger trigger) http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index a0a6e181..4f7b9c8 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -972,7 +972,7 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override - public boolean validateResourcePlan(String name) + public List<String> validateResourcePlan(String name) throws NoSuchObjectException, InvalidObjectException, MetaException { return objectStore.validateResourcePlan(name); } http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index a3c21fb..d6bf746 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -713,7 +713,13 @@ public class DDLTask extends Task<DDLWork> implements Serializable { private int alterResourcePlan(Hive db, AlterResourcePlanDesc desc) throws HiveException { if (desc.shouldValidate()) { - return db.validateResourcePlan(desc.getResourcePlanName()) ? 0 : 1; + List<String> errors = db.validateResourcePlan(desc.getResourcePlanName()); + try (DataOutputStream out = getOutputStream(desc.getResFile())) { + formatter.showErrors(out, errors); + } catch (IOException e) { + throw new HiveException(e); + }; + return 0; } WMResourcePlan resourcePlan = desc.getResourcePlan(); http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index f33855d..fbcd579 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -4799,7 +4799,7 @@ private void constructOneLBLocationMap(FileStatus fSta, } } - public boolean validateResourcePlan(String rpName) throws HiveException { + public List<String> validateResourcePlan(String rpName) throws HiveException { try { return getMSC().validateResourcePlan(rpName); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java index 2a568a7..a44adf6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/JsonMetaDataFormatter.java @@ -447,4 +447,23 @@ public class JsonMetaDataFormatter implements MetaDataFormatter { } } } + + @Override + public void showErrors(DataOutputStream out, List<String> errors) throws HiveException { + JsonGenerator generator = null; + try { + generator = new ObjectMapper().getJsonFactory().createJsonGenerator(out); + generator.writeStartArray(); + for (String error : errors) { + generator.writeString(error); + } + generator.writeEndArray(); + } catch (IOException e) { + throw new HiveException(e); + } finally { + if (generator != null) { + IOUtils.closeQuietly(generator); + } + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java index 405acdd..6ba474c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatter.java @@ -123,5 +123,8 @@ public interface MetaDataFormatter { public void showResourcePlans(DataOutputStream out, List<WMResourcePlan> resourcePlans) throws HiveException; + + public void showErrors(DataOutputStream out, List<String> errors) + throws HiveException; } http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java index a57e0ce..1b7b425 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java @@ -570,4 +570,14 @@ class TextMetaDataFormatter implements MetaDataFormatter { } } + public void showErrors(DataOutputStream out, List<String> errors) throws HiveException { + try { + for (String error : errors) { + out.write(error.getBytes("UTF-8")); + out.write(terminator); + } + } catch (IOException e) { + throw new HiveException(e); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index e5e1b53..a06a672 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -982,8 +982,16 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer { "Unexpected token in alter resource plan statement: " + child.getType()); } } - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - new AlterResourcePlanDesc(resourcePlan, rpName, validate, isEnableActive)), conf)); + AlterResourcePlanDesc desc = + new AlterResourcePlanDesc(resourcePlan, rpName, validate, isEnableActive); + if (validate) { + ctx.setResFile(ctx.getLocalTmpPath()); + desc.setResFile(ctx.getResFile().toString()); + } + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc), conf)); + if (validate) { + setFetchTask(createFetchTask(AlterResourcePlanDesc.getSchema())); + } } private void analyzeDropResourcePlan(ASTNode ast) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java index 43e6e33..d70d520 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterResourcePlanDesc.java @@ -32,6 +32,7 @@ public class AlterResourcePlanDesc extends DDLDesc implements Serializable { private String rpName; private boolean validate; private boolean isEnableActivate; + private String resFile; public AlterResourcePlanDesc() {} @@ -80,4 +81,17 @@ public class AlterResourcePlanDesc extends DDLDesc implements Serializable { public void setIsEnableActivate(boolean b) { this.isEnableActivate = b; } + + @Explain(displayName = "result file", explainLevels = { Level.EXTENDED }) + public String getResFile() { + return resFile; + } + + public void setResFile(String resFile) { + this.resFile = resFile; + } + + public static String getSchema() { + return "error#string"; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/test/queries/clientpositive/resourceplan.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/resourceplan.q b/ql/src/test/queries/clientpositive/resourceplan.q index beaf8db..002b21c 100644 --- a/ql/src/test/queries/clientpositive/resourceplan.q +++ b/ql/src/test/queries/clientpositive/resourceplan.q @@ -27,7 +27,7 @@ SHOW RESOURCE PLAN plan_1; SELECT * FROM SYS.WM_RESOURCEPLANS; -- Create and show plan_2. -CREATE RESOURCE PLAN plan_2 WITH QUERY_PARALLELISM=10; +CREATE RESOURCE PLAN plan_2 WITH QUERY_PARALLELISM=4; SHOW RESOURCE PLANS; SHOW RESOURCE PLAN plan_2; SELECT * FROM SYS.WM_RESOURCEPLANS; @@ -48,7 +48,7 @@ ALTER RESOURCE PLAN plan_1 RENAME TO plan_3; SELECT * FROM SYS.WM_RESOURCEPLANS; -- Change query parallelism, success. -ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 20; +ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 4; SELECT * FROM SYS.WM_RESOURCEPLANS; -- Will fail for now; there are no pools. @@ -194,9 +194,19 @@ CREATE POOL plan_2.default.c1 WITH ALLOC_FRACTION=0.3, QUERY_PARALLELISM=3, SCHEDULING_POLICY='priority'; CREATE POOL plan_2.default.c2 WITH - QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.7; + QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.2; -ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.2; +-- Cannot activate c1 + c2 = 0.5 +ALTER RESOURCE PLAN plan_2 VALIDATE; +ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE; + +ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1; + +-- Now we can activate. +ALTER RESOURCE PLAN plan_2 VALIDATE; +ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE; +ALTER RESOURCE PLAN plan_1 ACTIVATE; +ALTER RESOURCE PLAN plan_2 DISABLE; ALTER POOL plan_2.default SET path = def; SELECT * FROM SYS.WM_POOLS; http://git-wip-us.apache.org/repos/asf/hive/blob/0d832335/ql/src/test/results/clientpositive/llap/resourceplan.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index 3a3f5b9..093e5d5 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -3152,21 +3152,21 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_1 DISABLED NULL default -PREHOOK: query: CREATE RESOURCE PLAN plan_2 WITH QUERY_PARALLELISM=10 +PREHOOK: query: CREATE RESOURCE PLAN plan_2 WITH QUERY_PARALLELISM=4 PREHOOK: type: CREATE RESOURCEPLAN -POSTHOOK: query: CREATE RESOURCE PLAN plan_2 WITH QUERY_PARALLELISM=10 +POSTHOOK: query: CREATE RESOURCE PLAN plan_2 WITH QUERY_PARALLELISM=4 POSTHOOK: type: CREATE RESOURCEPLAN PREHOOK: query: SHOW RESOURCE PLANS PREHOOK: type: SHOW RESOURCEPLAN POSTHOOK: query: SHOW RESOURCE PLANS POSTHOOK: type: SHOW RESOURCEPLAN plan_1 DISABLED -plan_2 DISABLED 10 +plan_2 DISABLED 4 PREHOOK: query: SHOW RESOURCE PLAN plan_2 PREHOOK: type: SHOW RESOURCEPLAN POSTHOOK: query: SHOW RESOURCE PLAN plan_2 POSTHOOK: type: SHOW RESOURCEPLAN -plan_2 DISABLED 10 +plan_2 DISABLED 4 PREHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS PREHOOK: type: QUERY PREHOOK: Input: sys@wm_resourceplans @@ -3176,7 +3176,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_1 DISABLED NULL default -plan_2 DISABLED 10 default +plan_2 DISABLED 4 default FAILED: SemanticException Invalid set in create resource plan: TOK_DEFAULT_POOL PREHOOK: query: ALTER RESOURCE PLAN plan_1 RENAME TO plan_2 PREHOOK: type: ALTER RESOURCEPLAN @@ -3190,7 +3190,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_1 DISABLED NULL default -plan_2 DISABLED 10 default +plan_2 DISABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_1 RENAME TO plan_3 PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_1 RENAME TO plan_3 @@ -3203,11 +3203,11 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default +plan_2 DISABLED 4 default plan_3 DISABLED NULL default -PREHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 20 +PREHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 4 PREHOOK: type: ALTER RESOURCEPLAN -POSTHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 20 +POSTHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 4 POSTHOOK: type: ALTER RESOURCEPLAN PREHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS PREHOOK: type: QUERY @@ -3217,8 +3217,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED 20 default +plan_2 DISABLED 4 default +plan_3 DISABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 SET QUERY_PARALLELISM = 30, DEFAULT POOL = default1 PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. NoSuchObjectException(message:Cannot find pool: default1) @@ -3230,8 +3230,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED 20 default +plan_2 DISABLED 4 default +plan_3 DISABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE @@ -3254,8 +3254,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED 20 default +plan_2 DISABLED 4 default +plan_3 DISABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Resource plan plan_3 is disabled and should be enabled before activation (or in the same command)) @@ -3267,8 +3267,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED 20 default +plan_2 DISABLED 4 default +plan_3 DISABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE @@ -3281,8 +3281,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 DISABLED 20 default +plan_2 DISABLED 4 default +plan_3 DISABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE @@ -3295,8 +3295,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ENABLED 20 default +plan_2 DISABLED 4 default +plan_3 ENABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE @@ -3309,8 +3309,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE 20 default +plan_2 DISABLED 4 default +plan_3 ACTIVE 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ACTIVATE @@ -3323,8 +3323,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE 20 default +plan_2 DISABLED 4 default +plan_3 ACTIVE 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Resource plan plan_3 is active, activate another plan first.) @@ -3336,8 +3336,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE 20 default +plan_2 DISABLED 4 default +plan_3 ACTIVE 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:Resource plan plan_3 is active, activate another plan first.) @@ -3349,8 +3349,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 DISABLED 10 default -plan_3 ACTIVE 20 default +plan_2 DISABLED 4 default +plan_3 ACTIVE 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE @@ -3363,8 +3363,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ENABLED 10 default -plan_3 ACTIVE 20 default +plan_2 ENABLED 4 default +plan_3 ACTIVE 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_2 ACTIVATE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_2 ACTIVATE @@ -3377,8 +3377,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default -plan_3 ENABLED 20 default +plan_2 ACTIVE 4 default +plan_3 ENABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 ENABLE @@ -3391,8 +3391,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default -plan_3 ENABLED 20 default +plan_2 ACTIVE 4 default +plan_3 ENABLED 4 default PREHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE PREHOOK: type: ALTER RESOURCEPLAN POSTHOOK: query: ALTER RESOURCE PLAN plan_3 DISABLE @@ -3405,8 +3405,8 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default -plan_3 DISABLED 20 default +plan_2 ACTIVE 4 default +plan_3 DISABLED 4 default PREHOOK: query: DROP RESOURCE PLAN plan_2 PREHOOK: type: DROP RESOURCEPLAN FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Cannot drop an active resource plan) @@ -3422,7 +3422,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default +plan_2 ACTIVE 4 default PREHOOK: query: CREATE RESOURCE PLAN `table` PREHOOK: type: CREATE RESOURCEPLAN POSTHOOK: query: CREATE RESOURCE PLAN `table` @@ -3439,7 +3439,7 @@ POSTHOOK: query: SELECT * FROM SYS.WM_RESOURCEPLANS POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### -plan_2 ACTIVE 10 default +plan_2 ACTIVE 4 default table DISABLED 1 default PREHOOK: query: CREATE RESOURCE PLAN plan_1 PREHOOK: type: CREATE RESOURCEPLAN @@ -3569,7 +3569,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_1 ENABLED NULL default -plan_2 ACTIVE 10 default +plan_2 ACTIVE 4 default table DISABLED 1 default PREHOOK: query: DROP TRIGGER plan_1.trigger_2 PREHOOK: type: DROP TRIGGER @@ -3590,7 +3590,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_1 ACTIVE NULL default -plan_2 ENABLED 10 default +plan_2 ENABLED 4 default table DISABLED 1 default PREHOOK: query: DROP TRIGGER plan_1.trigger_2 PREHOOK: type: DROP TRIGGER @@ -3646,15 +3646,40 @@ POSTHOOK: query: CREATE POOL plan_2.default.c1 WITH ALLOC_FRACTION=0.3, QUERY_PARALLELISM=3, SCHEDULING_POLICY='priority' POSTHOOK: type: CREATE POOL PREHOOK: query: CREATE POOL plan_2.default.c2 WITH - QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.7 + QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.2 PREHOOK: type: CREATE POOL POSTHOOK: query: CREATE POOL plan_2.default.c2 WITH - QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.7 + QUERY_PARALLELISM=2, SCHEDULING_POLICY='fair', ALLOC_FRACTION=0.2 POSTHOOK: type: CREATE POOL -PREHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.2 +PREHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE +PREHOOK: type: ALTER RESOURCEPLAN +POSTHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE +POSTHOOK: type: ALTER RESOURCEPLAN +Sum of children pools' alloc fraction should be equal 1.0 got: 0.5 for pool: default +Sum of children pools' query parallelism: 5 is not equal to pool parallelism: 4 for pool: default +PREHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE +PREHOOK: type: ALTER RESOURCEPLAN +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. InvalidOperationException(message:ResourcePlan: plan_2 is invalid: [Sum of children pools' alloc fraction should be equal 1.0 got: 0.5 for pool: default, Sum of children pools' query parallelism: 5 is not equal to pool parallelism: 4 for pool: default]) +PREHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1 PREHOOK: type: ALTER POOL -POSTHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.2 +POSTHOOK: query: ALTER POOL plan_2.default.c2 SET ALLOC_FRACTION = 0.7, QUERY_PARALLELISM = 1 POSTHOOK: type: ALTER POOL +PREHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE +PREHOOK: type: ALTER RESOURCEPLAN +POSTHOOK: query: ALTER RESOURCE PLAN plan_2 VALIDATE +POSTHOOK: type: ALTER RESOURCEPLAN +PREHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE +PREHOOK: type: ALTER RESOURCEPLAN +POSTHOOK: query: ALTER RESOURCE PLAN plan_2 ENABLE ACTIVATE +POSTHOOK: type: ALTER RESOURCEPLAN +PREHOOK: query: ALTER RESOURCE PLAN plan_1 ACTIVATE +PREHOOK: type: ALTER RESOURCEPLAN +POSTHOOK: query: ALTER RESOURCE PLAN plan_1 ACTIVATE +POSTHOOK: type: ALTER RESOURCEPLAN +PREHOOK: query: ALTER RESOURCE PLAN plan_2 DISABLE +PREHOOK: type: ALTER RESOURCEPLAN +POSTHOOK: query: ALTER RESOURCE PLAN plan_2 DISABLE +POSTHOOK: type: ALTER RESOURCEPLAN PREHOOK: query: ALTER POOL plan_2.default SET path = def PREHOOK: type: ALTER POOL POSTHOOK: query: ALTER POOL plan_2.default SET path = def @@ -3670,7 +3695,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 priority -plan_2 def.c2 0.2 2 fair +plan_2 def.c2 0.7 1 fair table default 1.0 4 NULL PREHOOK: query: DROP POOL plan_2.default PREHOOK: type: DROP POOL @@ -3686,7 +3711,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 priority -plan_2 def.c2 0.2 2 fair +plan_2 def.c2 0.7 1 fair table default 1.0 4 NULL PREHOOK: query: CREATE POOL plan_2.child1.child2 WITH QUERY_PARALLELISM=2, SCHEDULING_POLICY='fcfs', ALLOC_FRACTION=0.8 @@ -3731,7 +3756,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 priority -plan_2 def.c2 0.2 2 fair +plan_2 def.c2 0.7 1 fair table default 1.0 4 NULL table table 0.0 1 random table table.pool1 0.9 3 priority @@ -3752,7 +3777,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 priority -plan_2 def.c2 0.2 2 fair +plan_2 def.c2 0.7 1 fair table default 1.0 4 NULL table table 0.0 1 random table table.pool 0.9 3 priority @@ -3772,7 +3797,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 priority -plan_2 def.c2 0.2 2 fair +plan_2 def.c2 0.7 1 fair table default 1.0 4 NULL table table 0.0 1 random table table.pool 0.9 3 priority @@ -3792,7 +3817,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 priority -plan_2 def.c2 0.2 2 fair +plan_2 def.c2 0.7 1 fair table default 1.0 4 NULL table table 0.0 1 random table table.pool 0.9 3 priority @@ -3807,7 +3832,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@wm_resourceplans #### A masked pattern was here #### plan_1 ACTIVE NULL default -plan_2 DISABLED 10 def +plan_2 DISABLED 4 def table DISABLED 1 default PREHOOK: query: ALTER RESOURCE PLAN `table` SET DEFAULT POOL = `table`.pool PREHOOK: type: ALTER RESOURCEPLAN @@ -3828,7 +3853,7 @@ POSTHOOK: Input: sys@wm_pools plan_1 default 1.0 4 NULL plan_2 def 1.0 4 NULL plan_2 def.c1 0.3 3 priority -plan_2 def.c2 0.2 2 fair +plan_2 def.c2 0.7 1 fair table table 0.0 1 random table table.pool 0.9 3 priority table table.pool.child1 0.3 1 random
