This is an automated email from the ASF dual-hosted git repository. anishek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 9ec1470 HIVE-23585: Retrieve replication instance metrics details (Aasha Medhi, reviewed by Pravin Kumar Sinha) 9ec1470 is described below commit 9ec1470027e59be322a0bb5166d289cc9412a9ce Author: Anishek Agarwal <anis...@gmail.com> AuthorDate: Thu Jun 18 09:19:39 2020 +0530 HIVE-23585: Retrieve replication instance metrics details (Aasha Medhi, reviewed by Pravin Kumar Sinha) --- .../upgrade/hive/hive-schema-4.0.0.hive.sql | 20 ++++++ .../upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql | 21 ++++++ .../clientpositive/replication_metrics_ingest.q | 40 ++++++++++++ .../llap/replication_metrics_ingest.q.out | 76 ++++++++++++++++++++++ .../results/clientpositive/llap/resourceplan.q.out | 4 ++ .../llap/strict_managed_tables_sysdb.q.out | 6 ++ .../test/results/clientpositive/llap/sysdb.q.out | 12 +++- 7 files changed, 178 insertions(+), 1 deletion(-) diff --git a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql index e7fe916..8fe0b17 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -1436,6 +1436,26 @@ SELECT DISTINCT HL.`HL_BLOCKEDBY_INT_ID` FROM SYS.`HIVE_LOCKS` AS HL; +CREATE EXTERNAL TABLE IF NOT EXISTS `REPLICATION_METRICS` ( + `SCHEDULED_EXECUTION_ID` bigint, + `POLICY_NAME` string, + `DUMP_EXECUTION_ID` bigint, + `METADATA` string, + `PROGRESS` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"RM_SCHEDULED_EXECUTION_ID\", + \"RM_POLICY\", + \"RM_DUMP_EXECUTION_ID\", + \"RM_METADATA\", + \"RM_PROGRESS\" +FROM \"REPLICATION_METRICS\"" +); + CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA; USE INFORMATION_SCHEMA; diff --git a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql index 3fb3f7a..7862f6d 100644 --- a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql @@ -497,6 +497,26 @@ SELECT DISTINCT HL.`HL_BLOCKEDBY_INT_ID` FROM SYS.`HIVE_LOCKS` AS HL; +CREATE EXTERNAL TABLE IF NOT EXISTS `REPLICATION_METRICS` ( + `SCHEDULED_EXECUTION_ID` bigint, + `POLICY_NAME` string, + `DUMP_EXECUTION_ID` bigint, + `METADATA` string, + `PROGRESS` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"RM_SCHEDULED_EXECUTION_ID\", + \"RM_POLICY\", + \"RM_DUMP_EXECUTION_ID\", + \"RM_METADATA\", + \"RM_PROGRESS\" +FROM \"REPLICATION_METRICS\"" +); + DROP TABLE IF EXISTS `VERSION`; CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.0.0' AS `SCHEMA_VERSION`, @@ -775,4 +795,5 @@ WHERE OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()); + SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0'; diff --git a/ql/src/test/queries/clientpositive/replication_metrics_ingest.q b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q new file mode 100644 index 0000000..7f0ef81 --- /dev/null +++ b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q @@ -0,0 +1,40 @@ +--! qt:authorizer +--! qt:scheduledqueryservice +--! qt:sysdb + +set hive.repl.rootdir=${system:test.tmp.dir}/repl; + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/repl/sentinel; +dfs -rmr ${system:test.tmp.dir}/repl; +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/repl; + +set user.name=hive_admin_user; +set role ADMIN; + +drop database if exists src cascade; +drop database if exists destination cascade; + +create database src with DBPROPERTIES ('repl.source.for' = '1,2,3'); + +create table src.t(id int, cnt int); + +-- add data to table +insert into src.t values(1,1); + +create scheduled query repl1 every 15 minutes as repl dump src +with ('hive.repl.rootdir'= '${system:test.tmp.dir}/repl'); + +alter scheduled query repl1 execute; + +!sleep 50; + +create scheduled query repl2 every 15 minutes as repl load src into destination +with ('hive.repl.rootdir'= '${system:test.tmp.dir}/repl'); + +alter scheduled query repl2 execute; + +!sleep 50; + +show databases; + +select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics; diff --git a/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out b/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out new file mode 100644 index 0000000..1380ccd --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out @@ -0,0 +1,76 @@ +#### A masked pattern was here #### +PREHOOK: query: set role ADMIN +PREHOOK: type: SHOW_ROLES +POSTHOOK: query: set role ADMIN +POSTHOOK: type: SHOW_ROLES +PREHOOK: query: drop database if exists src cascade +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists src cascade +POSTHOOK: type: DROPDATABASE +PREHOOK: query: drop database if exists destination cascade +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists destination cascade +POSTHOOK: type: DROPDATABASE +PREHOOK: query: create database src with DBPROPERTIES ('repl.source.for' = '1,2,3') +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:src +POSTHOOK: query: create database src with DBPROPERTIES ('repl.source.for' = '1,2,3') +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:src +PREHOOK: query: create table src.t(id int, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:src +PREHOOK: Output: src@t +POSTHOOK: query: create table src.t(id int, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:src +POSTHOOK: Output: src@t +PREHOOK: query: insert into src.t values(1,1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: src@t +POSTHOOK: query: insert into src.t values(1,1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: src@t +POSTHOOK: Lineage: t.cnt SCRIPT [] +POSTHOOK: Lineage: t.id SCRIPT [] +PREHOOK: query: create scheduled query repl1 every 15 minutes as repl dump src +#### A masked pattern was here #### +PREHOOK: type: CREATE SCHEDULED QUERY +POSTHOOK: query: create scheduled query repl1 every 15 minutes as repl dump src +#### A masked pattern was here #### +POSTHOOK: type: CREATE SCHEDULED QUERY +PREHOOK: query: alter scheduled query repl1 execute +PREHOOK: type: ALTER SCHEDULED QUERY +POSTHOOK: query: alter scheduled query repl1 execute +POSTHOOK: type: ALTER SCHEDULED QUERY +PREHOOK: query: create scheduled query repl2 every 15 minutes as repl load src into destination +#### A masked pattern was here #### +PREHOOK: type: CREATE SCHEDULED QUERY +POSTHOOK: query: create scheduled query repl2 every 15 minutes as repl load src into destination +#### A masked pattern was here #### +POSTHOOK: type: CREATE SCHEDULED QUERY +PREHOOK: query: alter scheduled query repl2 execute +PREHOOK: type: ALTER SCHEDULED QUERY +POSTHOOK: query: alter scheduled query repl2 execute +POSTHOOK: type: ALTER SCHEDULED QUERY +PREHOOK: query: show databases +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: show databases +POSTHOOK: type: SHOWDATABASES +default +destination +information_schema +src +sys +PREHOOK: query: select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics +PREHOOK: type: QUERY +PREHOOK: Input: sys@replication_metrics +#### A masked pattern was here #### +POSTHOOK: query: select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@replication_metrics +#### A masked pattern was here #### +1 repl1 0 +2 repl2 1 diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out index 24190d3..dddf147 100644 --- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out +++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out @@ -202,6 +202,10 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user sys partitions hive_test_user USER INSERT true -1 hive_test_user sys partitions hive_test_user USER SELECT true -1 hive_test_user sys partitions hive_test_user USER UPDATE true -1 hive_test_user +sys replication_metrics hive_test_user USER DELETE true -1 hive_test_user +sys replication_metrics hive_test_user USER INSERT true -1 hive_test_user +sys replication_metrics hive_test_user USER SELECT true -1 hive_test_user +sys replication_metrics hive_test_user USER UPDATE true -1 hive_test_user sys role_map hive_test_user USER DELETE true -1 hive_test_user sys role_map hive_test_user USER INSERT true -1 hive_test_user sys role_map hive_test_user USER SELECT true -1 hive_test_user diff --git a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out index 96bdecd..54de5a9 100644 --- a/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/strict_managed_tables_sysdb.q.out @@ -292,6 +292,10 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user sys partitions hive_test_user USER INSERT true -1 hive_test_user sys partitions hive_test_user USER SELECT true -1 hive_test_user sys partitions hive_test_user USER UPDATE true -1 hive_test_user +sys replication_metrics hive_test_user USER DELETE true -1 hive_test_user +sys replication_metrics hive_test_user USER INSERT true -1 hive_test_user +sys replication_metrics hive_test_user USER SELECT true -1 hive_test_user +sys replication_metrics hive_test_user USER UPDATE true -1 hive_test_user sys role_map hive_test_user USER DELETE true -1 hive_test_user sys role_map hive_test_user USER INSERT true -1 hive_test_user sys role_map hive_test_user USER SELECT true -1 hive_test_user @@ -480,6 +484,7 @@ PREHOOK: Output: sys@partition_keys PREHOOK: Output: sys@partition_params PREHOOK: Output: sys@partition_stats_view PREHOOK: Output: sys@partitions +PREHOOK: Output: sys@replication_metrics PREHOOK: Output: sys@role_map PREHOOK: Output: sys@roles PREHOOK: Output: sys@scheduled_executions @@ -539,6 +544,7 @@ POSTHOOK: Output: sys@partition_keys POSTHOOK: Output: sys@partition_params POSTHOOK: Output: sys@partition_stats_view POSTHOOK: Output: sys@partitions +POSTHOOK: Output: sys@replication_metrics POSTHOOK: Output: sys@role_map POSTHOOK: Output: sys@roles POSTHOOK: Output: sys@scheduled_executions diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out index 26e3200..c78fc24 100644 --- a/ql/src/test/results/clientpositive/llap/sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out @@ -240,6 +240,10 @@ sys partitions hive_test_user USER DELETE true -1 hive_test_user sys partitions hive_test_user USER INSERT true -1 hive_test_user sys partitions hive_test_user USER SELECT true -1 hive_test_user sys partitions hive_test_user USER UPDATE true -1 hive_test_user +sys replication_metrics hive_test_user USER DELETE true -1 hive_test_user +sys replication_metrics hive_test_user USER INSERT true -1 hive_test_user +sys replication_metrics hive_test_user USER SELECT true -1 hive_test_user +sys replication_metrics hive_test_user USER UPDATE true -1 hive_test_user sys role_map hive_test_user USER DELETE true -1 hive_test_user sys role_map hive_test_user USER INSERT true -1 hive_test_user sys role_map hive_test_user USER SELECT true -1 hive_test_user @@ -680,6 +684,11 @@ partitions part_id partitions part_name partitions sd_id partitions tbl_id +replication_metrics dump_execution_id +replication_metrics metadata +replication_metrics policy_name +replication_metrics progress +replication_metrics scheduled_execution_id role_map add_time role_map grant_option role_map grantor @@ -1179,7 +1188,7 @@ POSTHOOK: query: select count(*) from sds POSTHOOK: type: QUERY POSTHOOK: Input: sys@sds #### A masked pattern was here #### -72 +73 PREHOOK: query: select param_key, param_value from sd_params order by param_key, param_value limit 5 PREHOOK: type: QUERY PREHOOK: Input: sys@sd_params @@ -1585,6 +1594,7 @@ default sys partition_keys BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys partition_params BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys partition_stats_view VIEW NULL NULL NULL NULL NULL NO NO NULL default sys partitions BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL +default sys replication_metrics BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys role_map BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys roles BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys scheduled_executions BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL