This is an automated email from the ASF dual-hosted git repository. anishek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 7454393 HIVE-23668: Clean up Task for Hive Metrics (Aasha Medhi, reviewed by Pravin Kumar Sinha) 7454393 is described below commit 74543935c1be9f9dbd619dfac69eefaa7ba19986 Author: Anishek Agarwal <anis...@gmail.com> AuthorDate: Tue Jun 23 10:09:11 2020 +0530 HIVE-23668: Clean up Task for Hive Metrics (Aasha Medhi, reviewed by Pravin Kumar Sinha) --- .../hcatalog/listener/DummyRawStoreFailEvent.java | 5 ++ .../parse/TestScheduledReplicationScenarios.java | 2 +- .../hive/ql/parse/repl/metric/MetricSink.java | 2 +- .../hadoop/hive/metastore/conf/MetastoreConf.java | 12 +++- .../src/main/thrift/hive_metastore.thrift | 2 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 23 +++++++ .../org/apache/hadoop/hive/metastore/RawStore.java | 2 + .../metastore/ReplicationMetricsMaintTask.java | 77 ++++++++++++++++++++++ .../hadoop/hive/metastore/cache/CachedStore.java | 5 ++ .../hive/metastore/model/MReplicationMetrics.java | 9 +++ .../src/main/resources/package.jdo | 3 + .../src/main/sql/derby/hive-schema-4.0.0.derby.sql | 1 + .../sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql | 1 + .../src/main/sql/mssql/hive-schema-4.0.0.mssql.sql | 3 +- .../sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql | 3 +- .../src/main/sql/mysql/hive-schema-4.0.0.mysql.sql | 1 + .../sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql | 1 + .../main/sql/oracle/hive-schema-4.0.0.oracle.sql | 3 +- .../sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql | 3 +- .../sql/postgres/hive-schema-4.0.0.postgres.sql | 1 + .../postgres/upgrade-3.2.0-to-4.0.0.postgres.sql | 1 + .../metastore/DummyRawStoreControlledCommit.java | 5 ++ .../metastore/DummyRawStoreForJdoConnection.java | 5 ++ .../metastore/client/TestReplicationMetrics.java | 45 ++++++++++++- 24 files changed, 203 insertions(+), 12 deletions(-) diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 34fb318..a4b2f52 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1390,6 +1390,11 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + return objectStore.deleteReplicationMetrics(maxRetainSecs); + } + + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws MetaException, NoSuchObjectException { throw new RuntimeException("unimplemented"); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java index 017607e..fd0a214 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java @@ -58,7 +58,7 @@ import java.util.ArrayList; * TestScheduledReplicationScenarios - test scheduled replication . */ public class TestScheduledReplicationScenarios extends BaseReplicationScenariosAcidTables { - private static final long DEFAULT_PROBE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes + private static final long DEFAULT_PROBE_TIMEOUT = 5 * 60 * 1000L; // 5 minutes @BeforeClass public static void classLevelSetup() throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java index 88a7b07..a856c76 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java @@ -71,7 +71,7 @@ public final class MetricSink { long getFrequencyInSecs() { //Metastore conf is in minutes - return MetastoreConf.getLongVar(conf, MetastoreConf.ConfVars.REPL_METRICS_UPDATE_FREQUENCY) * 60; + return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.REPL_METRICS_UPDATE_FREQUENCY, TimeUnit.MINUTES) * 60; } public synchronized void tearDown() { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 02fb9cf..ca9cd7d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -981,9 +981,16 @@ public class MetastoreConf { + " and the frequency of persisting the metrics to persistent storage. " ), REPL_METRICS_UPDATE_FREQUENCY("metastore.repl.metrics.update.frequency", - "hive.repl.metrics.update.frequency", 1 /*1 minute */, + "hive.repl.metrics.update.frequency", 1L, TimeUnit.MINUTES /*1 minute */, "Frequency at which replication Metrics will be stored in persistent storage. " ), + REPL_METRICS_CLEANUP_FREQUENCY("metastore.repl.metrics.cleanup.frequency", + "hive.metastore.repl.metrics.cleanup.frequency", 1, TimeUnit.DAYS, + "Interval of scheduled metrics clean up task which removes metrics above max age; Max age is" + + " defined by the config metastore.repl.metrics.max.age. The max age should be greater than this frequency"), + REPL_METRICS_MAX_AGE("metastore.repl.metrics.max.age", + "hive.metastore.repl.metrics.max.age", 7, TimeUnit.DAYS, + "Maximal age of a replication metrics entry before it is removed."), SCHEMA_INFO_CLASS("metastore.schema.info.class", "hive.metastore.schema.info.class", "org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo", "Fully qualified class name for the metastore schema information class \n" @@ -1077,7 +1084,8 @@ public class MetastoreConf { EVENT_CLEANER_TASK_CLASS + "," + RUNTIME_STATS_CLEANER_TASK_CLASS + "," + "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask" + "," - + "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask", + + "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask" + "," + + "org.apache.hadoop.hive.metastore.ReplicationMetricsMaintTask", "Comma separated list of tasks that will be started in separate threads. These will " + "always be started, regardless of whether the metastore is running in embedded mode " + "or in server mode. They must implement " + METASTORE_TASK_THREAD_CLASS), diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 903b531..d6bfb89 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -2039,7 +2039,7 @@ struct ReplicationMetrics{ 2: required string policy, 3: required i64 dumpExecutionId, 4: optional string metadata, - 5: optional string progress, + 5: optional string progress } struct ReplicationMetricList{ diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 5af7169..c472e93 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -13104,6 +13104,7 @@ public class ObjectStore implements RawStore, Configurable { mReplicationMetrics.setDumpExecutionId(replicationMetric.getDumpExecutionId()); mReplicationMetrics.setScheduledExecutionId(replicationMetric.getScheduledExecutionId()); mReplicationMetrics.setPolicy(replicationMetric.getPolicy()); + mReplicationMetrics.setStartTime((int) (System.currentTimeMillis()/1000)); } if (!StringUtils.isEmpty(replicationMetric.getMetadata())) { mReplicationMetrics.setMetadata(replicationMetric.getMetadata()); @@ -13142,6 +13143,28 @@ public class ObjectStore implements RawStore, Configurable { } } + @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + if (maxRetainSecs < 0) { + LOG.debug("replication metrics deletion is disabled"); + return 0; + } + boolean committed = false; + Query q = null; + try { + openTransaction(); + int maxCreateTime = (int) ((System.currentTimeMillis() / 1000) - maxRetainSecs); + q = pm.newQuery(MReplicationMetrics.class); + q.setFilter("startTime <= maxCreateTime"); + q.declareParameters("long maxCreateTime"); + long deleted = q.deletePersistentAll(maxCreateTime); + committed = commitTransaction(); + return (int) deleted; + } finally { + rollbackAndCleanup(committed, q); + } + } + private ReplicationMetricList getMReplicationMetrics(String policy) { ReplicationMetricList ret = new ReplicationMetricList(); if (StringUtils.isEmpty(policy)) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 6534750..d456255 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -1857,6 +1857,8 @@ public interface RawStore extends Configurable { */ ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest replicationMetricsRequest); + int deleteReplicationMetrics(int maxRetainSecs); + int deleteScheduledExecutions(int maxRetainSecs); int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java new file mode 100644 index 0000000..4ba968f --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +/** + * Metastore task to remove old replication metrics. + */ +public class ReplicationMetricsMaintTask implements MetastoreTaskThread { + private static final Logger LOG = LoggerFactory.getLogger(ReplicationMetricsMaintTask.class); + + private Configuration conf; + + @Override + public long initialDelay(TimeUnit unit) { + // no delay before the first execution; + // after an ungracefull shutdown it might take time to notice that in-flight scheduled queries are not running anymore + return 0; + } + + @Override + public long runFrequency(TimeUnit unit) { + return MetastoreConf.getTimeVar(conf, ConfVars.REPL_METRICS_CLEANUP_FREQUENCY, + TimeUnit.DAYS); + } + + @Override + public void setConf(Configuration configuration) { + conf = configuration; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + try { + if (!MetastoreConf.getBoolVar(conf, ConfVars.SCHEDULED_QUERIES_ENABLED)) { + return; + } + RawStore ms = HiveMetaStore.HMSHandler.getMSForConf(conf); + int maxRetainSecs = (int) TimeUnit.DAYS.toSeconds(MetastoreConf.getTimeVar(conf, + ConfVars.REPL_METRICS_MAX_AGE, TimeUnit.DAYS)); + int deleteCnt = ms.deleteScheduledExecutions(maxRetainSecs); + if (deleteCnt > 0L){ + LOG.info("Number of deleted entries: " + deleteCnt); + } + } catch (Exception e) { + LOG.error("Exception while trying to delete: " + e.getMessage(), e); + } + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index c8e230b..16df3c3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -2900,6 +2900,11 @@ public class CachedStore implements RawStore, Configurable { } @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + return rawStore.deleteReplicationMetrics(maxRetainSecs); + } + + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws MetaException, NoSuchObjectException { return rawStore.getScheduledQuery(scheduleKey); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java index 463be25..5fe3129 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java @@ -29,6 +29,7 @@ public class MReplicationMetrics { private long dumpExecutionId; private String metadata; private String progress; + private int startTime; public MReplicationMetrics() { } @@ -82,4 +83,12 @@ public class MReplicationMetrics { public void setProgress(String progress) { this.progress = progress; } + + public int getStartTime() { + return startTime; + } + + public void setStartTime(int startTime) { + this.startTime = startTime; + } } diff --git a/standalone-metastore/metastore-server/src/main/resources/package.jdo b/standalone-metastore/metastore-server/src/main/resources/package.jdo index c88ed32..0252dd0 100644 --- a/standalone-metastore/metastore-server/src/main/resources/package.jdo +++ b/standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -1539,6 +1539,9 @@ <field name="progress"> <column name="RM_PROGRESS" jdbc-type="varchar" length="4000" allows-null="true"/> </field> + <field name="startTime"> + <column name="RM_START_TIME" jdbc-type="integer" allows-null="false"/> + </field> <index name="PolicyIndex"> <column name="RM_POLICY"/> </index> diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 811f2b1..c64805d 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -779,6 +779,7 @@ CREATE TABLE "APP"."REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer not null, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index aeea23f..a9c2cab 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -94,6 +94,7 @@ CREATE TABLE "APP"."REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer not null, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index cba72cd..a59b646 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -1326,7 +1326,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar(256) NOT NULL, "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(max), - "RM_PROGRESS" varchar(max) + "RM_PROGRESS" varchar(max), + "RM_START_TIME" integer NOT NULL ); -- Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index fa1250c..0826c1e 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -129,7 +129,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar(256) NOT NULL, "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(max), - "RM_PROGRESS" varchar(max) + "RM_PROGRESS" varchar(max), + "RM_START_TIME" integer NOT NULL ); -- Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index e036b31..0d0374c 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -1251,6 +1251,7 @@ CREATE TABLE REPLICATION_METRICS ( RM_DUMP_EXECUTION_ID bigint NOT NULL, RM_METADATA varchar(4000), RM_PROGRESS varchar(4000), + RM_START_TIME integer NOT NULL, PRIMARY KEY(RM_SCHEDULED_EXECUTION_ID) ); diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index c1c888b..acf09c1 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -101,6 +101,7 @@ CREATE TABLE REPLICATION_METRICS ( RM_DUMP_EXECUTION_ID bigint NOT NULL, RM_METADATA varchar(4000), RM_PROGRESS varchar(4000), + RM_START_TIME integer NOT NULL, PRIMARY KEY(RM_SCHEDULED_EXECUTION_ID) ); diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index c7bf0ba..bc757bf 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -1224,7 +1224,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar2(256) NOT NULL, "RM_DUMP_EXECUTION_ID" number NOT NULL, "RM_METADATA" varchar2(4000), - "RM_PROGRESS" varchar2(4000) + "RM_PROGRESS" varchar2(4000), + "RM_START_TIME" integer NOT NULL ); --Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index 6146710..84db436 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -98,7 +98,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar2(256) NOT NULL, "RM_DUMP_EXECUTION_ID" number NOT NULL, "RM_METADATA" varchar2(4000), - "RM_PROGRESS" varchar2(4000) + "RM_PROGRESS" varchar2(4000), + "RM_START_TIME" integer NOT NULL ); --Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index bdbe88f..9b03d73 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -1913,6 +1913,7 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer NOT NULL, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index f6caac8..53f671c 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -225,6 +225,7 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer NOT NULL, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index d7b2d65..a779813 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1339,6 +1339,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + return objectStore.deleteReplicationMetrics(maxRetainSecs); + } + + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws NoSuchObjectException { return objectStore.getScheduledQuery(scheduleKey); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 1958607..1cbc86f 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1322,6 +1322,11 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + throw new RuntimeException("unimplemented"); + } + + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) { throw new RuntimeException("unimplemented"); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java index f8c8e1f..d151d3a 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java @@ -70,7 +70,7 @@ public class TestReplicationMetrics extends MetaStoreClientTest { public void testAddMetrics() throws Exception { ObjectStore objStore = new ObjectStore(); objStore.setConf(metaStore.getConf()); - objStore.deleteRuntimeStats(0); + objStore.deleteReplicationMetrics(0); ReplicationMetricList replicationMetricList = new ReplicationMetricList(); List<ReplicationMetrics> replicationMetrics = new ArrayList<>(); replicationMetrics.add(createReplicationMetric("repl1", 1L)); @@ -128,7 +128,7 @@ public class TestReplicationMetrics extends MetaStoreClientTest { public void testUpdateMetrics() throws Exception { ObjectStore objStore = new ObjectStore(); objStore.setConf(metaStore.getConf()); - objStore.deleteRuntimeStats(0); + objStore.deleteReplicationMetrics(0); ReplicationMetricList replicationMetricList = new ReplicationMetricList(); List<ReplicationMetrics> replicationMetrics = new ArrayList<>(); replicationMetrics.add(createReplicationMetric("repl1", 1L)); @@ -193,7 +193,7 @@ public class TestReplicationMetrics extends MetaStoreClientTest { public void testGetMetricsByScheduleId() throws Exception { ObjectStore objStore = new ObjectStore(); objStore.setConf(metaStore.getConf()); - objStore.deleteRuntimeStats(0); + objStore.deleteReplicationMetrics(0); ReplicationMetricList replicationMetricList = new ReplicationMetricList(); List<ReplicationMetrics> replicationMetrics = new ArrayList<>(); replicationMetrics.add(createReplicationMetric("repl1", 1L)); @@ -244,6 +244,45 @@ public class TestReplicationMetrics extends MetaStoreClientTest { } + @Test + public void testDeleteMetrics() throws Exception { + ObjectStore objStore = new ObjectStore(); + objStore.setConf(metaStore.getConf()); + objStore.deleteReplicationMetrics(0); + ReplicationMetricList replicationMetricList = new ReplicationMetricList(); + List<ReplicationMetrics> replicationMetrics = new ArrayList<>(); + replicationMetrics.add(createReplicationMetric("repl1", 1L)); + replicationMetrics.add(createReplicationMetric("repl1", 2L)); + replicationMetricList.setReplicationMetricList(replicationMetrics); + objStore.addReplicationMetrics(replicationMetricList); + Thread.sleep(2000); + replicationMetrics = new ArrayList<>(); + replicationMetrics.add(createReplicationMetric("repl1", 3L)); + replicationMetricList.setReplicationMetricList(replicationMetrics); + objStore.addReplicationMetrics(replicationMetricList); + Thread.sleep(500); + + GetReplicationMetricsRequest getReplicationMetricsRequest = new GetReplicationMetricsRequest(); + getReplicationMetricsRequest.setPolicy("repl1"); + ReplicationMetricList actualList = client.getReplicationMetrics(getReplicationMetricsRequest); + assertEquals(3, actualList.getReplicationMetricListSize()); + //delete older metrics + objStore.deleteReplicationMetrics(2); + + getReplicationMetricsRequest = new GetReplicationMetricsRequest(); + getReplicationMetricsRequest.setPolicy("repl1"); + actualList = client.getReplicationMetrics(getReplicationMetricsRequest); + assertEquals(1, actualList.getReplicationMetricListSize()); + List<ReplicationMetrics> actualMetrics = actualList.getReplicationMetricList(); + ReplicationMetrics actualMetric0 = actualMetrics.get(0); + assertEquals("repl1", actualMetric0.getPolicy()); + assertEquals(3L, actualMetric0.getScheduledExecutionId()); + assertEquals(1, actualMetric0.getDumpExecutionId()); + assertEquals("metadata", actualMetric0.getMetadata()); + assertEquals("progress", actualMetric0.getProgress()); + + } + private ReplicationMetrics createReplicationMetric(String policyName, Long scheduleId) { ReplicationMetrics replicationMetrics = new ReplicationMetrics(); replicationMetrics.setPolicy(policyName);