Repository: hive Updated Branches: refs/heads/master 95d6d9b3d -> 577d32ef7
HIVE-19319 : RuntimeStats fixes (Zoltan Haindrich via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/577d32ef Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/577d32ef Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/577d32ef Branch: refs/heads/master Commit: 577d32ef7d4ab0bd464d3b42426bb5a974655616 Parents: 95d6d9b Author: Zoltan Haindrich <[email protected]> Authored: Mon Apr 30 13:24:55 2018 -0700 Committer: Ashutosh Chauhan <[email protected]> Committed: Mon Apr 30 13:28:42 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 5 +- .../listener/DummyRawStoreFailEvent.java | 8 +- .../hive/ql/plan/mapper/CachingStatsSource.java | 2 +- .../ql/plan/mapper/MetastoreStatsConnector.java | 42 +++- .../hive/ql/plan/mapper/StatsSources.java | 7 +- .../hadoop/hive/ql/reexec/ReOptimizePlugin.java | 8 +- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 57 +++++- .../gen/thrift/gen-cpp/hive_metastore_types.h | 14 +- .../metastore/api/GetRuntimeStatsRequest.java | 201 ++++++++++++++++++- .../metastore/api/hive_metastoreConstants.java | 7 +- .../src/gen/thrift/gen-php/metastore/Types.php | 50 ++++- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 36 ++++ .../gen/thrift/gen-rb/hive_metastore_types.rb | 7 +- .../hadoop/hive/metastore/HiveMetaStore.java | 2 +- .../hive/metastore/HiveMetaStoreClient.java | 8 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 3 +- .../hadoop/hive/metastore/ObjectStore.java | 75 ++++--- .../apache/hadoop/hive/metastore/RawStore.java | 5 +- .../hive/metastore/RuntimeStatsCleanerTask.java | 3 +- .../hive/metastore/cache/CachedStore.java | 8 +- .../hive/metastore/conf/MetastoreConf.java | 3 - .../src/main/thrift/hive_metastore.thrift | 2 + .../DummyRawStoreControlledCommit.java | 8 +- .../DummyRawStoreForJdoConnection.java | 4 +- .../HiveMetaStoreClientPreCatalog.java | 3 +- .../hive/metastore/client/TestRuntimeStats.java | 58 +++++- 26 files changed, 533 insertions(+), 93 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6e35653..09bc572 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4276,12 +4276,15 @@ public class HiveConf extends Configuration { + " hiveserver: runtime statistics are persisted in the hiveserver - all sessions share it\n" + " metastore: runtime statistics are persisted in the metastore as well"), + HIVE_QUERY_MAX_REEXECUTION_COUNT("hive.query.reexecution.max.count", 1, "Maximum number of re-executions for a single query."), HIVE_QUERY_REEXECUTION_ALWAYS_COLLECT_OPERATOR_STATS("hive.query.reexecution.always.collect.operator.stats", false, "If sessionstats are enabled; this option can be used to collect statistics all the time"), + HIVE_QUERY_REEXECUTION_STATS_CACHE_BATCH_SIZE("hive.query.reexecution.stats.cache.batch.size", -1, + "If runtime stats are stored in metastore; the maximal batch size per round during load."), HIVE_QUERY_REEXECUTION_STATS_CACHE_SIZE("hive.query.reexecution.stats.cache.size", 100_000, - "Size of the runtime statistics cache. Unit is: OperatorStat entry; a query plan consist ~100. See also: runtime.stats.max.entries"), + "Size of the runtime statistics cache. Unit is: OperatorStat entry; a query plan consist ~100."), HIVE_QUERY_RESULTS_CACHE_ENABLED("hive.query.results.cache.enabled", true, "If the query results cache is enabled. This will keep results of previously executed queries " + http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 8ecbaad..2e96983 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1205,13 +1205,13 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable { } @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { - return objectStore.getRuntimeStats(); + public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { + return objectStore.getRuntimeStats(maxEntries, maxCreateTime); } @Override - public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) { - return 0; + public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { + return objectStore.deleteRuntimeStats(maxRetainSecs); } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java index 2841638..6b440e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/CachingStatsSource.java @@ -49,7 +49,7 @@ public class CachingStatsSource implements StatsSource { @Override public boolean canProvideStatsFor(Class<?> clazz) { - if (Operator.class.isAssignableFrom(clazz)) { + if (cache.size() > 0 && Operator.class.isAssignableFrom(clazz)) { return true; } return false; http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java index 237c1cc..f9ee46d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/MetastoreStatsConnector.java @@ -50,7 +50,7 @@ class MetastoreStatsConnector implements StatsSource { private ExecutorService executor; - MetastoreStatsConnector(StatsSource ss) { + MetastoreStatsConnector(int cacheSize, int batchSize, StatsSource ss) { this.ss = ss; executor = Executors.newSingleThreadExecutor( new BasicThreadFactory.Builder() @@ -58,22 +58,43 @@ class MetastoreStatsConnector implements StatsSource { .daemon(true) .build()); - executor.submit(new RuntimeStatsLoader()); + executor.submit(new RuntimeStatsLoader(cacheSize, batchSize)); } private class RuntimeStatsLoader implements Runnable { + private int maxEntriesToLoad; + private int batchSize; + + public RuntimeStatsLoader(int maxEntriesToLoad, int batchSize) { + this.maxEntriesToLoad = maxEntriesToLoad; + if (batchSize <= 0) { + this.batchSize = -1; + } else { + this.batchSize = batchSize; + } + } + @Override public void run() { + int lastCreateTime = Integer.MAX_VALUE; + int loadedEntries = 0; try { - List<RuntimeStat> rs = Hive.get().getMSC().getRuntimeStats(); - for (RuntimeStat thriftStat : rs) { - try { - ss.putAll(decode(thriftStat)); - } catch (IOException e) { - logException("Exception while loading runtime stats", e); + do { + List<RuntimeStat> rs = Hive.get().getMSC().getRuntimeStats(batchSize, lastCreateTime); + if (rs.size() == 0) { + break; + } + for (RuntimeStat thriftStat : rs) { + loadedEntries += thriftStat.getWeight(); + lastCreateTime = Math.min(lastCreateTime, thriftStat.getCreateTime() - 1); + try { + ss.putAll(decode(thriftStat)); + } catch (IOException e) { + logException("Exception while loading runtime stats", e); + } } - } + } while (batchSize > 0 && loadedEntries < maxEntriesToLoad); } catch (TException | HiveException e) { logException("Exception while reading metastore runtime stats", e); } @@ -92,6 +113,9 @@ class MetastoreStatsConnector implements StatsSource { @Override public void putAll(Map<OpTreeSignature, OperatorStats> map) { + if (map.size() == 0) { + return; + } ss.putAll(map); executor.submit(new RuntimeStatsSubmitter(map)); } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java index 30b6a30..5a62046 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/mapper/StatsSources.java @@ -50,13 +50,14 @@ public class StatsSources { public static StatsSource getStatsSource(HiveConf conf) { String mode = conf.getVar(ConfVars.HIVE_QUERY_REEXECUTION_STATS_PERSISTENCE); int cacheSize = conf.getIntVar(ConfVars.HIVE_QUERY_REEXECUTION_STATS_CACHE_SIZE); + int batchSize = conf.getIntVar(ConfVars.HIVE_QUERY_REEXECUTION_STATS_CACHE_BATCH_SIZE); switch (mode) { case "query": return new MapBackedStatsSource(); case "hiveserver": return StatsSources.globalStatsSource(cacheSize); case "metastore": - return StatsSources.metastoreBackedStatsSource(StatsSources.globalStatsSource(cacheSize)); + return StatsSources.metastoreBackedStatsSource(cacheSize, batchSize, StatsSources.globalStatsSource(cacheSize)); default: throw new RuntimeException("Unknown StatsSource setting: " + mode); } @@ -111,9 +112,9 @@ public class StatsSources { return globalStatsSource; } - public static StatsSource metastoreBackedStatsSource(StatsSource parent) { + public static StatsSource metastoreBackedStatsSource(int cacheSize, int batchSize, StatsSource parent) { if (metastoreStatsConnector == null) { - metastoreStatsConnector = new MetastoreStatsConnector(parent); + metastoreStatsConnector = new MetastoreStatsConnector(cacheSize, batchSize, parent); } return metastoreStatsConnector; } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java index 8dc7387..09045af 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/reexec/ReOptimizePlugin.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.reexec; import java.util.Iterator; import java.util.List; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.Operator; @@ -30,7 +29,6 @@ import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.hooks.HookContext.HookType; import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper; -import org.apache.hadoop.hive.ql.plan.mapper.StatsSource; import org.apache.hadoop.hive.ql.plan.mapper.StatsSources; import org.apache.hadoop.hive.ql.stats.OperatorStatsReaderHook; import org.slf4j.Logger; @@ -96,8 +94,10 @@ public class ReOptimizePlugin implements IReExecutionPlugin { public void prepareToReExecute() { statsReaderHook.setCollectOnSuccess(true); retryPossible = false; - coreDriver.setStatsSource( - StatsSources.getStatsSourceContaining(coreDriver.getStatsSource(), coreDriver.getPlanMapper())); + if (!alwaysCollectStats) { + coreDriver.setStatsSource( + StatsSources.getStatsSourceContaining(coreDriver.getStatsSource(), coreDriver.getPlanMapper())); + } } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp index 2fab857..c6fadf8 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp @@ -30689,6 +30689,14 @@ GetRuntimeStatsRequest::~GetRuntimeStatsRequest() throw() { } +void GetRuntimeStatsRequest::__set_maxWeight(const int32_t val) { + this->maxWeight = val; +} + +void GetRuntimeStatsRequest::__set_maxCreateTime(const int32_t val) { + this->maxCreateTime = val; +} + uint32_t GetRuntimeStatsRequest::read(::apache::thrift::protocol::TProtocol* iprot) { apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); @@ -30701,6 +30709,8 @@ uint32_t GetRuntimeStatsRequest::read(::apache::thrift::protocol::TProtocol* ipr using ::apache::thrift::protocol::TProtocolException; + bool isset_maxWeight = false; + bool isset_maxCreateTime = false; while (true) { @@ -30708,12 +30718,37 @@ uint32_t GetRuntimeStatsRequest::read(::apache::thrift::protocol::TProtocol* ipr if (ftype == ::apache::thrift::protocol::T_STOP) { break; } - xfer += iprot->skip(ftype); + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->maxWeight); + isset_maxWeight = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->maxCreateTime); + isset_maxCreateTime = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } xfer += iprot->readFieldEnd(); } xfer += iprot->readStructEnd(); + if (!isset_maxWeight) + throw TProtocolException(TProtocolException::INVALID_DATA); + if (!isset_maxCreateTime) + throw TProtocolException(TProtocolException::INVALID_DATA); return xfer; } @@ -30722,6 +30757,14 @@ uint32_t GetRuntimeStatsRequest::write(::apache::thrift::protocol::TProtocol* op apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); xfer += oprot->writeStructBegin("GetRuntimeStatsRequest"); + xfer += oprot->writeFieldBegin("maxWeight", ::apache::thrift::protocol::T_I32, 1); + xfer += oprot->writeI32(this->maxWeight); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("maxCreateTime", ::apache::thrift::protocol::T_I32, 2); + xfer += oprot->writeI32(this->maxCreateTime); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -30729,20 +30772,24 @@ uint32_t GetRuntimeStatsRequest::write(::apache::thrift::protocol::TProtocol* op void swap(GetRuntimeStatsRequest &a, GetRuntimeStatsRequest &b) { using ::std::swap; - (void) a; - (void) b; + swap(a.maxWeight, b.maxWeight); + swap(a.maxCreateTime, b.maxCreateTime); } GetRuntimeStatsRequest::GetRuntimeStatsRequest(const GetRuntimeStatsRequest& other1159) { - (void) other1159; + maxWeight = other1159.maxWeight; + maxCreateTime = other1159.maxCreateTime; } GetRuntimeStatsRequest& GetRuntimeStatsRequest::operator=(const GetRuntimeStatsRequest& other1160) { - (void) other1160; + maxWeight = other1160.maxWeight; + maxCreateTime = other1160.maxCreateTime; return *this; } void GetRuntimeStatsRequest::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "GetRuntimeStatsRequest("; + out << "maxWeight=" << to_string(maxWeight); + out << ", " << "maxCreateTime=" << to_string(maxCreateTime); out << ")"; } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index cd78f58..6298893 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -13002,13 +13002,23 @@ class GetRuntimeStatsRequest { GetRuntimeStatsRequest(const GetRuntimeStatsRequest&); GetRuntimeStatsRequest& operator=(const GetRuntimeStatsRequest&); - GetRuntimeStatsRequest() { + GetRuntimeStatsRequest() : maxWeight(0), maxCreateTime(0) { } virtual ~GetRuntimeStatsRequest() throw(); + int32_t maxWeight; + int32_t maxCreateTime; - bool operator == (const GetRuntimeStatsRequest & /* rhs */) const + void __set_maxWeight(const int32_t val); + + void __set_maxCreateTime(const int32_t val); + + bool operator == (const GetRuntimeStatsRequest & rhs) const { + if (!(maxWeight == rhs.maxWeight)) + return false; + if (!(maxCreateTime == rhs.maxCreateTime)) + return false; return true; } bool operator != (const GetRuntimeStatsRequest &rhs) const { http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java index e5ce97e..593b7bf 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetRuntimeStatsRequest.java @@ -38,6 +38,8 @@ import org.slf4j.LoggerFactory; @org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class GetRuntimeStatsRequest implements org.apache.thrift.TBase<GetRuntimeStatsRequest, GetRuntimeStatsRequest._Fields>, java.io.Serializable, Cloneable, Comparable<GetRuntimeStatsRequest> { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("GetRuntimeStatsRequest"); + private static final org.apache.thrift.protocol.TField MAX_WEIGHT_FIELD_DESC = new org.apache.thrift.protocol.TField("maxWeight", org.apache.thrift.protocol.TType.I32, (short)1); + private static final org.apache.thrift.protocol.TField MAX_CREATE_TIME_FIELD_DESC = new org.apache.thrift.protocol.TField("maxCreateTime", org.apache.thrift.protocol.TType.I32, (short)2); private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>(); static { @@ -45,10 +47,13 @@ import org.slf4j.LoggerFactory; schemes.put(TupleScheme.class, new GetRuntimeStatsRequestTupleSchemeFactory()); } + private int maxWeight; // required + private int maxCreateTime; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { -; + MAX_WEIGHT((short)1, "maxWeight"), + MAX_CREATE_TIME((short)2, "maxCreateTime"); private static final Map<String, _Fields> byName = new HashMap<String, _Fields>(); @@ -63,6 +68,10 @@ import org.slf4j.LoggerFactory; */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { + case 1: // MAX_WEIGHT + return MAX_WEIGHT; + case 2: // MAX_CREATE_TIME + return MAX_CREATE_TIME; default: return null; } @@ -101,9 +110,18 @@ import org.slf4j.LoggerFactory; return _fieldName; } } + + // isset id assignments + private static final int __MAXWEIGHT_ISSET_ID = 0; + private static final int __MAXCREATETIME_ISSET_ID = 1; + private byte __isset_bitfield = 0; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.MAX_WEIGHT, new org.apache.thrift.meta_data.FieldMetaData("maxWeight", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.MAX_CREATE_TIME, new org.apache.thrift.meta_data.FieldMetaData("maxCreateTime", org.apache.thrift.TFieldRequirementType.REQUIRED, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetRuntimeStatsRequest.class, metaDataMap); } @@ -111,10 +129,24 @@ import org.slf4j.LoggerFactory; public GetRuntimeStatsRequest() { } + public GetRuntimeStatsRequest( + int maxWeight, + int maxCreateTime) + { + this(); + this.maxWeight = maxWeight; + setMaxWeightIsSet(true); + this.maxCreateTime = maxCreateTime; + setMaxCreateTimeIsSet(true); + } + /** * Performs a deep copy on <i>other</i>. */ public GetRuntimeStatsRequest(GetRuntimeStatsRequest other) { + __isset_bitfield = other.__isset_bitfield; + this.maxWeight = other.maxWeight; + this.maxCreateTime = other.maxCreateTime; } public GetRuntimeStatsRequest deepCopy() { @@ -123,15 +155,85 @@ import org.slf4j.LoggerFactory; @Override public void clear() { + setMaxWeightIsSet(false); + this.maxWeight = 0; + setMaxCreateTimeIsSet(false); + this.maxCreateTime = 0; + } + + public int getMaxWeight() { + return this.maxWeight; + } + + public void setMaxWeight(int maxWeight) { + this.maxWeight = maxWeight; + setMaxWeightIsSet(true); + } + + public void unsetMaxWeight() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAXWEIGHT_ISSET_ID); + } + + /** Returns true if field maxWeight is set (has been assigned a value) and false otherwise */ + public boolean isSetMaxWeight() { + return EncodingUtils.testBit(__isset_bitfield, __MAXWEIGHT_ISSET_ID); + } + + public void setMaxWeightIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXWEIGHT_ISSET_ID, value); + } + + public int getMaxCreateTime() { + return this.maxCreateTime; + } + + public void setMaxCreateTime(int maxCreateTime) { + this.maxCreateTime = maxCreateTime; + setMaxCreateTimeIsSet(true); + } + + public void unsetMaxCreateTime() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __MAXCREATETIME_ISSET_ID); + } + + /** Returns true if field maxCreateTime is set (has been assigned a value) and false otherwise */ + public boolean isSetMaxCreateTime() { + return EncodingUtils.testBit(__isset_bitfield, __MAXCREATETIME_ISSET_ID); + } + + public void setMaxCreateTimeIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXCREATETIME_ISSET_ID, value); } public void setFieldValue(_Fields field, Object value) { switch (field) { + case MAX_WEIGHT: + if (value == null) { + unsetMaxWeight(); + } else { + setMaxWeight((Integer)value); + } + break; + + case MAX_CREATE_TIME: + if (value == null) { + unsetMaxCreateTime(); + } else { + setMaxCreateTime((Integer)value); + } + break; + } } public Object getFieldValue(_Fields field) { switch (field) { + case MAX_WEIGHT: + return getMaxWeight(); + + case MAX_CREATE_TIME: + return getMaxCreateTime(); + } throw new IllegalStateException(); } @@ -143,6 +245,10 @@ import org.slf4j.LoggerFactory; } switch (field) { + case MAX_WEIGHT: + return isSetMaxWeight(); + case MAX_CREATE_TIME: + return isSetMaxCreateTime(); } throw new IllegalStateException(); } @@ -160,6 +266,24 @@ import org.slf4j.LoggerFactory; if (that == null) return false; + boolean this_present_maxWeight = true; + boolean that_present_maxWeight = true; + if (this_present_maxWeight || that_present_maxWeight) { + if (!(this_present_maxWeight && that_present_maxWeight)) + return false; + if (this.maxWeight != that.maxWeight) + return false; + } + + boolean this_present_maxCreateTime = true; + boolean that_present_maxCreateTime = true; + if (this_present_maxCreateTime || that_present_maxCreateTime) { + if (!(this_present_maxCreateTime && that_present_maxCreateTime)) + return false; + if (this.maxCreateTime != that.maxCreateTime) + return false; + } + return true; } @@ -167,6 +291,16 @@ import org.slf4j.LoggerFactory; public int hashCode() { List<Object> list = new ArrayList<Object>(); + boolean present_maxWeight = true; + list.add(present_maxWeight); + if (present_maxWeight) + list.add(maxWeight); + + boolean present_maxCreateTime = true; + list.add(present_maxCreateTime); + if (present_maxCreateTime) + list.add(maxCreateTime); + return list.hashCode(); } @@ -178,6 +312,26 @@ import org.slf4j.LoggerFactory; int lastComparison = 0; + lastComparison = Boolean.valueOf(isSetMaxWeight()).compareTo(other.isSetMaxWeight()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMaxWeight()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxWeight, other.maxWeight); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMaxCreateTime()).compareTo(other.isSetMaxCreateTime()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMaxCreateTime()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.maxCreateTime, other.maxCreateTime); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -198,12 +352,27 @@ import org.slf4j.LoggerFactory; StringBuilder sb = new StringBuilder("GetRuntimeStatsRequest("); boolean first = true; + sb.append("maxWeight:"); + sb.append(this.maxWeight); + first = false; + if (!first) sb.append(", "); + sb.append("maxCreateTime:"); + sb.append(this.maxCreateTime); + first = false; sb.append(")"); return sb.toString(); } public void validate() throws org.apache.thrift.TException { // check for required fields + if (!isSetMaxWeight()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'maxWeight' is unset! Struct:" + toString()); + } + + if (!isSetMaxCreateTime()) { + throw new org.apache.thrift.protocol.TProtocolException("Required field 'maxCreateTime' is unset! Struct:" + toString()); + } + // check for sub-struct validity } @@ -217,6 +386,8 @@ import org.slf4j.LoggerFactory; private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); @@ -241,6 +412,22 @@ import org.slf4j.LoggerFactory; break; } switch (schemeField.id) { + case 1: // MAX_WEIGHT + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.maxWeight = iprot.readI32(); + struct.setMaxWeightIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MAX_CREATE_TIME + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.maxCreateTime = iprot.readI32(); + struct.setMaxCreateTimeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -254,6 +441,12 @@ import org.slf4j.LoggerFactory; struct.validate(); oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(MAX_WEIGHT_FIELD_DESC); + oprot.writeI32(struct.maxWeight); + oprot.writeFieldEnd(); + oprot.writeFieldBegin(MAX_CREATE_TIME_FIELD_DESC); + oprot.writeI32(struct.maxCreateTime); + oprot.writeFieldEnd(); oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -271,11 +464,17 @@ import org.slf4j.LoggerFactory; @Override public void write(org.apache.thrift.protocol.TProtocol prot, GetRuntimeStatsRequest struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; + oprot.writeI32(struct.maxWeight); + oprot.writeI32(struct.maxCreateTime); } @Override public void read(org.apache.thrift.protocol.TProtocol prot, GetRuntimeStatsRequest struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; + struct.maxWeight = iprot.readI32(); + struct.setMaxWeightIsSet(true); + struct.maxCreateTime = iprot.readI32(); + struct.setMaxCreateTimeIsSet(true); } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java index c9d1051..1abedc9 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java @@ -6,11 +6,10 @@ */ package org.apache.hadoop.hive.metastore.api; -import java.lang.reflect.Modifier; -import java.lang.reflect.Field; import org.apache.thrift.scheme.IScheme; import org.apache.thrift.scheme.SchemeFactory; import org.apache.thrift.scheme.StandardScheme; + import org.apache.thrift.scheme.TupleScheme; import org.apache.thrift.protocol.TTupleProtocol; import org.apache.thrift.protocol.TProtocolException; @@ -35,9 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) [email protected] [email protected] -public class hive_metastoreConstants { [email protected] @org.apache.hadoop.classification.InterfaceStability.Stable public class hive_metastoreConstants { public static final String DDL_TIME = "transient_lastDdlTime"; http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php index c9ebfaf..9f6cc0e 100644 --- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -29846,12 +29846,36 @@ class RuntimeStat { class GetRuntimeStatsRequest { static $_TSPEC; + /** + * @var int + */ + public $maxWeight = null; + /** + * @var int + */ + public $maxCreateTime = null; - public function __construct() { + public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { self::$_TSPEC = array( + 1 => array( + 'var' => 'maxWeight', + 'type' => TType::I32, + ), + 2 => array( + 'var' => 'maxCreateTime', + 'type' => TType::I32, + ), ); } + if (is_array($vals)) { + if (isset($vals['maxWeight'])) { + $this->maxWeight = $vals['maxWeight']; + } + if (isset($vals['maxCreateTime'])) { + $this->maxCreateTime = $vals['maxCreateTime']; + } + } } public function getName() { @@ -29873,6 +29897,20 @@ class GetRuntimeStatsRequest { } switch ($fid) { + case 1: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->maxWeight); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->maxCreateTime); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -29886,6 +29924,16 @@ class GetRuntimeStatsRequest { public function write($output) { $xfer = 0; $xfer += $output->writeStructBegin('GetRuntimeStatsRequest'); + if ($this->maxWeight !== null) { + $xfer += $output->writeFieldBegin('maxWeight', TType::I32, 1); + $xfer += $output->writeI32($this->maxWeight); + $xfer += $output->writeFieldEnd(); + } + if ($this->maxCreateTime !== null) { + $xfer += $output->writeFieldBegin('maxCreateTime', TType::I32, 2); + $xfer += $output->writeI32($this->maxCreateTime); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index b1e577a..69579e2 100644 --- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -21332,10 +21332,22 @@ class RuntimeStat: return not (self == other) class GetRuntimeStatsRequest: + """ + Attributes: + - maxWeight + - maxCreateTime + """ thrift_spec = ( + None, # 0 + (1, TType.I32, 'maxWeight', None, None, ), # 1 + (2, TType.I32, 'maxCreateTime', None, None, ), # 2 ) + def __init__(self, maxWeight=None, maxCreateTime=None,): + self.maxWeight = maxWeight + self.maxCreateTime = maxCreateTime + def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) @@ -21345,6 +21357,16 @@ class GetRuntimeStatsRequest: (fname, ftype, fid) = iprot.readFieldBegin() if ftype == TType.STOP: break + if fid == 1: + if ftype == TType.I32: + self.maxWeight = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.maxCreateTime = iprot.readI32() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -21355,15 +21377,29 @@ class GetRuntimeStatsRequest: oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) return oprot.writeStructBegin('GetRuntimeStatsRequest') + if self.maxWeight is not None: + oprot.writeFieldBegin('maxWeight', TType.I32, 1) + oprot.writeI32(self.maxWeight) + oprot.writeFieldEnd() + if self.maxCreateTime is not None: + oprot.writeFieldBegin('maxCreateTime', TType.I32, 2) + oprot.writeI32(self.maxCreateTime) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): + if self.maxWeight is None: + raise TProtocol.TProtocolException(message='Required field maxWeight is unset!') + if self.maxCreateTime is None: + raise TProtocol.TProtocolException(message='Required field maxCreateTime is unset!') return def __hash__(self): value = 17 + value = (value * 31) ^ hash(self.maxWeight) + value = (value * 31) ^ hash(self.maxCreateTime) return value def __repr__(self): http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index 2687ce5..d7ebaaf 100644 --- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -4849,14 +4849,19 @@ end class GetRuntimeStatsRequest include ::Thrift::Struct, ::Thrift::Struct_Union + MAXWEIGHT = 1 + MAXCREATETIME = 2 FIELDS = { - + MAXWEIGHT => {:type => ::Thrift::Types::I32, :name => 'maxWeight'}, + MAXCREATETIME => {:type => ::Thrift::Types::I32, :name => 'maxCreateTime'} } def struct_fields; FIELDS; end def validate + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxWeight is unset!') unless @maxWeight + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field maxCreateTime is unset!') unless @maxCreateTime end ::Thrift::Struct.generate_accessors self http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 397a081..b0d5d52 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -8461,7 +8461,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { startFunction("get_runtime_stats"); Exception ex = null; try { - List<RuntimeStat> res = getMS().getRuntimeStats(); + List<RuntimeStat> res = getMS().getRuntimeStats(rqst.getMaxWeight(), rqst.getMaxCreateTime()); return res; } catch (MetaException e) { LOG.error("Caught exception", e); http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 1138ed3..e60c5e1 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -55,7 +55,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -3249,7 +3248,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { } @Override - public List<RuntimeStat> getRuntimeStats() throws TException { - return client.get_runtime_stats(new GetRuntimeStatsRequest()); + public List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException { + GetRuntimeStatsRequest req = new GetRuntimeStatsRequest(); + req.setMaxWeight(maxWeight); + req.setMaxCreateTime(maxCreateTime); + return client.get_runtime_stats(req); } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 72b814d..ebdf58f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -29,7 +29,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.annotation.NoReconnect; @@ -3648,6 +3647,6 @@ public interface IMetaStoreClient { void addRuntimeStat(RuntimeStat stat) throws TException; /** Reads runtime statistics. */ - List<RuntimeStat> getRuntimeStats() throws TException; + List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException; } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 2bd7c21..9c94d7e 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -55,8 +55,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; -import java.util.stream.Collectors; - import javax.jdo.JDOCanRetryException; import javax.jdo.JDODataStoreException; import javax.jdo.JDOException; @@ -11616,41 +11614,47 @@ public class ObjectStore implements RawStore, Configurable { public void addRuntimeStat(RuntimeStat stat) throws MetaException { LOG.debug("runtimeStat: " + stat); MRuntimeStat mStat = MRuntimeStat.fromThrift(stat); - pm.makePersistent(mStat); + boolean committed = false; + openTransaction(); + try { + pm.makePersistent(mStat); + committed = commitTransaction(); + } finally { + if (!committed) { + rollbackTransaction(); + } + } } @Override - public int deleteRuntimeStats(int maxRetainedWeight, int maxRetainSecs) throws MetaException { - List<MRuntimeStat> all = getMRuntimeStats(); - int retentionTime = 0; - if (maxRetainSecs >= 0) { - retentionTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs; + public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { + if (maxRetainSecs < 0) { + LOG.warn("runtime stats retention is disabled"); + return 0; } - if (maxRetainedWeight < 0) { - maxRetainedWeight = Integer.MAX_VALUE; - } - - Object maxIdToRemove = null; - long totalWeight = 0; - int deleted = 0; - for (MRuntimeStat mRuntimeStat : all) { - totalWeight += mRuntimeStat.getWeight(); - if (totalWeight > maxRetainedWeight || mRuntimeStat.getCreatedTime() < retentionTime) { - LOG.debug("removing runtime stat: " + mRuntimeStat); - pm.deletePersistent(mRuntimeStat); - deleted++; + boolean committed = false; + try { + openTransaction(); + int maxCreateTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs; + Query q = pm.newQuery(MRuntimeStat.class); + q.setFilter("createTime <= maxCreateTime"); + q.declareParameters("int maxCreateTime"); + long deleted = q.deletePersistentAll(maxCreateTime); + committed = commitTransaction(); + return (int) deleted; + } finally { + if (!committed) { + rollbackTransaction(); } } - return deleted; } @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { + public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { boolean committed = false; try { openTransaction(); - List<MRuntimeStat> mStats = getMRuntimeStats(); - List<RuntimeStat> stats = mStats.stream().map(MRuntimeStat::toThrift).collect(Collectors.toList()); + List<RuntimeStat> stats = getMRuntimeStats(maxEntries, maxCreateTime); committed = commitTransaction(); return stats; } finally { @@ -11660,12 +11664,27 @@ public class ObjectStore implements RawStore, Configurable { } } - private List<MRuntimeStat> getMRuntimeStats() { + private List<RuntimeStat> getMRuntimeStats(int maxEntries, int maxCreateTime) { Query<MRuntimeStat> query = pm.newQuery(MRuntimeStat.class); query.setOrdering("createTime descending"); + if (maxCreateTime > 0) { + query.setFilter("createTime < "+maxCreateTime); + } + if (maxEntries < 0) { + maxEntries = Integer.MAX_VALUE; + } + List<RuntimeStat> ret = new ArrayList<>(); List<MRuntimeStat> res = (List<MRuntimeStat>) query.execute(); - pm.retrieveAll(res); - return res; + int totalEntries = 0; + for (MRuntimeStat mRuntimeStat : res) { + pm.retrieve(mRuntimeStat); + totalEntries += mRuntimeStat.getWeight(); + ret.add(MRuntimeStat.toThrift(mRuntimeStat)); + if (totalEntries >= maxEntries) { + break; + } + } + return ret; } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 2c9f2e5..3719c39 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -1630,8 +1630,9 @@ public interface RawStore extends Configurable { void addRuntimeStat(RuntimeStat stat) throws MetaException; /** Reads runtime statistic entries. */ - List<RuntimeStat> getRuntimeStats() throws MetaException; + List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException; /** Removes outdated statistics. */ - int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException; + int deleteRuntimeStats(int maxRetainSecs) throws MetaException; + } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java index 202058e..ccec93d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java @@ -53,9 +53,8 @@ public class RuntimeStatsCleanerTask implements MetastoreTaskThread { try { RawStore ms = HiveMetaStore.HMSHandler.getMSForConf(conf); - int maxRetained = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.RUNTIME_STATS_MAX_ENTRIES); int maxRetainSecs=(int) MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.RUNTIME_STATS_MAX_AGE, TimeUnit.SECONDS); - int deleteCnt = ms.deleteRuntimeStats(maxRetained, maxRetainSecs); + int deleteCnt = ms.deleteRuntimeStats(maxRetainSecs); if (deleteCnt > 0L){ LOG.info("Number of deleted entries: " + deleteCnt); http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 92d000b..30d4bec 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -2477,12 +2477,12 @@ public class CachedStore implements RawStore, Configurable { } @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { - return rawStore.getRuntimeStats(); + public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { + return rawStore.getRuntimeStats(maxEntries, maxCreateTime); } @Override - public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException { - return rawStore.deleteRuntimeStats(maxRetained, maxRetainSecs); + public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { + return rawStore.deleteRuntimeStats(maxRetainSecs); } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 35aa40c..114d5da 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -583,9 +583,6 @@ public class MetastoreConf { TimeUnit.SECONDS, "Frequency at which timer task runs to remove outdated runtime stat entries."), RUNTIME_STATS_MAX_AGE("runtime.stats.max.age", "hive.metastore.runtime.stats.max.age", 86400 * 3, TimeUnit.SECONDS, "Stat entries which are older than this are removed."), - RUNTIME_STATS_MAX_ENTRIES("runtime.stats.max.entries", "hive.metastore.runtime.stats.max.entries", 100_000, - "Maximum number of runtime stats to keep; unit is operator stat infos - a complicated query has ~100 of these." - + "See also: hive.query.reexecution.stats.cache.size"), // Parameters for exporting metadata on table drop (requires the use of the) // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index c56a4f9..ccc3c93 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -1524,6 +1524,8 @@ struct RuntimeStat { } struct GetRuntimeStatsRequest { + 1: required i32 maxWeight, + 2: required i32 maxCreateTime } // Exceptions. http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index defc68f..3895c3d 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1171,12 +1171,12 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { - return objectStore.getRuntimeStats(); + public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { + return objectStore.getRuntimeStats(maxEntries, maxCreateTime); } @Override - public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException { - return objectStore.deleteRuntimeStats(maxRetained, maxRetainSecs); + public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { + return objectStore.deleteRuntimeStats(maxRetainSecs); } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 20c5d8a..cdfa001 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1157,12 +1157,12 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { + public List<RuntimeStat> getRuntimeStats(int maxEntries, int maxCreateTime) throws MetaException { return Collections.emptyList(); } @Override - public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException { + public int deleteRuntimeStats(int maxRetainSecs) throws MetaException { return 0; } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 8ae899f..4960e3b 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -53,7 +53,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -3359,7 +3358,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override - public List<RuntimeStat> getRuntimeStats() throws TException { + public List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException { throw new UnsupportedOperationException(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/577d32ef/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java index 2db7a8b..0b91ed4 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.metastore.client; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.api.RuntimeStat; import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; @@ -40,8 +41,10 @@ import static org.junit.Assert.assertNotNull; public class TestRuntimeStats extends MetaStoreClientTest { private final AbstractMetaStoreService metaStore; private IMetaStoreClient client; + private String metastoreName; public TestRuntimeStats(String name, AbstractMetaStoreService metaStore) throws Exception { + this.metastoreName = name; this.metaStore = metaStore; this.metaStore.start(); } @@ -60,14 +63,14 @@ public class TestRuntimeStats extends MetaStoreClientTest { @Test public void testRuntimeStatHandling() throws Exception { - List<RuntimeStat> rs0 = client.getRuntimeStats(); + List<RuntimeStat> rs0 = getRuntimeStats(); assertNotNull(rs0); assertEquals(0, rs0.size()); RuntimeStat stat = createStat(1); client.addRuntimeStat(stat); - List<RuntimeStat> rs1 = client.getRuntimeStats(); + List<RuntimeStat> rs1 = getRuntimeStats(); assertNotNull(rs1); assertEquals(1, rs1.size()); assertArrayEquals(stat.getPayload(), rs1.get(0).getPayload()); @@ -79,11 +82,59 @@ public class TestRuntimeStats extends MetaStoreClientTest { client.addRuntimeStat(createStat(3)); client.addRuntimeStat(createStat(4)); - List<RuntimeStat> rs2 = client.getRuntimeStats(); + List<RuntimeStat> rs2 = getRuntimeStats(); assertEquals(4, rs2.size()); } + @Test + public void testCleanup() throws Exception { + ObjectStore objStore = new ObjectStore(); + objStore.setConf(metaStore.getConf()); + objStore.deleteRuntimeStats(0); + objStore.addRuntimeStat(createStat(1)); + Thread.sleep(2000); + objStore.addRuntimeStat(createStat(2)); + int deleted = objStore.deleteRuntimeStats(1); + assertEquals(1, deleted); + + List<RuntimeStat> all = getRuntimeStats(); + assertEquals(1, all.size()); + assertEquals(2, all.get(0).getWeight()); + + } + + @Test + public void testReading() throws Exception { + ObjectStore objStore = new ObjectStore(); + objStore.setConf(metaStore.getConf()); + objStore.deleteRuntimeStats(0); + objStore.addRuntimeStat(createStat(1)); + Thread.sleep(1000); + objStore.addRuntimeStat(createStat(2)); + Thread.sleep(1000); + objStore.addRuntimeStat(createStat(3)); + + List<RuntimeStat> g0 = client.getRuntimeStats(3, -1); + assertEquals(1, g0.size()); + assertEquals(3, g0.get(0).getWeight()); + int ct = g0.get(0).getCreateTime(); + List<RuntimeStat> g1 = client.getRuntimeStats(3, ct); + + assertEquals(2, g1.size()); + assertEquals(2, g1.get(0).getWeight()); + assertEquals(1, g1.get(1).getWeight()); + int ct1 = g1.get(1).getCreateTime(); + List<RuntimeStat> g2 = client.getRuntimeStats(3, ct1); + + assertEquals(0, g2.size()); + + } + + private List<RuntimeStat> getRuntimeStats() throws Exception { + return client.getRuntimeStats(-1, -1); + } + private RuntimeStat createStat(int w) { byte[] payload = new byte[w]; @@ -97,4 +148,5 @@ public class TestRuntimeStats extends MetaStoreClientTest { return stat; } + }
