http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 184ecb6..125d5a7 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -55,7 +55,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; -import java.util.stream.Collectors; import javax.jdo.JDOCanRetryException; import javax.jdo.JDODataStoreException; @@ -84,6 +83,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; @@ -124,7 +124,6 @@ import org.apache.hadoop.hive.metastore.api.ResourceType; import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; @@ -187,7 +186,6 @@ import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege; import org.apache.hadoop.hive.metastore.model.MResourceUri; import org.apache.hadoop.hive.metastore.model.MRole; import org.apache.hadoop.hive.metastore.model.MRoleMap; -import org.apache.hadoop.hive.metastore.model.MRuntimeStat; import org.apache.hadoop.hive.metastore.model.MSchemaVersion; import org.apache.hadoop.hive.metastore.model.MSerDeInfo; import org.apache.hadoop.hive.metastore.model.MStorageDescriptor; @@ -212,6 +210,7 @@ import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; +import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; import org.datanucleus.AbstractNucleusContext; import org.datanucleus.ClassLoaderResolver; @@ -810,9 +809,7 @@ public class ObjectStore implements RawStore, Configurable { pm.makePersistent(mCat); committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -835,9 +832,7 @@ public class ObjectStore implements RawStore, Configurable { pm.makePersistent(mCat); committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -845,9 +840,7 @@ public class ObjectStore implements RawStore, Configurable { public Catalog getCatalog(String catalogName) throws NoSuchObjectException, MetaException { LOG.debug("Fetching catalog " + catalogName); MCatalog mCat = getMCatalog(catalogName); - if (mCat == null) { - throw new NoSuchObjectException("No catalog " + catalogName); - } + if (mCat == null) throw new NoSuchObjectException("No catalog " + catalogName); return mCatToCat(mCat); } @@ -881,15 +874,11 @@ public class ObjectStore implements RawStore, Configurable { openTransaction(); MCatalog mCat = getMCatalog(catalogName); pm.retrieve(mCat); - if (mCat == null) { - throw new NoSuchObjectException("No catalog " + catalogName); - } + if (mCat == null) throw new NoSuchObjectException("No catalog " + catalogName); pm.deletePersistent(mCat); committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -914,18 +903,14 @@ public class ObjectStore implements RawStore, Configurable { private MCatalog catToMCat(Catalog cat) { MCatalog mCat = new MCatalog(); mCat.setName(normalizeIdentifier(cat.getName())); - if (cat.isSetDescription()) { - mCat.setDescription(cat.getDescription()); - } + if (cat.isSetDescription()) mCat.setDescription(cat.getDescription()); mCat.setLocationUri(cat.getLocationUri()); return mCat; } private Catalog mCatToCat(MCatalog mCat) { Catalog cat = new Catalog(mCat.getName(), mCat.getLocationUri()); - if (mCat.getDescription() != null) { - cat.setDescription(mCat.getDescription()); - } + if (mCat.getDescription() != null) cat.setDescription(mCat.getDescription()); return cat; } @@ -1998,18 +1983,10 @@ public class ObjectStore implements RawStore, Configurable { } SerDeInfo serde = new SerDeInfo(ms.getName(), ms.getSerializationLib(), convertMap(ms.getParameters())); - if (ms.getDescription() != null) { - serde.setDescription(ms.getDescription()); - } - if (ms.getSerializerClass() != null) { - serde.setSerializerClass(ms.getSerializerClass()); - } - if (ms.getDeserializerClass() != null) { - serde.setDeserializerClass(ms.getDeserializerClass()); - } - if (ms.getSerdeType() > 0) { - serde.setSerdeType(SerdeType.findByValue(ms.getSerdeType())); - } + if (ms.getDescription() != null) serde.setDescription(ms.getDescription()); + if (ms.getSerializerClass() != null) serde.setSerializerClass(ms.getSerializerClass()); + if (ms.getDeserializerClass() != null) serde.setDeserializerClass(ms.getDeserializerClass()); + if (ms.getSerdeType() > 0) serde.setSerdeType(SerdeType.findByValue(ms.getSerdeType())); return serde; } @@ -3702,7 +3679,7 @@ public class ObjectStore implements RawStore, Configurable { @Override protected boolean canUseDirectSql(GetHelper<Integer> ctx) throws MetaException { return directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, filter); - } + }; @Override protected Integer getSqlResult(GetHelper<Integer> ctx) throws MetaException { @@ -10021,9 +9998,7 @@ public class ObjectStore implements RawStore, Configurable { pm.makePersistent(mSchema); committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -10042,17 +10017,11 @@ public class ObjectStore implements RawStore, Configurable { oldMSchema.setCompatibility(newSchema.getCompatibility().getValue()); oldMSchema.setValidationLevel(newSchema.getValidationLevel().getValue()); oldMSchema.setCanEvolve(newSchema.isCanEvolve()); - if (newSchema.isSetSchemaGroup()) { - oldMSchema.setSchemaGroup(newSchema.getSchemaGroup()); - } - if (newSchema.isSetDescription()) { - oldMSchema.setDescription(newSchema.getDescription()); - } + if (newSchema.isSetSchemaGroup()) oldMSchema.setSchemaGroup(newSchema.getSchemaGroup()); + if (newSchema.isSetDescription()) oldMSchema.setDescription(newSchema.getDescription()); committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -10066,9 +10035,7 @@ public class ObjectStore implements RawStore, Configurable { committed = commitTransaction(); return schema; } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -10087,9 +10054,7 @@ public class ObjectStore implements RawStore, Configurable { pm.retrieve(mSchema); return mSchema; } finally { - if (query != null) { - query.closeAll(); - } + if (query != null) query.closeAll(); } } @@ -10106,9 +10071,7 @@ public class ObjectStore implements RawStore, Configurable { } committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -10133,9 +10096,7 @@ public class ObjectStore implements RawStore, Configurable { pm.makePersistent(mSchemaVersion); committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction();; } } @@ -10152,17 +10113,11 @@ public class ObjectStore implements RawStore, Configurable { } // We only support changing the SerDe mapping and the state. - if (newVersion.isSetSerDe()) { - oldMSchemaVersion.setSerDe(convertToMSerDeInfo(newVersion.getSerDe())); - } - if (newVersion.isSetState()) { - oldMSchemaVersion.setState(newVersion.getState().getValue()); - } + if (newVersion.isSetSerDe()) oldMSchemaVersion.setSerDe(convertToMSerDeInfo(newVersion.getSerDe())); + if (newVersion.isSetState()) oldMSchemaVersion.setState(newVersion.getState().getValue()); committed = commitTransaction(); } finally { - if (!committed) { - commitTransaction(); - } + if (!committed) commitTransaction(); } } @@ -10177,9 +10132,7 @@ public class ObjectStore implements RawStore, Configurable { committed = commitTransaction(); return schemaVersion; } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction();; } } @@ -10199,15 +10152,11 @@ public class ObjectStore implements RawStore, Configurable { pm.retrieve(mSchemaVersion); if (mSchemaVersion != null) { pm.retrieveAll(mSchemaVersion.getCols()); - if (mSchemaVersion.getSerDe() != null) { - pm.retrieve(mSchemaVersion.getSerDe()); - } + if (mSchemaVersion.getSerDe() != null) pm.retrieve(mSchemaVersion.getSerDe()); } return mSchemaVersion; } finally { - if (query != null) { - query.closeAll(); - } + if (query != null) query.closeAll(); } } @@ -10231,9 +10180,7 @@ public class ObjectStore implements RawStore, Configurable { pm.retrieve(mSchemaVersion); if (mSchemaVersion != null) { pm.retrieveAll(mSchemaVersion.getCols()); - if (mSchemaVersion.getSerDe() != null) { - pm.retrieve(mSchemaVersion.getSerDe()); - } + if (mSchemaVersion.getSerDe() != null) pm.retrieve(mSchemaVersion.getSerDe()); } SchemaVersion version = mSchemaVersion == null ? null : convertToSchemaVersion(mSchemaVersion); committed = commitTransaction(); @@ -10259,15 +10206,11 @@ public class ObjectStore implements RawStore, Configurable { query.setOrdering("version descending"); List<MSchemaVersion> mSchemaVersions = query.setParameters(name, dbName, catName).executeList(); pm.retrieveAll(mSchemaVersions); - if (mSchemaVersions == null || mSchemaVersions.isEmpty()) { - return null; - } + if (mSchemaVersions == null || mSchemaVersions.isEmpty()) return null; List<SchemaVersion> schemaVersions = new ArrayList<>(mSchemaVersions.size()); for (MSchemaVersion mSchemaVersion : mSchemaVersions) { pm.retrieveAll(mSchemaVersion.getCols()); - if (mSchemaVersion.getSerDe() != null) { - pm.retrieve(mSchemaVersion.getSerDe()); - } + if (mSchemaVersion.getSerDe() != null) pm.retrieve(mSchemaVersion.getSerDe()); schemaVersions.add(convertToSchemaVersion(mSchemaVersion)); } committed = commitTransaction(); @@ -10289,12 +10232,8 @@ public class ObjectStore implements RawStore, Configurable { Query query = null; try { openTransaction(); - if (colName != null) { - colName = normalizeIdentifier(colName); - } - if (type != null) { - type = normalizeIdentifier(type); - } + if (colName != null) colName = normalizeIdentifier(colName); + if (type != null) type = normalizeIdentifier(type); Map<String, String> parameters = new HashMap<>(3); StringBuilder sql = new StringBuilder("select SCHEMA_VERSION_ID from " + "SCHEMA_VERSION, COLUMNS_V2 where SCHEMA_VERSION.CD_ID = COLUMNS_V2.CD_ID "); @@ -10320,16 +10259,12 @@ public class ObjectStore implements RawStore, Configurable { query = pm.newQuery("javax.jdo.query.SQL", sql.toString()); query.setClass(MSchemaVersion.class); List<MSchemaVersion> mSchemaVersions = query.setNamedParameters(parameters).executeList(); - if (mSchemaVersions == null || mSchemaVersions.isEmpty()) { - return Collections.emptyList(); - } + if (mSchemaVersions == null || mSchemaVersions.isEmpty()) return Collections.emptyList(); pm.retrieveAll(mSchemaVersions); List<SchemaVersion> schemaVersions = new ArrayList<>(mSchemaVersions.size()); for (MSchemaVersion mSchemaVersion : mSchemaVersions) { pm.retrieveAll(mSchemaVersion.getCols()); - if (mSchemaVersion.getSerDe() != null) { - pm.retrieve(mSchemaVersion.getSerDe()); - } + if (mSchemaVersion.getSerDe() != null) pm.retrieve(mSchemaVersion.getSerDe()); schemaVersions.add(convertToSchemaVersion(mSchemaVersion)); } committed = commitTransaction(); @@ -10356,9 +10291,7 @@ public class ObjectStore implements RawStore, Configurable { } committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -10375,9 +10308,7 @@ public class ObjectStore implements RawStore, Configurable { committed = commitTransaction(); return serde; } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction();; } } @@ -10391,9 +10322,7 @@ public class ObjectStore implements RawStore, Configurable { pm.retrieve(mSerDeInfo); return mSerDeInfo; } finally { - if (query != null) { - query.closeAll(); - } + if (query != null) query.closeAll(); } } @@ -10409,9 +10338,7 @@ public class ObjectStore implements RawStore, Configurable { pm.makePersistent(mSerde); committed = commitTransaction(); } finally { - if (!committed) { - rollbackTransaction(); - } + if (!committed) rollbackTransaction(); } } @@ -10428,9 +10355,7 @@ public class ObjectStore implements RawStore, Configurable { } private ISchema convertToISchema(MISchema mSchema) { - if (mSchema == null) { - return null; - } + if (mSchema == null) return null; ISchema schema = new ISchema(SchemaType.findByValue(mSchema.getSchemaType()), mSchema.getName(), mSchema.getDb().getCatalogName(), @@ -10438,12 +10363,8 @@ public class ObjectStore implements RawStore, Configurable { SchemaCompatibility.findByValue(mSchema.getCompatibility()), SchemaValidation.findByValue(mSchema.getValidationLevel()), mSchema.getCanEvolve()); - if (mSchema.getDescription() != null) { - schema.setDescription(mSchema.getDescription()); - } - if (mSchema.getSchemaGroup() != null) { - schema.setSchemaGroup(mSchema.getSchemaGroup()); - } + if (mSchema.getDescription() != null) schema.setDescription(mSchema.getDescription()); + if (mSchema.getSchemaGroup() != null) schema.setSchemaGroup(mSchema.getSchemaGroup()); return schema; } @@ -10464,33 +10385,19 @@ public class ObjectStore implements RawStore, Configurable { } private SchemaVersion convertToSchemaVersion(MSchemaVersion mSchemaVersion) throws MetaException { - if (mSchemaVersion == null) { - return null; - } + if (mSchemaVersion == null) return null; SchemaVersion schemaVersion = new SchemaVersion( new ISchemaName(mSchemaVersion.getiSchema().getDb().getCatalogName(), mSchemaVersion.getiSchema().getDb().getName(), mSchemaVersion.getiSchema().getName()), mSchemaVersion.getVersion(), mSchemaVersion.getCreatedAt(), convertToFieldSchemas(mSchemaVersion.getCols().getCols())); - if (mSchemaVersion.getState() > 0) { - schemaVersion.setState(SchemaVersionState.findByValue(mSchemaVersion.getState())); - } - if (mSchemaVersion.getDescription() != null) { - schemaVersion.setDescription(mSchemaVersion.getDescription()); - } - if (mSchemaVersion.getSchemaText() != null) { - schemaVersion.setSchemaText(mSchemaVersion.getSchemaText()); - } - if (mSchemaVersion.getFingerprint() != null) { - schemaVersion.setFingerprint(mSchemaVersion.getFingerprint()); - } - if (mSchemaVersion.getName() != null) { - schemaVersion.setName(mSchemaVersion.getName()); - } - if (mSchemaVersion.getSerDe() != null) { - schemaVersion.setSerDe(convertToSerDeInfo(mSchemaVersion.getSerDe())); - } + if (mSchemaVersion.getState() > 0) schemaVersion.setState(SchemaVersionState.findByValue(mSchemaVersion.getState())); + if (mSchemaVersion.getDescription() != null) schemaVersion.setDescription(mSchemaVersion.getDescription()); + if (mSchemaVersion.getSchemaText() != null) schemaVersion.setSchemaText(mSchemaVersion.getSchemaText()); + if (mSchemaVersion.getFingerprint() != null) schemaVersion.setFingerprint(mSchemaVersion.getFingerprint()); + if (mSchemaVersion.getName() != null) schemaVersion.setName(mSchemaVersion.getName()); + if (mSchemaVersion.getSerDe() != null) schemaVersion.setSerDe(convertToSerDeInfo(mSchemaVersion.getSerDe())); return schemaVersion; } @@ -11600,61 +11507,4 @@ public class ObjectStore implements RawStore, Configurable { rollbackAndCleanup(commited, (Query)null); } } - - @Override - public void addRuntimeStat(RuntimeStat stat) throws MetaException { - LOG.debug("runtimeStat: " + stat); - MRuntimeStat mStat = MRuntimeStat.fromThrift(stat); - pm.makePersistent(mStat); - } - - @Override - public int deleteRuntimeStats(int maxRetainedWeight, int maxRetainSecs) throws MetaException { - List<MRuntimeStat> all = getMRuntimeStats(); - int retentionTime = 0; - if (maxRetainSecs >= 0) { - retentionTime = (int) (System.currentTimeMillis() / 1000) - maxRetainSecs; - } - if (maxRetainedWeight < 0) { - maxRetainedWeight = Integer.MAX_VALUE; - } - - Object maxIdToRemove = null; - long totalWeight = 0; - int deleted = 0; - for (MRuntimeStat mRuntimeStat : all) { - totalWeight += mRuntimeStat.getWeight(); - if (totalWeight > maxRetainedWeight || mRuntimeStat.getCreatedTime() < retentionTime) { - LOG.debug("removing runtime stat: " + mRuntimeStat); - pm.deletePersistent(mRuntimeStat); - deleted++; - } - } - return deleted; - } - - @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { - boolean committed = false; - try { - openTransaction(); - List<MRuntimeStat> mStats = getMRuntimeStats(); - List<RuntimeStat> stats = mStats.stream().map(MRuntimeStat::toThrift).collect(Collectors.toList()); - committed = commitTransaction(); - return stats; - } finally { - if (!committed) { - rollbackTransaction(); - } - } - } - - private List<MRuntimeStat> getMRuntimeStats() { - Query<MRuntimeStat> query = pm.newQuery(MRuntimeStat.class); - query.setOrdering("createTime descending"); - List<MRuntimeStat> res = (List<MRuntimeStat>) query.execute(); - pm.retrieveAll(res); - return res; - } - }
http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 2c9f2e5..f6c46ee 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.metastore.api.AlterISchemaRequest; import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.ISchemaName; import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; @@ -37,6 +38,7 @@ import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -44,6 +46,7 @@ import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; import org.apache.hadoop.hive.metastore.api.ISchema; +import org.apache.hadoop.hive.metastore.api.ISchemaName; import org.apache.hadoop.hive.metastore.api.InvalidInputException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -63,8 +66,8 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; +import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; @@ -78,6 +81,7 @@ import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; import org.apache.hadoop.hive.metastore.api.WMMapping; import org.apache.hadoop.hive.metastore.api.WMNullablePool; import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; @@ -1625,13 +1629,4 @@ public interface RawStore extends Configurable { * @throws MetaException general database exception */ void addSerde(SerDeInfo serde) throws AlreadyExistsException, MetaException; - - /** Adds a RuntimeStat for persistence. */ - void addRuntimeStat(RuntimeStat stat) throws MetaException; - - /** Reads runtime statistic entries. */ - List<RuntimeStat> getRuntimeStats() throws MetaException; - - /** Removes outdated statistics. */ - int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException; } http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java deleted file mode 100644 index 202058e..0000000 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RuntimeStatsCleanerTask.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.metastore.RawStore; -import java.util.concurrent.TimeUnit; - -/** - * Metastore task to handle RuntimeStat related expiration. - */ -public class RuntimeStatsCleanerTask implements MetastoreTaskThread { - private static final Logger LOG = LoggerFactory.getLogger(RuntimeStatsCleanerTask.class); - - private Configuration conf; - - @Override - public long runFrequency(TimeUnit unit) { - return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.RUNTIME_STATS_CLEAN_FREQUENCY, unit); - } - - @Override - public void setConf(Configuration configuration) { - conf = configuration; - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public void run() { - - try { - RawStore ms = HiveMetaStore.HMSHandler.getMSForConf(conf); - int maxRetained = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.RUNTIME_STATS_MAX_ENTRIES); - int maxRetainSecs=(int) MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.RUNTIME_STATS_MAX_AGE, TimeUnit.SECONDS); - int deleteCnt = ms.deleteRuntimeStats(maxRetained, maxRetainSecs); - - if (deleteCnt > 0L){ - LOG.info("Number of deleted entries: " + deleteCnt); - } - } catch (Exception e) { - LOG.error("Exception while trying to delete: " + e.getMessage(), e); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 92d000b..ebdcbc2 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -89,7 +89,6 @@ import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; @@ -221,9 +220,8 @@ public class CachedStore implements RawStore, Configurable { LOG.info("Going to cache catalogs: " + org.apache.commons.lang.StringUtils.join(catalogsToCache, ", ")); List<Catalog> catalogs = new ArrayList<>(catalogsToCache.size()); - for (String catName : catalogsToCache) { + for (String catName : catalogsToCache) catalogs.add(rawStore.getCatalog(catName)); - } sharedCache.populateCatalogsInCache(catalogs); } catch (MetaException | NoSuchObjectException e) { LOG.warn("Failed to populate catalogs in cache, going to try again", e); @@ -2177,7 +2175,6 @@ public class CachedStore implements RawStore, Configurable { return rawStore.addNotNullConstraints(nns); } - @Override public List<String> addDefaultConstraints(List<SQLDefaultConstraint> nns) throws InvalidObjectException, MetaException { // TODO constraintCache @@ -2198,7 +2195,6 @@ public class CachedStore implements RawStore, Configurable { rawStore.createISchema(schema); } - @Override public List<ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName) throws MetaException, NoSuchObjectException { return rawStore.getPartitionColStatsForDatabase(catName, dbName); @@ -2470,19 +2466,4 @@ public class CachedStore implements RawStore, Configurable { sharedCache.resetCatalogCache(); setCachePrewarmedState(false); } - - @Override - public void addRuntimeStat(RuntimeStat stat) throws MetaException { - rawStore.addRuntimeStat(stat); - } - - @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { - return rawStore.getRuntimeStats(); - } - - @Override - public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException { - return rawStore.deleteRuntimeStats(maxRetained, maxRetainSecs); - } } http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 552eeca..59749e4 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.metastore.HiveAlterHandler; import org.apache.hadoop.hive.metastore.MaterializationsCacheCleanerTask; import org.apache.hadoop.hive.metastore.MaterializationsRebuildLockCleanerTask; import org.apache.hadoop.hive.metastore.MetastoreTaskThread; -import org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; @@ -578,15 +577,6 @@ public class MetastoreConf { "hive.metastore.materializations.invalidation.max.duration", 86400, TimeUnit.SECONDS, "Maximum duration for query producing a materialization. After this time, transaction" + "entries that are not relevant for materializations can be removed from invalidation cache."), - - RUNTIME_STATS_CLEAN_FREQUENCY("runtime.stats.clean.frequency", "hive.metastore.runtime.stats.clean.frequency", 3600, - TimeUnit.SECONDS, "Frequency at which timer task runs to remove outdated runtime stat entries."), - RUNTIME_STATS_MAX_AGE("runtime.stats.max.age", "hive.metastore.runtime.stats.max.age", 86400 * 3, TimeUnit.SECONDS, - "Stat entries which are older than this are removed."), - RUNTIME_STATS_MAX_ENTRIES("runtime.stats.max.entries", "hive.metastore.runtime.stats.max.entries", 100_000, - "Maximum number of runtime stats to keep; unit is operator stat infos - a complicated query has ~100 of these." - + "See also: hive.query.reexecution.stats.cache.size"), - // Parameters for exporting metadata on table drop (requires the use of the) // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener METADATA_EXPORT_LOCATION("metastore.metadata.export.location", "hive.metadata.export.location", @@ -742,10 +732,10 @@ public class MetastoreConf { + "The only supported special character right now is '/'. This flag applies only to quoted table names.\n" + "The default value is true."), TASK_THREADS_ALWAYS("metastore.task.threads.always", "metastore.task.threads.always", - EventCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName() + "," + + EventCleanerTask.class.getName() + "," + "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + MaterializationsCacheCleanerTask.class.getName() + "," + - MaterializationsRebuildLockCleanerTask.class.getName() + "," + RuntimeStatsCleanerTask.class.getName(), + MaterializationsRebuildLockCleanerTask.class.getName(), "Comma separated list of tasks that will be started in separate threads. These will " + "always be started, regardless of whether the metastore is running in embedded mode " + "or in server mode. They must implement " + MetastoreTaskThread.class.getName()), @@ -1137,22 +1127,16 @@ public class MetastoreConf { */ hiveSiteURL = findConfigFile(classLoader, "hive-site.xml"); } - if (hiveSiteURL != null) { - conf.addResource(hiveSiteURL); - } + if (hiveSiteURL != null) conf.addResource(hiveSiteURL); // Now add hivemetastore-site.xml. Again we add this before our own config files so that the // newer overrides the older. hiveMetastoreSiteURL = findConfigFile(classLoader, "hivemetastore-site.xml"); - if (hiveMetastoreSiteURL != null) { - conf.addResource(hiveMetastoreSiteURL); - } + if (hiveMetastoreSiteURL != null) conf.addResource(hiveMetastoreSiteURL); // Add in our conf file metastoreSiteURL = findConfigFile(classLoader, "metastore-site.xml"); - if (metastoreSiteURL != null) { - conf.addResource(metastoreSiteURL); - } + if (metastoreSiteURL != null) conf.addResource(metastoreSiteURL); // If a system property that matches one of our conf value names is set then use the value // it's set to to set our own conf value. @@ -1284,12 +1268,8 @@ public class MetastoreConf { public static Collection<String> getStringCollection(Configuration conf, ConfVars var) { assert var.defaultVal.getClass() == String.class; String val = conf.get(var.varname); - if (val == null) { - val = conf.get(var.hiveName, (String)var.defaultVal); - } - if (val == null) { - return Collections.emptySet(); - } + if (val == null) val = conf.get(var.hiveName, (String)var.defaultVal); + if (val == null) return Collections.emptySet(); return StringUtils.asSet(val.split(",")); } http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MRuntimeStat.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MRuntimeStat.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MRuntimeStat.java deleted file mode 100644 index 054ce7c..0000000 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MRuntimeStat.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore.model; - -import org.apache.hadoop.hive.metastore.api.RuntimeStat; - -/** - * Represents a runtime stat query entry. - * - * As a query may contain a large number of operatorstat entries; they are stored together in a single row in the metastore. - * The number of operator stat entries this entity has; is shown in the weight column. - */ -public class MRuntimeStat { - - private int createTime; - private int weight; - private byte[] payload; - - public static MRuntimeStat fromThrift(RuntimeStat stat) { - MRuntimeStat ret = new MRuntimeStat(); - ret.weight = stat.getWeight(); - ret.payload = stat.getPayload(); - ret.createTime = (int) (System.currentTimeMillis() / 1000); - return ret; - } - - public static RuntimeStat toThrift(MRuntimeStat stat) { - RuntimeStat ret = new RuntimeStat(); - ret.setWeight(stat.weight); - ret.setCreateTime(stat.createTime); - ret.setPayload(stat.payload); - return ret; - } - - public int getWeight() { - return weight; - } - - public int getCreatedTime() { - return createTime; - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/resources/package.jdo ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/resources/package.jdo b/standalone-metastore/src/main/resources/package.jdo index 221192e..9ddf598 100644 --- a/standalone-metastore/src/main/resources/package.jdo +++ b/standalone-metastore/src/main/resources/package.jdo @@ -1339,20 +1339,6 @@ <column name="SERDE_ID"/> </field> </class> - <class name="MRuntimeStat" identity-type="datastore" table="RUNTIME_STATS" detachable="true"> - <datastore-identity> - <column name="RS_ID"/> - </datastore-identity> - <field name="createTime"> - <column name="CREATE_TIME" jdbc-type="integer"/> - </field> - <field name="weight"> - <column name="WEIGHT" jdbc-type="integer"/> - </field> - <field name="payload"> - <column name="PAYLOAD" jdbc-type="BLOB" allows-null="true"/> - </field> - </class> </package> </jdo> http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index 48d28cb..adfa4c5 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -676,16 +676,6 @@ CREATE TABLE REPL_TXN_MAP ( RTM_TARGET_TXN_ID bigint NOT NULL, PRIMARY KEY (RTM_REPL_POLICY, RTM_SRC_TXN_ID) ); - -CREATE TABLE "APP"."RUNTIME_STATS" ( - "RS_ID" bigint primary key, - "CREATE_TIME" integer not null, - "WEIGHT" integer not null, - "PAYLOAD" BLOB -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index ed6c4cd..a75b740 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -238,16 +238,6 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); - -CREATE TABLE "APP"."RUNTIME_STATS" ( - "RS_ID" bigint primary key, - "CREATE_TIME" integer not null, - "WEIGHT" integer not null, - "PAYLOAD" BLOB -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index 6e31b16..91c581c 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -1230,15 +1230,6 @@ CREATE UNIQUE INDEX PART_TABLE_PK ON SEQUENCE_TABLE (SEQUENCE_NAME); INSERT INTO SEQUENCE_TABLE (SEQUENCE_NAME, NEXT_VAL) VALUES ('org.apache.hadoop.hive.metastore.model.MNotificationLog', 1); -CREATE TABLE RUNTIME_STATS ( - RS_ID bigint primary key, - CREATE_TIME bigint NOT NULL, - WEIGHT bigint NOT NULL, - PAYLOAD varbinary(max) -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index c2504d3..87f5884 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -306,15 +306,6 @@ PRIMARY KEY CLUSTERED CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); -CREATE TABLE RUNTIME_STATS ( - RS_ID bigint primary key, - CREATE_TIME bigint NOT NULL, - WEIGHT bigint NOT NULL, - PAYLOAD varbinary(max) -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index 4309911..7e2a57a 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -1154,16 +1154,6 @@ CREATE TABLE REPL_TXN_MAP ( PRIMARY KEY (RTM_REPL_POLICY, RTM_SRC_TXN_ID) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -CREATE TABLE RUNTIME_STATS ( - RS_ID bigint primary key, - CREATE_TIME bigint NOT NULL, - WEIGHT bigint NOT NULL, - PAYLOAD blob -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index e01b4da..5ba68ca 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -279,15 +279,6 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); -CREATE TABLE RUNTIME_STATS ( - RS_ID bigint primary key, - CREATE_TIME bigint NOT NULL, - WEIGHT bigint NOT NULL, - PAYLOAD blob -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index a45c7bb..f9e1a19 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -1124,16 +1124,6 @@ CREATE TABLE REPL_TXN_MAP ( PRIMARY KEY (RTM_REPL_POLICY, RTM_SRC_TXN_ID) ); -CREATE TABLE RUNTIME_STATS ( - RS_ID NUMBER primary key, - CREATE_TIME NUMBER(10) NOT NULL, - WEIGHT NUMBER(10) NOT NULL, - PAYLOAD BLOB -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - - -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index 327800b..a769d24 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -297,15 +297,6 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); -CREATE TABLE RUNTIME_STATS ( - RS_ID NUMBER primary key, - CREATE_TIME NUMBER(10) NOT NULL, - WEIGHT NUMBER(10) NOT NULL, - PAYLOAD BLOB -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index 2484744..6fed072 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -1811,17 +1811,6 @@ CREATE TABLE REPL_TXN_MAP ( PRIMARY KEY (RTM_REPL_POLICY, RTM_SRC_TXN_ID) ); - -CREATE TABLE RUNTIME_STATS ( - RS_ID bigint primary key, - CREATE_TIME bigint NOT NULL, - WEIGHT bigint NOT NULL, - PAYLOAD bytea -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - - -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index 63932a9..7b6b3b7 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -314,15 +314,6 @@ CREATE TABLE MIN_HISTORY_LEVEL ( CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); -CREATE TABLE RUNTIME_STATS ( - RS_ID bigint primary key, - CREATE_TIME bigint NOT NULL, - WEIGHT bigint NOT NULL, - PAYLOAD bytea -); - -CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='3.0.0', "VERSION_COMMENT"='Hive release version 3.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0'; http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index c56a4f9..12e4e40 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -1517,15 +1517,6 @@ struct GetSerdeRequest { 1: string serdeName } -struct RuntimeStat { - 1: optional i32 createTime, - 2: required i32 weight, - 3: required binary payload -} - -struct GetRuntimeStatsRequest { -} - // Exceptions. exception MetaException { @@ -2180,9 +2171,6 @@ service ThriftHiveMetastore extends fb303.FacebookService LockResponse get_lock_materialization_rebuild(1: string dbName, 2: string tableName, 3: i64 txnId) bool heartbeat_lock_materialization_rebuild(1: string dbName, 2: string tableName, 3: i64 txnId) - - void add_runtime_stats(1: RuntimeStat stat) throws(1:MetaException o1) - list<RuntimeStat> get_runtime_stats(1: GetRuntimeStatsRequest rqst) throws(1:MetaException o1) } // * Note about the DDL_TIME: When creating or altering a table or a partition, http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index defc68f..304f567 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -66,7 +66,6 @@ import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; @@ -281,7 +280,6 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { objectStore.updateCreationMetadata(catName, dbname, tablename, cm); } - @Override public List<String> getTables(String catName, String dbName, String pattern) throws MetaException { return objectStore.getTables(catName, dbName, pattern); } @@ -1094,7 +1092,6 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { return null; } - @Override public void createISchema(ISchema schema) throws AlreadyExistsException, MetaException, NoSuchObjectException { objectStore.createISchema(schema); @@ -1164,19 +1161,4 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { public void addSerde(SerDeInfo serde) throws AlreadyExistsException, MetaException { objectStore.addSerde(serde); } - - @Override - public void addRuntimeStat(RuntimeStat stat) throws MetaException { - objectStore.addRuntimeStat(stat); - } - - @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { - return objectStore.getRuntimeStats(); - } - - @Override - public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException { - return objectStore.deleteRuntimeStats(maxRetained, maxRetainSecs); - } } http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 20c5d8a..85c6727 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -64,7 +64,6 @@ import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; @@ -277,7 +276,6 @@ public class DummyRawStoreForJdoConnection implements RawStore { throws MetaException { } - @Override public List<String> getTables(String catName, String dbName, String pattern) throws MetaException { return Collections.emptyList(); } @@ -1082,7 +1080,6 @@ public class DummyRawStoreForJdoConnection implements RawStore { return null; } - @Override public void createISchema(ISchema schema) throws AlreadyExistsException, MetaException { } @@ -1151,18 +1148,4 @@ public class DummyRawStoreForJdoConnection implements RawStore { public void addSerde(SerDeInfo serde) throws AlreadyExistsException, MetaException { } - - @Override - public void addRuntimeStat(RuntimeStat stat) throws MetaException { - } - - @Override - public List<RuntimeStat> getRuntimeStats() throws MetaException { - return Collections.emptyList(); - } - - @Override - public int deleteRuntimeStats(int maxRetained, int maxRetainSecs) throws MetaException { - return 0; - } } http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index bf87cfc..cb51763 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -3352,14 +3352,4 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos public boolean heartbeatLockMaterializationRebuild(String dbName, String tableName, long txnId) throws TException { throw new UnsupportedOperationException(); } - - @Override - public void addRuntimeStat(RuntimeStat stat) throws TException { - throw new UnsupportedOperationException(); - } - - @Override - public List<RuntimeStat> getRuntimeStats() throws TException { - throw new UnsupportedOperationException(); - } } http://git-wip-us.apache.org/repos/asf/hive/blob/f0199500/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java deleted file mode 100644 index 2db7a8b..0000000 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestRuntimeStats.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore.client; - -import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; -import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import java.util.List; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; - -@RunWith(Parameterized.class) -@Category(MetastoreUnitTest.class) -public class TestRuntimeStats extends MetaStoreClientTest { - private final AbstractMetaStoreService metaStore; - private IMetaStoreClient client; - - public TestRuntimeStats(String name, AbstractMetaStoreService metaStore) throws Exception { - this.metaStore = metaStore; - this.metaStore.start(); - } - - @Before - public void setUp() throws Exception { - client = metaStore.getClient(); - - } - - @After - public void tearDown() throws Exception { - client.close(); - client = null; - } - - @Test - public void testRuntimeStatHandling() throws Exception { - List<RuntimeStat> rs0 = client.getRuntimeStats(); - assertNotNull(rs0); - assertEquals(0, rs0.size()); - - RuntimeStat stat = createStat(1); - client.addRuntimeStat(stat); - - List<RuntimeStat> rs1 = client.getRuntimeStats(); - assertNotNull(rs1); - assertEquals(1, rs1.size()); - assertArrayEquals(stat.getPayload(), rs1.get(0).getPayload()); - assertEquals(stat.getWeight(), rs1.get(0).getWeight()); - // server sets createtime - assertNotEquals(stat.getCreateTime(), rs1.get(0).getCreateTime()); - - client.addRuntimeStat(createStat(2)); - client.addRuntimeStat(createStat(3)); - client.addRuntimeStat(createStat(4)); - - List<RuntimeStat> rs2 = client.getRuntimeStats(); - assertEquals(4, rs2.size()); - - } - - private RuntimeStat createStat(int w) { - - byte[] payload = new byte[w]; - for (int i = 0; i < payload.length; i++) { - payload[i] = 'x'; - } - - RuntimeStat stat = new RuntimeStat(); - stat.setWeight(w); - stat.setPayload(payload); - return stat; - } - -}
