Repository: hive Updated Branches: refs/heads/master b4302bb7a -> cbe3228c2
HIVE-20765 : fetch partitions for txn stats validation in get_aggr_stats with one call (Sergey Shelukhin, reviewed by Ashutosh Chauhan, and Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cbe3228c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cbe3228c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cbe3228c Branch: refs/heads/master Commit: cbe3228c29e2db6ae5cffcb94cdcf79063859ffe Parents: b4302bb Author: sergey <[email protected]> Authored: Mon Oct 22 14:06:48 2018 -0700 Committer: sergey <[email protected]> Committed: Mon Oct 22 14:06:48 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/metastore/ObjectStore.java | 22 +++++++++++++------- .../metastore/PartitionProjectionEvaluator.java | 8 +++++-- 2 files changed, 20 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b98b4b4..ddd64e7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -9052,21 +9052,27 @@ public class ObjectStore implements RawStore, Configurable { return null; } - MTable table = getMTable(catName, dbName, tblName); + Table table = getTable(catName, dbName, tblName); boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters()); if (isTxn && !areTxnStatsSupported) { return null; } + GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec(); + fs.setFilterMode(PartitionFilterMode.BY_NAMES); + fs.setFilters(partNames); + GetPartitionsProjectionSpec ps = new GetPartitionsProjectionSpec(); + ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%'); + ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values")); + List<Partition> parts = getPartitionSpecsByFilterAndProjection(table, ps, fs); // Loop through the given "partNames" list // checking isolation-level-compliance of each partition column stats. - for (String partName : partNames) { - MPartition mpart = getMPartition( - catName, dbName, tblName, Warehouse.getPartValuesFromPartName(partName)); - if (!isCurrentStatsValidForTheQuery(mpart, writeIdList, false)) { - LOG.debug("The current metastore transactional partition column statistics " + - "for " + dbName + "." + tblName + "." + mpart.getPartitionName() + " is not valid " + - "for the current query."); + for (Partition part : parts) { + + if (!isCurrentStatsValidForTheQuery(part, part.getWriteId(), writeIdList, false)) { + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + LOG.debug("The current metastore transactional partition column statistics for " + dbName + + "." + tblName + "." + partName + " is not valid for the current query"); return null; } } http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java index e918a33..c3a0093 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java @@ -26,6 +26,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; + import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; @@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory; import javax.jdo.PersistenceManager; import javax.jdo.Query; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -369,8 +371,8 @@ public class PartitionProjectionEvaluator { int numColumns = buildQueryForSingleValuedFields(partitionIds, queryTextBuilder); String queryText = queryTextBuilder.toString(); - try (Query query = pm.newQuery("javax.jdo.query.SQL", queryText)) { - + Query<?> query = pm.newQuery("javax.jdo.query.SQL", queryText); + try { long start = LOG.isDebugEnabled() ? System.nanoTime() : 0; List<Object> sqlResult = MetastoreDirectSqlUtils.executeWithArray(query, null, queryText); long queryTime = LOG.isDebugEnabled() ? System.nanoTime() : 0; @@ -456,6 +458,8 @@ public class PartitionProjectionEvaluator { } catch (Exception e) { LOG.error("Exception received while getting partitions using projected fields", e); throw new MetaException(e.getMessage()); + } finally { + query.closeAll(); } }
