Repository: hive Updated Branches: refs/heads/master 6137ee5dd -> a81f53ac1
HIVE-20430: CachedStore: bug fixes for TestEmbeddedHiveMetaStore, TestRemoteHiveMetaStore, TestMiniLlapCliDriver, TestMiniTezCliDriver, TestMinimrCliDriver (Vaibhav Gumashta reviewed by Daniel Dai) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a81f53ac Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a81f53ac Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a81f53ac Branch: refs/heads/master Commit: a81f53ac17f4a5f0fd68fbdabe7a038b3612fd80 Parents: 6137ee5 Author: Vaibhav Gumashta <[email protected]> Authored: Tue Sep 25 12:04:39 2018 -0700 Committer: Vaibhav Gumashta <[email protected]> Committed: Tue Sep 25 12:11:01 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/exec/TestOperators.java | 1 + .../hive/metastore/cache/CachedStore.java | 130 +++++++++++-------- .../hive/metastore/cache/SharedCache.java | 8 +- 3 files changed, 81 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a81f53ac/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java index abf7198..c7cd4ad 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java @@ -394,6 +394,7 @@ public class TestOperators extends TestCase { // ensure that both of the partitions are in the complete list. String[] dirs = job.get("hive.complete.dir.list").split("\t"); assertEquals(2, dirs.length); + Arrays.sort(dirs); assertEquals(true, dirs[0].endsWith("/state=CA")); assertEquals(true, dirs[1].endsWith("/state=OR")); return super.getSplits(job, splits); http://git-wip-us.apache.org/repos/asf/hive/blob/a81f53ac/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 0445cbf..b9a5458 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -853,9 +853,7 @@ public class CachedStore implements RawStore, Configurable { } @Override - public Table getTable(String catName, String dbName, String tblName, - String validWriteIds) - throws MetaException { + public Table getTable(String catName, String dbName, String tblName, String validWriteIds) throws MetaException { catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); @@ -872,12 +870,28 @@ public class CachedStore implements RawStore, Configurable { return rawStore.getTable(catName, dbName, tblName, validWriteIds); } if (validWriteIds != null) { - tbl.setParameters(adjustStatsParamsForGet(tbl.getParameters(), - tbl.getParameters(), tbl.getWriteId(), validWriteIds)); + tbl.setParameters( + adjustStatsParamsForGet(tbl.getParameters(), tbl.getParameters(), tbl.getWriteId(), validWriteIds)); } tbl.unsetPrivileges(); tbl.setRewriteEnabled(tbl.isRewriteEnabled()); + if (tbl.getPartitionKeys() == null) { + // getTable call from ObjectStore returns an empty list + tbl.setPartitionKeys(new ArrayList<>()); + } + String tableType = tbl.getTableType(); + if (tableType == null) { + // for backwards compatibility with old metastore persistence + if (tbl.getViewOriginalText() != null) { + tableType = TableType.VIRTUAL_VIEW.toString(); + } else if ("TRUE".equals(tbl.getParameters().get("EXTERNAL"))) { + tableType = TableType.EXTERNAL_TABLE.toString(); + } else { + tableType = TableType.MANAGED_TABLE.toString(); + } + } + tbl.setTableType(tableType); return tbl; } @@ -1133,6 +1147,10 @@ public class CachedStore implements RawStore, Configurable { if (!isCachePrewarmed.get() || missSomeInCache) { return rawStore.getTableObjectsByName(catName, dbName, tblNames); } + Database db = sharedCache.getDatabaseFromCache(catName, dbName); + if (db == null) { + throw new UnknownDBException("Could not find database " + dbName); + } List<Table> tables = new ArrayList<>(); for (String tblName : tblNames) { tblName = normalizeIdentifier(tblName); @@ -1140,6 +1158,9 @@ public class CachedStore implements RawStore, Configurable { if (tbl == null) { tbl = rawStore.getTable(catName, dbName, tblName); } + if (tbl != null) { + tables.add(tbl); + } tables.add(tbl); } return tables; @@ -1155,14 +1176,10 @@ public class CachedStore implements RawStore, Configurable { } @Override - public List<String> listTableNamesByFilter(String catName, String dbName, String filter, - short max_tables) + // TODO: implement using SharedCache + public List<String> listTableNamesByFilter(String catName, String dbName, String filter, short max_tables) throws MetaException, UnknownDBException { - if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) { - return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables); - } - return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), - StringUtils.normalizeIdentifier(dbName), filter, max_tables); + return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables); } @Override @@ -1246,6 +1263,7 @@ public class CachedStore implements RawStore, Configurable { } @Override + // TODO: implement using SharedCache public List<Partition> getPartitionsByFilter(String catName, String dbName, String tblName, String filter, short maxParts) throws MetaException, NoSuchObjectException { @@ -1558,85 +1576,85 @@ public class CachedStore implements RawStore, Configurable { } @Override - public List<String> listPartitionNamesPs(String catName, String dbName, String tblName, - List<String> partVals, short maxParts) - throws MetaException, NoSuchObjectException { + public List<String> listPartitionNamesPs(String catName, String dbName, String tblName, List<String> partSpecs, + short maxParts) throws MetaException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { - return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts); + return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache - return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts); + return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts); } - List<String> partNames = new ArrayList<>(); + String partNameMatcher = getPartNameMatcher(table, partSpecs); + List<String> partitionNames = new ArrayList<>(); + List<Partition> allPartitions = sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts); int count = 0; - for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) { - boolean psMatch = true; - for (int i=0;i<partVals.size();i++) { - String psVal = partVals.get(i); - String partVal = part.getValues().get(i); - if (psVal!=null && !psVal.isEmpty() && !psVal.equals(partVal)) { - psMatch = false; - break; - } - } - if (!psMatch) { - continue; - } - if (maxParts == -1 || count < maxParts) { - partNames.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues())); + for (Partition part : allPartitions) { + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + if (partName.matches(partNameMatcher) && (maxParts == -1 || count < maxParts)) { + partitionNames.add(partName); count++; } } - return partNames; + return partitionNames; } @Override - public List<Partition> listPartitionsPsWithAuth(String catName, String dbName, String tblName, - List<String> partVals, short maxParts, String userName, List<String> groupNames) + public List<Partition> listPartitionsPsWithAuth(String catName, String dbName, String tblName, List<String> partSpecs, + short maxParts, String userName, List<String> groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { - return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName, - groupNames); + return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache - return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName, - groupNames); + return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames); } + String partNameMatcher = getPartNameMatcher(table, partSpecs); List<Partition> partitions = new ArrayList<>(); + List<Partition> allPartitions = sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts); int count = 0; - for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) { - boolean psMatch = true; - for (int i = 0; i < partVals.size(); i++) { - String psVal = partVals.get(i); - String partVal = part.getValues().get(i); - if (psVal != null && !psVal.isEmpty() && !psVal.equals(partVal)) { - psMatch = false; - break; - } - } - if (!psMatch) { - continue; - } - if (maxParts == -1 || count < maxParts) { - String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + for (Partition part : allPartitions) { + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + if (partName.matches(partNameMatcher) && (maxParts == -1 || count < maxParts)) { PrincipalPrivilegeSet privs = getPartitionPrivilegeSet(catName, dbName, tblName, partName, userName, groupNames); part.setPrivileges(privs); partitions.add(part); + count++; } } return partitions; } + + private String getPartNameMatcher(Table table, List<String> partSpecs) throws MetaException { + List<FieldSchema> partCols = table.getPartitionKeys(); + int numPartKeys = partCols.size(); + if (partSpecs.size() > numPartKeys) { + throw new MetaException( + "Incorrect number of partition values." + " numPartKeys=" + numPartKeys + ", partSpecs=" + partSpecs.size()); + } + partCols = partCols.subList(0, partSpecs.size()); + // Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... + // where partVal is either the escaped partition value given as input, + // or a regex of the form ".*" + // This works because the "=" and "/" separating key names and partition key/values + // are not escaped. + String partNameMatcher = Warehouse.makePartName(partCols, partSpecs, ".*"); + // add ".*" to the regex to match anything else afterwards the partial spec. + if (partSpecs.size() < numPartKeys) { + partNameMatcher += ".*"; + } + return partNameMatcher; + } // Note: ideally this should be above both CachedStore and ObjectStore. private Map<String, String> adjustStatsParamsForGet(Map<String, String> tableParams, @@ -1893,7 +1911,7 @@ public class CachedStore implements RawStore, Configurable { colStatsMap.put(colStatsAggregator, colStatsWithPartInfoList); } if (partsFoundForColumn == partNames.size()) { - partsFound++; + partsFound = partsFoundForColumn; } if (colStatsMap.size() < 1) { LOG.debug("No stats data found for: dbName={} tblName= {} partNames= {} colNames= ", dbName, http://git-wip-us.apache.org/repos/asf/hive/blob/a81f53ac/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 37c300e..c24e716 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -293,10 +293,10 @@ public class SharedCache { PartitionWrapper wrapper = partitionCache.remove(CacheUtils.buildPartitionCacheKey(partVal)); isPartitionCacheDirty.set(true); + part = CacheUtils.assemble(wrapper, sharedCache); if (wrapper.getSdHash() != null) { sharedCache.decrSd(wrapper.getSdHash()); } - part = CacheUtils.assemble(wrapper, sharedCache); // Remove col stats String partialKey = CacheUtils.buildPartitionCacheKey(partVal); Iterator<Entry<String, ColumnStatisticsObj>> iterator = @@ -461,7 +461,11 @@ public class SharedCache { public void removeTableColStats(String colName) { try { tableLock.writeLock().lock(); - tableColStatsCache.remove(colName); + if (colName == null) { + tableColStatsCache.clear(); + } else { + tableColStatsCache.remove(colName); + } isTableColStatsCacheDirty.set(true); } finally { tableLock.writeLock().unlock();
