Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1523148&r1=1523147&r2=1523148&view=diff ============================================================================== --- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original) +++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Sep 13 23:55:39 2013 @@ -76,7 +76,6 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; -import org.apache.hadoop.hive.metastore.api.SkewedValueList; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.index.HiveIndexHandler; @@ -1234,8 +1233,8 @@ public class Hive { org.apache.hadoop.hive.metastore.api.Partition newCreatedTpart = newTPart.getTPartition(); SkewedInfo skewedInfo = newCreatedTpart.getSd().getSkewedInfo(); /* Construct list bucketing location mappings from sub-directory name. */ - Map<SkewedValueList, String> skewedColValueLocationMaps = - constructListBucketingLocationMap(newPartPath, skewedInfo); + Map<List<String>, String> skewedColValueLocationMaps = constructListBucketingLocationMap( + newPartPath, skewedInfo); /* Add list bucketing location mappings. */ skewedInfo.setSkewedColValueLocationMaps(skewedColValueLocationMaps); newCreatedTpart.getSd().setSkewedInfo(skewedInfo); @@ -1268,8 +1267,7 @@ public class Hive { * @throws IOException */ private void walkDirTree(FileStatus fSta, FileSystem fSys, - Map<SkewedValueList, String> skewedColValueLocationMaps, - Path newPartPath, SkewedInfo skewedInfo) + Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo) throws IOException { /* Base Case. It's leaf. */ if (!fSta.isDir()) { @@ -1295,7 +1293,7 @@ private void walkDirTree(FileStatus fSta * @param skewedInfo */ private void constructOneLBLocationMap(FileStatus fSta, - Map<SkewedValueList, String> skewedColValueLocationMaps, + Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo) { Path lbdPath = fSta.getPath().getParent(); List<String> skewedValue = new ArrayList<String>(); @@ -1318,7 +1316,7 @@ private void constructOneLBLocationMap(F } if ((skewedValue.size() > 0) && (skewedValue.size() == skewedInfo.getSkewedColNames().size()) && !skewedColValueLocationMaps.containsKey(skewedValue)) { - skewedColValueLocationMaps.put(new SkewedValueList(skewedValue), lbdPath.toString()); + skewedColValueLocationMaps.put(skewedValue, lbdPath.toString()); } } @@ -1331,10 +1329,9 @@ private void constructOneLBLocationMap(F * @throws IOException * @throws FileNotFoundException */ - private Map<SkewedValueList, String> constructListBucketingLocationMap(Path newPartPath, + private Map<List<String>, String> constructListBucketingLocationMap(Path newPartPath, SkewedInfo skewedInfo) throws IOException, FileNotFoundException { - Map<SkewedValueList, String> skewedColValueLocationMaps = - new HashMap<SkewedValueList, String>(); + Map<List<String>, String> skewedColValueLocationMaps = new HashMap<List<String>, String>(); FileSystem fSys = newPartPath.getFileSystem(conf); walkDirTree(fSys.getFileStatus(newPartPath), fSys, skewedColValueLocationMaps, newPartPath, skewedInfo);
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1523148&r1=1523147&r2=1523148&view=diff ============================================================================== --- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java (original) +++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java Fri Sep 13 23:55:39 2013 @@ -40,7 +40,6 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.SkewedValueList; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; @@ -661,18 +660,18 @@ public class Partition implements Serial public void setSkewedValueLocationMap(List<String> valList, String dirName) throws HiveException { - Map<SkewedValueList, String> mappings = tPartition.getSd().getSkewedInfo() + Map<List<String>, String> mappings = tPartition.getSd().getSkewedInfo() .getSkewedColValueLocationMaps(); if (null == mappings) { - mappings = new HashMap<SkewedValueList, String>(); + mappings = new HashMap<List<String>, String>(); tPartition.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings); } // Add or update new mapping - mappings.put(new SkewedValueList(valList), dirName); + mappings.put(valList, dirName); } - public Map<SkewedValueList, String> getSkewedColValueLocationMaps() { + public Map<List<String>, String> getSkewedColValueLocationMaps() { return tPartition.getSd().getSkewedInfo().getSkewedColValueLocationMaps(); } } Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=1523148&r1=1523147&r2=1523148&view=diff ============================================================================== --- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (original) +++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Fri Sep 13 23:55:39 2013 @@ -45,7 +45,6 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; -import org.apache.hadoop.hive.metastore.api.SkewedValueList; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; @@ -145,7 +144,7 @@ public class Table implements Serializab SkewedInfo skewInfo = new SkewedInfo(); skewInfo.setSkewedColNames(new ArrayList<String>()); skewInfo.setSkewedColValues(new ArrayList<List<String>>()); - skewInfo.setSkewedColValueLocationMaps(new HashMap<SkewedValueList, String>()); + skewInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>()); sd.setSkewedInfo(skewInfo); } @@ -523,20 +522,20 @@ public class Table implements Serializab public void setSkewedValueLocationMap(List<String> valList, String dirName) throws HiveException { - Map<SkewedValueList, String> mappings = tTable.getSd().getSkewedInfo() + Map<List<String>, String> mappings = tTable.getSd().getSkewedInfo() .getSkewedColValueLocationMaps(); if (null == mappings) { - mappings = new HashMap<SkewedValueList, String>(); + mappings = new HashMap<List<String>, String>(); tTable.getSd().getSkewedInfo().setSkewedColValueLocationMaps(mappings); } // Add or update new mapping - mappings.put(new SkewedValueList(valList), dirName); + mappings.put(valList, dirName); } - public Map<SkewedValueList,String> getSkewedColValueLocationMaps() { + public Map<List<String>,String> getSkewedColValueLocationMaps() { return (tTable.getSd().getSkewedInfo() != null) ? tTable.getSd().getSkewedInfo() - .getSkewedColValueLocationMaps() : new HashMap<SkewedValueList, String>(); + .getSkewedColValueLocationMaps() : new HashMap<List<String>, String>(); } public void setSkewedColValues(List<List<String>> skewedValues) throws HiveException { Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java?rev=1523148&r1=1523147&r2=1523148&view=diff ============================================================================== --- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java (original) +++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MetaDataFormatUtils.java Fri Sep 13 23:55:39 2013 @@ -32,7 +32,6 @@ import org.apache.hadoop.hive.conf.HiveC import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.metastore.api.SkewedValueList; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.index.HiveIndex; import org.apache.hadoop.hive.ql.index.HiveIndex.IndexType; @@ -201,7 +200,7 @@ public final class MetaDataFormatUtils { formatOutput("Skewed Values:", skewedColValues.toString(), tableInfo); } - Map<SkewedValueList, String> skewedColMap = storageDesc.getSkewedInfo() + Map<List<String>, String> skewedColMap = storageDesc.getSkewedInfo() .getSkewedColValueLocationMaps(); if ((skewedColMap!=null) && (skewedColMap.size() > 0)) { formatOutput("Skewed Value to Path:", skewedColMap.toString(), @@ -209,8 +208,9 @@ public final class MetaDataFormatUtils { Map<List<String>, String> truncatedSkewedColMap = new HashMap<List<String>, String>(); // walk through existing map to truncate path so that test won't mask it // then we can verify location is right - for (Entry<SkewedValueList, String> entry : skewedColMap.entrySet()) { - truncatedSkewedColMap.put(entry.getKey().getSkewedValueList(), + Set<Entry<List<String>, String>> entries = skewedColMap.entrySet(); + for (Entry<List<String>, String> entry : entries) { + truncatedSkewedColMap.put(entry.getKey(), PlanUtils.removePrefixFromWarehouseConfig(entry.getValue())); } formatOutput("Skewed Value to Truncated Path:", Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java?rev=1523148&r1=1523147&r2=1523148&view=diff ============================================================================== --- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java (original) +++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java Fri Sep 13 23:55:39 2013 @@ -27,7 +27,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.metastore.api.SkewedValueList; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.optimizer.PrunerUtils; @@ -289,13 +288,12 @@ public class ListBucketingPruner impleme List<List<String>> uniqSkewedValues) throws SemanticException { // For each entry in dynamic-multi-dimension collection. List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column. - Map<SkewedValueList, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed + Map<List<String>, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed // map. assert ListBucketingPrunerUtils.isListBucketingPart(part) : part.getName() + " skewed metadata is corrupted. No skewed column and/or location mappings information."; List<List<String>> skewedValues = part.getSkewedColValues(); List<Boolean> nonSkewedValueMatchResult = new ArrayList<Boolean>(); - SkewedValueList skewedValueList = new SkewedValueList(); for (List<String> cell : collections) { // Walk through the tree to decide value. // Example: skewed column: C1, C2 ; @@ -309,9 +307,8 @@ public class ListBucketingPruner impleme /* It's valid case if a partition: */ /* 1. is defined with skewed columns and skewed values in metadata */ /* 2. doesn't have all skewed values within its data */ - skewedValueList.setSkewedValueList(cell); - if (mappings.get(skewedValueList) != null) { - selectedPaths.add(new Path(mappings.get(skewedValueList))); + if (mappings.get(cell) != null) { + selectedPaths.add(new Path(mappings.get(cell))); } } } else { Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1523148&r1=1523147&r2=1523148&view=diff ============================================================================== --- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original) +++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Fri Sep 13 23:55:39 2013 @@ -37,7 +37,6 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.SkewedValueList; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; @@ -981,31 +980,18 @@ public abstract class BaseSemanticAnalyz * @return */ protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames, - List<List<String>> skewedValues, Map<SkewedValueList, String> skewedColValueLocationMaps, + List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps, boolean isStoredAsSubDirectories, HiveConf conf) { ListBucketingCtx lbCtx = new ListBucketingCtx(); lbCtx.setSkewedColNames(skewedColNames); lbCtx.setSkewedColValues(skewedValues); - lbCtx.setLbLocationMap(convertSkewedValueListToSimpleList(skewedColValueLocationMaps)); + lbCtx.setLbLocationMap(skewedColValueLocationMaps); lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories); lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY); lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME); return lbCtx; } - // This is done to avoid the need of sending metastore jars to task nodes. - private Map<List<String>, String> convertSkewedValueListToSimpleList( - Map<SkewedValueList, String> skewedColValueLocationMaps) { - if (skewedColValueLocationMaps == null) { - return null; - } - Map<List<String>, String> converted = new HashMap<List<String>, String>(); - for (Map.Entry<SkewedValueList, String> entry : skewedColValueLocationMaps.entrySet()) { - converted.put(entry.getKey().getSkewedValueList(), entry.getValue()); - } - return converted; - } - /** * Given a ASTNode, return list of values. *
