Modified: hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1626364&r1=1626363&r2=1626364&view=diff ============================================================================== --- hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original) +++ hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Fri Sep 19 23:26:48 2014 @@ -37,6 +37,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import javax.security.auth.Subject; import javax.security.auth.login.LoginException; @@ -652,6 +653,17 @@ public class Hadoop20Shims implements Ha } @Override + public TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs, + FileStatus status) throws IOException { + TreeMap<Long, BlockLocation> offsetBlockMap = new TreeMap<Long, BlockLocation>(); + BlockLocation[] locations = getLocations(fs, status); + for (BlockLocation location : locations) { + offsetBlockMap.put(location.getOffset(), location); + } + return offsetBlockMap; + } + + @Override public void hflush(FSDataOutputStream stream) throws IOException { stream.sync(); }
Modified: hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1626364&r1=1626363&r2=1626364&view=diff ============================================================================== --- hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original) +++ hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Fri Sep 19 23:26:48 2014 @@ -27,6 +27,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; @@ -403,6 +404,17 @@ public class Hadoop20SShims extends Hado } @Override + public TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs, + FileStatus status) throws IOException { + TreeMap<Long, BlockLocation> offsetBlockMap = new TreeMap<Long, BlockLocation>(); + BlockLocation[] locations = getLocations(fs, status); + for (BlockLocation location : locations) { + offsetBlockMap.put(location.getOffset(), location); + } + return offsetBlockMap; + } + + @Override public void hflush(FSDataOutputStream stream) throws IOException { stream.sync(); } Modified: hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1626364&r1=1626363&r2=1626364&view=diff ============================================================================== --- hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original) +++ hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Fri Sep 19 23:26:48 2014 @@ -29,6 +29,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -511,6 +512,17 @@ public class Hadoop23Shims extends Hadoo } @Override + public TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs, + FileStatus status) throws IOException { + TreeMap<Long, BlockLocation> offsetBlockMap = new TreeMap<Long, BlockLocation>(); + BlockLocation[] locations = getLocations(fs, status); + for (BlockLocation location : locations) { + offsetBlockMap.put(location.getOffset(), location); + } + return offsetBlockMap; + } + + @Override public void hflush(FSDataOutputStream stream) throws IOException { stream.hflush(); } Modified: hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1626364&r1=1626363&r2=1626364&view=diff ============================================================================== --- hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original) +++ hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Fri Sep 19 23:26:48 2014 @@ -30,6 +30,7 @@ import java.security.PrivilegedException import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.TreeMap; import javax.security.auth.login.LoginException; @@ -477,6 +478,19 @@ public interface HadoopShims { FileStatus status) throws IOException; /** + * For the block locations returned by getLocations() convert them into a Treemap + * <Offset,blockLocation> by iterating over the list of blockLocation. + * Using TreeMap from offset to blockLocation, makes it O(logn) to get a particular + * block based upon offset. + * @param fs the file system + * @param status the file information + * @return TreeMap<Long, BlockLocation> + * @throws IOException + */ + TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs, + FileStatus status) throws IOException; + + /** * Flush and make visible to other users the changes to the given stream. * @param stream the stream to hflush. * @throws IOException
