Modified: hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1581982&r1=1581981&r2=1581982&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original) +++ hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Wed Mar 26 18:19:58 2014 @@ -21,16 +21,19 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; +import java.util.Arrays; import java.util.Comparator; import java.util.Iterator; import java.net.URI; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -360,29 +363,11 @@ public class Hadoop20SShims extends Hado } @Override - public Iterator<FileStatus> listLocatedStatus(final FileSystem fs, - final Path path, - final PathFilter filter - ) throws IOException { - return new Iterator<FileStatus>() { - private final FileStatus[] result = fs.listStatus(path, filter); - private int current = 0; - - @Override - public boolean hasNext() { - return current < result.length; - } - - @Override - public FileStatus next() { - return result[current++]; - } - - @Override - public void remove() { - throw new IllegalArgumentException("Not supported"); - } - }; + public List<FileStatus> listLocatedStatus(final FileSystem fs, + final Path path, + final PathFilter filter + ) throws IOException { + return Arrays.asList(fs.listStatus(path, filter)); } @Override @@ -392,6 +377,11 @@ public class Hadoop20SShims extends Hado } @Override + public void hflush(FSDataOutputStream stream) throws IOException { + stream.sync(); + } + + @Override public FileSystem createProxyFileSystem(FileSystem fs, URI uri) { return new ProxyFileSystem(fs, uri); }
Modified: hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1581982&r1=1581981&r2=1581982&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original) +++ hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Wed Mar 26 18:19:58 2014 @@ -22,8 +22,10 @@ import java.lang.Integer; import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.HashMap; import java.net.URI; @@ -34,6 +36,7 @@ import org.apache.commons.lang.StringUti import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -465,51 +468,19 @@ public class Hadoop23Shims extends Hadoo } @Override - public Iterator<FileStatus> listLocatedStatus(final FileSystem fs, - final Path path, - final PathFilter filter - ) throws IOException { - return new Iterator<FileStatus>() { - private final RemoteIterator<LocatedFileStatus> inner = - fs.listLocatedStatus(path); - private FileStatus next; - { - next = null; - while (inner.hasNext() && next == null) { - next = inner.next(); - if (filter != null && !filter.accept(next.getPath())) { - next = null; - } - } - } - - @Override - public boolean hasNext() { - return next != null; - } - - @Override - public FileStatus next() { - FileStatus result = next; - next = null; - try { - while (inner.hasNext() && next == null) { - next = inner.next(); - if (filter != null && !filter.accept(next.getPath())) { - next = null; - } - } - } catch (IOException ioe) { - throw new IllegalArgumentException("Iterator exception", ioe); - } - return result; + public List<FileStatus> listLocatedStatus(final FileSystem fs, + final Path path, + final PathFilter filter + ) throws IOException { + RemoteIterator<LocatedFileStatus> itr = fs.listLocatedStatus(path); + List<FileStatus> result = new ArrayList<FileStatus>(); + while(itr.hasNext()) { + FileStatus stat = itr.next(); + if (filter == null || filter.accept(stat.getPath())) { + result.add(stat); } - - @Override - public void remove() { - throw new IllegalArgumentException("Not supported"); - } - }; + } + return result; } @Override @@ -522,6 +493,11 @@ public class Hadoop23Shims extends Hadoo } } + @Override + public void hflush(FSDataOutputStream stream) throws IOException { + stream.hflush(); + } + class ProxyFileSystem23 extends ProxyFileSystem { public ProxyFileSystem23(FileSystem fs) { super(fs); Modified: hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1581982&r1=1581981&r2=1581982&view=diff ============================================================================== --- hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original) +++ hive/branches/branch-0.13/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Wed Mar 26 18:19:58 2014 @@ -38,6 +38,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -445,11 +446,11 @@ public interface HadoopShims { * @param fs the file system * @param path the directory name to get the status and block locations * @param filter a filter that needs to accept the file (or null) - * @return an iterator for the located file status objects + * @return an list for the located file status objects * @throws IOException */ - Iterator<FileStatus> listLocatedStatus(FileSystem fs, Path path, - PathFilter filter) throws IOException; + List<FileStatus> listLocatedStatus(FileSystem fs, Path path, + PathFilter filter) throws IOException; /** * For file status returned by listLocatedStatus, convert them into a list @@ -462,6 +463,13 @@ public interface HadoopShims { BlockLocation[] getLocations(FileSystem fs, FileStatus status) throws IOException; + /** + * Flush and make visible to other users the changes to the given stream. + * @param stream the stream to hflush. + * @throws IOException + */ + public void hflush(FSDataOutputStream stream) throws IOException; + public HCatHadoopShims getHCatShim(); public interface HCatHadoopShims {
