Repository: hadoop Updated Branches: refs/heads/HADOOP-12756 8346f9222 -> ad7308c94
HADOOP-13491. Fix several warnings from findbugs. Contributed by Genmao Yu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad7308c9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad7308c9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad7308c9 Branch: refs/heads/HADOOP-12756 Commit: ad7308c94e1aada358371ce11b9c92075b21c9d0 Parents: 8346f922 Author: Mingfei <[email protected]> Authored: Thu Aug 18 16:06:57 2016 +0800 Committer: Mingfei <[email protected]> Committed: Thu Aug 18 16:07:26 2016 +0800 ---------------------------------------------------------------------- .../fs/aliyun/oss/AliyunOSSFileSystem.java | 3 ++- .../fs/aliyun/oss/AliyunOSSInputStream.java | 6 ++--- .../fs/aliyun/oss/AliyunOSSOutputStream.java | 10 +++++--- .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 26 +++++++++++++++++++- 4 files changed, 37 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad7308c9/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java index 30ddf8c..6923b95 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.aliyun.oss; import static org.apache.hadoop.fs.aliyun.oss.Constants.*; +import com.aliyun.oss.ClientException; import com.aliyun.oss.common.auth.CredentialsProvider; import com.aliyun.oss.common.auth.DefaultCredentialProvider; import com.aliyun.oss.common.auth.DefaultCredentials; @@ -782,7 +783,7 @@ public class AliyunOSSFileSystem extends FileSystem { ossClient.completeMultipartUpload(completeMultipartUploadRequest); LOG.debug(completeMultipartUploadResult.getETag()); return true; - } catch (Exception e) { + } catch (OSSException | ClientException e) { AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, dstKey, uploadId); ossClient.abortMultipartUpload(abortMultipartUploadRequest); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad7308c9/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java index bcd00dc..b12e3f0 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java @@ -227,12 +227,12 @@ public class AliyunOSSInputStream extends FSInputStream { } @Override - public void seek(long pos) throws IOException { + public synchronized void seek(long pos) throws IOException { checkNotClosed(); if (position == pos) { return; } else if (pos > position && pos < position + partRemaining) { - wrappedStream.skip(pos - position); + AliyunOSSUtils.skipFully(wrappedStream, pos - position); position = pos; } else { reopen(pos); @@ -240,7 +240,7 @@ public class AliyunOSSInputStream extends FSInputStream { } @Override - public long getPos() throws IOException { + public synchronized long getPos() throws IOException { checkNotClosed(); return position; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad7308c9/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java index 589e014..654b81d 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java @@ -29,6 +29,8 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.List; +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSSException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -113,7 +115,9 @@ public class AliyunOSSOutputStream extends OutputStream { multipartUploadObject(); } } finally { - tmpFile.delete(); + if (!tmpFile.delete()) { + LOG.warn("Can not delete file: " + tmpFile); + } } } @@ -174,7 +178,7 @@ public class AliyunOSSOutputStream extends OutputStream { FileInputStream fis = new FileInputStream(object); try { long skipBytes = partSize * i; - fis.skip(skipBytes); + AliyunOSSUtils.skipFully(fis, skipBytes); long size = (partSize < dataLen - skipBytes) ? partSize : dataLen - skipBytes; UploadPartRequest uploadPartRequest = new UploadPartRequest(); @@ -198,7 +202,7 @@ public class AliyunOSSOutputStream extends OutputStream { CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest); LOG.debug(completeMultipartUploadResult.getETag()); - } catch (Exception e) { + } catch (OSSException | ClientException e) { AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, key, uploadId); ossClient.abortMultipartUpload(abortMultipartUploadRequest); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad7308c9/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java index 3f66a4f..9acde00 100644 --- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java +++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.aliyun.oss; import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URLDecoder; @@ -116,7 +117,7 @@ final public class AliyunOSSUtils { /** * Extract the user information details from a URI. - * @param name URI of the filesystem + * @param name URI of the filesystem. * @return a login tuple, possibly empty. */ public static UserInfo extractLoginDetails(URI name) { @@ -148,4 +149,27 @@ final public class AliyunOSSUtils { throw new RuntimeException(e); } } + + /** + * Skips the requested number of bytes or fail if there are not enough left. + * This allows for the possibility that {@link InputStream#skip(long)} may not + * skip as many bytes as requested (most likely because of reaching EOF). + * @param is the input stream to skip. + * @param n the number of bytes to skip. + * @throws IOException thrown when skipped less number of bytes. + */ + public static void skipFully(InputStream is, long n) throws IOException { + long total = 0; + long cur = 0; + + do { + cur = is.skip(n - total); + total += cur; + } while((total < n) && (cur > 0)); + + if (total < n) { + throw new IOException("Failed to skip " + n + " bytes, possibly due " + + "to EOF."); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
