subject:"hadoop git commit\: HADOOP\-13498. The number of multi\-part upload part should not bigger than 10000. Contributed by Genmao Yu."

[06/23] hadoop git commit: HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu.

2016-10-10 Thread drankye

HADOOP-13498. The number of multi-part upload part should not bigger than 
1. Contributed by Genmao Yu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdb77110
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdb77110
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdb77110

Branch: refs/heads/trunk
Commit: cdb77110e77b70ed0c1125b2a6a422a8c7c28ec7
Parents: 6bb741b
Author: Mingfei 
Authored: Wed Aug 24 10:09:37 2016 +0800
Committer: Mingfei 
Committed: Wed Sep 7 11:15:48 2016 +0800

--
 .../fs/aliyun/oss/AliyunOSSFileSystem.java  |  9 
 .../fs/aliyun/oss/AliyunOSSOutputStream.java| 23 ++--
 .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java| 15 +
 .../apache/hadoop/fs/aliyun/oss/Constants.java  |  4 +++-
 .../fs/aliyun/oss/TestOSSOutputStream.java  | 19 
 5 files changed, 53 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 99a60db..afe7242 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -71,7 +71,6 @@ import org.slf4j.LoggerFactory;
  * Aliyun OSS, used to access OSS blob system in a filesystem style.
  */
 public class AliyunOSSFileSystem extends FileSystem {
-
   private static final Logger LOG =
   LoggerFactory.getLogger(AliyunOSSFileSystem.class);
   private URI uri;
@@ -560,18 +559,18 @@ public class AliyunOSSFileSystem extends FileSystem {
* Used to create an empty file that represents an empty directory.
*
* @param bucket the bucket this directory belongs to
-   * @param objectName directory path
+   * @param key directory path
* @return true if directory successfully created
* @throws IOException
*/
-  private boolean mkdir(final String bucket, final String objectName)
+  private boolean mkdir(final String bucket, final String key)
   throws IOException {
-String dirName = objectName;
+String dirName = key;
 ObjectMetadata dirMeta = new ObjectMetadata();
 byte[] buffer = new byte[0];
 ByteArrayInputStream in = new ByteArrayInputStream(buffer);
 dirMeta.setContentLength(0);
-if (!objectName.endsWith("/")) {
+if (!key.endsWith("/")) {
   dirName += "/";
 }
 try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
index 654b81d..1e16df9 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -84,6 +84,9 @@ public class AliyunOSSOutputStream extends OutputStream {
 
 partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
 MULTIPART_UPLOAD_SIZE_DEFAULT);
+if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
+  partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
+}
 partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
 MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
 
@@ -151,6 +154,12 @@ public class AliyunOSSOutputStream extends OutputStream {
   private void multipartUploadObject() throws IOException {
 File object = tmpFile.getAbsoluteFile();
 long dataLen = object.length();
+long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
+int partNum = (int)(dataLen / realPartSize);
+if (dataLen % realPartSize != 0) {
+  partNum += 1;
+}
+
 InitiateMultipartUploadRequest initiateMultipartUploadRequest =
 new InitiateMultipartUploadRequest(bucketName, key);
 ObjectMetadata meta = new ObjectMetadata();
@@ -161,14 +170,6 @@ public class AliyunOSSOutputStream extends OutputStream {
 initiateMultipartUploadRequest.setObjectMetadata(meta);
 InitiateMultipartUploadResult initiateMultipartUploadResult =

[18/22] hadoop git commit: HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu.

2016-09-06 Thread shimingfei

HADOOP-13498. The number of multi-part upload part should not bigger than 
1. Contributed by Genmao Yu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdb77110
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdb77110
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdb77110

Branch: refs/heads/HADOOP-12756
Commit: cdb77110e77b70ed0c1125b2a6a422a8c7c28ec7
Parents: 6bb741b
Author: Mingfei 
Authored: Wed Aug 24 10:09:37 2016 +0800
Committer: Mingfei 
Committed: Wed Sep 7 11:15:48 2016 +0800

--
 .../fs/aliyun/oss/AliyunOSSFileSystem.java  |  9 
 .../fs/aliyun/oss/AliyunOSSOutputStream.java| 23 ++--
 .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java| 15 +
 .../apache/hadoop/fs/aliyun/oss/Constants.java  |  4 +++-
 .../fs/aliyun/oss/TestOSSOutputStream.java  | 19 
 5 files changed, 53 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 99a60db..afe7242 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -71,7 +71,6 @@ import org.slf4j.LoggerFactory;
  * Aliyun OSS, used to access OSS blob system in a filesystem style.
  */
 public class AliyunOSSFileSystem extends FileSystem {
-
   private static final Logger LOG =
   LoggerFactory.getLogger(AliyunOSSFileSystem.class);
   private URI uri;
@@ -560,18 +559,18 @@ public class AliyunOSSFileSystem extends FileSystem {
* Used to create an empty file that represents an empty directory.
*
* @param bucket the bucket this directory belongs to
-   * @param objectName directory path
+   * @param key directory path
* @return true if directory successfully created
* @throws IOException
*/
-  private boolean mkdir(final String bucket, final String objectName)
+  private boolean mkdir(final String bucket, final String key)
   throws IOException {
-String dirName = objectName;
+String dirName = key;
 ObjectMetadata dirMeta = new ObjectMetadata();
 byte[] buffer = new byte[0];
 ByteArrayInputStream in = new ByteArrayInputStream(buffer);
 dirMeta.setContentLength(0);
-if (!objectName.endsWith("/")) {
+if (!key.endsWith("/")) {
   dirName += "/";
 }
 try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb77110/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
index 654b81d..1e16df9 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -84,6 +84,9 @@ public class AliyunOSSOutputStream extends OutputStream {
 
 partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
 MULTIPART_UPLOAD_SIZE_DEFAULT);
+if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
+  partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
+}
 partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
 MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
 
@@ -151,6 +154,12 @@ public class AliyunOSSOutputStream extends OutputStream {
   private void multipartUploadObject() throws IOException {
 File object = tmpFile.getAbsoluteFile();
 long dataLen = object.length();
+long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
+int partNum = (int)(dataLen / realPartSize);
+if (dataLen % realPartSize != 0) {
+  partNum += 1;
+}
+
 InitiateMultipartUploadRequest initiateMultipartUploadRequest =
 new InitiateMultipartUploadRequest(bucketName, key);
 ObjectMetadata meta = new ObjectMetadata();
@@ -161,14 +170,6 @@ public class AliyunOSSOutputStream extends OutputStream {
 initiateMultipartUploadRequest.setObjectMetadata(meta);
 InitiateMultipartUploadResult initiateMultipartUploadResult =

hadoop git commit: HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu.

2016-08-23 Thread shimingfei

Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-12756 787750d1f -> aff1841d0


HADOOP-13498. The number of multi-part upload part should not bigger than 
1. Contributed by Genmao Yu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aff1841d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aff1841d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aff1841d

Branch: refs/heads/HADOOP-12756
Commit: aff1841d00d444b5422e633a6c966104ef704da9
Parents: 787750d
Author: Mingfei 
Authored: Wed Aug 24 10:09:37 2016 +0800
Committer: Mingfei 
Committed: Wed Aug 24 10:09:45 2016 +0800

--
 .../fs/aliyun/oss/AliyunOSSFileSystem.java  |  9 
 .../fs/aliyun/oss/AliyunOSSOutputStream.java| 23 ++--
 .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java| 15 +
 .../apache/hadoop/fs/aliyun/oss/Constants.java  |  4 +++-
 .../fs/aliyun/oss/TestOSSOutputStream.java  | 19 
 5 files changed, 53 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aff1841d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 99a60db..afe7242 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -71,7 +71,6 @@ import org.slf4j.LoggerFactory;
  * Aliyun OSS, used to access OSS blob system in a filesystem style.
  */
 public class AliyunOSSFileSystem extends FileSystem {
-
   private static final Logger LOG =
   LoggerFactory.getLogger(AliyunOSSFileSystem.class);
   private URI uri;
@@ -560,18 +559,18 @@ public class AliyunOSSFileSystem extends FileSystem {
* Used to create an empty file that represents an empty directory.
*
* @param bucket the bucket this directory belongs to
-   * @param objectName directory path
+   * @param key directory path
* @return true if directory successfully created
* @throws IOException
*/
-  private boolean mkdir(final String bucket, final String objectName)
+  private boolean mkdir(final String bucket, final String key)
   throws IOException {
-String dirName = objectName;
+String dirName = key;
 ObjectMetadata dirMeta = new ObjectMetadata();
 byte[] buffer = new byte[0];
 ByteArrayInputStream in = new ByteArrayInputStream(buffer);
 dirMeta.setContentLength(0);
-if (!objectName.endsWith("/")) {
+if (!key.endsWith("/")) {
   dirName += "/";
 }
 try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aff1841d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
index 654b81d..1e16df9 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -84,6 +84,9 @@ public class AliyunOSSOutputStream extends OutputStream {
 
 partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
 MULTIPART_UPLOAD_SIZE_DEFAULT);
+if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
+  partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
+}
 partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
 MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
 
@@ -151,6 +154,12 @@ public class AliyunOSSOutputStream extends OutputStream {
   private void multipartUploadObject() throws IOException {
 File object = tmpFile.getAbsoluteFile();
 long dataLen = object.length();
+long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
+int partNum = (int)(dataLen / realPartSize);
+if (dataLen % realPartSize != 0) {
+  partNum += 1;
+}
+
 InitiateMultipartUploadRequest initiateMultipartUploadRequest =
 new InitiateMultipartUploadRequest(bucketName, key);
 ObjectMetadata meta = new ObjectMetadata();
@@ -161,14 +170,6 @@ public class AliyunOSSOutputStream extends OutputStream {
 initiateMultipartUploadRequest.setObjectMetadata(meta);

[06/23] hadoop git commit: HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu.

[18/22] hadoop git commit: HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu.

hadoop git commit: HADOOP-13498. The number of multi-part upload part should not bigger than 10000. Contributed by Genmao Yu.

3 matches

Site Navigation

Mail list logo

Footer information