[incubator-pulsar] branch master updated: S3 offloader doesn't allow block size < 5MB (#1809)

[mmerli]

This is an automated email from the ASF dual-hosted git repository.

mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 4f076f2  S3 offloader doesn't allow block size < 5MB (#1809)
4f076f2 is described below

commit 4f076f2e77249473687869b03bfe996169784d3d
Author: Ivan Kelly <iv...@apache.org>
AuthorDate: Tue May 22 07:14:43 2018 +0200

    S3 offloader doesn't allow block size < 5MB (#1809)
    
    S3 doesn't allow multipart upload to be used with a block size less
    than 5MB. So we shouldn't allow our offloader to be configured with a
    value lower than 5MB.
    
    Master Issue: #1511
---
 conf/broker.conf                                         |  2 +-
 .../org/apache/pulsar/broker/ServiceConfiguration.java   |  3 ++-
 .../broker/s3offload/S3ManagedLedgerOffloader.java       |  3 +++
 .../broker/s3offload/S3ManagedLedgerOffloaderTest.java   | 16 ++++++++++++++++
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index 2a3d457..c977857 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -496,7 +496,7 @@ s3ManagedLedgerOffloadBucket=
 # For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for 
testing)
 s3ManagedLedgerOffloadServiceEndpoint=
 
-# For Amazon S3 ledger offload, Max block size in bytes.
+# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB 
minimum)
 s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864
 
 # For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default)
diff --git 
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
 
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 0cdaec8..8a764cf 100644
--- 
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++ 
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -489,7 +489,8 @@ public class ServiceConfiguration implements 
PulsarConfiguration {
     private String s3ManagedLedgerOffloadServiceEndpoint = null;
 
     // For Amazon S3 ledger offload, Max block size in bytes.
-    private int s3ManagedLedgerOffloadMaxBlockSizeInBytes = 64 * 1024 * 1024;
+    @FieldContext(minValue = 5242880) // 5MB
+    private int s3ManagedLedgerOffloadMaxBlockSizeInBytes = 64 * 1024 * 1024; 
// 64MB
 
     // For Amazon S3 ledger offload, Read buffer size in bytes.
     @FieldContext(minValue = 1024)
diff --git 
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
 
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
index 3459919..ee82532 100644
--- 
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
+++ 
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
@@ -76,6 +76,9 @@ public class S3ManagedLedgerOffloader implements 
LedgerOffloader {
         if (Strings.isNullOrEmpty(bucket)) {
             throw new PulsarServerException("s3ManagedLedgerOffloadBucket 
cannot be empty if s3 offload enabled");
         }
+        if (maxBlockSize < 5*1024*1024) {
+            throw new 
PulsarServerException("s3ManagedLedgerOffloadMaxBlockSizeInBytes cannot be less 
than 5MB");
+        }
 
         AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
         if (!Strings.isNullOrEmpty(endpoint)) {
diff --git 
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
 
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
index 21ceb97..5e465b6 100644
--- 
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
+++ 
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
@@ -160,6 +160,22 @@ class S3ManagedLedgerOffloaderTest extends S3TestBase {
     }
 
     @Test
+    public void testSmallBlockSizeConfigured() throws Exception {
+        ServiceConfiguration conf = new ServiceConfiguration();
+        
conf.setManagedLedgerOffloadDriver(S3ManagedLedgerOffloader.DRIVER_NAME);
+        conf.setS3ManagedLedgerOffloadRegion("eu-west-1");
+        conf.setS3ManagedLedgerOffloadBucket(BUCKET);
+        conf.setS3ManagedLedgerOffloadMaxBlockSizeInBytes(1024);
+
+        try {
+            S3ManagedLedgerOffloader.create(conf, scheduler);
+            Assert.fail("Should have thrown exception");
+        } catch (PulsarServerException pse) {
+            // correct
+        }
+    }
+
+    @Test
     public void testOffloadAndRead() throws Exception {
         ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 3);
         LedgerOffloader offloader = new S3ManagedLedgerOffloader(s3client, 
BUCKET, scheduler,

-- 
To stop receiving notification emails like this one, please contact
mme...@apache.org.