merlimat closed pull request #1809: S3 offloader doesn't allow block size < 5MB
URL: https://github.com/apache/incubator-pulsar/pull/1809
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/conf/broker.conf b/conf/broker.conf
index 39523cd3cb..10dac30ca0 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -492,7 +492,7 @@ s3ManagedLedgerOffloadBucket=
# For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for
testing)
s3ManagedLedgerOffloadServiceEndpoint=
-# For Amazon S3 ledger offload, Max block size in bytes.
+# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB
minimum)
s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864
# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default)
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index d28ff4c846..b64af74b75 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -485,7 +485,8 @@
private String s3ManagedLedgerOffloadServiceEndpoint = null;
// For Amazon S3 ledger offload, Max block size in bytes.
- private int s3ManagedLedgerOffloadMaxBlockSizeInBytes = 64 * 1024 * 1024;
+ @FieldContext(minValue = 5242880) // 5MB
+ private int s3ManagedLedgerOffloadMaxBlockSizeInBytes = 64 * 1024 * 1024;
// 64MB
// For Amazon S3 ledger offload, Read buffer size in bytes.
@FieldContext(minValue = 1024)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
index 7a73a3bbc8..7aa0412b5c 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java
@@ -74,6 +74,9 @@ public static S3ManagedLedgerOffloader
create(ServiceConfiguration conf,
if (Strings.isNullOrEmpty(bucket)) {
throw new PulsarServerException("s3ManagedLedgerOffloadBucket
cannot be empty is s3 offload enabled");
}
+ if (maxBlockSize < 5*1024*1024) {
+ throw new
PulsarServerException("s3ManagedLedgerOffloadMaxBlockSizeInBytes cannot be less
than 5MB");
+ }
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
if (!Strings.isNullOrEmpty(endpoint)) {
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
index ed44ddc586..a1aa115402 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloaderTest.java
@@ -159,6 +159,22 @@ public void testNoBucketConfigured() throws Exception {
}
}
+ @Test
+ public void testSmallBlockSizeConfigured() throws Exception {
+ ServiceConfiguration conf = new ServiceConfiguration();
+
conf.setManagedLedgerOffloadDriver(S3ManagedLedgerOffloader.DRIVER_NAME);
+ conf.setS3ManagedLedgerOffloadRegion("eu-west-1");
+ conf.setS3ManagedLedgerOffloadBucket(BUCKET);
+ conf.setS3ManagedLedgerOffloadMaxBlockSizeInBytes(1024);
+
+ try {
+ S3ManagedLedgerOffloader.create(conf, scheduler);
+ Assert.fail("Should have thrown exception");
+ } catch (PulsarServerException pse) {
+ // correct
+ }
+ }
+
@Test
public void testOffloadAndRead() throws Exception {
ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 3);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
