(pulsar) 02/02: [fix][offload] Increase file upload limit from 2048MiB to 4096MiB for GCP/GCS offloading (#22554)

[lhotari]

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/pulsar.git

commit 5c09c20b33c8e9563df82b4f510fe1d7b519f33c
Author: Lari Hotari <lhot...@users.noreply.github.com>
AuthorDate: Mon Apr 22 18:13:45 2024 +0300

    [fix][offload] Increase file upload limit from 2048MiB to 4096MiB for 
GCP/GCS offloading (#22554)
    
    (cherry picked from commit e81f37000ec212676c5daffa17faad8fc604ff77)
---
 conf/broker.conf                                              | 11 ++++++-----
 .../pulsar/common/policies/data/OffloadPoliciesImpl.java      |  7 ++++---
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/conf/broker.conf b/conf/broker.conf
index ec0974dca20..34f7ab017e9 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -1657,10 +1657,10 @@ s3ManagedLedgerOffloadBucket=
 # For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for 
testing)
 s3ManagedLedgerOffloadServiceEndpoint=
 
-# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB 
minimum)
+# For Amazon S3 ledger offload, Max block size in bytes. (64MiB by default, 
5MiB minimum)
 s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864
 
-# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default)
+# For Amazon S3 ledger offload, Read buffer size in bytes (1MiB by default)
 s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576
 
 # For Google Cloud Storage ledger offload, region where offload bucket is 
located.
@@ -1670,10 +1670,11 @@ gcsManagedLedgerOffloadRegion=
 # For Google Cloud Storage ledger offload, Bucket to place offloaded ledger 
into
 gcsManagedLedgerOffloadBucket=
 
-# For Google Cloud Storage ledger offload, Max block size in bytes. (64MB by 
default, 5MB minimum)
-gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864
+# For Google Cloud Storage ledger offload, Max block size in bytes. (128MiB by 
default, 5MiB minimum)
+# Since JClouds limits the maximum number of blocks to 32, the maximum size of 
a ledger is 32 times the block size.
+gcsManagedLedgerOffloadMaxBlockSizeInBytes=134217728
 
-# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MB by 
default)
+# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MiB by 
default)
 gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576
 
 # For Google Cloud Storage, path to json file containing service account 
credentials.
diff --git 
a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java
 
b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java
index 51e181811c2..6c40aa3f2ed 100644
--- 
a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java
+++ 
b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java
@@ -79,8 +79,9 @@ public class OffloadPoliciesImpl implements Serializable, 
OffloadPolicies {
         }
     }
 
-    public static final int DEFAULT_MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 
1024;   // 64MB
-    public static final int DEFAULT_READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024;   
   // 1MB
+    public static final int DEFAULT_MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 
1024;   // 64MiB
+    public static final int DEFAULT_GCS_MAX_BLOCK_SIZE_IN_BYTES = 128 * 1024 * 
1024;   // 128MiB
+    public static final int DEFAULT_READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024;   
   // 1MiB
     public static final int DEFAULT_OFFLOAD_MAX_THREADS = 2;
     public static final int DEFAULT_OFFLOAD_MAX_PREFETCH_ROUNDS = 1;
     public static final String DEFAULT_OFFLOADER_DIRECTORY = "./offloaders";
@@ -163,7 +164,7 @@ public class OffloadPoliciesImpl implements Serializable, 
OffloadPolicies {
     private String gcsManagedLedgerOffloadBucket = null;
     @Configuration
     @JsonProperty(access = JsonProperty.Access.READ_WRITE)
-    private Integer gcsManagedLedgerOffloadMaxBlockSizeInBytes = 
DEFAULT_MAX_BLOCK_SIZE_IN_BYTES;
+    private Integer gcsManagedLedgerOffloadMaxBlockSizeInBytes = 
DEFAULT_GCS_MAX_BLOCK_SIZE_IN_BYTES;
     @Configuration
     @JsonProperty(access = JsonProperty.Access.READ_WRITE)
     private Integer gcsManagedLedgerOffloadReadBufferSizeInBytes = 
DEFAULT_READ_BUFFER_SIZE_IN_BYTES;