sijie closed pull request #2376: adding tiered storage confs to endpoint
URL: https://github.com/apache/incubator-pulsar/pull/2376
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git 
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java 
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
index f6319aeaa7..f5240c989d 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java
@@ -62,7 +62,7 @@
 import org.apache.pulsar.broker.loadbalance.LoadSheddingTask;
 import org.apache.pulsar.broker.loadbalance.impl.LoadManagerShared;
 import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.broker.offload.TieredStorageConfigurationData;
+import org.apache.pulsar.common.offload.TieredStorageConfigurationData;
 import org.apache.pulsar.broker.offload.impl.BlobStoreManagedLedgerOffloader;
 import org.apache.pulsar.broker.service.BrokerService;
 import org.apache.pulsar.broker.service.Topic;
@@ -660,14 +660,13 @@ public LedgerOffloader getManagedLedgerOffloader() {
     static final String METADATA_SOFTWARE_VERSION_KEY = 
"S3ManagedLedgerOffloaderSoftwareVersion";
     static final String METADATA_SOFTWARE_GITSHA_KEY = 
"S3ManagedLedgerOffloaderSoftwareGitSha";
 
-
     public synchronized LedgerOffloader 
createManagedLedgerOffloader(ServiceConfiguration conf)
             throws PulsarServerException {
         if (conf.getManagedLedgerOffloadDriver() != null
             && 
BlobStoreManagedLedgerOffloader.driverSupported(conf.getManagedLedgerOffloadDriver()))
 {
             try {
                 return BlobStoreManagedLedgerOffloader.create(
-                    getTieredStorageConf(conf),
+                    getTieredStorageConf(),
                     ImmutableMap.of(
                         METADATA_SOFTWARE_VERSION_KEY.toLowerCase(), 
PulsarBrokerVersionStringUtils.getNormalizedVersionString(),
                         METADATA_SOFTWARE_GITSHA_KEY.toLowerCase(), 
PulsarBrokerVersionStringUtils.getGitSha()
@@ -681,23 +680,23 @@ public synchronized LedgerOffloader 
createManagedLedgerOffloader(ServiceConfigur
         }
     }
 
-    private static TieredStorageConfigurationData 
getTieredStorageConf(ServiceConfiguration serverConf) {
+    public TieredStorageConfigurationData getTieredStorageConf() {
         TieredStorageConfigurationData tsConf = new 
TieredStorageConfigurationData();
         // generic settings
-        
tsConf.setManagedLedgerOffloadDriver(serverConf.getManagedLedgerOffloadDriver());
-        
tsConf.setManagedLedgerOffloadMaxThreads(serverConf.getManagedLedgerOffloadMaxThreads());
+        
tsConf.setManagedLedgerOffloadDriver(config.getManagedLedgerOffloadDriver());
+        
tsConf.setManagedLedgerOffloadMaxThreads(config.getManagedLedgerOffloadMaxThreads());
         // s3 settings
-        
tsConf.setS3ManagedLedgerOffloadRegion(serverConf.getS3ManagedLedgerOffloadRegion());
-        
tsConf.setS3ManagedLedgerOffloadBucket(serverConf.getS3ManagedLedgerOffloadBucket());
-        
tsConf.setS3ManagedLedgerOffloadServiceEndpoint(serverConf.getS3ManagedLedgerOffloadServiceEndpoint());
-        
tsConf.setS3ManagedLedgerOffloadMaxBlockSizeInBytes(serverConf.getS3ManagedLedgerOffloadMaxBlockSizeInBytes());
-        
tsConf.setS3ManagedLedgerOffloadReadBufferSizeInBytes(serverConf.getS3ManagedLedgerOffloadReadBufferSizeInBytes());
+        
tsConf.setS3ManagedLedgerOffloadRegion(config.getS3ManagedLedgerOffloadRegion());
+        
tsConf.setS3ManagedLedgerOffloadBucket(config.getS3ManagedLedgerOffloadBucket());
+        
tsConf.setS3ManagedLedgerOffloadServiceEndpoint(config.getS3ManagedLedgerOffloadServiceEndpoint());
+        
tsConf.setS3ManagedLedgerOffloadMaxBlockSizeInBytes(config.getS3ManagedLedgerOffloadMaxBlockSizeInBytes());
+        
tsConf.setS3ManagedLedgerOffloadReadBufferSizeInBytes(config.getS3ManagedLedgerOffloadReadBufferSizeInBytes());
         // gcs settings
-        
tsConf.setGcsManagedLedgerOffloadRegion(serverConf.getGcsManagedLedgerOffloadRegion());
-        
tsConf.setGcsManagedLedgerOffloadBucket(serverConf.getGcsManagedLedgerOffloadBucket());
-        
tsConf.setGcsManagedLedgerOffloadServiceAccountKeyFile(serverConf.getGcsManagedLedgerOffloadServiceAccountKeyFile());
-        
tsConf.setGcsManagedLedgerOffloadMaxBlockSizeInBytes(serverConf.getGcsManagedLedgerOffloadMaxBlockSizeInBytes());
-        
tsConf.setGcsManagedLedgerOffloadReadBufferSizeInBytes(serverConf.getGcsManagedLedgerOffloadReadBufferSizeInBytes());
+        
tsConf.setGcsManagedLedgerOffloadRegion(config.getGcsManagedLedgerOffloadRegion());
+        
tsConf.setGcsManagedLedgerOffloadBucket(config.getGcsManagedLedgerOffloadBucket());
+        
tsConf.setGcsManagedLedgerOffloadServiceAccountKeyFile(config.getGcsManagedLedgerOffloadServiceAccountKeyFile());
+        
tsConf.setGcsManagedLedgerOffloadMaxBlockSizeInBytes(config.getGcsManagedLedgerOffloadMaxBlockSizeInBytes());
+        
tsConf.setGcsManagedLedgerOffloadReadBufferSizeInBytes(config.getGcsManagedLedgerOffloadReadBufferSizeInBytes());
         return tsConf;
     }
 
@@ -971,7 +970,8 @@ private void startWorkerService() throws 
InterruptedException, IOException, Keep
             InternalConfigurationData internalConf = new 
InternalConfigurationData(
                     this.getConfiguration().getZookeeperServers(),
                     this.getConfiguration().getConfigurationStoreServers(),
-                    new ClientConfiguration().getZkLedgersRootPath());
+                    new ClientConfiguration().getZkLedgersRootPath(),
+                    getTieredStorageConf());
 
             URI dlogURI;
             try {
diff --git 
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
 
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
index cab707e90c..618a0c2a9d 100644
--- 
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
+++ 
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java
@@ -193,7 +193,7 @@ public InternalConfigurationData 
getInternalConfigurationData() {
         return new InternalConfigurationData(
             pulsar().getConfiguration().getZookeeperServers(),
             pulsar().getConfiguration().getConfigurationStoreServers(),
-            conf.getZkLedgersRootPath());
+            conf.getZkLedgersRootPath(), pulsar().getTieredStorageConf());
     }
 
 }
diff --git 
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java 
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java
index 1f0058c74c..3c3b78ef27 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java
@@ -66,6 +66,7 @@
 import org.apache.pulsar.broker.web.RestException;
 import org.apache.pulsar.common.conf.InternalConfigurationData;
 import org.apache.pulsar.common.naming.NamespaceName;
+import org.apache.pulsar.common.offload.TieredStorageConfigurationData;
 import org.apache.pulsar.common.policies.data.AuthAction;
 import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
 import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType;
@@ -209,7 +210,7 @@ void internalConfiguration() throws Exception {
         InternalConfigurationData expectedData = new InternalConfigurationData(
             pulsar.getConfiguration().getZookeeperServers(),
             pulsar.getConfiguration().getConfigurationStoreServers(),
-            new ClientConfiguration().getZkLedgersRootPath());
+            new ClientConfiguration().getZkLedgersRootPath(), new 
TieredStorageConfigurationData());
 
         assertEquals(brokers.getInternalConfigurationData(), expectedData);
     }
diff --git 
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
 
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
index 20bb4074b6..af655aca35 100644
--- 
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
+++ 
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java
@@ -770,7 +770,7 @@ public void 
testNonPersistentBrokerModeRejectPersistentTopic(String loadManagerN
      *
      * @throws Exception
      */
-    @Test
+    @Test(invocationCount=100, skipFailedInvocations=true)
     public void testMsgDropStat() throws Exception {
 
         int defaultNonPersistentMessageRate = 
conf.getMaxConcurrentNonPersistentMessagePerConnection();
diff --git 
a/pulsar-common/src/main/java/org/apache/pulsar/common/conf/InternalConfigurationData.java
 
b/pulsar-common/src/main/java/org/apache/pulsar/common/conf/InternalConfigurationData.java
index 4a038a31db..87c4a168c4 100644
--- 
a/pulsar-common/src/main/java/org/apache/pulsar/common/conf/InternalConfigurationData.java
+++ 
b/pulsar-common/src/main/java/org/apache/pulsar/common/conf/InternalConfigurationData.java
@@ -18,61 +18,27 @@
  */
 package org.apache.pulsar.common.conf;
 
-import com.google.common.base.MoreObjects;
-import java.util.Objects;
+import lombok.Data;
+import org.apache.pulsar.common.offload.TieredStorageConfigurationData;
 
+@Data
 public class InternalConfigurationData {
 
     private String zookeeperServers;
     private String configurationStoreServers;
     private String ledgersRootPath;
-
+    private TieredStorageConfigurationData tieredStorageConfigurationData;
+    
     public InternalConfigurationData() {
     }
 
     public InternalConfigurationData(String zookeeperServers,
                                      String configurationStoreServers,
-                                     String ledgersRootPath) {
+                                     String ledgersRootPath,
+                                     TieredStorageConfigurationData 
tieredStorageConfigurationData) {
         this.zookeeperServers = zookeeperServers;
         this.configurationStoreServers = configurationStoreServers;
         this.ledgersRootPath = ledgersRootPath;
+        this.tieredStorageConfigurationData = tieredStorageConfigurationData;
     }
-
-    public String getZookeeperServers() {
-        return zookeeperServers;
-    }
-
-    public String getConfigurationStoreServers() {
-        return configurationStoreServers;
-    }
-
-    public String getLedgersRootPath() {
-        return ledgersRootPath;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof InternalConfigurationData)) {
-            return false;
-        }
-        InternalConfigurationData other = (InternalConfigurationData) obj;
-        return Objects.equals(zookeeperServers, other.zookeeperServers)
-            && Objects.equals(configurationStoreServers, 
other.configurationStoreServers)
-            && Objects.equals(ledgersRootPath, other.ledgersRootPath);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(zookeeperServers, configurationStoreServers, 
ledgersRootPath);
-    }
-
-    @Override
-    public String toString() {
-        return MoreObjects.toStringHelper(this)
-            .add("zookeeperServers", zookeeperServers)
-            .add("configurationStoreServers", configurationStoreServers)
-            .add("ledgersRootPath", ledgersRootPath)
-            .toString();
-    }
-
 }
diff --git 
a/tiered-storage/jcloud/src/main/java/org/apache/pulsar/broker/offload/TieredStorageConfigurationData.java
 
b/pulsar-common/src/main/java/org/apache/pulsar/common/offload/TieredStorageConfigurationData.java
similarity index 90%
rename from 
tiered-storage/jcloud/src/main/java/org/apache/pulsar/broker/offload/TieredStorageConfigurationData.java
rename to 
pulsar-common/src/main/java/org/apache/pulsar/common/offload/TieredStorageConfigurationData.java
index c7ef3fd322..ff9bce662d 100644
--- 
a/tiered-storage/jcloud/src/main/java/org/apache/pulsar/broker/offload/TieredStorageConfigurationData.java
+++ 
b/pulsar-common/src/main/java/org/apache/pulsar/common/offload/TieredStorageConfigurationData.java
@@ -16,9 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.pulsar.broker.offload;
+package org.apache.pulsar.common.offload;
 
 import java.io.Serializable;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Data;
 
 /**
@@ -64,6 +66,8 @@
 
     // For Google Cloud Storage, path to json file containing service account 
credentials.
     // For more details, see the "Service Accounts" section of 
https://support.google.com/googleapi/answer/6158849
+    // Add @JsonIgnore so that this field will not be serialized and returned 
via any endpoints e.g. {@link 
org.apache.pulsar.broker.admin.impl.BrokerBase#getInternalConfigurationData}
+    @JsonIgnore
     private String gcsManagedLedgerOffloadServiceAccountKeyFile = null;
 
 }
diff --git 
a/tiered-storage/jcloud/src/main/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloader.java
 
b/tiered-storage/jcloud/src/main/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloader.java
index b0345fae35..43911c6480 100644
--- 
a/tiered-storage/jcloud/src/main/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloader.java
+++ 
b/tiered-storage/jcloud/src/main/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloader.java
@@ -42,7 +42,7 @@
 import org.apache.pulsar.broker.offload.BlockAwareSegmentInputStream;
 import org.apache.pulsar.broker.offload.OffloadIndexBlock;
 import org.apache.pulsar.broker.offload.OffloadIndexBlockBuilder;
-import org.apache.pulsar.broker.offload.TieredStorageConfigurationData;
+import org.apache.pulsar.common.offload.TieredStorageConfigurationData;
 import org.jclouds.Constants;
 import org.jclouds.ContextBuilder;
 import org.jclouds.blobstore.BlobStore;
diff --git 
a/tiered-storage/jcloud/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
 
b/tiered-storage/jcloud/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
index 0999d63768..bb8ef4df2e 100644
--- 
a/tiered-storage/jcloud/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
+++ 
b/tiered-storage/jcloud/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
@@ -53,7 +53,7 @@
 import org.apache.bookkeeper.mledger.LedgerOffloader;
 import org.apache.bookkeeper.util.ZkUtils;
 import org.apache.pulsar.broker.offload.BlobStoreTestBase;
-import org.apache.pulsar.broker.offload.TieredStorageConfigurationData;
+import org.apache.pulsar.common.offload.TieredStorageConfigurationData;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.MockZooKeeper;
 import org.apache.zookeeper.data.ACL;


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

Reply via email to