sijie closed pull request #2281: Issue #2165: add System property config to run
real S3 and GCS test for offloader
URL: https://github.com/apache/incubator-pulsar/pull/2281
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/BlobStoreTestBase.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/BlobStoreTestBase.java
index d1474f95fa..8e21c471ea 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/BlobStoreTestBase.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/BlobStoreTestBase.java
@@ -36,16 +36,41 @@
@BeforeMethod
public void start() throws Exception {
- context =
ContextBuilder.newBuilder("transient").build(BlobStoreContext.class);
- blobStore = context.getBlobStore();
- boolean create = blobStore.createContainerInLocation(null, BUCKET);
-
- log.debug("TestBase Create Bucket: {}, in blobStore, result: {}",
BUCKET, create);
+ if (Boolean.parseBoolean(System.getProperty("testRealAWS", "false"))) {
+ log.info("TestReal AWS S3, bucket: {}", BUCKET);
+ // To use this, must config credentials using "aws_access_key_id"
as S3ID,
+ // and "aws_secret_access_key" as S3Key. And bucket should exist
in default region. e.g.
+ // props.setProperty("S3ID", "AXXXXXXQ");
+ // props.setProperty("S3Key", "HXXXXXĂ");
+ context = ContextBuilder.newBuilder("aws-s3")
+ .credentials(System.getProperty("S3ID"),
System.getProperty("S3Key"))
+ .build(BlobStoreContext.class);
+ blobStore = context.getBlobStore();
+ // To use this, ~/.aws must be configured with credentials and a
default region
+ //s3client = AmazonS3ClientBuilder.standard().build();
+ } else if (Boolean.parseBoolean(System.getProperty("testRealGCS",
"false"))) {
+ log.info("TestReal GCS, bucket: {}", BUCKET);
+ // To use this, must config credentials using "client_email" as
GCSID and "private_key" as GCSKey.
+ // And bucket should exist in default region. e.g.
+ // props.setProperty("GCSID",
"[email protected]");
+ // props.setProperty("GCSKey", "XXXXXX");
+ context = ContextBuilder.newBuilder("google-cloud-storage")
+ .credentials(System.getProperty("GCSID"),
System.getProperty("GCSKey"))
+ .build(BlobStoreContext.class);
+ blobStore = context.getBlobStore();
+ } else {
+ context =
ContextBuilder.newBuilder("transient").build(BlobStoreContext.class);
+ blobStore = context.getBlobStore();
+ boolean create = blobStore.createContainerInLocation(null, BUCKET);
+ log.debug("TestBase Create Bucket: {}, in blobStore, result: {}",
BUCKET, create);
+ }
}
@AfterMethod
public void tearDown() {
- if (blobStore != null) {
+ if (blobStore != null &&
+ (!Boolean.parseBoolean(System.getProperty("testRealGCS", "false"))
&&
+ !Boolean.parseBoolean(System.getProperty("testRealGCS",
"false")))) {
blobStore.deleteContainer(BUCKET);
}
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
index 86b4657384..f1e43c2144 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/offload/impl/BlobStoreManagedLedgerOffloaderTest.java
@@ -59,6 +59,7 @@
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
+import org.testng.collections.Maps;
@Slf4j
class BlobStoreManagedLedgerOffloaderTest extends BlobStoreTestBase {
@@ -131,8 +132,8 @@ public void testBucketDoesNotExist() throws Exception {
offloader.offload(buildReadHandle(), UUID.randomUUID(), new
HashMap<>()).get();
Assert.fail("Shouldn't be able to add to bucket");
} catch (ExecutionException e) {
- log.error("Exception: ", e.getMessage());
- Assert.assertTrue(e.getMessage().contains("not found"));
+ log.error("Exception: ", e);
+ Assert.assertTrue(e.getMessage().toLowerCase().contains("not
found"));
}
}
@@ -520,8 +521,11 @@ public void testReadUnknownDataVersion() throws Exception {
String dataKey = dataBlockOffloadKey(toWrite.getId(), uuid);
- Map<String, String> userMeta = blobStore.blobMetadata(BUCKET,
dataKey).getUserMetadata();
-
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY,
String.valueOf(-12345));
+ // Here it will return a Immutable map.
+ Map<String, String> immutableMap = blobStore.blobMetadata(BUCKET,
dataKey).getUserMetadata();
+ Map<String, String> userMeta = Maps.newHashMap();
+ userMeta.putAll(immutableMap);
+
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY.toLowerCase(),
String.valueOf(-12345));
blobStore.copyBlob(BUCKET, dataKey, BUCKET, dataKey,
CopyOptions.builder().userMetadata(userMeta).build());
try (ReadHandle toRead = offloader.readOffloaded(toWrite.getId(),
uuid).get()) {
@@ -533,7 +537,7 @@ public void testReadUnknownDataVersion() throws Exception {
Assert.assertTrue(e.getCause().getMessage().contains("Error
reading from BlobStore"));
}
-
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY,
String.valueOf(12345));
+
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY.toLowerCase(),
String.valueOf(12345));
blobStore.copyBlob(BUCKET, dataKey, BUCKET, dataKey,
CopyOptions.builder().userMetadata(userMeta).build());
try (ReadHandle toRead = offloader.readOffloaded(toWrite.getId(),
uuid).get()) {
@@ -555,8 +559,11 @@ public void testReadUnknownIndexVersion() throws Exception
{
String indexKey = indexBlockOffloadKey(toWrite.getId(), uuid);
- Map<String, String> userMeta = blobStore.blobMetadata(BUCKET,
indexKey).getUserMetadata();
-
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY,
String.valueOf(-12345));
+ // Here it will return a Immutable map.
+ Map<String, String> immutableMap = blobStore.blobMetadata(BUCKET,
indexKey).getUserMetadata();
+ Map<String, String> userMeta = Maps.newHashMap();
+ userMeta.putAll(immutableMap);
+
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY.toLowerCase(),
String.valueOf(-12345));
blobStore.copyBlob(BUCKET, indexKey, BUCKET, indexKey,
CopyOptions.builder().userMetadata(userMeta).build());
try {
@@ -567,7 +574,7 @@ public void testReadUnknownIndexVersion() throws Exception {
Assert.assertTrue(e.getCause().getMessage().contains("Invalid
object version"));
}
-
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY,
String.valueOf(12345));
+
userMeta.put(BlobStoreManagedLedgerOffloader.METADATA_FORMAT_VERSION_KEY.toLowerCase(),
String.valueOf(12345));
blobStore.copyBlob(BUCKET, indexKey, BUCKET, indexKey,
CopyOptions.builder().userMetadata(userMeta).build());
try {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
