This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 177e1856cd Fix GCS based ingestion if bucket name contains underscores
(#12445)
177e1856cd is described below
commit 177e1856cdaf3e5aa7a5754d129fc1945b9f7c72
Author: Tejaswini Bandlamudi <[email protected]>
AuthorDate: Thu Apr 21 09:22:35 2022 +0530
Fix GCS based ingestion if bucket name contains underscores (#12445)
GCP allows bucket names to contain underscores. When a location in such a
bucket
is mapped to `java.net.URI`, `URI.getHost()` returns null. `URI.getHost()`
is used as
the bucket name in `CloudObjectLocation`, leading to an NPE.
This commit uses `URI.getAuthority()` as the bucket name if `URI.getHost()`
is null.
---
.../druid/data/input/impl/CloudObjectLocation.java | 2 +-
.../data/input/impl/CloudObjectLocationTest.java | 28 ++++++++++++++++------
2 files changed, 22 insertions(+), 8 deletions(-)
diff --git
a/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java
b/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java
index ccc6916633..c2cdd07e77 100644
---
a/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java
+++
b/core/src/main/java/org/apache/druid/data/input/impl/CloudObjectLocation.java
@@ -71,7 +71,7 @@ public class CloudObjectLocation
public CloudObjectLocation(URI uri)
{
- this(uri.getHost(), uri.getPath());
+ this(uri.getHost() != null ? uri.getHost() : uri.getAuthority(),
uri.getPath());
}
/**
diff --git
a/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java
b/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java
index c7cd9957f4..f79b3d5b70 100644
---
a/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java
+++
b/core/src/test/java/org/apache/druid/data/input/impl/CloudObjectLocationTest.java
@@ -115,13 +115,27 @@ public class CloudObjectLocationTest
}
@Test
- public void testInvalidBucketName()
+ public void testBucketNameWithoutUnderscores()
{
- expectedException.expect(NullPointerException.class);
- expectedException.expectMessage("bucket name cannot be null. Please verify
if bucket name adheres to naming rules");
- // Underscore(_) character is not valid for bucket names
- CloudObjectLocation invalidBucket1 = new
CloudObjectLocation("test_bucket", "path/to/path");
- CloudObjectLocation invalidBucket2 = new
CloudObjectLocation(invalidBucket1.toUri(SCHEME));
- Assert.assertEquals("test_bucket", new
CloudObjectLocation(invalidBucket2.toUri(SCHEME)));
+ CloudObjectLocation gsValidBucket = new
CloudObjectLocation(URI.create("gs://1test.bucket-value/path/to/path"));
+ Assert.assertEquals("1test.bucket-value", gsValidBucket.getBucket());
+ Assert.assertEquals("path/to/path", gsValidBucket.getPath());
+
+ CloudObjectLocation s3ValidBucket = new
CloudObjectLocation(URI.create("s3://2test.bucket-value/path/to/path"));
+ Assert.assertEquals("2test.bucket-value", s3ValidBucket.getBucket());
+ Assert.assertEquals("path/to/path", s3ValidBucket.getPath());
+ }
+
+ @Test
+ public void testBucketNameWithUnderscores()
+ {
+ // Underscore(_) character is allowed for bucket names by GCP
+ CloudObjectLocation gsValidBucket = new
CloudObjectLocation(URI.create("gs://test_bucket/path/to/path"));
+ Assert.assertEquals("test_bucket", gsValidBucket.getBucket());
+ Assert.assertEquals("path/to/path", gsValidBucket.getPath());
+
+ CloudObjectLocation s3ValidBucket = new
CloudObjectLocation(URI.create("s3://test_bucket/path/to/path"));
+ Assert.assertEquals("test_bucket", s3ValidBucket.getBucket());
+ Assert.assertEquals("path/to/path", s3ValidBucket.getPath());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
