This is an automated email from the ASF dual-hosted git repository.
elserj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-filesystem.git
The following commit(s) were added to refs/heads/master by this push:
new 8ba3a91 HBASE-27042. Remove S3Guard awareness from HBoss
8ba3a91 is described below
commit 8ba3a9138d0d9fee8cba7a21c8ee542eb2ffe490
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Mon May 16 14:22:29 2022 +0100
HBASE-27042. Remove S3Guard awareness from HBoss
* Cut HBoss test code which would explicitly disable S3Guard
-obsolete and breaks the build on forthcoming hadoop releases.
* Also cut all references in docs and XML.
* EmbeddedS3 to implement deleteObject() and so support
AWS SDKs 1.11.1026+, which expects this (hadoop 3.3.2+).
* Maven test runs to provide full stack traces on failures,
so runtime failures can be debugged and fixed.
Change-Id: Iab10cfc6780cea064ea116efc1c177332d780514
Signed-off-by: Josh Elser <els...@apache.org>
---
.../main/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java | 6 ++++++
hbase-oss/README.md | 12 +++++-------
.../src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java | 5 -----
hbase-oss/src/test/resources/core-site.xml | 12 ------------
pom.xml | 1 +
5 files changed, 12 insertions(+), 24 deletions(-)
diff --git
a/hadoop-testutils/src/main/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java
b/hadoop-testutils/src/main/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java
index 2284a6d..864b661 100644
--- a/hadoop-testutils/src/main/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java
+++ b/hadoop-testutils/src/main/java/org/apache/hadoop/hbase/oss/EmbeddedS3.java
@@ -23,6 +23,7 @@ import com.amazonaws.services.s3.AbstractAmazonS3;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsResult;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
@@ -145,6 +146,11 @@ public class EmbeddedS3 {
bucket.remove(key);
}
+ public void deleteObject(DeleteObjectRequest request) {
+ LOG.debug("deleteObject: {}", request);
+ bucket.remove(request.getKey());
+ }
+
public DeleteObjectsResult deleteObjects(DeleteObjectsRequest request) {
for (DeleteObjectsRequest.KeyVersion keyVersion : request.getKeys()) {
String key = keyVersion.getKey();
diff --git a/hbase-oss/README.md b/hbase-oss/README.md
index 9beb529..99c3c3d 100644
--- a/hbase-oss/README.md
+++ b/hbase-oss/README.md
@@ -74,19 +74,17 @@ You may also want to configure:
An implementation based on Amazon's DynamoDB lock library was considered but
was not completed due to the lack of an efficient way to traverse the tree and
-discover locks on child nodes. The benefit is that S3Guard is required for s3a
-use and as such there's a dependency on DynamoDB anyway.
+discover locks on child nodes.
## Storage Implementations
Currently HBOSS is primarily designed for and exclusively tested with Hadoop's
-s3a client against Amazon S3. *S3Guard must be enabled, which is available in
-Hadoop 2.9.0, 3.0.0, and higher*.
+s3a client against Amazon S3.
-Both the use of S3Guard and Zookeeper for locking (i.e. Zookeeper) have
+The use of Zookeeper for locking (i.e. Zookeeper) have
implications for other clients that are not configured to share the same
-metadata store and Zookeeper ensemble. Ideally, all clients should be have the
-same configuration in these respects. Read-only clients may not share these
+Zookeeper ensemble. Ideally, all clients should have the
+same configuration. Read-only clients may not share these
resources with the HBase processes, but they will not have the added safety
provided by these features. Clients that do not share these resources and
modify
data can compromise the correctness of HBase.
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
index f2e7011..b283715 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/TestUtils.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.oss;
import static org.apache.hadoop.fs.s3a.Constants.S3_CLIENT_FACTORY_IMPL;
-import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
import static org.apache.hadoop.hbase.oss.Constants.DATA_URI;
import java.net.URI;
@@ -29,7 +28,6 @@ import java.net.URL;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore;
import org.apache.hadoop.hbase.oss.sync.EmbeddedZK;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.yetus.audience.InterfaceAudience;
@@ -69,9 +67,6 @@ public class TestUtils {
public static boolean usingEmbeddedS3 = false;
public static void conditionalStart(Configuration conf) {
- if (StringUtils.isEmpty(conf.get(S3_METADATA_STORE_IMPL))) {
- conf.set(S3_METADATA_STORE_IMPL, LocalMetadataStore.class.getName());
- }
boolean notConfigured = StringUtils.isEmpty(conf.get(DATA_URI));
if (notConfigured) {
diff --git a/hbase-oss/src/test/resources/core-site.xml
b/hbase-oss/src/test/resources/core-site.xml
index a46594f..045d91a 100644
--- a/hbase-oss/src/test/resources/core-site.xml
+++ b/hbase-oss/src/test/resources/core-site.xml
@@ -42,18 +42,6 @@
</property>
</configuration>
- S3Guard will also be configured to the LocalMetadataStore implementation
- by default. You may wish to configure DynamoDBMetadataStore for integration
- or if S3Guard is not needed (S3-compatible appliance, for instance), you
- can disable it by configuring NullMetadataStore explicitly (see below).
- -->
-
- <!--
- <property>
- <name>fs.s3a.metadatastore.impl</name>
- <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
- <description>Uncomment to disable S3Guard entirely</description>
- </property>
-->
<include xmlns="http://www.w3.org/2001/XInclude"; href="auth-keys.xml">
diff --git a/pom.xml b/pom.xml
index ced3c49..c1db1bd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.1</version>
<configuration>
+ <trimStackTrace>false</trimStackTrace>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<reuseForks>false</reuseForks>
<systemProperties>
