[hbase] branch branch-2.4 updated: HBASE-26196 Support configuration override for remote cluster of HFileOutputFormat locality sensitive (#3582)

[stack]

This is an automated email from the ASF dual-hosted git repository.

stack pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
     new 6cc26c2  HBASE-26196 Support configuration override for remote cluster 
of HFileOutputFormat locality sensitive (#3582)
6cc26c2 is described below

commit 6cc26c2487b471a294e0c4dbeed68884cd39ad46
Author: bitterfox <yoshida.shi...@linecorp.com>
AuthorDate: Tue Aug 17 02:26:25 2021 +0900

    HBASE-26196 Support configuration override for remote cluster of 
HFileOutputFormat locality sensitive (#3582)
    
    Signed-off-by: stack <st...@apache.org>
---
 .../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 26 +++++++++++++++++++---
 .../hbase/mapreduce/TestHFileOutputFormat2.java    |  9 ++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
index 02b5768..2187de0 100644
--- 
a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
+++ 
b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
@@ -31,6 +31,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -176,12 +177,14 @@ public class HFileOutputFormat2
   static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY =
           "hbase.mapreduce.use.multi.table.hfileoutputformat";
 
+  public static final String REMOTE_CLUSTER_CONF_PREFIX =
+    "hbase.hfileoutputformat.remote.cluster.";
   public static final String REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY =
-    "hbase.hfileoutputformat.remote.cluster.zookeeper.quorum";
+    REMOTE_CLUSTER_CONF_PREFIX + "zookeeper.quorum";
   public static final String REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY =
-    "hbase.hfileoutputformat.remote.cluster.zookeeper." + 
HConstants.CLIENT_PORT_STR;
+    REMOTE_CLUSTER_CONF_PREFIX + "zookeeper." + HConstants.CLIENT_PORT_STR;
   public static final String REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY =
-    "hbase.hfileoutputformat.remote.cluster." + 
HConstants.ZOOKEEPER_ZNODE_PARENT;
+    REMOTE_CLUSTER_CONF_PREFIX + HConstants.ZOOKEEPER_ZNODE_PARENT;
 
   public static final String STORAGE_POLICY_PROPERTY = 
HStore.BLOCK_STORAGE_POLICY_KEY;
   public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = 
STORAGE_POLICY_PROPERTY + ".";
@@ -379,6 +382,23 @@ public class HFileOutputFormat2
           newConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parent);
         }
 
+        for (Entry<String, String> entry : conf) {
+          String key = entry.getKey();
+          if (REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY.equals(key) ||
+              REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY.equals(key) ||
+              REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY.equals(key)) {
+            // Handled them above
+            continue;
+          }
+
+          if (entry.getKey().startsWith(REMOTE_CLUSTER_CONF_PREFIX)) {
+            String originalKey = 
entry.getKey().substring(REMOTE_CLUSTER_CONF_PREFIX.length());
+            if (!originalKey.isEmpty()) {
+              newConf.set(originalKey, entry.getValue());
+            }
+          }
+        }
+
         return newConf;
       }
 
diff --git 
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
 
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index f94df25..f4cc385 100644
--- 
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++ 
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HStore;
 import org.apache.hadoop.hbase.regionserver.TestHRegionFileSystem;
 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
+import org.apache.hadoop.hbase.security.SecurityConstants;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
@@ -1612,6 +1613,11 @@ public class TestHFileOutputFormat2  {
       assertEquals(confB.get(HConstants.ZOOKEEPER_ZNODE_PARENT),
         
jobConf.get(HFileOutputFormat2.REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY));
 
+      String bSpecificConfigKey = "my.override.config.for.b";
+      String bSpecificConfigValue = "b-specific-value";
+      jobConf.set(HFileOutputFormat2.REMOTE_CLUSTER_CONF_PREFIX + 
bSpecificConfigKey,
+        bSpecificConfigValue);
+
       FileOutputFormat.setOutputPath(job, testDir);
 
       assertFalse(util.getTestFileSystem().exists(testDir));
@@ -1629,6 +1635,9 @@ public class TestHFileOutputFormat2  {
           config.get(HConstants.ZOOKEEPER_CLIENT_PORT));
         assertEquals(confB.get(HConstants.ZOOKEEPER_ZNODE_PARENT),
           config.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
+
+        assertEquals(bSpecificConfigValue,
+          config.get(bSpecificConfigKey));
       }
     } finally {
       utilB.deleteTable(tableName);