This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2bdec6f HIVE-26015: CREATE HBase table fails when SERDEPROPERTIES
contain special characters (Steve Carlin, reviewed by Alessandro Solimando,
Stamatis Zampetakis)
2bdec6f is described below
commit 2bdec6f63d217cb42720fd7fe5cee804a2e5803c
Author: Steve Carlin <[email protected]>
AuthorDate: Tue Mar 8 12:29:04 2022 -0800
HIVE-26015: CREATE HBase table fails when SERDEPROPERTIES contain special
characters (Steve Carlin, reviewed by Alessandro Solimando, Stamatis Zampetakis)
Fields in the Serde Properties can have a hash tag (#) in it, so the
URI needs to be URLEncoded.
Closes #3084
---
.../hadoop/hive/hbase/HBaseStorageHandler.java | 31 ++++++---
.../hadoop/hive/hbase/TestHBaseStorageHandler.java | 76 ++++++++++++++++++++++
2 files changed, 99 insertions(+), 8 deletions(-)
diff --git
a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
index 302c09c..03d455f 100644
---
a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
+++
b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.hive.hbase;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
@@ -293,14 +295,27 @@ public class HBaseStorageHandler extends
DefaultStorageHandler
public URI getURIForAuth(Table table) throws URISyntaxException {
Map<String, String> tableProperties =
HiveCustomStorageHandlerUtils.getTableProperties(table);
hbaseConf = getConf();
- String hbase_host = tableProperties.containsKey(HBASE_HOST_NAME)?
tableProperties.get(HBASE_HOST_NAME) : hbaseConf.get(HBASE_HOST_NAME);
- String hbase_port = tableProperties.containsKey(HBASE_CLIENT_PORT)?
tableProperties.get(HBASE_CLIENT_PORT) : hbaseConf.get(HBASE_CLIENT_PORT);
- String table_name =
tableProperties.getOrDefault(HBaseSerDe.HBASE_TABLE_NAME, null);
- String column_family =
tableProperties.getOrDefault(HBaseSerDe.HBASE_COLUMNS_MAPPING, null);
- if (column_family != null)
- return new
URI(HBASE_PREFIX+"//"+hbase_host+":"+hbase_port+"/"+table_name+"/"+column_family);
- else
- return new
URI(HBASE_PREFIX+"//"+hbase_host+":"+hbase_port+"/"+table_name);
+ String hbase_host = tableProperties.getOrDefault(HBASE_HOST_NAME,
+ hbaseConf.get(HBASE_HOST_NAME));
+ String hbase_port = tableProperties.getOrDefault(HBASE_CLIENT_PORT,
+ hbaseConf.get(HBASE_CLIENT_PORT));
+ String table_name =
encodeString(tableProperties.getOrDefault(HBaseSerDe.HBASE_TABLE_NAME,
+ null));
+ String column_family = encodeString(tableProperties.getOrDefault(
+ HBaseSerDe.HBASE_COLUMNS_MAPPING, null));
+ String URIString = HBASE_PREFIX + "//" + hbase_host + ":" + hbase_port +
"/" + table_name;
+ if (column_family != null) {
+ URIString += "/" + column_family;
+ }
+ return new URI(URIString);
+ }
+
+ private static String encodeString(String rawString) throws
URISyntaxException {
+ try {
+ return rawString != null ? URLEncoder.encode(rawString, "UTF-8"): null;
+ } catch (UnsupportedEncodingException e) {
+ throw new URISyntaxException(rawString, "Could not URLEncode string");
+ }
}
/**
diff --git
a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java
b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java
index 8c8702a..b12df94 100644
---
a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java
+++
b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseStorageHandler.java
@@ -17,9 +17,16 @@
*/
package org.apache.hadoop.hive.hbase;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.mapred.JobConf;
import org.junit.Assert;
@@ -46,6 +53,58 @@ public class TestHBaseStorageHandler {
jobConfToConfigure.get("hbase.some.fake.option.from.xml.file") !=
null);
}
+ @Test
+ public void testGetUriForAuthEmptyTableDefaultHostPort() throws
URISyntaxException {
+ Table table = createMockTable(new HashMap<>());
+ URI uri = checkURIForAuth(table);
+ // If there is no tablename provided, the default "null" is still
+ // written out. At the time this test was written, this was the current
+ // behavior, so I left this test as/is. Need to research if a null
+ // table can be provided here.
+ Assert.assertEquals("hbase://localhost:2181/null", uri.toString());
+ }
+
+ @Test
+ public void testGetUriForAuthEmptyTable() throws URISyntaxException {
+ Map<String, String> serdeParams = new HashMap<>();
+ serdeParams.put("hbase.zookeeper.quorum", "testhost");
+ serdeParams.put("hbase.zookeeper.property.clientPort", "8765");
+ URI uri = checkURIForAuth(createMockTable(serdeParams));
+ Assert.assertEquals("hbase://testhost:8765/null", uri.toString());
+ }
+
+ @Test
+ public void testGetUriForAuthWithTable() throws URISyntaxException {
+ Map<String, String> serdeParams = new HashMap<>();
+ serdeParams.put("hbase.zookeeper.quorum", "testhost");
+ serdeParams.put("hbase.zookeeper.property.clientPort", "8765");
+ serdeParams.put("hbase.table.name", "mytbl");
+ URI uri = checkURIForAuth(createMockTable(serdeParams));
+ Assert.assertEquals("hbase://testhost:8765/mytbl", uri.toString());
+ }
+
+ @Test
+ public void testGetUriForAuthWithTableAndColumns() throws URISyntaxException
{
+ Map<String, String> serdeParams = new HashMap<>();
+ serdeParams.put("hbase.zookeeper.quorum", "testhost");
+ serdeParams.put("hbase.zookeeper.property.clientPort", "8765");
+ serdeParams.put("hbase.table.name", "mytbl");
+ serdeParams.put("hbase.columns.mapping", "mycolumns");
+ URI uri = checkURIForAuth(createMockTable(serdeParams));
+ Assert.assertEquals("hbase://testhost:8765/mytbl/mycolumns",
uri.toString());
+ }
+
+ @Test
+ public void testGetUriForAuthWithTableAndEncodedColumns() throws
URISyntaxException {
+ Map<String, String> serdeParams = new HashMap<>();
+ serdeParams.put("hbase.zookeeper.quorum", "testhost");
+ serdeParams.put("hbase.zookeeper.property.clientPort", "8765");
+ serdeParams.put("hbase.table.name", "my#tbl");
+ serdeParams.put("hbase.columns.mapping", "myco#lumns");
+ URI uri = checkURIForAuth(createMockTable(serdeParams));
+ Assert.assertEquals("hbase://testhost:8765/my%23tbl/myco%23lumns",
uri.toString());
+ }
+
private TableDesc getHBaseTableDesc() {
TableDesc tableDesc = Mockito.mock(TableDesc.class);
Properties properties = new Properties();
@@ -56,4 +115,21 @@ public class TestHBaseStorageHandler {
Mockito.when(tableDesc.getProperties()).thenReturn(properties);
return tableDesc;
}
+
+ private static URI checkURIForAuth(Table table) throws URISyntaxException {
+ HBaseStorageHandler hbaseStorageHandler = new HBaseStorageHandler();
+ hbaseStorageHandler.setConf(new JobConf(new HiveConf()));
+ return hbaseStorageHandler.getURIForAuth(table);
+ }
+
+ private static Table createMockTable(Map<String, String> serdeParams) {
+ Table table = new Table();
+ StorageDescriptor sd = new StorageDescriptor();
+ SerDeInfo sdi = new SerDeInfo();
+ sdi.setParameters(serdeParams);
+ sd.setSerdeInfo(sdi);
+ table.setSd(sd);
+ table.setParameters(new HashMap<>());
+ return table;
+ }
}
