HIVE-13527: Using deprecated APIs in HBase client causes zookeeper connection leaks (Naveen Gangam via Chaoyu Tang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/74197688 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/74197688 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/74197688 Branch: refs/heads/llap Commit: 74197688f18d9a6bee6393cfe1d7ab635beb3db5 Parents: 287f045 Author: ctang <[email protected]> Authored: Sat Apr 23 20:50:00 2016 -0400 Committer: ctang <[email protected]> Committed: Sat Apr 23 20:50:00 2016 -0400 ---------------------------------------------------------------------- .../hive/hbase/HiveHBaseTableInputFormat.java | 33 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/74197688/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java index 88d1865..d9db624 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java @@ -26,6 +26,9 @@ import java.util.Map; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; @@ -84,6 +87,7 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase static final Logger LOG = LoggerFactory.getLogger(HiveHBaseTableInputFormat.class); private static final Object hbaseTableMonitor = new Object(); + private Connection conn = null; @Override public RecordReader<ImmutableBytesWritable, ResultWritable> getRecordReader( @@ -94,7 +98,11 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase HBaseSplit hbaseSplit = (HBaseSplit) split; TableSplit tableSplit = hbaseSplit.getTableSplit(); - setHTable(HiveHBaseInputFormatUtil.getTable(jobConf)); + if (conn == null) { + conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf)); + } + initializeTable(conn, tableSplit.getTable()); + setScan(HiveHBaseInputFormatUtil.getScan(jobConf)); Job job = new Job(jobConf); @@ -107,6 +115,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase recordReader.initialize(tableSplit, tac); } catch (InterruptedException e) { closeTable(); // Free up the HTable connections + if (conn != null) { + conn.close(); + conn = null; + } throw new IOException("Failed to initialize RecordReader", e); } @@ -116,6 +128,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase public void close() throws IOException { recordReader.close(); closeTable(); + if (conn != null) { + conn.close(); + conn = null; + } } @Override @@ -442,7 +458,12 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase } String hbaseTableName = jobConf.get(HBaseSerDe.HBASE_TABLE_NAME); - setHTable(new HTable(HBaseConfiguration.create(jobConf), Bytes.toBytes(hbaseTableName))); + if (conn == null) { + conn = ConnectionFactory.createConnection(HBaseConfiguration.create(jobConf)); + } + TableName tableName = TableName.valueOf(hbaseTableName); + initializeTable(conn, tableName); + String hbaseColumnsMapping = jobConf.get(HBaseSerDe.HBASE_COLUMNS_MAPPING); boolean doColumnRegexMatching = jobConf.getBoolean(HBaseSerDe.HBASE_COLUMNS_REGEX_MATCHING, true); @@ -509,6 +530,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase return results; } finally { closeTable(); + if (conn != null) { + conn.close(); + conn = null; + } } } @@ -516,6 +541,10 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase protected void finalize() throws Throwable { try { closeTable(); + if (conn != null) { + conn.close(); + conn = null; + } } finally { super.finalize(); }
