Repository: kylin Updated Branches: refs/heads/1.4-rc 922c9c41f -> 8c0de89a6
KYLIN-1740: HDFS storage leak in HBaseResourceStore.deleteResourceImpl() in case of large cell of dictionary or table snapshot Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8c0de89a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8c0de89a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8c0de89a Branch: refs/heads/1.4-rc Commit: 8c0de89a6ad22c2b408c076ea2105c2eecdb2211 Parents: 922c9c4 Author: kyotoYaho <[email protected]> Authored: Tue Jun 21 10:36:16 2016 +0800 Committer: kyotoYaho <[email protected]> Committed: Tue Jun 21 10:36:16 2016 +0800 ---------------------------------------------------------------------- .../kylin/storage/hbase/HBaseResourceStore.java | 41 +++++++++++++++----- 1 file changed, 32 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8c0de89a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index a389240..c0367bf 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -267,9 +267,28 @@ public class HBaseResourceStore extends ResourceStore { protected void deleteResourceImpl(String resPath) throws IOException { HTableInterface table = getConnection().getTable(getAllInOneTableName()); try { + boolean hdfsResourceExist = false; + Result result = internalGetFromHTable(table,resPath,true,false); + if (result != null) { + byte[] value = result.getValue(B_FAMILY, B_COLUMN); + if (value != null && value.length == 0) { + hdfsResourceExist = true; + } + } + Delete del = new Delete(Bytes.toBytes(resPath)); table.delete(del); table.flushCommits(); + + if (hdfsResourceExist) { // remove hdfs cell value + Path redirectPath = bigCellHDFSPath(resPath); + Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); + FileSystem fileSystem = FileSystem.get(hconf); + + if (fileSystem.exists(redirectPath)) { + fileSystem.delete(redirectPath, true); + } + } } finally { IOUtils.closeQuietly(table); } @@ -281,10 +300,19 @@ public class HBaseResourceStore extends ResourceStore { } private Result getFromHTable(String path, boolean fetchContent, boolean fetchTimestamp) throws IOException { + HTableInterface table = getConnection().getTable(getAllInOneTableName()); + try { + return internalGetFromHTable(table,path,fetchContent,fetchTimestamp); + } finally { + IOUtils.closeQuietly(table); + } + } + + private Result internalGetFromHTable(HTableInterface table, String path, boolean fetchContent, boolean fetchTimestamp) throws IOException { byte[] rowkey = Bytes.toBytes(path); Get get = new Get(rowkey); - + if (!fetchContent && !fetchTimestamp) { get.setCheckExistenceOnly(true); } else { @@ -294,14 +322,9 @@ public class HBaseResourceStore extends ResourceStore { get.addColumn(B_FAMILY, B_COLUMN_TS); } - HTableInterface table = getConnection().getTable(getAllInOneTableName()); - try { - Result result = table.get(get); - boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists())); - return exists ? result : null; - } finally { - IOUtils.closeQuietly(table); - } + Result result = table.get(get); + boolean exists = result != null && (!result.isEmpty() || (result.getExists() != null && result.getExists())); + return exists ? result : null; } private byte[] plusZero(byte[] startRow) {
