Repository: kylin
Updated Branches:
  refs/heads/1.4-rc 922c9c41f -> 8c0de89a6


KYLIN-1740: HDFS storage leak in HBaseResourceStore.deleteResourceImpl() in 
case of large cell of dictionary or table snapshot


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8c0de89a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8c0de89a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8c0de89a

Branch: refs/heads/1.4-rc
Commit: 8c0de89a6ad22c2b408c076ea2105c2eecdb2211
Parents: 922c9c4
Author: kyotoYaho <[email protected]>
Authored: Tue Jun 21 10:36:16 2016 +0800
Committer: kyotoYaho <[email protected]>
Committed: Tue Jun 21 10:36:16 2016 +0800

----------------------------------------------------------------------
 .../kylin/storage/hbase/HBaseResourceStore.java | 41 +++++++++++++++-----
 1 file changed, 32 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/8c0de89a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index a389240..c0367bf 100644
--- 
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ 
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -267,9 +267,28 @@ public class HBaseResourceStore extends ResourceStore {
     protected void deleteResourceImpl(String resPath) throws IOException {
         HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
         try {
+            boolean hdfsResourceExist = false;
+            Result result = internalGetFromHTable(table,resPath,true,false);
+            if (result != null) {
+                byte[] value = result.getValue(B_FAMILY, B_COLUMN);
+                if (value != null && value.length == 0) {
+                    hdfsResourceExist = true;
+                }
+            }
+
             Delete del = new Delete(Bytes.toBytes(resPath));
             table.delete(del);
             table.flushCommits();
+
+            if (hdfsResourceExist) { // remove hdfs cell value
+                Path redirectPath = bigCellHDFSPath(resPath);
+                Configuration hconf = 
HBaseConnection.getCurrentHBaseConfiguration();
+                FileSystem fileSystem = FileSystem.get(hconf);
+
+                if (fileSystem.exists(redirectPath)) {
+                    fileSystem.delete(redirectPath, true);
+                }
+            }
         } finally {
             IOUtils.closeQuietly(table);
         }
@@ -281,10 +300,19 @@ public class HBaseResourceStore extends ResourceStore {
     }
 
     private Result getFromHTable(String path, boolean fetchContent, boolean 
fetchTimestamp) throws IOException {
+        HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
+        try {
+            return 
internalGetFromHTable(table,path,fetchContent,fetchTimestamp);
+        } finally {
+            IOUtils.closeQuietly(table);
+        }
+    }
+
+    private Result internalGetFromHTable(HTableInterface table, String path, 
boolean fetchContent, boolean fetchTimestamp) throws IOException {
         byte[] rowkey = Bytes.toBytes(path);
 
         Get get = new Get(rowkey);
-        
+
         if (!fetchContent && !fetchTimestamp) {
             get.setCheckExistenceOnly(true);
         } else {
@@ -294,14 +322,9 @@ public class HBaseResourceStore extends ResourceStore {
                 get.addColumn(B_FAMILY, B_COLUMN_TS);
         }
 
-        HTableInterface table = 
getConnection().getTable(getAllInOneTableName());
-        try {
-            Result result = table.get(get);
-            boolean exists = result != null && (!result.isEmpty() || 
(result.getExists() != null && result.getExists()));
-            return exists ? result : null;
-        } finally {
-            IOUtils.closeQuietly(table);
-        }
+        Result result = table.get(get);
+        boolean exists = result != null && (!result.isEmpty() || 
(result.getExists() != null && result.getExists()));
+        return exists ? result : null;
     }
 
     private byte[] plusZero(byte[] startRow) {

Reply via email to