This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch sync
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c5e46bbf93e6ba6092cab3bd364f19680d23ac30
Author: Jiatao Tao <245915...@qq.com>
AuthorDate: Sat Mar 3 09:25:01 2018 +0800
KYLIN-3234, add a API to ResourceStore that can recursively
---
.../common/persistence/FileResourceStore.java | 29 ++++++++++++++++------
.../common/persistence/HDFSResourceStore.java | 5 +++-
.../kylin/common/persistence/ResourceStore.java | 12 +++++++--
.../kylin/storage/hbase/HBaseResourceStore.java | 24 ++++++++++++------
4 files changed, 52 insertions(+), 18 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
index 12c8aba..6ac4536 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java
@@ -25,11 +25,14 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.NavigableSet;
import java.util.TreeSet;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -56,16 +59,28 @@ public class FileResourceStore extends ResourceStore {
}
@Override
- protected NavigableSet<String> listResourcesImpl(String folderPath) throws
IOException {
+ protected NavigableSet<String> listResourcesImpl(String folderPath,
boolean recursive) throws IOException {
synchronized (FileResourceStore.class) {
- String[] names = file(folderPath).list();
- if (names == null) // not a directory
- return null;
-
TreeSet<String> r = new TreeSet<>();
+ File file = file(folderPath);
+ String[] names = file.list();
+ // not a directory
+ if (names == null)
+ // fixme should return empty set, like HBase implement.
+ return null;
String prefix = folderPath.endsWith("/") ? folderPath : folderPath
+ "/";
- for (String n : names) {
- r.add(prefix + n);
+ if (recursive) {
+ Collection<File> files = FileUtils.listFiles(file, null, true);
+ for (File f : files) {
+ String path = f.getAbsolutePath();
+ String[] split = path.split(prefix);
+ Preconditions.checkArgument(split.length == 2);
+ r.add(prefix + split[1]);
+ }
+ } else {
+ for (String n : names) {
+ r.add(prefix + n);
+ }
}
return r;
}
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/HDFSResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/HDFSResourceStore.java
index bfc3a72..7a3a933 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/HDFSResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/HDFSResourceStore.java
@@ -88,7 +88,10 @@ public class HDFSResourceStore extends ResourceStore {
}
@Override
- protected NavigableSet<String> listResourcesImpl(String folderPath) throws
IOException {
+ protected NavigableSet<String> listResourcesImpl(String folderPath,
boolean recursive) throws IOException {
+ if (recursive) {
+ throw new IllegalArgumentException("Not support fullPath yet");
+ }
Path p = getRealHDFSPath(folderPath);
if (!fs.exists(p) || !fs.isDirectory(p)) {
return null;
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 629e12e..a6b3337 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -128,13 +128,21 @@ abstract public class ResourceStore {
*/
final public NavigableSet<String> listResources(String folderPath) throws
IOException {
String path = norm(folderPath);
- return listResourcesImpl(path);
+ return listResourcesImpl(path, false);
+ }
+
+ /**
+ * List resources and its full path, only support HBase now.
+ */
+ final public NavigableSet<String> listResourcesRecursively(String
folderPath) throws IOException {
+ String path = norm(folderPath);
+ return listResourcesImpl(path, true);
}
/**
* return null if given path is not a folder or not exists
*/
- abstract protected NavigableSet<String> listResourcesImpl(String
folderPath) throws IOException;
+ abstract protected NavigableSet<String> listResourcesImpl(String
folderPath, boolean recursive) throws IOException;
protected String createMetaStoreUUID() throws IOException {
return UUID.randomUUID().toString();
diff --git
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 1f83e3e..3762437 100644
---
a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -118,15 +118,23 @@ public class HBaseResourceStore extends ResourceStore {
}
@Override
- protected NavigableSet<String> listResourcesImpl(String folderPath) throws
IOException {
+ protected NavigableSet<String> listResourcesImpl(String folderPath,
boolean recursive) throws IOException {
final TreeSet<String> result = new TreeSet<>();
-
- visitFolder(folderPath, new KeyOnlyFilter(), new FolderVisitor() {
- @Override
- public void visit(String childPath, String fullPath, Result
hbaseResult) {
- result.add(childPath);
- }
- });
+ if (recursive) {
+ visitFolder(folderPath, new KeyOnlyFilter(), new FolderVisitor() {
+ @Override
+ public void visit(String childPath, String fullPath, Result
hbaseResult) {
+ result.add(fullPath);
+ }
+ });
+ } else {
+ visitFolder(folderPath, new KeyOnlyFilter(), new FolderVisitor() {
+ @Override
+ public void visit(String childPath, String fullPath, Result
hbaseResult) {
+ result.add(childPath);
+ }
+ });
+ }
// return null to indicate not a folder
return result.isEmpty() ? null : result;
}
--
To stop receiving notification emails like this one, please contact
liy...@apache.org.
