KYLIN-1504 Use NavigableSet<String> instead ArrayList<String> to keep rowkey original order instead of urgly using Collections.sort
Signed-off-by: Yang Li <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fb694a6a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fb694a6a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fb694a6a Branch: refs/heads/master Commit: fb694a6a6516b6e4976d8ede8e88fa40726aa7ce Parents: 5f8ab07 Author: Hao Chen <[email protected]> Authored: Thu Mar 17 17:54:08 2016 +0800 Committer: Yang Li <[email protected]> Committed: Sun Mar 20 15:16:49 2016 +0800 ---------------------------------------------------------------------- .../common/persistence/FileResourceStore.java | 8 ++-- .../kylin/common/persistence/ResourceStore.java | 7 ++-- .../kylin/common/persistence/ResourceTool.java | 7 ++-- .../persistence/LocalFileResourceStoreTest.java | 3 +- .../common/MetadataVersionRefresher.java | 5 +-- .../org/apache/kylin/cube/CubeManagerTest.java | 29 +++++++------- .../apache/kylin/dict/DictionaryManager.java | 42 ++++++++------------ .../kylin/dict/lookup/SnapshotManager.java | 5 ++- .../org/apache/kylin/job/dao/ExecutableDao.java | 31 +++++++-------- .../engine/mr/steps/MetadataCleanupJob.java | 15 +++---- .../storage/hbase/ITHBaseResourceStoreTest.java | 3 +- .../apache/kylin/rest/service/JobService.java | 1 + .../kylin/storage/hbase/HBaseResourceStore.java | 40 +++---------------- 13 files changed, 79 insertions(+), 117 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java index 49ff441..1ab659f 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.NavigableSet; +import java.util.TreeSet; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -49,12 +51,12 @@ public class FileResourceStore extends ResourceStore { } @Override - protected ArrayList<String> listResourcesImpl(String resPath) throws IOException { + protected NavigableSet<String> listResourcesImpl(String resPath) throws IOException { String[] names = file(resPath).list(); if (names == null) // not a directory return null; - ArrayList<String> r = new ArrayList<String>(names.length); + TreeSet<String> r = new TreeSet<>(); String prefix = resPath.endsWith("/") ? resPath : resPath + "/"; for (String n : names) { r.add(prefix + n); @@ -75,7 +77,7 @@ public class FileResourceStore extends ResourceStore { try { String commonPrefix = StringUtils.getCommonPrefix(rangeEnd, rangeStart); commonPrefix = commonPrefix.substring(0, commonPrefix.lastIndexOf("/") + 1); - final ArrayList<String> resources = listResourcesImpl(commonPrefix); + final NavigableSet<String> resources = listResourcesImpl(commonPrefix); for (String resource : resources) { if (resource.compareTo(rangeStart) >= 0 && resource.compareTo(rangeEnd) <= 0) { if (existsImpl(resource)) { http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index 88ee553..4e9e904 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -26,6 +26,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.io.IOUtils; @@ -116,12 +117,12 @@ abstract public class ResourceStore { * return a list of child resources & folders under given path, return null * if given path is not a folder */ - final public ArrayList<String> listResources(String resPath) throws IOException { + final public NavigableSet<String> listResources(String resPath) throws IOException { resPath = norm(resPath); return listResourcesImpl(resPath); } - abstract protected ArrayList<String> listResourcesImpl(String resPath) throws IOException; + abstract protected NavigableSet<String> listResourcesImpl(String resPath) throws IOException; /** * return true if a resource exists, return false in case of folder or @@ -278,7 +279,7 @@ abstract public class ResourceStore { } public void scanRecursively(String path, Visitor visitor) throws IOException { - ArrayList<String> children = listResources(path); + NavigableSet<String> children = listResources(path); if (children != null) { for (String child : children) scanRecursively(child, visitor); http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java index b11fe1a..56f855c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java @@ -24,6 +24,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; +import java.util.NavigableSet; import org.apache.commons.io.IOUtils; import org.apache.kylin.common.KylinConfig; @@ -99,7 +100,7 @@ public class ResourceTool { public static void list(KylinConfig config, String path) throws IOException { ResourceStore store = ResourceStore.getStore(config); - ArrayList<String> result = store.listResources(path); + NavigableSet<String> result = store.listResources(path); System.out.println("" + result); } @@ -119,7 +120,7 @@ public class ResourceTool { } public static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException { - ArrayList<String> children = src.listResources(path); + NavigableSet<String> children = src.listResources(path); // case of resource (not a folder) if (children == null) { @@ -169,7 +170,7 @@ public class ResourceTool { } private static void resetR(ResourceStore store, String path) throws IOException { - ArrayList<String> children = store.listResources(path); + NavigableSet<String> children = store.listResources(path); if (children == null) { // path is a resource (not a folder) if (matchFilter(path)) { store.deleteResource(path); http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java index a62a578..7ba5329 100644 --- a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java @@ -27,6 +27,7 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.NavigableSet; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; @@ -91,7 +92,7 @@ public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase { } // list - ArrayList<String> list; + NavigableSet<String> list; list = store.listResources(dir1); assertTrue(list.contains(path1)); http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java index 50833e3..02f084c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java @@ -21,10 +21,9 @@ package org.apache.kylin.cube.upgrade.common; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; import java.util.List; +import java.util.NavigableSet; -import org.apache.kylin.common.KylinVersion; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.metadata.MetadataConstants; import org.slf4j.Logger; @@ -73,7 +72,7 @@ public class MetadataVersionRefresher { } public static void collectFiles(ResourceStore src, String path, List<String> ret) throws IOException { - ArrayList<String> children = src.listResources(path); + NavigableSet<String> children = src.listResources(path); // case of resource (not a folder) if (children == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 15c6deb..18a8150 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -18,17 +18,6 @@ package org.apache.kylin.cube; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.TimeZone; - -import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.JsonUtil; @@ -41,6 +30,12 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.Iterator; +import java.util.List; +import java.util.NavigableSet; + +import static org.junit.Assert.*; + /** * @author yangli9 */ @@ -130,12 +125,16 @@ public class CubeManagerTest extends LocalFileMetadataTestCase { @Test public void testGetAllCubes() throws Exception { final ResourceStore store = ResourceStore.getStore(getTestConfig()); - final ArrayList<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT); + final NavigableSet<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT); + final Iterator<String> iterator = cubePath.iterator(); + final String firstPath = iterator.next(); + final String secondPath = iterator.next(); + final String lastPath = cubePath.last(); assertTrue(cubePath.size() > 1); - Collections.sort(cubePath); - final List<CubeInstance> cubes = store.getAllResources(cubePath.get(0), cubePath.get(cubePath.size() - 1), CubeInstance.class, CubeManager.CUBE_SERIALIZER); + + final List<CubeInstance> cubes = store.getAllResources(firstPath, lastPath, CubeInstance.class, CubeManager.CUBE_SERIALIZER); assertEquals(cubePath.size(), cubes.size()); - assertEquals(cubePath.size() - 1, store.getAllResources(cubePath.get(1), cubePath.get(cubePath.size() - 1), CubeInstance.class, CubeManager.CUBE_SERIALIZER).size()); + assertEquals(cubePath.size() - 1, store.getAllResources(secondPath, lastPath, CubeInstance.class, CubeManager.CUBE_SERIALIZER).size()); } @Test http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index b7c95da..b3ed2ea 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -18,17 +18,8 @@ package org.apache.kylin.dict; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - +import com.google.common.cache.*; +import com.google.common.collect.Lists; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.dimension.Dictionary; @@ -43,12 +34,15 @@ import org.apache.kylin.source.SourceFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; -import com.google.common.collect.Lists; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.NavigableSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; public class DictionaryManager { @@ -175,7 +169,7 @@ public class DictionaryManager { private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList<String> existings = store.listResources(dictInfo.getResourceDir()); + NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null) return null; @@ -318,13 +312,12 @@ public class DictionaryManager { private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList<String> existings = store.listResources(dictInfo.getResourceDir()); + NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir()); if (existings == null || existings.isEmpty()) { return null; } - Collections.sort(existings); - final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(existings.get(0), existings.get(existings.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); + final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(existings.first(), existings.last(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); TableSignature input = dictInfo.getInput(); @@ -338,13 +331,12 @@ public class DictionaryManager { private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException { ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList<String> dictInfos = store.listResources(dictInfo.getResourceDir()); + NavigableSet<String> dictInfos = store.listResources(dictInfo.getResourceDir()); if (dictInfos == null || dictInfos.isEmpty()) { return null; } - Collections.sort(dictInfos); - final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.get(0), dictInfos.get(dictInfos.size() - 1), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); + final List<DictionaryInfo> allResources = MetadataManager.getInstance(config).getStore().getAllResources(dictInfos.first(), dictInfos.last(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER); DictionaryInfo largestDict = null; for (DictionaryInfo dictionaryInfo : allResources) { @@ -373,7 +365,7 @@ public class DictionaryManager { info.setSourceColumn(srcCol); ResourceStore store = MetadataManager.getInstance(config).getStore(); - ArrayList<String> existings = store.listResources(info.getResourceDir()); + NavigableSet<String> existings = store.listResources(info.getResourceDir()); if (existings == null) return; http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java index 1123c2d..b9c17dd 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java @@ -20,6 +20,7 @@ package org.apache.kylin.dict.lookup; import java.io.IOException; import java.util.ArrayList; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.kylin.common.KylinConfig; @@ -139,7 +140,7 @@ public class SnapshotManager { private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); - ArrayList<String> existings = store.listResources(resourceDir); + NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; @@ -157,7 +158,7 @@ public class SnapshotManager { private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); - ArrayList<String> existings = store.listResources(resourceDir); + NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java index d063ee7..493555d 100644 --- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java +++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.kylin.common.KylinConfig; @@ -95,13 +96,12 @@ public class ExecutableDao { public List<ExecutableOutputPO> getJobOutputs() throws PersistentException { try { - ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); + NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); if (resources == null || resources.isEmpty()) { return Collections.emptyList(); } - Collections.sort(resources); - String rangeStart = resources.get(0); - String rangeEnd = resources.get(resources.size() - 1); + String rangeStart = resources.first(); + String rangeEnd = resources.last(); return store.getAllResources(rangeStart, rangeEnd, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -111,13 +111,12 @@ public class ExecutableDao { public List<ExecutableOutputPO> getJobOutputs(long timeStartInMillis, long timeEndInMillis) throws PersistentException { try { - ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); + NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); if (resources == null || resources.isEmpty()) { return Collections.emptyList(); } - Collections.sort(resources); - String rangeStart = resources.get(0); - String rangeEnd = resources.get(resources.size() - 1); + String rangeStart = resources.first(); + String rangeEnd = resources.last(); return store.getAllResources(rangeStart, rangeEnd, timeStartInMillis, timeEndInMillis, ExecutableOutputPO.class, JOB_OUTPUT_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -127,13 +126,12 @@ public class ExecutableDao { public List<ExecutablePO> getJobs() throws PersistentException { try { - final List<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); + final NavigableSet<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); if (jobIds == null || jobIds.isEmpty()) { return Collections.emptyList(); } - Collections.sort(jobIds); - String rangeStart = jobIds.get(0); - String rangeEnd = jobIds.get(jobIds.size() - 1); + String rangeStart = jobIds.first(); + String rangeEnd = jobIds.last(); return store.getAllResources(rangeStart, rangeEnd, ExecutablePO.class, JOB_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -143,13 +141,12 @@ public class ExecutableDao { public List<ExecutablePO> getJobs(long timeStartInMillis, long timeEndInMillis) throws PersistentException { try { - final List<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); + final NavigableSet<String> jobIds = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); if (jobIds == null || jobIds.isEmpty()) { return Collections.emptyList(); } - Collections.sort(jobIds); - String rangeStart = jobIds.get(0); - String rangeEnd = jobIds.get(jobIds.size() - 1); + String rangeStart = jobIds.first(); + String rangeEnd = jobIds.last(); return store.getAllResources(rangeStart, rangeEnd, timeStartInMillis, timeEndInMillis, ExecutablePO.class, JOB_SERIALIZER); } catch (IOException e) { logger.error("error get all Jobs:", e); @@ -159,7 +156,7 @@ public class ExecutableDao { public List<String> getJobIds() throws PersistentException { try { - ArrayList<String> resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); + NavigableSet<String> resources = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); if (resources == null) { return Collections.emptyList(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java index 99f8a9f..b88a170 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MetadataCleanupJob.java @@ -18,10 +18,7 @@ package org.apache.kylin.engine.mr.steps; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; +import java.util.*; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; @@ -114,11 +111,11 @@ public class MetadataCleanupJob extends AbstractHadoopJob { // two level resources, snapshot tables and cube statistics for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) { - ArrayList<String> snapshotTables = getStore().listResources(resourceRoot); + NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot); if (snapshotTables != null) { for (String snapshotTable : snapshotTables) { - ArrayList<String> snapshotNames = getStore().listResources(snapshotTable); + NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable); if (snapshotNames != null) for (String snapshot : snapshotNames) { if (!activeResourceList.contains(snapshot)) { @@ -131,13 +128,13 @@ public class MetadataCleanupJob extends AbstractHadoopJob { } // three level resources, only dictionaries - ArrayList<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT); + NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT); for (String table : dictTables) { - ArrayList<String> tableColNames = getStore().listResources(table); + NavigableSet<String> tableColNames = getStore().listResources(table); if (tableColNames != null) for (String tableCol : tableColNames) { - ArrayList<String> dictionaries = getStore().listResources(tableCol); + NavigableSet<String> dictionaries = getStore().listResources(tableCol); if (dictionaries != null) for (String dict : dictionaries) if (!activeResourceList.contains(dict)) { http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java index c560bb8..dccdca9 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITHBaseResourceStoreTest.java @@ -27,6 +27,7 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.NavigableSet; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -133,7 +134,7 @@ public class ITHBaseResourceStoreTest extends HBaseMetadataTestCase { } // list - ArrayList<String> list; + NavigableSet<String> list; list = store.listResources(dir1); assertTrue(list.contains(path1)); http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/server/src/main/java/org/apache/kylin/rest/service/JobService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/JobService.java b/server/src/main/java/org/apache/kylin/rest/service/JobService.java index cd8eef9..d0c29a4 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/JobService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/JobService.java @@ -90,6 +90,7 @@ public class JobService extends BasicService { return jobs.subList(offset, offset + limit); } + public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) { Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); http://git-wip-us.apache.org/repos/asf/kylin/blob/fb694a6a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index 6d77240..bbb5e21 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -22,11 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; @@ -41,10 +37,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.filter.CompareFilter; -import org.apache.hadoop.hbase.filter.FilterList; -import org.apache.hadoop.hbase.filter.KeyOnlyFilter; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.*; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; @@ -67,22 +60,9 @@ public class HBaseResourceStore extends ResourceStore { private static final String COLUMN_TS = "t"; private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS); - private static final Map<String, String> TABLE_SUFFIX_MAP = new LinkedHashMap<String, String>(); - - static { - TABLE_SUFFIX_MAP.put(CUBE_RESOURCE_ROOT + "/", "_cube"); - TABLE_SUFFIX_MAP.put(DICT_RESOURCE_ROOT + "/", "_dict"); - TABLE_SUFFIX_MAP.put("/invertedindex/", "_invertedindex"); - TABLE_SUFFIX_MAP.put(PROJECT_RESOURCE_ROOT + "/", "_proj"); - TABLE_SUFFIX_MAP.put(SNAPSHOT_RESOURCE_ROOT + "/", "_table_snapshot"); - TABLE_SUFFIX_MAP.put("", ""); // DEFAULT CASE - } - final String tableNameBase; final String hbaseUrl; - // final Map<String, String> tableNameMap; // path prefix ==> HBase table name - private HConnection getConnection() throws IOException { return HBaseConnection.get(hbaseUrl); } @@ -97,15 +77,6 @@ public class HBaseResourceStore extends ResourceStore { hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); createHTableIfNeeded(getAllInOneTableName()); - - // tableNameMap = new LinkedHashMap<String, String>(); - // for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) { - // String pathPrefix = entry.getKey(); - // String tableName = tableNameBase + entry.getValue(); - // tableNameMap.put(pathPrefix, tableName); - // createHTableIfNeeded(tableName); - // } - } private void createHTableIfNeeded(String tableName) throws IOException { @@ -117,14 +88,14 @@ public class HBaseResourceStore extends ResourceStore { } @Override - protected ArrayList<String> listResourcesImpl(String resPath) throws IOException { + protected NavigableSet<String> listResourcesImpl(String resPath) throws IOException { assert resPath.startsWith("/"); String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/"; byte[] startRow = Bytes.toBytes(lookForPrefix); byte[] endRow = Bytes.toBytes(lookForPrefix); endRow[endRow.length - 1]++; - ArrayList<String> result = new ArrayList<String>(); + TreeSet<String> result = new TreeSet<>(); HTableInterface table = getConnection().getTable(getAllInOneTableName()); Scan scan = new Scan(startRow, endRow); @@ -136,8 +107,7 @@ public class HBaseResourceStore extends ResourceStore { assert path.startsWith(lookForPrefix); int cut = path.indexOf('/', lookForPrefix.length()); String child = cut < 0 ? path : path.substring(0, cut); - if (result.contains(child) == false) - result.add(child); + result.add(child); } } finally { IOUtils.closeQuietly(table);
