KYLIN-1127 Drop getProject()/setProject() in IRealization. Pass test on CacheService.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3ad57284 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3ad57284 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3ad57284 Branch: refs/heads/2.x-staging Commit: 3ad5728497f695b28dbea9f232d80d1e06e51d0f Parents: 86a2b6e Author: Li, Yang <yang...@ebay.com> Authored: Thu Nov 12 14:45:37 2015 +0800 Committer: Li, Yang <yang...@ebay.com> Committed: Thu Nov 12 14:45:37 2015 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeDescManager.java | 2 +- .../org/apache/kylin/cube/CubeInstance.java | 13 +-------- .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../kylin/metadata/project/ProjectManager.java | 20 -------------- .../metadata/realization/IRealization.java | 4 --- .../kylin/storage/hybrid/HybridInstance.java | 11 -------- .../apache/kylin/invertedindex/IIInstance.java | 10 ------- .../apache/kylin/invertedindex/IIManager.java | 3 +- .../apache/kylin/rest/service/CacheService.java | 29 ++++++++++++++++---- .../apache/kylin/rest/service/CubeService.java | 1 - 10 files changed, 28 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java index c50836c..53cd00f 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java @@ -229,7 +229,7 @@ public class CubeDescManager { * @return * @throws IOException */ - public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException { + public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException { // Validate CubeDesc if (desc.getUuid() == null || desc.getName() == null) { throw new IllegalArgumentException(); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index f8e37f3..2208136 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -52,7 +52,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, private static final int COST_WEIGHT_DIMENSION = 10; private static final int COST_WEIGHT_INNER_JOIN = 100; - public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) { + public static CubeInstance create(String cubeName, CubeDesc cubeDesc) { CubeInstance cubeInstance = new CubeInstance(); cubeInstance.setConfig(cubeDesc.getConfig()); @@ -62,7 +62,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, cubeInstance.setSegments(new ArrayList<CubeSegment>()); cubeInstance.setStatus(RealizationStatusEnum.DISABLED); cubeInstance.updateRandomUuid(); - cubeInstance.setProjectName(projectName); return cubeInstance; } @@ -90,8 +89,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, @JsonProperty("create_time_utc") private long createTimeUTC; - private String projectName; - // default constructor for jackson public CubeInstance() { } @@ -378,14 +375,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return Lists.newArrayList(getDescriptor().listAllColumns()); } - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - @Override public long getDateRangeStart() { List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 89873d2..4592b15 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -248,7 +248,7 @@ public class CubeManager implements IRealizationProvider { logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'"); // save cube resource - CubeInstance cube = CubeInstance.create(cubeName, projectName, desc); + CubeInstance cube = CubeInstance.create(cubeName, desc); cube.setOwner(owner); updateCubeWithRetry(new CubeUpdate(cube), 0); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java index 8304128..37241c3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java @@ -20,7 +20,6 @@ package org.apache.kylin.metadata.project; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -36,7 +35,6 @@ import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.realization.IRealization; -import org.apache.kylin.metadata.realization.RealizationRegistry; import org.apache.kylin.metadata.realization.RealizationType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +103,6 @@ public class ProjectManager { for (String path : paths) { reloadProjectLocalAt(path); } - wireProjectAndRealizations(projectMap.values()); logger.debug("Loaded " + projectMap.size() + " Project(s)"); } @@ -129,23 +126,6 @@ public class ProjectManager { return projectInstance; } - private void wireProjectAndRealizations(Collection<ProjectInstance> projectInstances) { - if (projectInstances.isEmpty()) - return; - - RealizationRegistry registry = RealizationRegistry.getInstance(config); - for (ProjectInstance projectInstance : projectInstances) { - for (RealizationEntry realization : projectInstance.getRealizationEntries()) { - IRealization rel = registry.getRealization(realization.getType(), realization.getRealization()); - if (rel != null) { - rel.setProjectName(projectInstance.getName()); - } else { - logger.warn("Realization '" + realization + "' defined under project '" + projectInstance + "' is not found"); - } - } - } - } - public List<ProjectInstance> listAllProjects() { return new ArrayList<ProjectInstance>(projectMap.values()); } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index 8c9258a..ccfa956 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -65,10 +65,6 @@ public interface IRealization extends IStorageAware { public String getCanonicalName(); - public String getProjectName(); - - public void setProjectName(String prjName); - public long getDateRangeStart(); public long getDateRangeEnd(); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 4049fcd..2a6e5d3 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -66,7 +66,6 @@ public class HybridInstance extends RootPersistentEntity implements IRealization private long dateRangeStart; private long dateRangeEnd; private boolean isReady = false; - private String projectName; private boolean initiated = false; private final static Logger logger = LoggerFactory.getLogger(HybridInstance.class); @@ -226,16 +225,6 @@ public class HybridInstance extends RootPersistentEntity implements IRealization return getType() + "[name=" + name + "]"; } - @Override - public String getProjectName() { - return projectName; - } - - @Override - public void setProjectName(String prjName) { - projectName = prjName; - } - public KylinConfig getConfig() { return config; } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java index 20f289c..0ba07ee 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java @@ -81,8 +81,6 @@ public class IIInstance extends RootPersistentEntity implements IRealization, IB @JsonProperty("create_time_utc") private long createTimeUTC; - private String projectName; - private static final int COST_WEIGHT_LOOKUP_TABLE = 1; private static final int COST_WEIGHT_INNER_JOIN = 2; @@ -314,14 +312,6 @@ public class IIInstance extends RootPersistentEntity implements IRealization, IB return getDescriptor().getMeasures(); } - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - @Override public long getDateRangeStart() { List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY); http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java index b6dfdf1..06aeb04 100644 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java +++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java @@ -134,7 +134,8 @@ public class IIManager implements IRealizationProvider { this.updateII(ii); - String projectName = (null == ii.getProjectName()) ? ProjectInstance.DEFAULT_PROJECT_NAME : ii.getProjectName(); + // FIXME need to pass in project name + String projectName = ProjectInstance.DEFAULT_PROJECT_NAME; ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner()); return ii; } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/server/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java index 8371907..0150b2e 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -118,7 +118,7 @@ public class CacheService extends BasicService { } } - protected void removeOLAPDataSource(String project) { + private static void removeOLAPDataSource(String project) { logger.info("removeOLAPDataSource is called for project " + project); if (StringUtils.isEmpty(project)) throw new IllegalArgumentException("removeOLAPDataSource: project name not given"); @@ -186,8 +186,7 @@ public class CacheService extends BasicService { getCubeDescManager().reloadCubeDescLocal(cacheKey); break; case PROJECT: - getProjectManager().reloadProjectLocal(cacheKey); - removeOLAPDataSource(cacheKey); + reloadProjectCache(cacheKey); break; case INVERTED_INDEX: //II update does not need to update storage cache because it is dynamic already @@ -234,7 +233,7 @@ public class CacheService extends BasicService { private void rebuildCubeCache(String cubeName) { CubeInstance cube = getCubeManager().reloadCubeLocal(cubeName); getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cubeName); - getProjectManager().clearL2Cache(); + reloadProjectCache(getProjectManager().findProjects(RealizationType.CUBE, cubeName)); //clean query related cache first if (cube != null) { cleanDataCache(cube.getUuid()); @@ -278,11 +277,29 @@ public class CacheService extends BasicService { if (cube == null) { cube = getCubeManager().getCube(cubeName); } - + getCubeManager().removeCubeLocal(cubeName); - + getHybridManager().reloadHybridInstanceByChild(RealizationType.CUBE, cubeName); + reloadProjectCache(getProjectManager().findProjects(RealizationType.CUBE, cubeName)); + if (cube != null) { cleanDataCache(cube.getUuid()); } } + + private void reloadProjectCache(List<ProjectInstance> projects) { + for (ProjectInstance prj : projects) { + reloadProjectCache(prj.getName()); + } + } + + private void reloadProjectCache(String projectName) { + try { + getProjectManager().reloadProjectLocal(projectName); + } catch (IOException ex) { + logger.warn("Failed to reset project cache", ex); + } + removeOLAPDataSource(projectName); + } + } http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3ad57284/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index bcbd9f8..e9d446f 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -397,7 +397,6 @@ public class CubeService extends BasicService { } try { - CubeUpdate cubeBuilder = new CubeUpdate(cube); cubeBuilder.setStatus(RealizationStatusEnum.READY); return getCubeManager().updateCube(cubeBuilder);
