KYLIN-1277 Upgrade tool to put old-version cube and new-version cube into a hybrid model
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/88cb7b92 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/88cb7b92 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/88cb7b92 Branch: refs/heads/2.0-rc Commit: 88cb7b92fd082e33f5ddb3e1b1717e692d5a0f35 Parents: 34466c0 Author: lidongsjtu <don...@ebay.com> Authored: Mon Jan 4 10:09:05 2016 +0800 Committer: lidongsjtu <don...@ebay.com> Committed: Mon Jan 4 23:03:16 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeInstance.java | 15 ++ .../java/org/apache/kylin/cube/CubeManager.java | 15 ++ .../org/apache/kylin/cube/model/CubeDesc.java | 21 ++ .../cube/upgrade/v2/CubeMetadataUpgradeV2.java | 2 +- .../kylin/metadata/model/DataModelDesc.java | 31 ++- .../kylin/metadata/model/PartitionDesc.java | 9 + .../metadata/project/RealizationEntry.java | 7 + .../storage/hybrid/ExtendCubeToHybridTool.java | 196 +++++++++++++++++++ .../kylin/storage/hybrid/HybridInstance.java | 19 ++ 9 files changed, 306 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 41c1f41..6139a2e 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -427,4 +427,19 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return getDescriptor().getEngineType(); } + public static CubeInstance getCopyOf(CubeInstance cubeInstance) { + CubeInstance newCube = new CubeInstance(); + newCube.setName(cubeInstance.getName()); + newCube.setSegments(cubeInstance.getSegments()); + newCube.setDescName(cubeInstance.getDescName()); + newCube.setConfig(cubeInstance.getConfig()); + newCube.setStatus(cubeInstance.getStatus()); + newCube.setOwner(cubeInstance.getOwner()); + newCube.setVersion(cubeInstance.getVersion()); + newCube.setCost(cubeInstance.getCost()); + newCube.setCreateTimeUTC(System.currentTimeMillis()); + newCube.updateRandomUuid(); + return newCube; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index 50dc166..04a447c 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -254,6 +254,21 @@ public class CubeManager implements IRealizationProvider { return cube; } + public CubeInstance createCube(CubeInstance cube, String projectName, String owner) throws IOException { + logger.info("Creating cube '" + projectName + "-->" + cube.getName() + "' from instance object. '"); + + // save cube resource + cube.setOwner(owner); + + updateCubeWithRetry(new CubeUpdate(cube), 0); + ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cube.getName(), projectName, owner); + + if (listener != null) + listener.afterCubeCreate(cube); + + return cube; + } + public CubeInstance updateCube(CubeUpdate update) throws IOException { CubeInstance cube = updateCubeWithRetry(update, 0); http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 57179ae..3c2527a 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -856,4 +856,25 @@ public class CubeDesc extends RootPersistentEntity { return result; } + public static CubeDesc getCopyOf(CubeDesc cubeDesc) { + CubeDesc newCubeDesc = new CubeDesc(); + newCubeDesc.setName(cubeDesc.getName()); + newCubeDesc.setModelName(cubeDesc.getModelName()); + newCubeDesc.setDescription(cubeDesc.getDescription()); + newCubeDesc.setNullStrings(cubeDesc.getNullStrings()); + newCubeDesc.setDimensions(cubeDesc.getDimensions()); + newCubeDesc.setMeasures(cubeDesc.getMeasures()); + newCubeDesc.setRowkey(cubeDesc.getRowkey()); + newCubeDesc.setHBaseMapping(cubeDesc.getHBaseMapping()); + newCubeDesc.setSignature(cubeDesc.getSignature()); + newCubeDesc.setNotifyList(cubeDesc.getNotifyList()); + newCubeDesc.setStatusNeedNotify(cubeDesc.getStatusNeedNotify()); + newCubeDesc.setAutoMergeTimeRanges(cubeDesc.getAutoMergeTimeRanges()); + newCubeDesc.setRetentionRange(cubeDesc.getRetentionRange()); + newCubeDesc.setEngineType(cubeDesc.getEngineType()); + newCubeDesc.setStorageType(cubeDesc.getStorageType()); + newCubeDesc.setConfig(cubeDesc.getConfig()); + newCubeDesc.updateRandomUuid(); + return newCubeDesc; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java index f11d95c..54e1ffb 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java @@ -109,8 +109,8 @@ public class CubeMetadataUpgradeV2 { List<CubeDesc> cubeDescs = cubeDescManager.listAllDesc(); for (CubeDesc cubeDesc : cubeDescs) { if (ArrayUtils.isEmpty(models) || ArrayUtils.contains(models, cubeDesc.getModelName())) { - upgradeCubeDescSignature(cubeDesc); upgradeDataModelDesc(cubeDesc); + upgradeCubeDescSignature(cubeDesc); } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index db53bc5..1fb96b7 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -322,12 +322,27 @@ public class DataModelDesc extends RootPersistentEntity { public String[] getMetrics() { return metrics; } - - public void setDimensions(List<DimensionDesc> dimensions) { - this.dimensions = dimensions; - } - - public void setMetrics(String[] metrics) { - this.metrics = metrics; - } + + public void setDimensions(List<DimensionDesc> dimensions) { + this.dimensions = dimensions; + } + + public void setMetrics(String[] metrics) { + this.metrics = metrics; + } + + public static DataModelDesc getCopyOf(DataModelDesc dataModelDesc) { + DataModelDesc newDataModelDesc = new DataModelDesc(); + newDataModelDesc.setName(dataModelDesc.getName()); + newDataModelDesc.setCapacity(dataModelDesc.getCapacity()); + newDataModelDesc.setDescription(dataModelDesc.getDescription()); + newDataModelDesc.setDimensions(dataModelDesc.getDimensions()); + newDataModelDesc.setFilterCondition(dataModelDesc.getFilterCondition()); + newDataModelDesc.setFactTable(dataModelDesc.getFactTable()); + newDataModelDesc.setLookups(dataModelDesc.getLookups()); + newDataModelDesc.setMetrics(dataModelDesc.getMetrics()); + newDataModelDesc.setPartitionDesc(PartitionDesc.getCopyOf(dataModelDesc.getPartitionDesc())); + newDataModelDesc.updateRandomUuid(); + return newDataModelDesc; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index 5f3468c..6194df3 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -196,4 +196,13 @@ public class PartitionDesc { return builder.toString(); } } + + public static PartitionDesc getCopyOf(PartitionDesc partitionDesc) { + PartitionDesc newPartDesc = new PartitionDesc(); + newPartDesc.setCubePartitionType(partitionDesc.getCubePartitionType()); + newPartDesc.setPartitionDateColumn(partitionDesc.getPartitionDateColumn()); + newPartDesc.setPartitionDateFormat(partitionDesc.getPartitionDateFormat()); + newPartDesc.setPartitionDateStart(partitionDesc.getPartitionDateStart()); + return newPartDesc; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java index 5cf1e8d..84e5fd5 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/RealizationEntry.java @@ -78,4 +78,11 @@ public class RealizationEntry { public String toString() { return "" + type.name() + "." + realization; } + + public static RealizationEntry create(RealizationType type, String realization) { + RealizationEntry entry = new RealizationEntry(); + entry.setRealization(realization); + entry.setType(type); + return entry; + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-storage/src/main/java/org/apache/kylin/storage/hybrid/ExtendCubeToHybridTool.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/ExtendCubeToHybridTool.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/ExtendCubeToHybridTool.java new file mode 100644 index 0000000..f2b67b9 --- /dev/null +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/ExtendCubeToHybridTool.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.storage.hybrid; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.util.DateFormat; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.IEngineAware; +import org.apache.kylin.metadata.model.IStorageAware; +import org.apache.kylin.metadata.project.ProjectManager; +import org.apache.kylin.metadata.project.RealizationEntry; +import org.apache.kylin.metadata.realization.RealizationType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + * Created by dongli on 12/29/15. + */ +public class ExtendCubeToHybridTool { + private static final Logger logger = LoggerFactory.getLogger(ExtendCubeToHybridTool.class); + + private KylinConfig kylinConfig; + private CubeManager cubeManager; + private CubeDescManager cubeDescManager; + private MetadataManager metadataManager; + private ResourceStore store; + + public ExtendCubeToHybridTool() { + this.kylinConfig = KylinConfig.getInstanceFromEnv(); + this.store = ResourceStore.getStore(kylinConfig); + this.cubeManager = CubeManager.getInstance(kylinConfig); + this.cubeDescManager = CubeDescManager.getInstance(kylinConfig); + this.metadataManager = MetadataManager.getInstance(kylinConfig); + } + + public static void main(String[] args) throws Exception { + if (args.length != 2 && args.length != 3) { + System.out.println("Usage: ExtendCubeToHybridTool project cube [partition_date]"); + return; + } + + ExtendCubeToHybridTool tool = new ExtendCubeToHybridTool(); + + String projectName = args[0]; + String cubeName = args[1]; + String partitionDate = args.length == 3 ? args[2] : null; + + try { + tool.createFromCube(projectName, cubeName, partitionDate); + tool.verify(); + logger.info("Job Finished."); + } catch (Exception e) { + e.printStackTrace(); + logger.error("Job Aborted.", e.getMessage()); + } + } + + private boolean validateCubeInstance(CubeInstance cubeInstance) { + if (cubeInstance == null) { + logger.error("This cube does not exist."); + return false; + } + if (cubeInstance.getSegments().isEmpty()) { + logger.error("No segments in this cube, no need to extend."); + return false; + } + return true; + } + + public void createFromCube(String projectName, String cubeName, String partitionDateStr) throws IOException { + logger.info("Create hybrid for cube[" + cubeName + "], project[" + projectName + "], partition_date[" + partitionDateStr + "]."); + + CubeInstance cubeInstance = cubeManager.getCube(cubeName); + if (!validateCubeInstance(cubeInstance)) { + return; + } + + CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cubeInstance.getDescName()); + DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName()); + String owner = cubeInstance.getOwner(); + String dateFormat = dataModelDesc.getPartitionDesc().getPartitionDateFormat(); + long partitionDate = partitionDateStr != null ? DateFormat.stringToMillis(partitionDateStr, dateFormat) : 0; + + // get new name for old cube and cube_desc + String newCubeDescName = rename(cubeDesc.getName()); + String newCubeInstanceName = rename(cubeInstance.getName()); + while (cubeDescManager.getCubeDesc(newCubeDescName) != null) + newCubeDescName = rename(newCubeDescName); + while (cubeManager.getCube(newCubeInstanceName) != null) + newCubeInstanceName = rename(newCubeInstanceName); + + // create new cube_instance for old segments + CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance); + newCubeInstance.setName(newCubeInstanceName); + newCubeInstance.setDescName(newCubeDescName); + newCubeInstance.updateRandomUuid(); + Iterator<CubeSegment> segmentIterator = newCubeInstance.getSegments().iterator(); + CubeSegment currentSeg = null; + while (segmentIterator.hasNext()) { + currentSeg = segmentIterator.next(); + if (partitionDateStr != null && (currentSeg.getDateRangeStart() >= partitionDate || currentSeg.getDateRangeEnd() > partitionDate)) { + segmentIterator.remove(); + logger.info("CubeSegment[" + currentSeg + "] was removed."); + } + } + if (partitionDateStr != null && partitionDate != currentSeg.getDateRangeEnd()) { + logger.error("PartitionDate must be end date of one segment."); + return; + } + if (currentSeg != null && partitionDateStr == null) + partitionDate = currentSeg.getDateRangeEnd(); + + cubeManager.createCube(newCubeInstance, projectName, owner); + logger.info("CubeInstance was saved at: " + newCubeInstance.getResourcePath()); + + // create new cube for old segments + CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc); + newCubeDesc.setName(newCubeDescName); + newCubeDesc.updateRandomUuid(); + newCubeDesc.init(kylinConfig, metadataManager.getAllTablesMap()); + newCubeDesc.setPartitionDateEnd(partitionDate); + newCubeDesc.calculateSignature(); + cubeDescManager.createCubeDesc(newCubeDesc); + logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath()); + + // update old cube_desc to new-version metadata + cubeDesc.setPartitionDateStart(partitionDate); + cubeDesc.setEngineType(IEngineAware.ID_MR_V2); + cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE); + cubeDesc.calculateSignature(); + cubeDescManager.updateCubeDesc(cubeDesc); + logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath()); + + // clear segments for old cube + cubeInstance.setSegments(new ArrayList<CubeSegment>()); + store.putResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER); + logger.info("CubeInstance was saved at: " + cubeInstance.getResourcePath()); + + // create hybrid model for these two cubes + List<RealizationEntry> realizationEntries = Lists.newArrayListWithCapacity(2); + realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cubeInstance.getName())); + realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, newCubeInstance.getName())); + HybridInstance hybridInstance = HybridInstance.create(kylinConfig, cubeInstance.getName(), realizationEntries); + store.putResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER); + ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner); + logger.info("HybridInstance was saved at: " + hybridInstance.getResourcePath()); + } + + private void verify() { + CubeDescManager.clearCache(); + CubeDescManager.getInstance(kylinConfig); + + CubeManager.clearCache(); + CubeManager.getInstance(kylinConfig); + + ProjectManager.clearCache(); + ProjectManager.getInstance(kylinConfig); + + HybridManager.clearCache(); + HybridManager.getInstance(kylinConfig); + } + + private String rename(String origName) { + return origName + "_1"; + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/88cb7b92/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 2a6e5d3..ff8609b 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -53,6 +53,10 @@ public class HybridInstance extends RootPersistentEntity implements IRealization @JsonProperty("name") private String name; + public void setRealizationEntries(List<RealizationEntry> realizationEntries) { + this.realizationEntries = realizationEntries; + } + @JsonProperty("realizations") private List<RealizationEntry> realizationEntries; @@ -74,6 +78,17 @@ public class HybridInstance extends RootPersistentEntity implements IRealization return realizationEntries; } + public static HybridInstance create(KylinConfig config, String name, List<RealizationEntry> realizationEntries) { + HybridInstance hybridInstance = new HybridInstance(); + + hybridInstance.setConfig(config); + hybridInstance.setName(name); + hybridInstance.setRealizationEntries(realizationEntries); + hybridInstance.updateRandomUuid(); + + return hybridInstance; + } + private void init() { if (initiated == true) return; @@ -259,6 +274,10 @@ public class HybridInstance extends RootPersistentEntity implements IRealization return realizations; } + public String getResourcePath() { + return concatResourcePath(name); + } + public static String concatResourcePath(String hybridName) { return ResourceStore.HYBRID_RESOURCE_ROOT + "/" + hybridName + ".json"; }
