KYLIN-1495 add the concept of signature compatible
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/98ac225f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/98ac225f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/98ac225f Branch: refs/heads/master Commit: 98ac225f20716593769b6c6b8d14c6b9f70bd77c Parents: bb01b18 Author: Hongbin Ma <[email protected]> Authored: Wed Mar 16 16:10:48 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Wed Mar 16 23:42:24 2016 +0800 ---------------------------------------------------------------------- build/bin/upgrade_metadata_v_1_5_0.sh | 46 --- build/bin/upgrade_metadata_v_1_5_1.sh | 41 +++ .../org/apache/kylin/common/KylinVersion.java | 69 ++++- .../org/apache/kylin/common/util/BasicTest.java | 1 + .../org/apache/kylin/cube/model/CubeDesc.java | 8 +- .../upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java | 284 ------------------- .../V1_5_0/CubeMetadataUpgrade_v_1_5_0.java | 150 ---------- .../upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java | 284 +++++++++++++++++++ .../V1_5_1/CubeMetadataUpgrade_v_1_5_1.java | 150 ++++++++++ .../common/MetadataVersionRefresher.java | 7 +- .../entry/CubeMetadataUpgradeEntry_v_1_5_0.java | 46 --- .../entry/CubeMetadataUpgradeEntry_v_1_5_1.java | 46 +++ .../v1_4_0/CubeMetadataUpgrade_v_1_4_0.java | 2 +- 13 files changed, 601 insertions(+), 533 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/build/bin/upgrade_metadata_v_1_5_0.sh ---------------------------------------------------------------------- diff --git a/build/bin/upgrade_metadata_v_1_5_0.sh b/build/bin/upgrade_metadata_v_1_5_0.sh deleted file mode 100644 index b60204b..0000000 --- a/build/bin/upgrade_metadata_v_1_5_0.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# the script will upgrade Kylin 1.0 ~ 1.3 compatible metadata store to 1.5 compatible metadata store -# the approach is 1) upgrade to 1.4 compatible 2) upgrade from 1.4 compatible to 1.5 compatible - -dir=$(dirname ${0}) -source ${dir}/check-env.sh - -# start command -if [ "$#" -ne 1 ] -then - echo "usage: upgrade_metadata_v_1_5_0.sh current_metadata_store_dump_path" - exit 1 -fi - - -echo "=====Upgrade Cube metadata to 1.4 compatible =====" -$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.v1_4_0.CubeMetadataUpgrade_v_1_4_0 $1 - -echo "=====Upgrade Cube metadata to 1.5 compatible =====" -$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.V1_5_0.CubeMetadataUpgrade_v_1_5_0 $1 - -echo "=====Refresh all cubes' signature =====" -$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.cli.CubeSignatureRefresher - -echo "======Deploy coprocessor=======" -$KYLIN_HOME/bin/kylin.sh org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI $KYLIN_HOME/lib/kylin-coprocessor-2.0-SNAPSHOT.jar all - -echo "==============End==============" http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/build/bin/upgrade_metadata_v_1_5_1.sh ---------------------------------------------------------------------- diff --git a/build/bin/upgrade_metadata_v_1_5_1.sh b/build/bin/upgrade_metadata_v_1_5_1.sh new file mode 100644 index 0000000..6b9701e --- /dev/null +++ b/build/bin/upgrade_metadata_v_1_5_1.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# the script will upgrade Kylin 1.0 ~ 1.3 compatible metadata store to 1.5 compatible metadata store +# the approach is 1) upgrade to 1.4 compatible 2) upgrade from 1.4 compatible to 1.5 compatible + +dir=$(dirname ${0}) +source ${dir}/check-env.sh + +# start command +if [ "$#" -ne 1 ] +then + echo "usage: upgrade_metadata_v_1_5_1.sh current_metadata_store_dump_path" + exit 1 +fi + + +echo "=====Upgrade Cube metadata to 1.4 compatible =====" +$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.entry.CubeMetadataUpgradeEntry_v_1_5_1 $1 + + +echo "======Deploy coprocessor=======" +$KYLIN_HOME/bin/kylin.sh org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI $KYLIN_HOME/lib/kylin-coprocessor-2.0-SNAPSHOT.jar all + +echo "==============End==============" http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java index 0571492..a006e1c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java @@ -16,15 +16,44 @@ */ package org.apache.kylin.common; +import java.util.HashSet; +import java.util.Set; + +import javax.annotation.Nullable; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; + public class KylinVersion { + + static class Version { + public int major; + public int minor; + public int revision; + + public Version(String version) { + String[] splits = version.split("\\."); + major = Integer.parseInt(splits[0]); + minor = Integer.parseInt(splits[1]); + revision = Integer.parseInt(splits[2]); + } + } + /** * Require MANUAL updating kylin version per ANY upgrading. */ private static final String CURRENT_KYLIN_VERSION = "1.5.1"; + private static final Set<String> SIGNATURE_INCOMPATIBLE_REVISIONS = new HashSet<String>(); + + static { + SIGNATURE_INCOMPATIBLE_REVISIONS.add("1.5.1"); + } + /** * Get current Kylin version - * + * <p/> * Currently the implementation is reading directly from constant variable * * @return current kylin version in String @@ -32,4 +61,42 @@ public class KylinVersion { public static String getCurrentVersion() { return CURRENT_KYLIN_VERSION; } + + public static boolean isCompatibleWith(String version) { + Version v = new Version(version); + Version current = new Version(CURRENT_KYLIN_VERSION); + if (current.major != v.major || current.minor != v.minor) { + return false; + } else { + return true; + } + } + + public static boolean isSignatureCompatibleWith(String version) { + if (!isCompatibleWith(version)) { + return false; + } + final Version v = new Version(version); + boolean signatureIncompatible = Iterables.any(Iterables.filter( + + Iterables.transform(SIGNATURE_INCOMPATIBLE_REVISIONS, new Function<String, Version>() { + @Nullable + @Override + public Version apply(@Nullable String input) { + return new Version(input); + } + }), new Predicate<Version>() { + @Override + public boolean apply(@Nullable Version input) { + return v.major == input.major && v.minor == input.minor; + } + }), new Predicate<Version>() { + @Override + public boolean apply(@Nullable Version input) { + return input.revision > v.revision; + } + }); + + return !signatureIncompatible; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java index f1f5aa4..8fcb66a 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/BasicTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.apache.kylin.common.KylinVersion; import org.junit.Ignore; import org.junit.Test; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 105aa82..704af60 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -439,8 +439,8 @@ public class CubeDesc extends RootPersistentEntity { } public boolean checkSignature() { - if (!KylinVersion.getCurrentVersion().equals(getVersion())) { - logger.info("checkSignature on {} is skipped as the its version is {} (not latest but compatible) ", getName(), getVersion()); + if (KylinVersion.isCompatibleWith(getVersion()) && !KylinVersion.isSignatureCompatibleWith(getVersion())) { + logger.info("checkSignature on {} is skipped as the its version is {} (not signature compatible but compatible) ", getName(), getVersion()); return true; } @@ -469,7 +469,9 @@ public class CubeDesc extends RootPersistentEntity { .append(JsonUtil.writeValueAsString(this.engineType)).append("|")// .append(JsonUtil.writeValueAsString(this.storageType)).append("|"); - byte[] signature = md.digest(sigString.toString().toLowerCase().getBytes()); + String signatureInput = sigString.toString().replaceAll("\\s+", "").toLowerCase(); + + byte[] signature = md.digest(signatureInput.getBytes()); String ret = new String(Base64.encodeBase64(signature)); return ret; } catch (NoSuchAlgorithmException | JsonProcessingException e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java deleted file mode 100644 index 3892a48..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.upgrade.V1_5_0; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.commons.lang3.StringUtils; -import org.apache.kylin.common.persistence.JsonSerializer; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.Serializer; -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.cube.model.AggregationGroup; -import org.apache.kylin.cube.model.SelectRule; -import org.apache.kylin.cube.model.v1_4_0.CubeDesc; -import org.apache.kylin.cube.model.v1_4_0.DimensionDesc; -import org.apache.kylin.cube.model.v1_4_0.HBaseMappingDesc; -import org.apache.kylin.cube.model.v1_4_0.RowKeyColDesc; -import org.apache.kylin.cube.model.v1_4_0.RowKeyDesc; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -public class CubeDescUpgrade_v_1_5_0 { - - @SuppressWarnings("unused") - private static final Logger logger = LoggerFactory.getLogger(CubeDescUpgrade_v_1_5_0.class); - - private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<>(CubeDesc.class); - - private ResourceStore store; - private String resourcePath; - - private List<String[]> oldHierarchies = Lists.newArrayList(); - private List<String> oldMandatories = Lists.newArrayList(); - private String[][] oldAggGroup = null; - private Set<String> allRowKeyCols = newIgnoreCaseSet(null); - - public CubeDescUpgrade_v_1_5_0(String resourcePath, ResourceStore resourceStore) { - this.resourcePath = resourcePath; - this.store = resourceStore; - } - - public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException { - CubeDesc oldModel = loadOldCubeDesc(resourcePath); - - org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc(); - copyUnChangedProperties(oldModel, newModel); - upgradeDimension(oldModel, newModel); - upgradeRowKey(oldModel, newModel); - upgradeHBaseMapping(oldModel, newModel); - upgradeAggregationGroup(newModel);//must do at last - - return newModel; - } - - private CubeDesc loadOldCubeDesc(String path) throws IOException { - CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer); - - if (StringUtils.isBlank(ndesc.getName())) { - throw new IllegalStateException("CubeDesc name must not be blank"); - } - - return ndesc; - } - - private Set<String> newIgnoreCaseSet(Set<String> input) { - Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - if (input != null) - ret.addAll(input); - return ret; - } - - private String[] toArray(Set<String> input) { - return input.toArray(new String[input.size()]); - } - - private boolean rowKeyColExistsInMultipleAggGroup() { - if (oldAggGroup == null) - return false; - - int total = 0; - Set<String> overall = newIgnoreCaseSet(null); - for (String[] group : oldAggGroup) { - Set<String> temp = newIgnoreCaseSet(null); - for (String entry : group) { - - overall.add(entry); - temp.add(entry); - } - total += temp.size(); - } - return overall.size() != total; - } - - private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) { - - List<AggregationGroup> aggs = Lists.newArrayList(); - if (oldAggGroup == null || oldAggGroup.length == 0) { - oldAggGroup = new String[1][]; - oldAggGroup[0] = toArray(allRowKeyCols); - } - - if (rowKeyColExistsInMultipleAggGroup()) { - throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!"); - } - - Set<String> visited = newIgnoreCaseSet(null); - - for (String[] group : oldAggGroup) { - AggregationGroup agg = new AggregationGroup(); - - Set<String> remaining = newIgnoreCaseSet(allRowKeyCols); - remaining.removeAll(visited); - - Set<String> joint = newIgnoreCaseSet(remaining); - joint.removeAll(oldMandatories); - - Set<String> groupAsSet = newIgnoreCaseSet(null); - for (String entry : group) { - groupAsSet.add(entry); - } - visited.addAll(groupAsSet); - joint.removeAll(groupAsSet); - - List<String> mandatories = Lists.newArrayList(); - List<String[]> hierarchies = Lists.newArrayList(); - - for (String s : oldMandatories) { - mandatories.add(s); - } - - for (String[] h : oldHierarchies) { - if (groupAsSet.containsAll(Arrays.asList(h))) { - hierarchies.add(h); - } - } - - agg.setIncludes(toArray(remaining)); - - SelectRule selectRule = new SelectRule(); - selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]); - if (joint.size() != 0) { - selectRule.joint_dims = new String[1][]; - selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]); - } else { - selectRule.joint_dims = new String[0][]; - } - selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]); - agg.setSelectRule(selectRule); - - aggs.add(agg); - - } - newModel.setAggregationGroups(aggs); - } - - private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - List<DimensionDesc> oldDimensions = oldModel.getDimensions(); - if (oldDimensions == null) { - throw new IllegalArgumentException("dimensions is null"); - } - List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList(); - - for (DimensionDesc oldDim : oldDimensions) { - if (oldDim.isDerived()) { - org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc(); - - newDim.setName(oldDim.getName()); - newDim.setTable(oldDim.getTable()); - newDim.setColumn("{FK}"); - newDim.setDerived(oldDim.getDerived()); - - newDimensions.add(newDim); - } else { - if (oldDim.isHierarchy()) { - oldHierarchies.add(oldDim.getColumn()); - } - - for (String columnStr : oldDim.getColumn()) { - org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc(); - - newDim.setName(oldDim.getName()); - newDim.setTable(oldDim.getTable()); - newDim.setColumn(columnStr); - newDim.setDerived(null); - - newDimensions.add(newDim); - } - } - } - - newModel.setDimensions(newDimensions); - } - - private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - RowKeyDesc oldRowKey = oldModel.getRowkey(); - if (oldModel == null) { - throw new IllegalArgumentException("RowKeyDesc is null"); - } - - if (oldRowKey.getRowKeyColumns() == null) { - throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null"); - } - - org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc(); - org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length]; - int index = 0; - for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) { - org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc(); - - allRowKeyCols.add(oldRowKeyCol.getColumn()); - if (oldRowKeyCol.isMandatory()) { - oldMandatories.add(oldRowKeyCol.getColumn()); - } - - newRowKeyCol.setColumn(oldRowKeyCol.getColumn()); - if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) { - newRowKeyCol.setEncoding("dict"); - } else if (oldRowKeyCol.getLength() > 0) { - newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength()); - } else { - throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength()); - } - cols[index++] = newRowKeyCol; - } - oldAggGroup = oldRowKey.getAggregationGroups(); - - newRowKey.setRowkeyColumns(cols); - newModel.setRowkey(newRowKey); - } - - private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping(); - try { - - ByteArrayOutputStream os = new ByteArrayOutputStream(); - JsonUtil.writeValueIndent(os, hbaseMappingDesc); - byte[] blob = os.toByteArray(); - ByteArrayInputStream is = new ByteArrayInputStream(blob); - org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class); - newModel.setHbaseMapping(newHBaseMappingDesc); - - } catch (IOException e) { - throw new RuntimeException("error when copying HBaseMappingDesc"); - } - } - - private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { - newModel.setUuid(oldModel.getUuid()); - newModel.setName(oldModel.getName()); - newModel.setDescription(oldModel.getDescription()); - newModel.setMeasures(oldModel.getMeasures()); - newModel.setNullStrings(oldModel.getNullStrings()); - newModel.setModelName(oldModel.getModelName()); - newModel.setNotifyList(oldModel.getNotifyList()); - newModel.setLastModified(oldModel.getLastModified()); - newModel.setStorageType(oldModel.getStorageType()); - newModel.setEngineType(oldModel.getEngineType()); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java deleted file mode 100644 index a1390f3..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeMetadataUpgrade_v_1_5_0.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.upgrade.V1_5_0; - -import java.io.IOException; -import java.util.List; - -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.cube.CubeDescManager; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.cube.upgrade.common.CubeMetadataUpgrade; -import org.apache.kylin.cube.upgrade.common.MetadataVersionRefresher; -import org.apache.kylin.metadata.model.IEngineAware; -import org.apache.kylin.metadata.model.IStorageAware; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * In 1.0, 1.1, 1.2 and 1.3 release there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1, - * that upgrades metadata store from kylin-0.7 compatible to 1.0 ~ 1.3 compatible. The major difference is that we split - * cube desc to cube desc + model desc - * - * Metadata are backward compatible from 1.0 ~ 1.3. - * - * In 1.4.x there is a CubeMetadataUpgradeV2 which is responsible for upgrading the metadata store - * from 1.0 ~ 1.3 compatible to 1.4.x compatible. The major actions in that is revising cube desc signature and upgrade model desc. - * Since the Apache Kylin community never officially make a 1.4.x release. Normal users can ignore it. - * - * This CubeMetadataUpgrade_v_1_5 upgrades metadata store from 1.0 ~ 1.3 compatible to 1.5 compatible. - * the major differences are: - * 1) changes from 1.0 ~ 1.3 compatible to 1.4.x compatible - * 2) brand new definition of partial cubes to allow users to select cuboids more flexibly. https://issues.apache.org/jira/browse/KYLIN-242 - * - * Notice: - * 1) From Kylin 1.5.0, every metadata store entity will bind with release version number. See RootPersistentEntity.version - * 2) From Kylin 1.5.0, the CubeMetadataUpgrade class will be named after version number. As with CubeMetadataUpgrade_v_1_5_0 - * 3) For details on how to upgrade from prior 1.5.0 to 1.5.0 compatible please visit http://kylin.apache.org/docs15/howto/howto_upgrade.html. - */ -public class CubeMetadataUpgrade_v_1_5_0 extends CubeMetadataUpgrade { - private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgrade_v_1_5_0.class); - - public CubeMetadataUpgrade_v_1_5_0(String newMetadataUrl) { - super(newMetadataUrl); - } - - public void upgradeNonCompatibleMeta() { - upgradeCubeDesc(); - } - - public void upgradeCompatibleMeta() { - upgradeVersion(); - clear(); - upgradeEngineTypeStorageType(); - upgradeSignature(); - } - - private void upgradeVersion() { - MetadataVersionRefresher refresher = new MetadataVersionRefresher(this.store); - try { - refresher.refresh(); - } catch (IOException e) { - throw new RuntimeException("Failed to upgrade version number", e); - } - } - - private void upgradeCubeDesc() { - List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); - for (String path : paths) { - logger.info("CubeMetadataUpgrade_v_1_5_0 handling in upgradeCubeDesc {}", path); - - try { - CubeDescUpgrade_v_1_5_0 upgrade = new CubeDescUpgrade_v_1_5_0(path, store); - CubeDesc ndesc = upgrade.upgrade(); - - ResourceStore.getStore(config).putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); - updatedResources.add(ndesc.getResourcePath()); - } catch (Exception e) { - logger.error("error", e); - errorMsgs.add("upgradeCubeDesc at '" + path + "' failed: " + e.getLocalizedMessage()); - } - } - } - - private void upgradeSignature() { - CubeDescManager cubeDescManager = CubeDescManager.getInstance(config); - List<CubeDesc> cubeDescs = cubeDescManager.listAllDesc(); - for (CubeDesc cubeDesc : cubeDescs) { - logger.info("CubeMetadataUpgrade_v_1_5_0 handling in upgradeSignature {}", cubeDesc.getName()); - upgradeSignature(cubeDesc); - } - } - - private void upgradeSignature(CubeDesc cubeDesc) { - try { - String calculatedSign = cubeDesc.calculateSignature(); - if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign))) { - cubeDesc.setSignature(calculatedSign); - store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); - updatedResources.add(cubeDesc.getResourcePath()); - } - } catch (Exception e) { - logger.error("error", e); - errorMsgs.add("upgradeSignature [" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); - } - } - - // Update engine_type and storage_type to v2, if the cube has no segments. - private void upgradeEngineTypeStorageType() { - CubeManager cubeManager = CubeManager.getInstance(config); - List<CubeInstance> cubes = cubeManager.listAllCubes(); - for (CubeInstance cube : cubes) { - try { - org.apache.kylin.cube.model.CubeDesc cubeDesc = cube.getDescriptor(); - if (cube.getFirstSegment() == null && cubeDesc != null && cubeDesc.getStorageType() == IStorageAware.ID_HBASE && cubeDesc.getEngineType() == IEngineAware.ID_MR_V1) { - logger.info("CubeMetadataUpgrade_v_1_5_0 handling in upgradeEngineTypeStorageType {}", cube.getName()); - - cubeDesc.setEngineType(IEngineAware.ID_MR_V2); - cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE); - - store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); - updatedResources.add(cubeDesc.getResourcePath()); - } else { - logger.info("CubeDesc {}'s storage type and engine type will not be upgraded because they're not empty", cubeDesc.getName()); - } - } catch (Exception e) { - logger.error("error", e); - errorMsgs.add("upgradeEngineTypeStorageType [" + cube.getName() + "] failed: " + e.getLocalizedMessage()); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java new file mode 100644 index 0000000..82a26e0 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeDescUpgrade_v_1_5_1.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.V1_5_1; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.lang3.StringUtils; +import org.apache.kylin.common.persistence.JsonSerializer; +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.JsonUtil; +import org.apache.kylin.cube.model.AggregationGroup; +import org.apache.kylin.cube.model.SelectRule; +import org.apache.kylin.cube.model.v1_4_0.CubeDesc; +import org.apache.kylin.cube.model.v1_4_0.DimensionDesc; +import org.apache.kylin.cube.model.v1_4_0.HBaseMappingDesc; +import org.apache.kylin.cube.model.v1_4_0.RowKeyColDesc; +import org.apache.kylin.cube.model.v1_4_0.RowKeyDesc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class CubeDescUpgrade_v_1_5_1 { + + @SuppressWarnings("unused") + private static final Logger logger = LoggerFactory.getLogger(CubeDescUpgrade_v_1_5_1.class); + + private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<>(CubeDesc.class); + + private ResourceStore store; + private String resourcePath; + + private List<String[]> oldHierarchies = Lists.newArrayList(); + private List<String> oldMandatories = Lists.newArrayList(); + private String[][] oldAggGroup = null; + private Set<String> allRowKeyCols = newIgnoreCaseSet(null); + + public CubeDescUpgrade_v_1_5_1(String resourcePath, ResourceStore resourceStore) { + this.resourcePath = resourcePath; + this.store = resourceStore; + } + + public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException { + CubeDesc oldModel = loadOldCubeDesc(resourcePath); + + org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc(); + copyUnChangedProperties(oldModel, newModel); + upgradeDimension(oldModel, newModel); + upgradeRowKey(oldModel, newModel); + upgradeHBaseMapping(oldModel, newModel); + upgradeAggregationGroup(newModel);//must do at last + + return newModel; + } + + private CubeDesc loadOldCubeDesc(String path) throws IOException { + CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer); + + if (StringUtils.isBlank(ndesc.getName())) { + throw new IllegalStateException("CubeDesc name must not be blank"); + } + + return ndesc; + } + + private Set<String> newIgnoreCaseSet(Set<String> input) { + Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + if (input != null) + ret.addAll(input); + return ret; + } + + private String[] toArray(Set<String> input) { + return input.toArray(new String[input.size()]); + } + + private boolean rowKeyColExistsInMultipleAggGroup() { + if (oldAggGroup == null) + return false; + + int total = 0; + Set<String> overall = newIgnoreCaseSet(null); + for (String[] group : oldAggGroup) { + Set<String> temp = newIgnoreCaseSet(null); + for (String entry : group) { + + overall.add(entry); + temp.add(entry); + } + total += temp.size(); + } + return overall.size() != total; + } + + private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) { + + List<AggregationGroup> aggs = Lists.newArrayList(); + if (oldAggGroup == null || oldAggGroup.length == 0) { + oldAggGroup = new String[1][]; + oldAggGroup[0] = toArray(allRowKeyCols); + } + + if (rowKeyColExistsInMultipleAggGroup()) { + throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!"); + } + + Set<String> visited = newIgnoreCaseSet(null); + + for (String[] group : oldAggGroup) { + AggregationGroup agg = new AggregationGroup(); + + Set<String> remaining = newIgnoreCaseSet(allRowKeyCols); + remaining.removeAll(visited); + + Set<String> joint = newIgnoreCaseSet(remaining); + joint.removeAll(oldMandatories); + + Set<String> groupAsSet = newIgnoreCaseSet(null); + for (String entry : group) { + groupAsSet.add(entry); + } + visited.addAll(groupAsSet); + joint.removeAll(groupAsSet); + + List<String> mandatories = Lists.newArrayList(); + List<String[]> hierarchies = Lists.newArrayList(); + + for (String s : oldMandatories) { + mandatories.add(s); + } + + for (String[] h : oldHierarchies) { + if (groupAsSet.containsAll(Arrays.asList(h))) { + hierarchies.add(h); + } + } + + agg.setIncludes(toArray(remaining)); + + SelectRule selectRule = new SelectRule(); + selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]); + if (joint.size() != 0) { + selectRule.joint_dims = new String[1][]; + selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]); + } else { + selectRule.joint_dims = new String[0][]; + } + selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]); + agg.setSelectRule(selectRule); + + aggs.add(agg); + + } + newModel.setAggregationGroups(aggs); + } + + private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { + List<DimensionDesc> oldDimensions = oldModel.getDimensions(); + if (oldDimensions == null) { + throw new IllegalArgumentException("dimensions is null"); + } + List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList(); + + for (DimensionDesc oldDim : oldDimensions) { + if (oldDim.isDerived()) { + org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc(); + + newDim.setName(oldDim.getName()); + newDim.setTable(oldDim.getTable()); + newDim.setColumn("{FK}"); + newDim.setDerived(oldDim.getDerived()); + + newDimensions.add(newDim); + } else { + if (oldDim.isHierarchy()) { + oldHierarchies.add(oldDim.getColumn()); + } + + for (String columnStr : oldDim.getColumn()) { + org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc(); + + newDim.setName(oldDim.getName()); + newDim.setTable(oldDim.getTable()); + newDim.setColumn(columnStr); + newDim.setDerived(null); + + newDimensions.add(newDim); + } + } + } + + newModel.setDimensions(newDimensions); + } + + private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { + RowKeyDesc oldRowKey = oldModel.getRowkey(); + if (oldModel == null) { + throw new IllegalArgumentException("RowKeyDesc is null"); + } + + if (oldRowKey.getRowKeyColumns() == null) { + throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null"); + } + + org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc(); + org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length]; + int index = 0; + for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) { + org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc(); + + allRowKeyCols.add(oldRowKeyCol.getColumn()); + if (oldRowKeyCol.isMandatory()) { + oldMandatories.add(oldRowKeyCol.getColumn()); + } + + newRowKeyCol.setColumn(oldRowKeyCol.getColumn()); + if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) { + newRowKeyCol.setEncoding("dict"); + } else if (oldRowKeyCol.getLength() > 0) { + newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength()); + } else { + throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength()); + } + cols[index++] = newRowKeyCol; + } + oldAggGroup = oldRowKey.getAggregationGroups(); + + newRowKey.setRowkeyColumns(cols); + newModel.setRowkey(newRowKey); + } + + private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { + HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping(); + try { + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + JsonUtil.writeValueIndent(os, hbaseMappingDesc); + byte[] blob = os.toByteArray(); + ByteArrayInputStream is = new ByteArrayInputStream(blob); + org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class); + newModel.setHbaseMapping(newHBaseMappingDesc); + + } catch (IOException e) { + throw new RuntimeException("error when copying HBaseMappingDesc"); + } + } + + private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { + newModel.setUuid(oldModel.getUuid()); + newModel.setName(oldModel.getName()); + newModel.setDescription(oldModel.getDescription()); + newModel.setMeasures(oldModel.getMeasures()); + newModel.setNullStrings(oldModel.getNullStrings()); + newModel.setModelName(oldModel.getModelName()); + newModel.setNotifyList(oldModel.getNotifyList()); + newModel.setLastModified(oldModel.getLastModified()); + newModel.setStorageType(oldModel.getStorageType()); + newModel.setEngineType(oldModel.getEngineType()); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java new file mode 100644 index 0000000..8a85629 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_1/CubeMetadataUpgrade_v_1_5_1.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.V1_5_1; + +import java.io.IOException; +import java.util.List; + +import org.apache.kylin.common.persistence.ResourceStore; +import org.apache.kylin.cube.CubeDescManager; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.model.CubeDesc; +import org.apache.kylin.cube.upgrade.common.CubeMetadataUpgrade; +import org.apache.kylin.cube.upgrade.common.MetadataVersionRefresher; +import org.apache.kylin.metadata.model.IEngineAware; +import org.apache.kylin.metadata.model.IStorageAware; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * In 1.0, 1.1, 1.2 and 1.3 release there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1, + * that upgrades metadata store from kylin-0.7 compatible to 1.0 ~ 1.3 compatible. The major difference is that we split + * cube desc to cube desc + model desc + * + * Metadata are backward compatible from 1.0 ~ 1.3. + * + * In 1.4.x there is a CubeMetadataUpgradeV2 which is responsible for upgrading the metadata store + * from 1.0 ~ 1.3 compatible to 1.4.x compatible. The major actions in that is revising cube desc signature and upgrade model desc. + * Since the Apache Kylin community never officially make a 1.4.x release. Normal users can ignore it. + * + * This CubeMetadataUpgrade_v_1_5 upgrades metadata store from 1.0 ~ 1.3 compatible to 1.5 compatible. + * the major differences are: + * 1) changes from 1.0 ~ 1.3 compatible to 1.4.x compatible + * 2) brand new definition of partial cubes to allow users to select cuboids more flexibly. https://issues.apache.org/jira/browse/KYLIN-242 + * + * Notice: + * 1) From Kylin 1.5.0, every metadata store entity will bind with release version number. See RootPersistentEntity.version + * 2) From Kylin 1.5.1, the CubeMetadataUpgrade class will be named after version number. As with CubeMetadataUpgrade_v_1_5_1 + * 3) For details on how to upgrade from prior 1.5.0 to 1.5 compatible please visit http://kylin.apache.org/docs15/howto/howto_upgrade.html. + */ +public class CubeMetadataUpgrade_v_1_5_1 extends CubeMetadataUpgrade { + private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgrade_v_1_5_1.class); + + public CubeMetadataUpgrade_v_1_5_1(String newMetadataUrl) { + super(newMetadataUrl); + } + + public void upgradeNonCompatibleMeta() { + upgradeCubeDesc(); + } + + public void upgradeCompatibleMeta() { + upgradeVersion(); + clear(); + upgradeEngineTypeStorageType(); + upgradeSignature(); + } + + private void upgradeVersion() { + MetadataVersionRefresher refresher = new MetadataVersionRefresher(this.store, "1.5.1"); + try { + refresher.refresh(); + } catch (IOException e) { + throw new RuntimeException("Failed to upgrade version number", e); + } + } + + private void upgradeCubeDesc() { + List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); + for (String path : paths) { + logger.info("CubeMetadataUpgrade_v_1_5_1 handling in upgradeCubeDesc {}", path); + + try { + CubeDescUpgrade_v_1_5_1 upgrade = new CubeDescUpgrade_v_1_5_1(path, store); + CubeDesc ndesc = upgrade.upgrade(); + + ResourceStore.getStore(config).putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(ndesc.getResourcePath()); + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("upgradeCubeDesc at '" + path + "' failed: " + e.getLocalizedMessage()); + } + } + } + + private void upgradeSignature() { + CubeDescManager cubeDescManager = CubeDescManager.getInstance(config); + List<CubeDesc> cubeDescs = cubeDescManager.listAllDesc(); + for (CubeDesc cubeDesc : cubeDescs) { + logger.info("CubeMetadataUpgrade_v_1_5_1 handling in upgradeSignature {}", cubeDesc.getName()); + upgradeSignature(cubeDesc); + } + } + + private void upgradeSignature(CubeDesc cubeDesc) { + try { + String calculatedSign = cubeDesc.calculateSignature(); + if (cubeDesc.getSignature() == null || (!cubeDesc.getSignature().equals(calculatedSign))) { + cubeDesc.setSignature(calculatedSign); + store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(cubeDesc.getResourcePath()); + } + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("upgradeSignature [" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage()); + } + } + + // Update engine_type and storage_type to v2, if the cube has no segments. + private void upgradeEngineTypeStorageType() { + CubeManager cubeManager = CubeManager.getInstance(config); + List<CubeInstance> cubes = cubeManager.listAllCubes(); + for (CubeInstance cube : cubes) { + try { + org.apache.kylin.cube.model.CubeDesc cubeDesc = cube.getDescriptor(); + if (cube.getFirstSegment() == null && cubeDesc != null && cubeDesc.getStorageType() == IStorageAware.ID_HBASE && cubeDesc.getEngineType() == IEngineAware.ID_MR_V1) { + logger.info("CubeMetadataUpgrade_v_1_5_1 handling in upgradeEngineTypeStorageType {}", cube.getName()); + + cubeDesc.setEngineType(IEngineAware.ID_MR_V2); + cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE); + + store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER); + updatedResources.add(cubeDesc.getResourcePath()); + } else { + logger.info("CubeDesc {}'s storage type and engine type will not be upgraded because they're not empty", cubeDesc.getName()); + } + } catch (Exception e) { + logger.error("error", e); + errorMsgs.add("upgradeEngineTypeStorageType [" + cube.getName() + "] failed: " + e.getLocalizedMessage()); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java index 511edb8..50833e3 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/MetadataVersionRefresher.java @@ -44,9 +44,12 @@ public class MetadataVersionRefresher { private static final Logger logger = LoggerFactory.getLogger(MetadataVersionRefresher.class); private ResourceStore store; + private String version; - public MetadataVersionRefresher(ResourceStore resourceStore) { + public MetadataVersionRefresher(ResourceStore resourceStore, String version) { this.store = resourceStore; + this.version = version; + } public void refresh() throws IOException { @@ -60,7 +63,7 @@ public class MetadataVersionRefresher { if (path.endsWith(MetadataConstants.FILE_SURFIX) && !(path.startsWith(ResourceStore.DICT_RESOURCE_ROOT) || path.startsWith(ResourceStore.SNAPSHOT_RESOURCE_ROOT))) { logger.info("Updating metadata version of path {}", path); ObjectNode objectNode = (ObjectNode) mapper.readTree(this.store.getResource(path).inputStream); - objectNode.put("version", KylinVersion.getCurrentVersion()); + objectNode.put("version", version); ByteArrayOutputStream baos = new ByteArrayOutputStream(); mapper.writeValue(baos, objectNode); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java deleted file mode 100644 index b4a0286..0000000 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_0.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.cube.upgrade.entry; - -import org.apache.kylin.cube.upgrade.V1_5_0.CubeMetadataUpgrade_v_1_5_0; -import org.apache.kylin.cube.upgrade.v1_4_0.CubeMetadataUpgrade_v_1_4_0; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CubeMetadataUpgradeEntry_v_1_5_0 { - private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgradeEntry_v_1_5_0.class); - - public static void main(String[] args) { - if (!(args != null && (args.length == 1))) { - System.out.println("Usage: java CubeMetadataUpgradeEntry_v_1_5_0 <metadata_export_folder>"); - System.out.println(", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); - return; - } - - try { - CubeMetadataUpgrade_v_1_4_0.upgradeOrVerify(CubeMetadataUpgrade_v_1_4_0.class, args, true, false); - CubeMetadataUpgrade_v_1_5_0.upgradeOrVerify(CubeMetadataUpgrade_v_1_5_0.class, new String[] { args[0] + "_workspace" }, false, true); - } catch (Exception e) { - logger.error("something went wrong when upgrading, suggest to roll back metadata", e); - return; - } - - logger.info("The metadata upgrade is complete locally. You need to upload the metadata to you actual metadata store to verify locally. You need to upload the metadata to you actual metadata store to verify."); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java new file mode 100644 index 0000000..ba391b4 --- /dev/null +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/entry/CubeMetadataUpgradeEntry_v_1_5_1.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.cube.upgrade.entry; + +import org.apache.kylin.cube.upgrade.V1_5_1.CubeMetadataUpgrade_v_1_5_1; +import org.apache.kylin.cube.upgrade.v1_4_0.CubeMetadataUpgrade_v_1_4_0; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CubeMetadataUpgradeEntry_v_1_5_1 { + private static final Logger logger = LoggerFactory.getLogger(CubeMetadataUpgradeEntry_v_1_5_1.class); + + public static void main(String[] args) { + if (!(args != null && (args.length == 1))) { + System.out.println("Usage: java CubeMetadataUpgradeEntry_v_1_5_1 <metadata_export_folder>"); + System.out.println(", where metadata_export_folder is the folder containing your current metadata's dump (Upgrade program will not modify it directly, relax."); + return; + } + + try { + CubeMetadataUpgrade_v_1_4_0.upgradeOrVerify(CubeMetadataUpgrade_v_1_4_0.class, args, true, false); + CubeMetadataUpgrade_v_1_5_1.upgradeOrVerify(CubeMetadataUpgrade_v_1_5_1.class, new String[] { args[0] + "_workspace" }, false, true); + } catch (Exception e) { + logger.error("something went wrong when upgrading, suggest to roll back metadata", e); + return; + } + + logger.info("The metadata upgrade is complete locally. You need to upload the metadata to you actual metadata store to verify locally. You need to upload the metadata to you actual metadata store to verify."); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/98ac225f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java index 04f09d8..007b868 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java @@ -45,7 +45,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; /** - * please check doc on CubeMetadataUpgrade_v_1_5_0 + * please check doc on CubeMetadataUpgrade_v_1_5_1 */ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade {
