KYLIN-1495 bug fixes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d52a8caa Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d52a8caa Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d52a8caa Branch: refs/heads/master Commit: d52a8caa42fd5993742e466175ff6cc201065aee Parents: 98f8e0a Author: Hongbin Ma <[email protected]> Authored: Thu Mar 17 16:20:05 2016 +0800 Committer: Hongbin Ma <[email protected]> Committed: Thu Mar 17 16:20:05 2016 +0800 ---------------------------------------------------------------------- build/bin/upgrade_metadata_v_1_5_1.sh | 41 -------------------- .../kylin/rest/controller/CubeController.java | 23 +++++++---- .../apache/kylin/rest/service/CubeService.java | 6 ++- .../hbase/cube/v1/CubeSegmentTupleIterator.java | 2 +- .../hbase/util/DeployCoprocessorCLI.java | 17 +++++--- .../hbase/steps/SandboxMetastoreCLI.java | 2 +- 6 files changed, 33 insertions(+), 58 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/build/bin/upgrade_metadata_v_1_5_1.sh ---------------------------------------------------------------------- diff --git a/build/bin/upgrade_metadata_v_1_5_1.sh b/build/bin/upgrade_metadata_v_1_5_1.sh deleted file mode 100644 index 6b9701e..0000000 --- a/build/bin/upgrade_metadata_v_1_5_1.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# the script will upgrade Kylin 1.0 ~ 1.3 compatible metadata store to 1.5 compatible metadata store -# the approach is 1) upgrade to 1.4 compatible 2) upgrade from 1.4 compatible to 1.5 compatible - -dir=$(dirname ${0}) -source ${dir}/check-env.sh - -# start command -if [ "$#" -ne 1 ] -then - echo "usage: upgrade_metadata_v_1_5_1.sh current_metadata_store_dump_path" - exit 1 -fi - - -echo "=====Upgrade Cube metadata to 1.4 compatible =====" -$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.entry.CubeMetadataUpgradeEntry_v_1_5_1 $1 - - -echo "======Deploy coprocessor=======" -$KYLIN_HOME/bin/kylin.sh org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI $KYLIN_HOME/lib/kylin-coprocessor-2.0-SNAPSHOT.jar all - -echo "==============End==============" http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java index 410b2bd..6f8ad51 100644 --- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -380,19 +380,26 @@ public class CubeController extends BasicController { // Check if the cube is editable isCubeDescFreeEditable = cubeService.isCubeDescFreeEditable(desc); - //cube renaming is not allowed - if (!cubeRequest.getCubeName().equalsIgnoreCase(CubeService.getCubeNameFromDesc(desc.getName()))) { - String error = "Cube Desc renaming is not allowed: " + desc.getName(); - updateRequest(cubeRequest, false, error); - return cubeRequest; - } - String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject(); try { CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName()); + + if (cube == null) { + String error = "The cube named " + cubeRequest.getCubeName() + " does not exist "; + updateRequest(cubeRequest, false, error); + return cubeRequest; + } + + //cube renaming is not allowed + if (!cube.getDescriptor().getName().equalsIgnoreCase(desc.getName())) { + String error = "Cube Desc renaming is not allowed: desc.getName(): " + desc.getName() + ", cubeRequest.getCubeName(): " + cubeRequest.getCubeName(); + updateRequest(cubeRequest, false, error); + return cubeRequest; + } + oldCubeDesc = cube.getDescriptor(); if (isCubeDescFreeEditable || oldCubeDesc.consistentWith(desc)) { - desc = cubeService.updateCubeAndDesc(cube, desc, projectName); + desc = cubeService.updateCubeAndDesc(cube, desc, projectName, true); } else { logger.warn("Won't update the cube desc due to inconsistency"); updateRequest(cubeRequest, false, "CubeDesc " + desc.getName() + " is inconsistent with existing. Try purge that cube first or avoid updating key cube desc fields."); http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/server/src/main/java/org/apache/kylin/rest/service/CubeService.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java index 6ef796c..eb15beb 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java @@ -244,7 +244,7 @@ public class CubeService extends BasicService { } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')") - public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException { + public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException, JobException { final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING)); if (!cubingJobs.isEmpty()) { @@ -252,7 +252,8 @@ public class CubeService extends BasicService { } try { - if (!cube.getDescriptor().consistentWith(desc)) { + //double check again + if (!forceUpdate && !cube.getDescriptor().consistentWith(desc) ) { throw new IllegalStateException("cube's desc is not consistent with the new desc"); } @@ -594,6 +595,7 @@ public class CubeService extends BasicService { } private void keepCubeRetention(String cubeName) { + logger.info("checking keepCubeRetention"); CubeInstance cube = getCubeManager().getCube(cubeName); CubeDesc desc = cube.getDescriptor(); if (desc.getRetentionRange() > 0) { http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java index 909de39..d7ea2a0 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java @@ -210,7 +210,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator { private void logScan(HBaseKeyRange keyRange) { StringBuilder info = new StringBuilder(); - info.append(" Scan hbase table ").append(tableName).append(": "); + info.append("Scan hbase table ").append(tableName).append(": "); if (keyRange.getCuboid().requirePostAggregation()) { info.append(" cuboid require post aggregation, from "); } else { http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java index 5bca721..8bf06b7 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java @@ -70,21 +70,28 @@ public class DeployCoprocessorCLI { private static final Logger logger = LoggerFactory.getLogger(DeployCoprocessorCLI.class); public static void main(String[] args) throws IOException { + + if (args == null || args.length <= 1) { + printUsageAndExit(); + } + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); FileSystem fileSystem = FileSystem.get(hconf); HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf); - String localCoprocessorJar = new File(args[0]).getAbsolutePath(); + String localCoprocessorJar; + if ("default".equals(args[0])) { + localCoprocessorJar = kylinConfig.getCoprocessorLocalJar(); + } else { + localCoprocessorJar = new File(args[0]).getAbsolutePath(); + } + logger.info("Identify coprocessor jar " + localCoprocessorJar); List<String> tableNames = getHTableNames(kylinConfig); logger.info("Identify tables " + tableNames); - if (args.length <= 1) { - printUsageAndExit(); - } - String filterType = args[1].toLowerCase(); if (filterType.equals("-table")) { tableNames = filterByTables(tableNames, Arrays.asList(args).subList(2, args.length)); http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java index 30889ab..b12451a 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java @@ -47,7 +47,7 @@ public class SandboxMetastoreCLI { throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2"); } - if (args.length < 2) { + if (args.length < 1) { printUsage(); return; }
