KYLIN-1157 copy ACL when migrate cube
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/0069fad3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/0069fad3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/0069fad3 Branch: refs/heads/2.0-rc Commit: 0069fad30b8e8beec6565510617848ee894e0277 Parents: 23d63cf Author: shaofengshi <shaofeng...@apache.org> Authored: Tue Nov 24 11:46:36 2015 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Tue Nov 24 13:43:37 2015 +0800 ---------------------------------------------------------------------- .../storage/hbase/util/CubeMigrationCLI.java | 67 ++++++++++++++------ 1 file changed, 48 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/0069fad3/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java index 6a4d5e3..a50b1eb 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java @@ -18,24 +18,20 @@ package org.apache.kylin.storage.hbase.util; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - +import com.google.common.base.Preconditions; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.Serializer; +import org.apache.kylin.common.util.Bytes; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.cube.CubeSegment; @@ -54,10 +50,14 @@ import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.realization.IRealizationConstants; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.storage.hbase.HBaseConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; /** * <p/> @@ -81,20 +81,20 @@ public class CubeMigrationCLI { public static void main(String[] args) throws IOException, InterruptedException { - if (args.length != 6) { + if (args.length != 7) { usage(); System.exit(1); } - moveCube(args[0], args[1], args[2], args[3], args[4], args[5]); + moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6]); } private static void usage() { - System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName overwriteIfExists realExecute"); - System.out.println(" srcKylinConfigUri: The KylinConfig of the cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n"); - } + System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot overwriteIfExists realExecute"); + System.out.println(" srcKylinConfigUri: The KylinConfig of the cubeâs source \n" + "dstKylinConfigUri: The KylinConfig of the cubeâs new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n"); + } - public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { + public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { srcConfig = srcCfg; srcStore = ResourceStore.getStore(srcConfig); @@ -127,6 +127,9 @@ public class CubeMigrationCLI { renameFoldersInHdfs(cube); changeHtableHost(cube); addCubeIntoProject(cubeName, projectName, cube.getModelName()); + if (Boolean.parseBoolean(copyAcl) == true) { + copyACL(cube); + } if (realExecute.equalsIgnoreCase("true")) { doOpts(); @@ -135,8 +138,8 @@ public class CubeMigrationCLI { } } - public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { - moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, overwriteIfExists, realExecute); + public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String overwriteIfExists, String realExecute) throws IOException, InterruptedException { + moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, overwriteIfExists, realExecute); } private static String checkAndGetHbaseUrl() { @@ -184,6 +187,10 @@ public class CubeMigrationCLI { } } + private static void copyACL(CubeInstance cube) { + operations.add(new Opt(OptType.COPY_ACL, new Object[] { cube.getUuid() })); + } + private static void copyFilesInMetaStore(CubeInstance cube, String overwriteIfExists) throws IOException { List<String> metaItems = new ArrayList<String>(); @@ -233,7 +240,7 @@ public class CubeMigrationCLI { } private static enum OptType { - COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST + COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, COPY_ACL } private static class Opt { @@ -386,6 +393,28 @@ public class CubeMigrationCLI { logger.info("Project instance for " + projectName + " is corrected"); break; } + case COPY_ACL: { + String cubeId = (String) opt.params[0]; + HTableInterface srcAclHtable = null; + HTableInterface destAclHtable = null; + try { + srcAclHtable = HBaseConnection.get(srcConfig.getMetadataUrl()).getTable(srcConfig.getMetadataUrlPrefix() + "_acl"); + destAclHtable = HBaseConnection.get(dstConfig.getMetadataUrl()).getTable(dstConfig.getMetadataUrlPrefix() + "_acl"); + + Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId))); + + for (Cell cell : result.listCells()) { + Put put = new Put(Bytes.toBytes(cubeId)); + put.add(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), CellUtil.cloneValue(cell)); + destAclHtable.put(put); + } + destAclHtable.flushCommits(); + } finally { + IOUtils.closeQuietly(srcAclHtable); + IOUtils.closeQuietly(destAclHtable); + } + break; + } } }
