zhangayqian commented on a change in pull request #1611:
URL: https://github.com/apache/kylin/pull/1611#discussion_r610474387
##########
File path: tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
##########
@@ -86,82 +98,113 @@
protected String dstProject;
protected String srcHdfsWorkDir;
protected String dstHdfsWorkDir;
+ private boolean migrateSegment;
+ private boolean realExecute;
+ private boolean purgeAndDisable;
+ private OptionsHelper optHelper;
private static final String ACL_PREFIX = "/acl/";
private static final String GLOBAL_DICT_PREFIX = "/dict/global_dict/";
- public static void main(String[] args) throws IOException,
InterruptedException {
+ private static final Option OPTION_SRC_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s source").create("srcConfig");
+ private static final Option OPTION_DST_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s new home").create("dstConfig");
+ private static final Option OPTION_ALL_CUBES =
OptionBuilder.isRequired(false).withDescription("migrate all cubes meta from
source cluster").create("allCubes");
+ private static final Option OPTION_CUBE =
OptionBuilder.isRequired(false).hasArg().withDescription("Cube name to
migrate").create("cube");
+ private static final Option OPTION_DST_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("cube's new project
home, if not set, keep the same as source cluster").create("dstProject");
+ private static final Option OPTION_SRC_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("source project to
migrate").create("srcProject");
+ private static final Option OPTION_COPY_ACL =
OptionBuilder.isRequired(false).hasArg().withDescription("copy
ACL").create("copyAcl");
+ private static final Option OPTION_PURGE_AND_DISABLE =
OptionBuilder.isRequired(false).withDescription("purge source cluster
data").create("purgeAndDisable");
+ private static final Option OPTION_OVERWRITE =
OptionBuilder.isRequired(false).withDescription("overwrite target cluster's
meta if exists").create("overwriteIfExists");
+ private static final Option OPTION_EXECUTE =
OptionBuilder.isRequired(false).withDescription("execute
migration").create("execute");
Review comment:
Maybe the parameter `execute ` should be called `realMigrate`, so that
users can better to understand the meaning of this parameter. And its default
value can be set to `true`.
##########
File path: tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
##########
@@ -86,82 +98,113 @@
protected String dstProject;
protected String srcHdfsWorkDir;
protected String dstHdfsWorkDir;
+ private boolean migrateSegment;
+ private boolean realExecute;
+ private boolean purgeAndDisable;
+ private OptionsHelper optHelper;
private static final String ACL_PREFIX = "/acl/";
private static final String GLOBAL_DICT_PREFIX = "/dict/global_dict/";
- public static void main(String[] args) throws IOException,
InterruptedException {
+ private static final Option OPTION_SRC_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s source").create("srcConfig");
+ private static final Option OPTION_DST_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s new home").create("dstConfig");
+ private static final Option OPTION_ALL_CUBES =
OptionBuilder.isRequired(false).withDescription("migrate all cubes meta from
source cluster").create("allCubes");
+ private static final Option OPTION_CUBE =
OptionBuilder.isRequired(false).hasArg().withDescription("Cube name to
migrate").create("cube");
+ private static final Option OPTION_DST_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("cube's new project
home, if not set, keep the same as source cluster").create("dstProject");
+ private static final Option OPTION_SRC_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("source project to
migrate").create("srcProject");
+ private static final Option OPTION_COPY_ACL =
OptionBuilder.isRequired(false).hasArg().withDescription("copy
ACL").create("copyAcl");
+ private static final Option OPTION_PURGE_AND_DISABLE =
OptionBuilder.isRequired(false).withDescription("purge source cluster
data").create("purgeAndDisable");
+ private static final Option OPTION_OVERWRITE =
OptionBuilder.isRequired(false).withDescription("overwrite target cluster's
meta if exists").create("overwriteIfExists");
+ private static final Option OPTION_EXECUTE =
OptionBuilder.isRequired(false).withDescription("execute
migration").create("execute");
+ private static final Option OPTION_MIGRATE_SEGMENTS =
OptionBuilder.isRequired(false).withDescription("migrate segment
data").create("migrateSegment");
+
+ public static void main(String[] args) throws Exception {
+ CubeMigrationCLI cli = new CubeMigrationCLI();
+ cli.init(args);
+ cli.moveCube();
+ }
+ public void init(String[] args) {
+ optHelper = new OptionsHelper();
CubeMigrationCLI cli = new CubeMigrationCLI();
- if (args.length != 8 && args.length != 9) {
- cli.usage();
+ try {
+ optHelper.parseOptions(cli.getOptions(), args);
+ } catch (Exception e) {
+ logger.error("failed to parse arguments", e);
+ optHelper.printUsage("CubeMigrationCLI", cli.getOptions());
System.exit(1);
}
- if (args.length == 8) {
- cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5],
args[6], args[7]);
- } else if (args.length == 9) {
- cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5],
args[6], args[7], args[8]);
- }
+ doAclCopy = optHelper.hasOption(OPTION_COPY_ACL);
+ doOverwrite = optHelper.hasOption(OPTION_OVERWRITE);
+ doMigrateSegment = optHelper.hasOption(OPTION_MIGRATE_SEGMENTS);
+ purgeAndDisable = optHelper.hasOption(OPTION_PURGE_AND_DISABLE);
+ srcConfig =
KylinConfig.createInstanceFromUri(optHelper.getOptionValue(OPTION_SRC_CONFIG));
+ srcStore = ResourceStore.getStore(srcConfig);
+ dstConfig =
KylinConfig.createInstanceFromUri(optHelper.getOptionValue(OPTION_DST_CONFIG));
+ dstStore = ResourceStore.getStore(dstConfig);
+ migrateSegment = optHelper.hasOption(OPTION_MIGRATE_SEGMENTS);
Review comment:
What is the difference between `migrateSegment ` and `doMigrateSegment `?
##########
File path: tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
##########
@@ -86,82 +98,113 @@
protected String dstProject;
protected String srcHdfsWorkDir;
protected String dstHdfsWorkDir;
+ private boolean migrateSegment;
+ private boolean realExecute;
+ private boolean purgeAndDisable;
+ private OptionsHelper optHelper;
private static final String ACL_PREFIX = "/acl/";
private static final String GLOBAL_DICT_PREFIX = "/dict/global_dict/";
- public static void main(String[] args) throws IOException,
InterruptedException {
+ private static final Option OPTION_SRC_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s source").create("srcConfig");
+ private static final Option OPTION_DST_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s new home").create("dstConfig");
+ private static final Option OPTION_ALL_CUBES =
OptionBuilder.isRequired(false).withDescription("migrate all cubes meta from
source cluster").create("allCubes");
+ private static final Option OPTION_CUBE =
OptionBuilder.isRequired(false).hasArg().withDescription("Cube name to
migrate").create("cube");
+ private static final Option OPTION_DST_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("cube's new project
home, if not set, keep the same as source cluster").create("dstProject");
+ private static final Option OPTION_SRC_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("source project to
migrate").create("srcProject");
+ private static final Option OPTION_COPY_ACL =
OptionBuilder.isRequired(false).hasArg().withDescription("copy
ACL").create("copyAcl");
+ private static final Option OPTION_PURGE_AND_DISABLE =
OptionBuilder.isRequired(false).withDescription("purge source cluster
data").create("purgeAndDisable");
+ private static final Option OPTION_OVERWRITE =
OptionBuilder.isRequired(false).withDescription("overwrite target cluster's
meta if exists").create("overwriteIfExists");
+ private static final Option OPTION_EXECUTE =
OptionBuilder.isRequired(false).withDescription("execute
migration").create("execute");
+ private static final Option OPTION_MIGRATE_SEGMENTS =
OptionBuilder.isRequired(false).withDescription("migrate segment
data").create("migrateSegment");
Review comment:
In addition to being used as a metadata upgrade tool, CubeMigrationCLI
may also be used for metadata migration between different clusters of the same
version.
Therefore, a parameter should be added to determine whether the user is
upgrading kylin 2/3 metadata to kylin 4.0. It can be called `upgradeMeta`.
When `upgradeMeta` is set to `true`, then parameter `doMigrateSegment` cannot
be set to `true`.
##########
File path: tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
##########
@@ -86,82 +98,113 @@
protected String dstProject;
protected String srcHdfsWorkDir;
protected String dstHdfsWorkDir;
+ private boolean migrateSegment;
+ private boolean realExecute;
+ private boolean purgeAndDisable;
+ private OptionsHelper optHelper;
private static final String ACL_PREFIX = "/acl/";
private static final String GLOBAL_DICT_PREFIX = "/dict/global_dict/";
- public static void main(String[] args) throws IOException,
InterruptedException {
+ private static final Option OPTION_SRC_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s source").create("srcConfig");
+ private static final Option OPTION_DST_CONFIG =
OptionBuilder.isRequired(true).hasArg().withDescription("The KylinConfig of the
cube’s new home").create("dstConfig");
+ private static final Option OPTION_ALL_CUBES =
OptionBuilder.isRequired(false).withDescription("migrate all cubes meta from
source cluster").create("allCubes");
+ private static final Option OPTION_CUBE =
OptionBuilder.isRequired(false).hasArg().withDescription("Cube name to
migrate").create("cube");
+ private static final Option OPTION_DST_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("cube's new project
home, if not set, keep the same as source cluster").create("dstProject");
+ private static final Option OPTION_SRC_PROJECT =
OptionBuilder.isRequired(false).hasArg().withDescription("source project to
migrate").create("srcProject");
+ private static final Option OPTION_COPY_ACL =
OptionBuilder.isRequired(false).hasArg().withDescription("copy
ACL").create("copyAcl");
+ private static final Option OPTION_PURGE_AND_DISABLE =
OptionBuilder.isRequired(false).withDescription("purge source cluster
data").create("purgeAndDisable");
+ private static final Option OPTION_OVERWRITE =
OptionBuilder.isRequired(false).withDescription("overwrite target cluster's
meta if exists").create("overwriteIfExists");
+ private static final Option OPTION_EXECUTE =
OptionBuilder.isRequired(false).withDescription("execute
migration").create("execute");
+ private static final Option OPTION_MIGRATE_SEGMENTS =
OptionBuilder.isRequired(false).withDescription("migrate segment
data").create("migrateSegment");
+
+ public static void main(String[] args) throws Exception {
+ CubeMigrationCLI cli = new CubeMigrationCLI();
+ cli.init(args);
+ cli.moveCube();
+ }
+ public void init(String[] args) {
+ optHelper = new OptionsHelper();
CubeMigrationCLI cli = new CubeMigrationCLI();
- if (args.length != 8 && args.length != 9) {
- cli.usage();
+ try {
+ optHelper.parseOptions(cli.getOptions(), args);
+ } catch (Exception e) {
+ logger.error("failed to parse arguments", e);
+ optHelper.printUsage("CubeMigrationCLI", cli.getOptions());
System.exit(1);
}
- if (args.length == 8) {
- cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5],
args[6], args[7]);
- } else if (args.length == 9) {
- cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5],
args[6], args[7], args[8]);
- }
+ doAclCopy = optHelper.hasOption(OPTION_COPY_ACL);
+ doOverwrite = optHelper.hasOption(OPTION_OVERWRITE);
+ doMigrateSegment = optHelper.hasOption(OPTION_MIGRATE_SEGMENTS);
+ purgeAndDisable = optHelper.hasOption(OPTION_PURGE_AND_DISABLE);
+ srcConfig =
KylinConfig.createInstanceFromUri(optHelper.getOptionValue(OPTION_SRC_CONFIG));
+ srcStore = ResourceStore.getStore(srcConfig);
+ dstConfig =
KylinConfig.createInstanceFromUri(optHelper.getOptionValue(OPTION_DST_CONFIG));
+ dstStore = ResourceStore.getStore(dstConfig);
+ migrateSegment = optHelper.hasOption(OPTION_MIGRATE_SEGMENTS);
+ realExecute = optHelper.hasOption(OPTION_EXECUTE);
+ dstProject = optHelper.getOptionValue(OPTION_DST_PROJECT);
+ conf = HadoopUtil.getCurrentConfiguration();
}
- protected void usage() {
- System.out.println(
- "Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri
cubeName projectName copyAclOrNot purgeOrNot overwriteIfExists realExecute
migrateSegmentOrNot");
- System.out.println("srcKylinConfigUri: The KylinConfig of the cube’s
source \n"
- + "dstKylinConfigUri: The KylinConfig of the cube’s new home
\n"
- + "cubeName: the name of cube to be migrated. \n"
- + "projectName: The target project in the target
environment.(Make sure it exist) \n"
- + "copyAclOrNot: true or false: whether copy cube ACL to
target environment. \n"
- + "purgeOrNot: true or false: whether purge the cube from src
server after the migration. \n"
- + "overwriteIfExists: overwrite cube if it already exists in
the target environment. \n"
- + "realExecute: if false, just print the operations to take,
if true, do the real migration. \n"
- + "migrateSegmentOrNot:(optional) true or false: whether copy
segment data to target environment. \n");
-
- }
+ public void moveCube() throws Exception {
+ conf = HadoopUtil.getCurrentConfiguration();
- public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName,
String projectName, String copyAcl,
- String purgeAndDisable, String overwriteIfExists,
String realExecute)
- throws IOException, InterruptedException {
+ if (optHelper.hasOption(OPTION_CUBE)) {
+ CubeManager cubeManager = CubeManager.getInstance(srcConfig);
+ moveSingleCube(optHelper.getOptionValue(OPTION_CUBE), dstProject,
cubeManager);
Review comment:
`dstProject ` here may be`null` and may not have been created in
dstKylin.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
