This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 80ff2285027d1c55aa07bb07c0afd64c823a71ea Author: zhennzhang <zhennzh...@ebay.com> AuthorDate: Fri Jan 13 11:50:43 2023 +0800 KYLIN-5391 Kylin metadata tool for read specific file --- build/bin/metastore.sh | 43 ++++++++++- .../java/org/apache/kylin/tool/MetadataTool.java | 84 +++++++++++++++++++++- .../org/apache/kylin/tool/MetadataToolTest.java | 32 +++++++++ 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh index 05c998eae1..9e791eea71 100755 --- a/build/bin/metastore.sh +++ b/build/bin/metastore.sh @@ -26,6 +26,8 @@ fi function help { echo "usage: metastore.sh backup METADATA_BACKUP_PATH(the default path is KYLIN_HOME/meta_backups/)" + echo " metastore.sh fetch TARGET_FILE_PATH METADATA_FETCH_PATH(the default path is KYLIN_HOME/meta_fetch/)" + echo " metastore.sh list TARGET_FOLDER_PATH" echo " metastore.sh restore METADATA_RESTORE_PATH [--after-truncate]" echo " metastore.sh backup-project PROJECT_NAME METADATA_BACKUP_PATH(the default path is KYLIN_HOME/meta_backups/)" echo " metastore.sh restore-project PROJECT_NAME METADATA_RESTORE_PATH [--after-truncate]" @@ -44,6 +46,18 @@ function printBackupResult() { fi } +function printFetchResult() { + error=$1 + if [[ $error == 0 ]]; then + if [[ -z "$path" ]]; then + path="\${KYLIN_HOME}/meta_fetch" + fi + echo -e "${YELLOW} Fetch at local dist succeed.${RESTORE}" + else + echo -e "${YELLOW} Fetch failed.${RESTORE}" + fi +} + function printRestoreResult() { error=$1 @@ -134,6 +148,34 @@ then ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.MetadataTool ${BACKUP_OPTS} printBackupResult $? +elif [ "$1" == "fetch" ] +then + FETCH_OPTS="-fetch" + if [ $# -eq 2 ]; then + _file=$2 + FETCH_OPTS="${FETCH_OPTS} -target ${_file}" + elif [ $# -eq 3 ]; then + _file=$2 + path=`cd $3 && pwd -P` + check_path_empty ${path} + FETCH_OPTS="${FETCH_OPTS} -target ${_file} -dir ${path}" + else + help + fi + + ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.MetadataTool ${FETCH_OPTS} + printFetchResult $? + +elif [ "$1" == "list" ] +then + if [ $# -eq 2 ]; then + _folder=$2 + LIST_OPTS="-list -target ${_folder}" + else + help + fi + ${KYLIN_HOME}/bin/kylin.sh org.apache.kylin.tool.MetadataTool ${LIST_OPTS} + elif [ "$1" == "restore" ] then if [ $# -eq 2 ]; then @@ -171,4 +213,3 @@ else help fi - diff --git a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java index 66aa1d8b3d..daa8aed307 100644 --- a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java +++ b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java @@ -87,6 +87,12 @@ public class MetadataTool extends ExecutableApplication { private static final Option OPERATE_COMPRESS = OptionBuilder.getInstance() .withDescription("Backup compressed metadata to HDFS path").isRequired(false).create("compress"); + private static final Option OPERATE_FETCH = OptionBuilder.getInstance() + .withDescription("Fetch part of metadata to local path").isRequired(false).create("fetch"); + + private static final Option OPERATE_LIST = OptionBuilder.getInstance() + .withDescription("List children of target folder").isRequired(false).create("list"); + private static final Option OPERATE_RESTORE = OptionBuilder.getInstance() .withDescription("Restore metadata from local path or HDFS path").isRequired(false).create("restore"); @@ -100,6 +106,9 @@ public class MetadataTool extends ExecutableApplication { private static final Option OPTION_PROJECT = OptionBuilder.getInstance().hasArg().withArgName("PROJECT_NAME") .withDescription("Specify project level backup and restore (optional)").isRequired(false).create("project"); + private static final Option OPTION_TARGET = OptionBuilder.getInstance().hasArg().withArgName("TARGET_FILE") + .withDescription("Specify part of metadata for fetch to local path").isRequired(false).create("target"); + private static final Option FOLDER_NAME = OptionBuilder.getInstance().hasArg().withArgName("FOLDER_NAME") .withDescription("Specify the folder name for backup").isRequired(false).create("folder"); @@ -116,6 +125,9 @@ public class MetadataTool extends ExecutableApplication { @Getter private String backupPath; + @Getter + private String fetchPath; + MetadataTool() { kylinConfig = KylinConfig.getInstanceFromEnv(); this.options = new Options(); @@ -154,7 +166,8 @@ public class MetadataTool extends ExecutableApplication { val optionsHelper = new OptionsHelper(); optionsHelper.parseOptions(tool.getOptions(), args); boolean isBackup = optionsHelper.hasOption(OPERATE_BACKUP); - if (isBackup && ScreenPrintUtil.isMainThread()) { + boolean isFetch = optionsHelper.hasOption(OPERATE_FETCH); + if ((isBackup || isFetch) && ScreenPrintUtil.isMainThread()) { config.setProperty("kylin.env.metadata.only-for-read", "true"); } val resourceStore = ResourceStore.getKylinMetaStore(config); @@ -258,12 +271,15 @@ public class MetadataTool extends ExecutableApplication { final OptionGroup optionGroup = new OptionGroup(); optionGroup.setRequired(true); optionGroup.addOption(OPERATE_BACKUP); + optionGroup.addOption(OPERATE_FETCH); + optionGroup.addOption(OPERATE_LIST); optionGroup.addOption(OPERATE_RESTORE); options.addOptionGroup(optionGroup); options.addOption(OPTION_DIR); options.addOption(OPTION_PROJECT); options.addOption(FOLDER_NAME); + options.addOption(OPTION_TARGET); options.addOption(OPERATE_COMPRESS); options.addOption(OPTION_EXCLUDE_TABLE_EXD); options.addOption(OPTION_AFTER_TRUNCATE); @@ -305,6 +321,10 @@ public class MetadataTool extends ExecutableApplication { } } + } else if (optionsHelper.hasOption(OPERATE_FETCH)) { + fetch(optionsHelper); + } else if (optionsHelper.hasOption(OPERATE_LIST)) { + list(optionsHelper); } else if (optionsHelper.hasOption(OPERATE_RESTORE)) { restore(optionsHelper, optionsHelper.hasOption(OPTION_AFTER_TRUNCATE)); } else { @@ -330,6 +350,68 @@ public class MetadataTool extends ExecutableApplication { } } + private void fetch(OptionsHelper optionsHelper) throws Exception { + var path = optionsHelper.getOptionValue(OPTION_DIR); + var folder = optionsHelper.getOptionValue(FOLDER_NAME); + val excludeTableExd = optionsHelper.hasOption(OPTION_EXCLUDE_TABLE_EXD); + val target = optionsHelper.getOptionValue(OPTION_TARGET); + if (StringUtils.isBlank(path)) { + path = KylinConfigBase.getKylinHome() + File.separator + "meta_fetch"; + } + if (StringUtils.isEmpty(folder)) { + folder = LocalDateTime.now(Clock.systemDefaultZone()).format(DATE_TIME_FORMATTER) + "_fetch"; + } + if (target == null) { + System.out.println("target file must be set with fetch mode"); + } else { + fetchPath = StringUtils.appendIfMissing(path, "/") + folder; + // currently do not support compress with fetch + val fetchMetadataUrl = getMetadataUrl(fetchPath, false); + val fetchConfig = KylinConfig.createKylinConfig(kylinConfig); + fetchConfig.setMetadataUrl(fetchMetadataUrl); + abortIfAlreadyExists(fetchPath); + logger.info("The fetch metadataUrl is {} and backup path is {}", fetchMetadataUrl, fetchPath); + + try (val fetchResourceStore = ResourceStore.getKylinMetaStore(fetchConfig)) { + + val fetchMetadataStore = fetchResourceStore.getMetadataStore(); + + String targetPath = target.startsWith("/") ? target.substring(1) : target; + + logger.info("start to copy target file {} from ResourceStore.", target); + UnitOfWork.doInTransactionWithRetry( + UnitOfWorkParams.builder().readonly(true).unitName(target).processor(() -> { + copyResourceStore("/" + targetPath, resourceStore, fetchResourceStore, true, excludeTableExd); + // uuid + val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG); + fetchResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(), + uuid.getTimestamp(), -1); + return null; + }).build()); + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("metadata task is interrupt"); + } + logger.info("start to fetch target file {}", target); + + // fetchResourceStore is read-only, currently we don't do any write operation on it. + // fetchResourceStore.deleteResource(ResourceStore.METASTORE_TRASH_RECORD); + fetchMetadataStore.dump(fetchResourceStore); + logger.info("fetch successfully at {}", fetchPath); + } + } + } + + private NavigableSet<String> list(OptionsHelper optionsHelper) throws Exception { + val target = optionsHelper.getOptionValue(OPTION_TARGET); + var res = resourceStore.listResources(target); + if (res == null) { + System.out.printf("%s is not exist%n", target); + } else { + System.out.println("" + res); + } + return res; + } + private void backup(OptionsHelper optionsHelper) throws Exception { val project = optionsHelper.getOptionValue(OPTION_PROJECT); var path = optionsHelper.getOptionValue(OPTION_DIR); diff --git a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java index b82216ab9f..d8ddd329aa 100644 --- a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java +++ b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java @@ -124,6 +124,38 @@ public class MetadataToolTest extends NLocalFileMetadataTestCase { return tool; } + @Test + public void testFetchTargetFile() throws IOException { + val junitFolder = temporaryFolder.getRoot(); + val tool = new MetadataTool(getTestConfig()); + // test case for fetching a specific file + tool.execute(new String[] { + "-fetch", "-target", "default/table/DEFAULT.STREAMING_TABLE.json", "-dir", junitFolder.getAbsolutePath(), "-folder", "target_fetch" + }); + //test case for fetching a folder + tool.execute(new String[] { + "-fetch", "-target", "_global", "-dir", junitFolder.getAbsolutePath(), "-folder", "target_fetch_global" + }); + + Assertions.assertThat(junitFolder.listFiles()).hasSize(2); + val archiveFolder = junitFolder.listFiles()[1]; + val globleFolder = junitFolder.listFiles()[0]; + Assertions.assertThat(archiveFolder).exists(); + + Assertions.assertThat(archiveFolder.list()).isNotEmpty().containsOnly("default", "UUID"); + + val projectFolder = findFile(archiveFolder.listFiles(), f -> f.getName().equals("default")); + assertProjectFolder(projectFolder, globleFolder); + } + + @Test + public void testListFile() { + val tool = new MetadataTool(getTestConfig()); + tool.execute(new String[] { + "-list", "-target", "default" + }); + } + @Test public void testBackupProject() throws IOException { val junitFolder = temporaryFolder.getRoot();