Repository: kylin Updated Branches: refs/heads/2.x-staging ea95e6b0d -> 109e1daf8
KYLIN-1327 Tool for batch updating host information of htables Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2789868 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2789868 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2789868 Branch: refs/heads/2.x-staging Commit: b2789868ac697424409080414f1396f491a1385b Parents: ea95e6b Author: lidongsjtu <don...@ebay.com> Authored: Mon Jan 18 16:18:51 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Tue Jan 19 17:15:43 2016 +0800 ---------------------------------------------------------------------- .../storage/hbase/util/UpdateHTableHostCLI.java | 192 +++++++++++++++++++ 1 file changed, 192 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/b2789868/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java new file mode 100644 index 0000000..1ff98a4 --- /dev/null +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/UpdateHTableHostCLI.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.storage.hbase.util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeSegment; +import org.apache.kylin.invertedindex.IIInstance; +import org.apache.kylin.invertedindex.IIManager; +import org.apache.kylin.invertedindex.IISegment; +import org.apache.kylin.metadata.model.SegmentStatusEnum; +import org.apache.kylin.metadata.realization.IRealizationConstants; +import org.apache.kylin.metadata.realization.RealizationStatusEnum; +import org.apache.kylin.storage.hbase.HBaseConnection; + +import com.google.common.collect.Lists; + +/** + * Created by dongli on 1/18/16. + */ +public class UpdateHTableHostCLI { + private static final Log logger = LogFactory.getLog(UpdateHTableHostCLI.class); + private List<String> updatedResources = Lists.newArrayList(); + private List<String> errorMsgs = Lists.newArrayList(); + + private List<String> htables; + private HBaseAdmin hbaseAdmin; + private KylinConfig kylinConfig; + + public UpdateHTableHostCLI(List<String> htables) throws IOException { + this.htables = htables; + this.hbaseAdmin = new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration()); + this.kylinConfig = KylinConfig.getInstanceFromEnv(); + } + + public static void main(String args[]) throws Exception { + if (args.length < 1) { + printUsageAndExit(); + } + + List<String> tableNames = getHTableNames(KylinConfig.getInstanceFromEnv()); + String filterType = args[0].toLowerCase(); + if (filterType.equals("-table")) { + tableNames = filterByTables(tableNames, Arrays.asList(args).subList(1, args.length)); + } else if (filterType.equals("-cube")) { + tableNames = filterByCubes(tableNames, Arrays.asList(args).subList(1, args.length)); + } else if (!filterType.equals("-all")) { + printUsageAndExit(); + } + + UpdateHTableHostCLI updateHTableHostCLI = new UpdateHTableHostCLI(tableNames); + updateHTableHostCLI.execute(); + + logger.info("================================================================="); + logger.info("Run UpdateHTableHostCLI completed;"); + + if (!updateHTableHostCLI.updatedResources.isEmpty()) { + logger.info("Following resources are updated successfully:"); + for (String s : updateHTableHostCLI.updatedResources) { + logger.info(s); + } + } else { + logger.warn("No resource updated."); + } + + if (!updateHTableHostCLI.errorMsgs.isEmpty()) { + logger.info("Here are the error/warning messages, you may need to check:"); + for (String s : updateHTableHostCLI.errorMsgs) { + logger.warn(s); + } + } else { + logger.info("No error or warning messages; The update succeeds."); + } + + logger.info("================================================================="); + } + + private static void printUsageAndExit() { + logger.info("Usage: exec -all|-cube cubeA,cubeB|-table tableA,tableB"); + System.exit(0); + } + + private static List<String> getHTableNames(KylinConfig config) { + CubeManager cubeMgr = CubeManager.getInstance(config); + + ArrayList<String> result = new ArrayList<String>(); + for (CubeInstance cube : cubeMgr.listAllCubes()) { + for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { + String tableName = seg.getStorageLocationIdentifier(); + if (!StringUtils.isBlank(tableName)) { + result.add(tableName); + System.out.println("added new table: " + tableName); + } + } + } + + for (IIInstance ii : IIManager.getInstance(config).listAllIIs()) { + if (ii.getStatus() == RealizationStatusEnum.READY) { + for (IISegment seg : ii.getSegments()) {//streaming segment is never "READY" + String tableName = seg.getStorageLocationIdentifier(); + if (!StringUtils.isBlank(tableName)) { + result.add(tableName); + System.out.println("added new table: " + tableName); + } + } + } + } + + return result; + } + + private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) { + CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); + List<String> result = Lists.newArrayList(); + for (String c : cubeNames) { + c = c.trim(); + if (c.endsWith(",")) + c = c.substring(0, c.length() - 1); + + CubeInstance cubeInstance = cubeManager.getCube(c); + for (CubeSegment segment : cubeInstance.getSegments()) { + String tableName = segment.getStorageLocationIdentifier(); + if (allTableNames.contains(tableName)) { + result.add(tableName); + } + } + } + return result; + } + + private static List<String> filterByTables(List<String> allTableNames, List<String> tableNames) { + List<String> result = Lists.newArrayList(); + for (String t : tableNames) { + t = t.trim(); + if (t.endsWith(",")) + t = t.substring(0, t.length() - 1); + + if (allTableNames.contains(t)) { + result.add(t); + } + } + return result; + } + + private void updateHtable(String tableName) throws IOException { + HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName)); + hbaseAdmin.disableTable(tableName); + desc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix()); + hbaseAdmin.modifyTable(tableName, desc); + hbaseAdmin.enableTable(tableName); + } + + public void execute() { + for (String htable : htables) { + try { + updateHtable(htable); + updatedResources.add(htable); + } catch (IOException ex) { + ex.printStackTrace(); + errorMsgs.add("Update HTable[" + htable + "] failed: " + ex.getMessage()); + } + } + } +}
