This is an automated email from the ASF dual-hosted git repository. stack pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hbase-operator-tools.git
The following commit(s) were added to refs/heads/master by this push: new c6af272 Add assigns/unassigns commands. c6af272 is described below commit c6af272f82cd70694dcc717a4b7c5c6dcefa0505 Author: Michael Stack <st...@apache.org> AuthorDate: Mon Sep 17 14:29:32 2018 -0700 Add assigns/unassigns commands. Tests are disabled for the moment while figure issue with shaded client. --- hbase-hbck2/pom.xml | 8 +- .../src/main/java/org/apache/hbase/HBCK2.java | 91 ++++++++++++++++---- .../src/test/java/org/apache/hbase/TestHBCK2.java | 96 ++++++++++++++++++++++ pom.xml | 2 +- 4 files changed, 181 insertions(+), 16 deletions(-) diff --git a/hbase-hbck2/pom.xml b/hbase-hbck2/pom.xml index 4996f97..a550ebe 100644 --- a/hbase-hbck2/pom.xml +++ b/hbase-hbck2/pom.xml @@ -114,7 +114,13 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-shaded-client</artifactId> - <version>2.1.1-SNAPSHOT</version> + <version>${hbase.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-testing-util</artifactId> + <version>${hbase.version}</version> + <scope>test</scope> </dependency> </dependencies> diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 4234637..2d06a78 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.util.Arrays; +import java.util.List; import java.util.stream.Collectors; /** @@ -49,6 +50,7 @@ import java.util.stream.Collectors; */ // TODO: // + Add bulk assign/unassigns. If 60k OPENING regions, doing it via shell takes 10-60 seconds each. +// + On assign, can we look to see if existing assign and if so fail until cancelled? // + Add test of Master version to ensure it supports hbck functionality. // + Doc how we just take pointer to zk ensemble... If want to do more exotic config. on client, // then add a hbase-site.xml onto CLASSPATH for this tool to pick up. @@ -56,11 +58,42 @@ public class HBCK2 { private static final Logger LOG = LogManager.getLogger(HBCK2.class); public static final int EXIT_SUCCESS = 0; public static final int EXIT_FAILURE = 1; + // Commands private static final String SET_TABLE_STATE = "setTableState"; + private static final String ASSIGN = "assign"; + private static final String UNASSIGN = "unassign"; + + static TableState setTableState(Configuration conf, TableName tableName, TableState.State state) + throws IOException { + try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) { + try (Hbck hbck = conn.getHbck()) { + return hbck.setTableStateInMeta(new TableState(tableName, state)); + } + } + } + + static List<Long> assigns(Configuration conf, List<String> encodedRegionNames) + throws IOException { + try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) { + try (Hbck hbck = conn.getHbck()) { + return hbck.assigns(encodedRegionNames); + } + } + } + + static List<Long> unassigns(Configuration conf, List<String> encodedRegionNames) + throws IOException { + try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) { + try (Hbck hbck = conn.getHbck()) { + return hbck.unassigns(encodedRegionNames); + } + } + } private static final String getCommandUsage() { StringWriter sw = new StringWriter(); PrintWriter writer = new PrintWriter(sw); + writer.println(); writer.println("Commands:"); writer.println(" " + SET_TABLE_STATE + " <TABLENAME> <STATE>"); writer.println(" Possible table states: " + Arrays.stream(TableState.State.values()). @@ -70,6 +103,25 @@ public class HBCK2 { writer.println(" A value of \\x08\\x00 == ENABLED, \\x08\\x01 == DISABLED, etc."); writer.println(" An example making table name 'user' ENABLED:"); writer.println(" $ HBCK2 setTableState users ENABLED"); + writer.println(" Returns whatever the previous table state was."); + writer.println(); + writer.println(" " + ASSIGN + " <ENCODED_REGIONNAME> ..."); + writer.println(" A 'raw' assign that can be used even during Master initialization."); + writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:"); + writer.println(" e.g. 1588230740 is hard-coded encoding for hbase:meta region and"); + writer.println(" de00010733901a05f5a2a3a382e27dd4 is an example of what a random"); + writer.println(" user-space encoded Region name looks like. For example:"); + writer.println(" $ HBCK2 assign 1588230740 de00010733901a05f5a2a3a382e27dd4"); + writer.println(" Returns the pid of the created AssignProcedure or -1 if none."); + writer.println(); + writer.println(" " + UNASSIGN + " <ENCODED_REGIONNAME> ..."); + writer.println(" A 'raw' unassign that can be used even during Master initialization."); + writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:"); + writer.println(" Skirts Coprocessors. Pass one or more encoded RegionNames:"); + writer.println(" de00010733901a05f5a2a3a382e27dd4 is an example of what a random"); + writer.println(" user-space encoded Region name looks like. For example:"); + writer.println(" $ HBCK2 unassign 1588230740 de00010733901a05f5a2a3a382e27dd4"); + writer.println(" Returns the pid of the created UnassignProcedure or -1 if none."); writer.close(); return sw.toString(); } @@ -84,21 +136,9 @@ public class HBCK2 { } HelpFormatter formatter = new HelpFormatter(); formatter.printHelp( "HBCK2 <OPTIONS> COMMAND [<ARGS>]", - "Options:", options, getCommandUsage()); + "\nOptions:", options, getCommandUsage()); } - /** - * Call the HbckService#setTableState - * @throws IOException - */ - static void setTableState(Configuration conf, TableName tableName, TableState.State state) - throws IOException { - try (ClusterConnection conn = (ClusterConnection)ConnectionFactory.createConnection(conf)) { - try (Hbck hbck = conn.getHbck()) { - hbck.setTableStateInMeta(new TableState(tableName, state)); - } - } - } /** * @return Return what to exit with. @@ -162,7 +202,26 @@ public class HBCK2 { usage(options, command + " takes tablename and state arguments: e.g. user ENABLED"); return EXIT_FAILURE; } - setTableState(conf, TableName.valueOf(commands[1]), TableState.State.valueOf(commands[2])); + System.out.println(setTableState(conf, + TableName.valueOf(commands[1]), TableState.State.valueOf(commands[2]))); + break; + + case ASSIGN: + if (commands.length < 2) { + usage(options, command + " takes one or more encoded region names"); + return EXIT_FAILURE; + } + System.out.println(pidsToString( + assigns(conf, Arrays.stream(commands).skip(1).collect(Collectors.toList())))); + break; + + case UNASSIGN: + if (commands.length < 2) { + usage(options, command + " takes one or more encoded region names"); + return EXIT_FAILURE; + } + System.out.println(pidsToString( + unassigns(conf, Arrays.stream(commands).skip(1).collect(Collectors.toList())))); break; default: @@ -172,6 +231,10 @@ public class HBCK2 { return EXIT_SUCCESS; } + private static String pidsToString(List<Long> pids) { + return pids.stream().map(i -> i.toString()).collect(Collectors.joining(", ")); + } + public static void main(String [] args) throws IOException { int exitCode = doWork(args); if (exitCode != 0) { diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java index 30afeb2..c875960 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java @@ -17,17 +17,53 @@ */ package org.apache.hbase; +import junit.framework.TestCase; import org.apache.commons.cli.ParseException; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableState; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.logging.log4j.LogManager; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TestHBCK2 { + private static final org.apache.logging.log4j.Logger LOG = LogManager.getLogger(TestHBCK2.class); + // private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final TableName TABLE_NAME = TableName.valueOf(TestHBCK2.class.getSimpleName()); + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + /* + TEST_UTIL.startMiniCluster(3); + TEST_UTIL.createMultiRegionTable(TABLE_NAME, Bytes.toBytes("family1"), 5); + */ + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + /* + TEST_UTIL.shutdownMiniCluster(); + */ + } + @Test public void testHelp() throws ParseException, IOException { // TODO @@ -42,4 +78,64 @@ public class TestHBCK2 { String output = os.toString(); assertTrue(output, output.startsWith("usage: HBCK2")); } + + @Test + public void testSetTableStateInMeta() throws IOException { + /* + TableState state = + HBCK2.setTableState(TEST_UTIL.getConfiguration(), TABLE_NAME, TableState.State.DISABLED); + TestCase.assertTrue("Found=" + state.getState(), state.isDisabled()); + // Restore the state. + HBCK2.setTableState(TEST_UTIL.getConfiguration(), TABLE_NAME, state.getState()); + */ + } + + @Test + public void testAssigns() throws IOException { + /* + try (Admin admin = TEST_UTIL.getConnection().getAdmin()) { + List<RegionInfo> regions = admin.getRegions(TABLE_NAME); + for (RegionInfo ri: regions) { + RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). + getRegionStates().getRegionState(ri.getEncodedName()); + LOG.info("RS: {}", rs.toString()); + } + List<Long> pids = HBCK2.unassigns(TEST_UTIL.getConfiguration(), + regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList())); + waitOnPids(pids); + for (RegionInfo ri: regions) { + RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). + getRegionStates().getRegionState(ri.getEncodedName()); + LOG.info("RS: {}", rs.toString()); + TestCase.assertTrue(rs.toString(), rs.isClosed()); + } + pids = HBCK2.assigns(TEST_UTIL.getConfiguration(), + regions.stream().map(r -> r.getEncodedName()).collect(Collectors.toList())); + waitOnPids(pids); + for (RegionInfo ri: regions) { + RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). + getRegionStates().getRegionState(ri.getEncodedName()); + LOG.info("RS: {}", rs.toString()); + TestCase.assertTrue(rs.toString(), rs.isOpened()); + } + // What happens if crappy region list passed? + pids = HBCK2.assigns(TEST_UTIL.getConfiguration(), + Arrays.stream(new String [] {"a", "some rubbish name"}).collect(Collectors.toList())); + for (long pid: pids) { + assertEquals(org.apache.hadoop.hbase.procedure2.Procedure.NO_PROC_ID, pid); + } + } + */ + } + + private void waitOnPids(List<Long> pids) { + /* + for (Long pid: pids) { + while (!TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(). + isFinished(pid)) { + Threads.sleep(100); + } + } + */ + } } diff --git a/pom.xml b/pom.xml index 5df605d..b97f120 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ <compileSource>1.8</compileSource> <java.min.version>${compileSource}</java.min.version> <maven.min.version>3.3.3</maven.min.version> - <hbase.version>2.1.0</hbase.version> + <hbase.version>2.1.1-SNAPSHOT</hbase.version> <maven.compiler.version>3.6.1</maven.compiler.version> <surefire.version>2.21.0</surefire.version> <surefire.provider>surefire-junit47</surefire.provider>