Repository: hbase Updated Branches: refs/heads/HBASE-7912 5db8995f4 -> d269a1419
HBASE-15590 Add ACL for requesting table backup Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d269a141 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d269a141 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d269a141 Branch: refs/heads/HBASE-7912 Commit: d269a141926a1bc414bef3662f4b18da03947d46 Parents: 5db8995 Author: tedyu <[email protected]> Authored: Wed May 11 07:36:35 2016 -0700 Committer: tedyu <[email protected]> Committed: Wed May 11 07:36:35 2016 -0700 ---------------------------------------------------------------------- .../BaseMasterAndRegionObserver.java | 14 +++++++++++ .../hbase/coprocessor/BaseMasterObserver.java | 14 +++++++++++ .../hbase/coprocessor/MasterObserver.java | 26 ++++++++++++++++++++ .../hbase/master/MasterCoprocessorHost.java | 24 ++++++++++++++++++ .../hadoop/hbase/master/MasterRpcServices.java | 20 ++++++++++++--- .../hadoop/hbase/master/MasterServices.java | 2 +- .../hbase/security/access/AccessController.java | 14 +++++++++++ .../visibility/VisibilityController.java | 14 +++++++++++ .../hbase/coprocessor/TestMasterObserver.java | 15 +++++++++++ .../security/access/TestAccessController.java | 17 +++++++++++++ 10 files changed, 156 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java index 428840c..5fea0c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterAndRegionObserver.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Admin; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.util.Pair; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) @InterfaceStability.Evolving @@ -435,6 +437,18 @@ public class BaseMasterAndRegionObserver extends BaseRegionObserver } @Override + public void preBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final String targetRootDir, + final int workers, final long bandwidth) throws IOException { + } + + @Override + public void postBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final Pair<Long, String> pair) + throws IOException { + } + + @Override public void preBalance(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException { } http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java index 7da63bf..0466193 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseMasterObserver.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Admin; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.util.Pair; @InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.CONFIG}) @InterfaceStability.Evolving @@ -438,6 +440,18 @@ public class BaseMasterObserver implements MasterObserver { } @Override + public void preBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final String targetRootDir, + final int workers, final long bandwidth) throws IOException { + } + + @Override + public void postBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final Pair<Long, String> pair) + throws IOException { + } + + @Override public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx, final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException { return false; http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 230f4ce..21af755 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Admin; @@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.util.Pair; /** * Defines coprocessor hooks for interacting with operations on the @@ -798,6 +800,30 @@ public interface MasterObserver extends Coprocessor { throws IOException; /** + * Called prior to backing up tables + * @param ctx the coprocessor instance's environment + * @param type the type of backup + * @param tablesList list of tables to backup + * @param targetRootDir root directory + * @param workers number of parallel workers + * @param bandwidth bandwidth per worker in MB per sec + */ + void preBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final String targetRootDir, + final int workers, final long bandwidth) throws IOException; + + /** + * Called after backing up tables + * @param ctx the coprocessor instance's environment + * @param type the type of backup + * @param tablesList list of tables to backup + * @param pair the pair of procedure Id and backup Id + */ + void postBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final Pair<Long, String> pair) + throws IOException; + + /** * Called prior to setting split / merge switch * @param ctx the coprocessor instance's environment * @param newValue the new value submitted in the call http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 53582cd..93c4ca2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; @@ -45,6 +46,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.util.Pair; /** * Provides the coprocessor framework and environment for master oriented @@ -772,6 +774,28 @@ public class MasterCoprocessorHost }); } + public boolean preBackupTables(final BackupType type, final List<TableName> tablesList, + final String targetRootDir, final int workers, final long bandwidth) throws IOException { + return execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() { + @Override + public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx) + throws IOException { + oserver.preBackupTables(ctx, type, tablesList, targetRootDir, workers, bandwidth); + } + }); + } + + public void postBackupTables(final BackupType type, final List<TableName> tablesList, + final Pair<Long, String> pair) throws IOException { + execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() { + @Override + public void call(MasterObserver oserver, ObserverContext<MasterCoprocessorEnvironment> ctx) + throws IOException { + oserver.postBackupTables(ctx, type, tablesList, pair); + } + }); + } + public boolean preSetSplitOrMergeEnabled(final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException { return execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() { http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 930bfd6..53486cf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1062,9 +1062,23 @@ public class MasterRpcServices extends RSRpcServices for (HBaseProtos.TableName table : request.getTablesList()) { tablesList.add(ProtobufUtil.toTableName(table)); } - Pair<Long, String> pair = master.backupTables( - BackupType.valueOf(request.getType().name()), tablesList, request.getTargetRootDir(), - (int)request.getWorkers(), request.getBandwidth()); + BackupType type = BackupType.valueOf(request.getType().name()); + boolean bypass = false; + if (master.cpHost != null) { + bypass = master.cpHost.preBackupTables(type, tablesList, request.getTargetRootDir(), + (int)request.getWorkers(), request.getBandwidth()); + } + Pair<Long, String> pair; + if (!bypass) { + pair = master.backupTables( + type, tablesList, request.getTargetRootDir(), + (int)request.getWorkers(), request.getBandwidth()); + if (master.cpHost != null) { + master.cpHost.postBackupTables(type, tablesList, pair); + } + } else { + pair = new Pair<>(-1L, ""); + } return response.setProcId(pair.getFirst()).setBackupId(pair.getSecond()).build(); } catch (IOException e) { throw new ServiceException(e); http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 3557bb4..e89bf8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -186,7 +186,7 @@ public interface MasterServices extends Server { * @param type whether the backup is full or incremental * @param tableList list of tables to backup * @param targetRootDir root dir for saving the backup - * @param workers number of paralle workers. -1 - system defined + * @param workers number of parallel workers. -1 - system defined * @param bandwidth bandwidth per worker in MB per sec. -1 - unlimited * @return pair of procedure Id and backupId * @throws IOException http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index f81ab8c..6ce984b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagRewriteCell; import org.apache.hadoop.hbase.TagUtil; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Append; @@ -1259,6 +1260,19 @@ public class AccessController extends BaseMasterAndRegionObserver } @Override + public void preBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final String targetRootDir, + final int workers, final long bandwidth) throws IOException { + requirePermission("backupTables", Action.ADMIN); + } + + @Override + public void postBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final Pair<Long, String> pair) + throws IOException { + } + + @Override public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx, final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException { requirePermission("setSplitOrMergeEnabled", Action.ADMIN); http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java index 6e2f8ed..519e84e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagRewriteCell; import org.apache.hadoop.hbase.TagType; import org.apache.hadoop.hbase.TagUtil; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Append; @@ -103,6 +104,7 @@ import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.Superusers; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessController; +import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -309,6 +311,18 @@ public class VisibilityController extends BaseMasterAndRegionObserver implements } @Override + public void preBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final String targetRootDir, + final int workers, final long bandwidth) throws IOException { + } + + @Override + public void postBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final Pair<Long, String> pair) + throws IOException { + } + + @Override public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx, final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException { return false; http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java index 5b0d202..ccef3be 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; @@ -64,9 +65,11 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescripto import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -345,6 +348,18 @@ public class TestMasterObserver { } @Override + public void preBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, + final String targetRootDir, final int workers, final long bandwidth) throws IOException { + } + + @Override + public void postBackupTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, + final BackupType type, final List<TableName> tablesList, final Pair<Long, String> pair) + throws IOException { + } + + @Override public boolean preSetSplitOrMergeEnabled(final ObserverContext<MasterCoprocessorEnvironment> ctx, final boolean newValue, final Admin.MasterSwitchType switchType) throws IOException { return false; http://git-wip-us.apache.org/repos/asf/hbase/blob/d269a141/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 7317abb..f6b36cd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.ArrayBackedTag; +import org.apache.hadoop.hbase.backup.BackupType; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Connection; @@ -723,6 +724,22 @@ public class TestAccessController extends SecureTestUtil { } @Test (timeout=180000) + public void testbackupTables() throws Exception { + AccessTestAction action = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preBackupTables(ObserverContext.createAndPrepare(CP_ENV, null), + BackupType.FULL, null, null, 0, 0); + return null; + } + }; + + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, + USER_GROUP_WRITE, USER_GROUP_CREATE); + } + + @Test (timeout=180000) public void testBalance() throws Exception { AccessTestAction action = new AccessTestAction() { @Override
