http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index 347d01d..683d840 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -48,9 +48,9 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; @@ -69,6 +69,7 @@ public class CloneSnapshotProcedure private HTableDescriptor hTableDescriptor; private SnapshotDescription snapshot; + private boolean restoreAcl; private List<HRegionInfo> newRegions = null; private Map<String, Pair<String, String> > parentsToChildrenPairMap = new HashMap<>(); @@ -83,6 +84,11 @@ public class CloneSnapshotProcedure public CloneSnapshotProcedure() { } + public CloneSnapshotProcedure(final MasterProcedureEnv env, + final HTableDescriptor hTableDescriptor, final SnapshotDescription snapshot) { + this(env, hTableDescriptor, snapshot, false); + } + /** * Constructor * @param env MasterProcedureEnv @@ -90,10 +96,12 @@ public class CloneSnapshotProcedure * @param snapshot snapshot to clone from */ public CloneSnapshotProcedure(final MasterProcedureEnv env, - final HTableDescriptor hTableDescriptor, final SnapshotDescription snapshot) { + final HTableDescriptor hTableDescriptor, final SnapshotDescription snapshot, + final boolean restoreAcl) { super(env); this.hTableDescriptor = hTableDescriptor; this.snapshot = snapshot; + this.restoreAcl = restoreAcl; getMonitorStatus(); } @@ -109,6 +117,14 @@ public class CloneSnapshotProcedure return monitorStatus; } + private void restoreSnapshotAcl(MasterProcedureEnv env) throws IOException { + Configuration conf = env.getMasterServices().getConfiguration(); + if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null + && SnapshotDescriptionUtils.isSecurityAvailable(conf)) { + RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, hTableDescriptor.getTableName(), conf); + } + } + @Override protected Flow executeFromState(final MasterProcedureEnv env, final CloneSnapshotState state) throws InterruptedException { @@ -138,6 +154,10 @@ public class CloneSnapshotProcedure break; case CLONE_SNAPSHOT_UPDATE_DESC_CACHE: CreateTableProcedure.updateTableDescCache(env, getTableName()); + setNextState(CloneSnapshotState.CLONE_SNAPHOST_RESTORE_ACL); + break; + case CLONE_SNAPHOST_RESTORE_ACL: + restoreSnapshotAcl(env); setNextState(CloneSnapshotState.CLONE_SNAPSHOT_POST_OPERATION); break; case CLONE_SNAPSHOT_POST_OPERATION:
http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java index f8c9d8f..21709f8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java @@ -48,9 +48,9 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreSnapshotState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; @@ -69,6 +69,7 @@ public class RestoreSnapshotProcedure private Map<String, Pair<String, String>> parentsToChildrenPairMap = new HashMap<>(); private SnapshotDescription snapshot; + private boolean restoreAcl; // Monitor private MonitoredTask monitorStatus = null; @@ -81,6 +82,10 @@ public class RestoreSnapshotProcedure public RestoreSnapshotProcedure() { } + public RestoreSnapshotProcedure(final MasterProcedureEnv env, + final HTableDescriptor hTableDescriptor, final SnapshotDescription snapshot) { + this(env, hTableDescriptor, snapshot, false); + } /** * Constructor * @param env MasterProcedureEnv @@ -91,12 +96,14 @@ public class RestoreSnapshotProcedure public RestoreSnapshotProcedure( final MasterProcedureEnv env, final HTableDescriptor hTableDescriptor, - final SnapshotDescription snapshot) { + final SnapshotDescription snapshot, + final boolean restoreAcl) { super(env); // This is the new schema we are going to write out as this modification. this.modifiedHTableDescriptor = hTableDescriptor; // Snapshot information this.snapshot = snapshot; + this.restoreAcl = restoreAcl; // Monitor getMonitorStatus(); @@ -140,6 +147,10 @@ public class RestoreSnapshotProcedure break; case RESTORE_SNAPSHOT_UPDATE_META: updateMETA(env); + setNextState(RestoreSnapshotState.RESTORE_SNAPSHOT_RESTORE_ACL); + break; + case RESTORE_SNAPSHOT_RESTORE_ACL: + restoreSnapshotAcl(env); return Flow.NO_MORE_STATE; default: throw new UnsupportedOperationException("unhandled state=" + state); @@ -474,6 +485,16 @@ public class RestoreSnapshotProcedure monitorStatus.getCompletionTimestamp() - monitorStatus.getStartTime()); } + private void restoreSnapshotAcl(final MasterProcedureEnv env) throws IOException { + if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null + && SnapshotDescriptionUtils + .isSecurityAvailable(env.getMasterServices().getConfiguration())) { + // restore acl of snapshot to table. + RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, TableName.valueOf(snapshot.getTable()), + env.getMasterServices().getConfiguration()); + } + } + /** * Make sure that region states of the region list is in OFFLINE state. * @param env MasterProcedureEnv http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java index 5f86e08..c23ca6d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/DisabledTableSnapshotHandler.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotManifest; import org.apache.hadoop.hbase.util.FSUtils; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java index 73cd4d7..8d6568f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/EnabledTableSnapshotHandler.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.procedure.Procedure; import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; import org.apache.hadoop.hbase.util.Pair; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java index de46268..0448f92 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index 55d58e0..b81c7db 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -67,10 +67,10 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameStringPair; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription.Type; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription.Type; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; @@ -680,7 +680,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable */ private long cloneSnapshot(final SnapshotDescription reqSnapshot, final TableName tableName, final SnapshotDescription snapshot, final HTableDescriptor snapshotTableDesc, - final NonceKey nonceKey) throws IOException { + final NonceKey nonceKey, final boolean restoreAcl) throws IOException { MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); HTableDescriptor htd = new HTableDescriptor(tableName, snapshotTableDesc); if (cpHost != null) { @@ -688,7 +688,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable } long procId; try { - procId = cloneSnapshot(snapshot, htd, nonceKey); + procId = cloneSnapshot(snapshot, htd, nonceKey, restoreAcl); } catch (IOException e) { LOG.error("Exception occurred while cloning the snapshot " + snapshot.getName() + " as table " + tableName.getNameAsString(), e); @@ -712,7 +712,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable * @return procId the ID of the clone snapshot procedure */ synchronized long cloneSnapshot(final SnapshotDescription snapshot, - final HTableDescriptor hTableDescriptor, final NonceKey nonceKey) + final HTableDescriptor hTableDescriptor, final NonceKey nonceKey, final boolean restoreAcl) throws HBaseSnapshotException { TableName tableName = hTableDescriptor.getTableName(); @@ -728,8 +728,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable try { long procId = master.getMasterProcedureExecutor().submitProcedure( - new CloneSnapshotProcedure( - master.getMasterProcedureExecutor().getEnvironment(), hTableDescriptor, snapshot), + new CloneSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(), + hTableDescriptor, snapshot, restoreAcl), nonceKey); this.restoreTableToProcIdMap.put(tableName, procId); return procId; @@ -747,8 +747,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable * @param nonceKey unique identifier to prevent duplicated RPC * @throws IOException */ - public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey) - throws IOException { + public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey, + final boolean restoreAcl) throws IOException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir); @@ -777,28 +777,30 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable // Execute the restore/clone operation long procId; if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) { - procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey); + procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, + restoreAcl); } else { - procId = cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey); + procId = + cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl); } return procId; } /** - * Restore the specified snapshot. - * The restore will fail if the destination table has a snapshot or restore in progress. - * + * Restore the specified snapshot. The restore will fail if the destination table has a snapshot + * or restore in progress. * @param reqSnapshot Snapshot Descriptor from request * @param tableName table to restore * @param snapshot Snapshot Descriptor * @param snapshotTableDesc Table Descriptor * @param nonceKey unique identifier to prevent duplicated RPC + * @param restoreAcl true to restore acl of snapshot * @return procId the ID of the restore snapshot procedure * @throws IOException */ private long restoreSnapshot(final SnapshotDescription reqSnapshot, final TableName tableName, final SnapshotDescription snapshot, final HTableDescriptor snapshotTableDesc, - final NonceKey nonceKey) throws IOException { + final NonceKey nonceKey, final boolean restoreAcl) throws IOException { MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost(); if (master.getTableStateManager().isTableState( @@ -815,7 +817,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable long procId; try { - procId = restoreSnapshot(snapshot, snapshotTableDesc, nonceKey); + procId = restoreSnapshot(snapshot, snapshotTableDesc, nonceKey, restoreAcl); } catch (IOException e) { LOG.error("Exception occurred while restoring the snapshot " + snapshot.getName() + " as table " + tableName.getNameAsString(), e); @@ -831,16 +833,16 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable } /** - * Restore the specified snapshot. - * The restore will fail if the destination table has a snapshot or restore in progress. - * + * Restore the specified snapshot. The restore will fail if the destination table has a snapshot + * or restore in progress. * @param snapshot Snapshot Descriptor * @param hTableDescriptor Table Descriptor * @param nonceKey unique identifier to prevent duplicated RPC + * @param restoreAcl true to restore acl of snapshot * @return procId the ID of the restore snapshot procedure */ private synchronized long restoreSnapshot(final SnapshotDescription snapshot, - final HTableDescriptor hTableDescriptor, final NonceKey nonceKey) + final HTableDescriptor hTableDescriptor, final NonceKey nonceKey, final boolean restoreAcl) throws HBaseSnapshotException { final TableName tableName = hTableDescriptor.getTableName(); @@ -856,8 +858,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable try { long procId = master.getMasterProcedureExecutor().submitProcedure( - new RestoreSnapshotProcedure( - master.getMasterProcedureExecutor().getEnvironment(), hTableDescriptor, snapshot), + new RestoreSnapshotProcedure(master.getMasterProcedureExecutor().getEnvironment(), + hTableDescriptor, snapshot, restoreAcl), nonceKey); this.restoreTableToProcIdMap.put(tableName, procId); return procId; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java index 123758f..fce4eaa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java @@ -48,7 +48,7 @@ import org.apache.hadoop.hbase.master.locking.LockManager; import org.apache.hadoop.hbase.master.locking.LockProcedure; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 91fb44b..a4a7537 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -158,7 +158,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceCall; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionLoad; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.StoreSequenceId; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.FlushDescriptor; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java index 248ccdc..9c42e4d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/FlushSnapshotSubprocedure.java @@ -28,10 +28,10 @@ import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.procedure.ProcedureMember; import org.apache.hadoop.hbase.procedure.Subprocedure; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager.SnapshotSubprocedurePool; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils; /** http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java index 7b43c3d..a11844d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java @@ -51,10 +51,10 @@ import org.apache.hadoop.hbase.procedure.RegionServerProcedureManager; import org.apache.hadoop.hbase.procedure.Subprocedure; import org.apache.hadoop.hbase.procedure.SubprocedureFactory; import org.apache.hadoop.hbase.procedure.ZKProcedureMemberRpcs; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.KeeperException; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java index c1aad93..b719eba 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlLists.java @@ -461,7 +461,7 @@ public class AccessControlLists { return allPerms; } - static ListMultimap<String, TablePermission> getTablePermissions(Configuration conf, + public static ListMultimap<String, TablePermission> getTablePermissions(Configuration conf, TableName tableName) throws IOException { return getPermissions(conf, tableName != null ? tableName.getName() : null, null); } http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index 8a6eb96..aa0c094 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -113,8 +113,8 @@ import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.util.ByteRange; import org.apache.hadoop.hbase.util.Bytes; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java index e3ad951..e80410f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshot.java @@ -53,7 +53,7 @@ import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.WALLink; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mob.MobUtils; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.util.AbstractHBaseTool; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 63839c4..683c404 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -29,10 +29,12 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ThreadPoolExecutor; +import com.google.common.collect.ListMultimap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -46,13 +48,17 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.security.access.AccessControlClient; +import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil; +import org.apache.hadoop.hbase.security.access.TablePermission; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; @@ -825,4 +831,25 @@ public class RestoreSnapshotHelper { } return metaChanges; } + + public static void restoreSnapshotAcl(SnapshotDescription snapshot, TableName newTableName, + Configuration conf) throws IOException { + if (snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null) { + LOG.info("Restore snapshot acl to table. snapshot: " + snapshot + ", table: " + newTableName); + ListMultimap<String, TablePermission> perms = + ShadedAccessControlUtil.toUserTablePermissions(snapshot.getUsersAndPermissions()); + try (Connection conn = ConnectionFactory.createConnection(conf)) { + for (Entry<String, TablePermission> e : perms.entries()) { + String user = e.getKey(); + TablePermission perm = e.getValue(); + perm.setTableName(newTableName); + AccessControlClient.grant(conn, perm.getTableName(), user, perm.getFamily(), + perm.getQualifier(), perm.getActions()); + } + } catch (Throwable e) { + throw new IOException("Grant acl into newly creatd table failed. snapshot: " + snapshot + + ", table: " + newTableName, e); + } + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java index 0a482d8..c471337 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDescriptionUtils.java @@ -17,10 +17,11 @@ */ package org.apache.hadoop.hbase.snapshot; -import java.io.FileNotFoundException; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.Collections; +import com.google.common.collect.ListMultimap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -30,11 +31,17 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil; +import org.apache.hadoop.hbase.security.access.TablePermission; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.snapshot.SnapshotManifestV2; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; @@ -247,10 +254,10 @@ public final class SnapshotDescriptionUtils { * {@link SnapshotDescription}. */ public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf) - throws IllegalArgumentException { + throws IllegalArgumentException, IOException { if (!snapshot.hasTable()) { throw new IllegalArgumentException( - "Descriptor doesn't apply to a table, so we can't build it."); + "Descriptor doesn't apply to a table, so we can't build it."); } // set the creation time, if one hasn't been set @@ -263,6 +270,11 @@ public final class SnapshotDescriptionUtils { builder.setCreationTime(time); snapshot = builder.build(); } + + // set the acl to snapshot if security feature is enabled. + if (isSecurityAvailable(conf)) { + snapshot = writeAclToSnapshotDescription(snapshot, conf); + } return snapshot; } @@ -366,4 +378,26 @@ public final class SnapshotDescriptionUtils { if (!snapshot.hasOwner()) return false; return snapshot.getOwner().equals(user.getShortName()); } + + public static boolean isSecurityAvailable(Configuration conf) throws IOException { + try (Connection conn = ConnectionFactory.createConnection(conf)) { + try (Admin admin = conn.getAdmin()) { + return admin.tableExists(AccessControlLists.ACL_TABLE_NAME); + } + } + } + + private static SnapshotDescription writeAclToSnapshotDescription(SnapshotDescription snapshot, + Configuration conf) throws IOException { + ListMultimap<String, TablePermission> perms = + User.runAsLoginUser(new PrivilegedExceptionAction<ListMultimap<String, TablePermission>>() { + @Override + public ListMultimap<String, TablePermission> run() throws Exception { + return AccessControlLists.getTablePermissions(conf, + TableName.valueOf(snapshot.getTable())); + } + }); + return snapshot.toBuilder() + .setUsersAndPermissions(ShadedAccessControlUtil.toUserTablePermissions(perms)).build(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java index eb96438..d3f1cbc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotInfo.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.util.StringUtils; @@ -144,7 +145,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { private AtomicLong nonSharedHfilesArchiveSize = new AtomicLong(); private AtomicLong logSize = new AtomicLong(); - private final HBaseProtos.SnapshotDescription snapshot; + private final SnapshotProtos.SnapshotDescription snapshot; private final TableName snapshotTable; private final Configuration conf; private final FileSystem fs; @@ -159,7 +160,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { } SnapshotStats(final Configuration conf, final FileSystem fs, - final HBaseProtos.SnapshotDescription snapshot) { + final SnapshotProtos.SnapshotDescription snapshot) { this.snapshot = snapshot; this.snapshotTable = TableName.valueOf(snapshot.getTable()); this.conf = conf; @@ -234,7 +235,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { * with other snapshots and tables * * This is only calculated when - * {@link #getSnapshotStats(Configuration, HBaseProtos.SnapshotDescription, Map)} + * {@link #getSnapshotStats(Configuration, SnapshotProtos.SnapshotDescription, Map)} * is called with a non-null Map */ public long getNonSharedArchivedStoreFilesSize() { @@ -413,7 +414,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { return false; } - HBaseProtos.SnapshotDescription snapshotDesc = + SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc); return true; @@ -423,7 +424,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { * Dump the {@link SnapshotDescription} */ private void printInfo() { - HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); + SnapshotProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); System.out.println("Snapshot Info"); System.out.println("----------------------------------------"); @@ -457,7 +458,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { } // Collect information about hfiles and logs in the snapshot - final HBaseProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); + final SnapshotProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); final String table = snapshotDesc.getTable(); final SnapshotDescription desc = ProtobufUtil.createSnapshotDesc(snapshotDesc); final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc); @@ -552,7 +553,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { */ public static SnapshotStats getSnapshotStats(final Configuration conf, final SnapshotDescription snapshot) throws IOException { - HBaseProtos.SnapshotDescription snapshotDesc = + SnapshotProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); return getSnapshotStats(conf, snapshotDesc, null); } @@ -565,7 +566,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, - final HBaseProtos.SnapshotDescription snapshotDesc, + final SnapshotProtos.SnapshotDescription snapshotDesc, final Map<Path, Integer> filesMap) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); @@ -598,7 +599,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); List<SnapshotDescription> snapshotLists = new ArrayList<>(snapshots.length); for (FileStatus snapshotDirStat: snapshots) { - HBaseProtos.SnapshotDescription snapshotDesc = + SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()); snapshotLists.add(ProtobufUtil.createSnapshotDesc(snapshotDesc)); } @@ -621,7 +622,7 @@ public final class SnapshotInfo extends AbstractHBaseTool { final ConcurrentHashMap<Path, Integer> filesMap, final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize, final AtomicLong uniqueHFilesMobSize) throws IOException { - HBaseProtos.SnapshotDescription snapshotDesc = + SnapshotProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); Path rootDir = FSUtils.getRootDir(conf); final FileSystem fs = FileSystem.get(rootDir.toUri(), conf); http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index 4e838ad..c5b24e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -46,8 +46,8 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java index 46893f9..1fa9dc5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java index 567f42d..ccd3a7f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV2.java @@ -42,7 +42,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.FSUtils; @@ -127,7 +127,7 @@ public final class SnapshotManifestV2 { } static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf, - final Executor executor,final FileSystem fs, final Path snapshotDir, + final Executor executor, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final int manifestSizeLimit) throws IOException { FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() { @Override http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java index 7a2bfe6..4b49be1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotReferenceUtil.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.HFileArchiveUtil; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp b/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp index 58f74f4..434784d 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/snapshotsStats.jsp @@ -26,10 +26,11 @@ import="org.apache.hadoop.fs.Path" import="org.apache.hadoop.hbase.HBaseConfiguration" import="org.apache.hadoop.hbase.master.HMaster" - import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription" import="org.apache.hadoop.hbase.snapshot.SnapshotInfo" import="org.apache.hadoop.hbase.TableName" - import="org.apache.hadoop.util.StringUtils" %> + import="org.apache.hadoop.util.StringUtils" + import="org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription" +%> <% HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER); Configuration conf = master.getConfiguration(); http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java new file mode 100644 index 0000000..5d6920a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithAcl.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessControlConstants; +import org.apache.hadoop.hbase.security.access.AccessController; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hadoop.hbase.security.access.SecureTestUtil; +import org.apache.hadoop.hbase.testclassification.ClientTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.IOException; + +@Category({ MediumTests.class, ClientTests.class }) +public class TestSnapshotWithAcl extends SecureTestUtil { + + public TableName TEST_TABLE = TableName.valueOf("TestSnapshotWithAcl"); + + private static final int ROW_COUNT = 30000; + + private static byte[] TEST_FAMILY = Bytes.toBytes("f1"); + private static byte[] TEST_QUALIFIER = Bytes.toBytes("cq"); + private static byte[] TEST_ROW = Bytes.toBytes(0); + private static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static Configuration conf; + private static HBaseAdmin admin = null; + + // user is table owner. will have all permissions on table + private static User USER_OWNER; + // user with rw permissions on column family. + private static User USER_RW; + // user with read-only permissions + private static User USER_RO; + // user with none permissions + private static User USER_NONE; + + static class AccessReadAction implements AccessTestAction { + + private TableName tableName; + + public AccessReadAction(TableName tableName) { + this.tableName = tableName; + } + + @Override + public Object run() throws Exception { + Get g = new Get(TEST_ROW); + g.addFamily(TEST_FAMILY); + try (Connection conn = ConnectionFactory.createConnection(conf)) { + try (Table t = conn.getTable(tableName)) { + t.get(g); + } + } + return null; + } + }; + + static class AccessWriteAction implements AccessTestAction { + private TableName tableName; + + public AccessWriteAction(TableName tableName) { + this.tableName = tableName; + } + + @Override + public Object run() throws Exception { + Put p = new Put(TEST_ROW); + p.addColumn(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(0)); + try (Connection conn = ConnectionFactory.createConnection(conf)) { + try (Table t = conn.getTable(tableName)) { + t.put(p); + } + } + return null; + } + } + + + @BeforeClass + public static void setupBeforeClass() throws Exception { + conf = TEST_UTIL.getConfiguration(); + // Enable security + enableSecurity(conf); + conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName()); + // Verify enableSecurity sets up what we require + verifyConfiguration(conf); + // Enable EXEC permission checking + conf.setBoolean(AccessControlConstants.EXEC_PERMISSION_CHECKS_KEY, true); + TEST_UTIL.startMiniCluster(); + MasterCoprocessorHost cpHost = + TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterCoprocessorHost(); + cpHost.load(AccessController.class, Coprocessor.PRIORITY_HIGHEST, conf); + + USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]); + USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]); + USER_RO = User.createUserForTesting(conf, "rouser", new String[0]); + USER_NONE = User.createUserForTesting(conf, "usernone", new String[0]); + } + + @Before + public void setUp() throws Exception { + admin = TEST_UTIL.getHBaseAdmin(); + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); + HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); + hcd.setMaxVersions(100); + htd.addFamily(hcd); + htd.setOwner(USER_OWNER); + admin.createTable(htd, new byte[][] { Bytes.toBytes("s") }); + TEST_UTIL.waitTableEnabled(TEST_TABLE); + + grantOnTable(TEST_UTIL, USER_RW.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ, Permission.Action.WRITE); + + grantOnTable(TEST_UTIL, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ); + } + + private void loadData() throws IOException { + try (Connection conn = ConnectionFactory.createConnection(conf)) { + try (Table t = conn.getTable(TEST_TABLE)) { + for (int i = 0; i < ROW_COUNT; i++) { + Put put = new Put(Bytes.toBytes(i)); + put.addColumn(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i)); + t.put(put); + } + } + } + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private void verifyRows(TableName tableName) throws IOException { + try (Connection conn = ConnectionFactory.createConnection(conf)) { + try (Table t = conn.getTable(tableName)) { + try (ResultScanner scanner = t.getScanner(new Scan())) { + Result result; + int rowCount = 0; + while ((result = scanner.next()) != null) { + byte[] value = result.getValue(TEST_FAMILY, TEST_QUALIFIER); + Assert.assertArrayEquals(value, Bytes.toBytes(rowCount++)); + } + Assert.assertEquals(rowCount, ROW_COUNT); + } + } + } + } + + @Test + public void testRestoreSnapshot() throws Exception { + verifyAllowed(new AccessReadAction(TEST_TABLE), USER_OWNER, USER_RO, USER_RW); + verifyDenied(new AccessReadAction(TEST_TABLE), USER_NONE); + verifyAllowed(new AccessWriteAction(TEST_TABLE), USER_OWNER, USER_RW); + verifyDenied(new AccessWriteAction(TEST_TABLE), USER_RO, USER_NONE); + + loadData(); + verifyRows(TEST_TABLE); + + String snapshotName1 = "testSnapshot1"; + admin.snapshot(snapshotName1, TEST_TABLE); + + // clone snapshot with restoreAcl true. + TableName tableName1 = TableName.valueOf("tableName1"); + admin.cloneSnapshot(snapshotName1, tableName1, true); + verifyRows(tableName1); + verifyAllowed(new AccessReadAction(tableName1), USER_OWNER, USER_RO, USER_RW); + verifyDenied(new AccessReadAction(tableName1), USER_NONE); + verifyAllowed(new AccessWriteAction(tableName1), USER_OWNER, USER_RW); + verifyDenied(new AccessWriteAction(tableName1), USER_RO, USER_NONE); + + // clone snapshot with restoreAcl false. + TableName tableName2 = TableName.valueOf("tableName2"); + admin.cloneSnapshot(snapshotName1, tableName2, false); + verifyRows(tableName2); + verifyAllowed(new AccessReadAction(tableName2), USER_OWNER); + verifyDenied(new AccessReadAction(tableName2), USER_NONE, USER_RO, USER_RW); + verifyAllowed(new AccessWriteAction(tableName2), USER_OWNER); + verifyDenied(new AccessWriteAction(tableName2), USER_RO, USER_RW, USER_NONE); + + // remove read permission for USER_RO. + revokeFromTable(TEST_UTIL, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ); + verifyAllowed(new AccessReadAction(TEST_TABLE), USER_OWNER, USER_RW); + verifyDenied(new AccessReadAction(TEST_TABLE), USER_RO, USER_NONE); + verifyAllowed(new AccessWriteAction(TEST_TABLE), USER_OWNER, USER_RW); + verifyDenied(new AccessWriteAction(TEST_TABLE), USER_RO, USER_NONE); + + // restore snapshot with restoreAcl false. + admin.disableTable(TEST_TABLE); + admin.restoreSnapshot(snapshotName1, false, false); + admin.enableTable(TEST_TABLE); + verifyAllowed(new AccessReadAction(TEST_TABLE), USER_OWNER, USER_RW); + verifyDenied(new AccessReadAction(TEST_TABLE), USER_RO, USER_NONE); + verifyAllowed(new AccessWriteAction(TEST_TABLE), USER_OWNER, USER_RW); + verifyDenied(new AccessWriteAction(TEST_TABLE), USER_RO, USER_NONE); + + // restore snapshot with restoreAcl true. + admin.disableTable(TEST_TABLE); + admin.restoreSnapshot(snapshotName1, false, true); + admin.enableTable(TEST_TABLE); + verifyAllowed(new AccessReadAction(TEST_TABLE), USER_OWNER, USER_RO, USER_RW); + verifyDenied(new AccessReadAction(TEST_TABLE), USER_NONE); + verifyAllowed(new AccessWriteAction(TEST_TABLE), USER_OWNER, USER_RW); + verifyDenied(new AccessWriteAction(TEST_TABLE), USER_RO, USER_NONE); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java index 3b80406..1e6d717 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java @@ -64,10 +64,10 @@ import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableDescriptorsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableNamesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.testclassification.CoprocessorTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java index 9c6b132..8f88af7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler; import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteSnapshotRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse; @@ -50,6 +49,7 @@ import org.apache.hadoop.hbase.regionserver.CompactedHFilesDischarger; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java index d8221bb..54efe76 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCloneSnapshotProcedure.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -50,7 +51,7 @@ public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase { protected final byte[] CF = Bytes.toBytes("cf1"); - private static HBaseProtos.SnapshotDescription snapshot = null; + private static SnapshotProtos.SnapshotDescription snapshot = null; @After @Override @@ -60,7 +61,7 @@ public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase { snapshot = null; } - private HBaseProtos.SnapshotDescription getSnapshot() throws Exception { + private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception { if (snapshot == null) { final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot"); long tid = System.currentTimeMillis(); @@ -102,7 +103,7 @@ public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase { final HTableDescriptor htd = createHTableDescriptor(clonedTableName, CF); // take the snapshot - HBaseProtos.SnapshotDescription snapshotDesc = getSnapshot(); + SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); long procId = ProcedureTestingUtility.submitAndWait( procExec, new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); @@ -115,7 +116,7 @@ public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase { @Test(timeout=60000) public void testCloneSnapshotToSameTable() throws Exception { // take the snapshot - HBaseProtos.SnapshotDescription snapshotDesc = getSnapshot(); + SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final TableName clonedTableName = TableName.valueOf(snapshotDesc.getTable()); @@ -137,7 +138,7 @@ public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase { final HTableDescriptor htd = createHTableDescriptor(clonedTableName, CF); // take the snapshot - HBaseProtos.SnapshotDescription snapshotDesc = getSnapshot(); + SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); @@ -161,7 +162,7 @@ public class TestCloneSnapshotProcedure extends TestTableDDLProcedureBase { final HTableDescriptor htd = createHTableDescriptor(clonedTableName, CF); // take the snapshot - HBaseProtos.SnapshotDescription snapshotDesc = getSnapshot(); + SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java index 479b206..9141e0f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestRestoreSnapshotProcedure.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreSnapshotState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; @@ -67,7 +68,7 @@ public class TestRestoreSnapshotProcedure extends TestTableDDLProcedureBase { protected final int rowCountCF4 = 40; protected final int rowCountCF1addition = 10; - private HBaseProtos.SnapshotDescription snapshot = null; + private SnapshotProtos.SnapshotDescription snapshot = null; private HTableDescriptor snapshotHTD = null; @Rule http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java index 1dd1c3a..1d1a936 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java @@ -167,7 +167,7 @@ public class SecureTestUtil { * To indicate the action was not allowed, either throw an AccessDeniedException * or return an empty list of KeyValues. */ - static interface AccessTestAction extends PrivilegedExceptionAction<Object> { } + protected static interface AccessTestAction extends PrivilegedExceptionAction<Object> { } /** This fails only in case of ADE or empty list for any of the actions. */ public static void verifyAllowed(User user, AccessTestAction... actions) throws Exception { http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 8bf2c5c..bcf8670 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -117,8 +117,8 @@ import org.apache.hadoop.hbase.security.Superusers; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.shaded.ipc.protobuf.generated.TestProcedureProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; @@ -131,7 +131,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import org.mockito.Mockito; import com.google.protobuf.BlockingRpcChannel; import com.google.protobuf.RpcCallback; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java index 08c8107..040dfa6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestWithDisabledAuthorization.java @@ -57,7 +57,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.CompareFilter; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.regionserver.Region; @@ -67,6 +66,7 @@ import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.Permission.Action; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java index 3c10dee..9098277 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse; @@ -127,7 +128,7 @@ public final class SnapshotTestingUtils { * Make sure that there is only one snapshot returned from the master */ public static void assertOneSnapshotThatMatches(Admin admin, - HBaseProtos.SnapshotDescription snapshot) throws IOException { + SnapshotProtos.SnapshotDescription snapshot) throws IOException { assertOneSnapshotThatMatches(admin, snapshot.getName(), TableName.valueOf(snapshot.getTable())); } @@ -159,20 +160,19 @@ public final class SnapshotTestingUtils { } public static void confirmSnapshotValid(HBaseTestingUtility testUtil, - HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family) + SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family) throws IOException { MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem(); - confirmSnapshotValid(snapshotDescriptor, tableName, family, - mfs.getRootDir(), testUtil.getAdmin(), mfs.getFileSystem()); + confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(), + testUtil.getAdmin(), mfs.getFileSystem()); } /** * Confirm that the snapshot contains references to all the files that should * be in the snapshot. */ - public static void confirmSnapshotValid( - HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName, - byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) + public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor, + TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) throws IOException { ArrayList nonEmptyTestFamilies = new ArrayList(1); nonEmptyTestFamilies.add(testFamily); @@ -184,7 +184,7 @@ public final class SnapshotTestingUtils { * Confirm that the snapshot has no references files but only metadata. */ public static void confirmEmptySnapshotValid( - HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName, + SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) throws IOException { ArrayList emptyTestFamilies = new ArrayList(1); @@ -200,7 +200,7 @@ public final class SnapshotTestingUtils { * by the MasterSnapshotVerifier, at the end of the snapshot operation. */ public static void confirmSnapshotValid( - HBaseProtos.SnapshotDescription snapshotDescriptor, TableName tableName, + SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies, Path rootDir, Admin admin, FileSystem fs) throws IOException { final Configuration conf = admin.getConfiguration(); @@ -210,7 +210,7 @@ public final class SnapshotTestingUtils { snapshotDescriptor, rootDir); assertTrue(fs.exists(snapshotDir)); - HBaseProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + SnapshotProtos.SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); // Extract regions and families with store files final Set<byte[]> snapshotFamilies = new TreeSet<>(Bytes.BYTES_COMPARATOR); @@ -272,7 +272,7 @@ public final class SnapshotTestingUtils { * @throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException */ public static void waitForSnapshotToComplete(HMaster master, - HBaseProtos.SnapshotDescription snapshot, long sleep) + SnapshotProtos.SnapshotDescription snapshot, long sleep) throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException { final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder() .setSnapshot(snapshot).build(); @@ -426,7 +426,7 @@ public final class SnapshotTestingUtils { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, mfs.getRootDir()); - HBaseProtos.SnapshotDescription snapshotDesc = + SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); final TableName table = TableName.valueOf(snapshotDesc.getTable()); @@ -476,7 +476,7 @@ public final class SnapshotTestingUtils { public static class SnapshotBuilder { private final RegionData[] tableRegions; - private final HBaseProtos.SnapshotDescription desc; + private final SnapshotProtos.SnapshotDescription desc; private final HTableDescriptor htd; private final Configuration conf; private final FileSystem fs; @@ -486,7 +486,7 @@ public final class SnapshotTestingUtils { public SnapshotBuilder(final Configuration conf, final FileSystem fs, final Path rootDir, final HTableDescriptor htd, - final HBaseProtos.SnapshotDescription desc, final RegionData[] tableRegions) + final SnapshotProtos.SnapshotDescription desc, final RegionData[] tableRegions) throws IOException { this.fs = fs; this.conf = conf; @@ -503,7 +503,7 @@ public final class SnapshotTestingUtils { return this.htd; } - public HBaseProtos.SnapshotDescription getSnapshotDescription() { + public SnapshotProtos.SnapshotDescription getSnapshotDescription() { return this.desc; } @@ -527,7 +527,7 @@ public final class SnapshotTestingUtils { .build()); } - private Path[] addRegion(final HBaseProtos.SnapshotDescription desc) throws IOException { + private Path[] addRegion(final SnapshotProtos.SnapshotDescription desc) throws IOException { if (this.snapshotted == tableRegions.length) { throw new UnsupportedOperationException("No more regions in the table"); } @@ -668,7 +668,7 @@ public final class SnapshotTestingUtils { HTableDescriptor htd = createHtd(tableName); RegionData[] regions = createTable(htd, numRegions); - HBaseProtos.SnapshotDescription desc = HBaseProtos.SnapshotDescription.newBuilder() + SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder() .setTable(htd.getNameAsString()) .setName(snapshotName) .setVersion(version) http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java index cc055a5..b875320 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java index 86405dc..2882120 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.client.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -274,9 +275,9 @@ public class TestFlushSnapshotFromClient { @Test public void testAsyncFlushSnapshot() throws Exception { - HBaseProtos.SnapshotDescription snapshot = HBaseProtos.SnapshotDescription.newBuilder() + SnapshotProtos.SnapshotDescription snapshot = SnapshotProtos.SnapshotDescription.newBuilder() .setName("asyncSnapshot").setTable(TABLE_NAME.getNameAsString()) - .setType(HBaseProtos.SnapshotDescription.Type.FLUSH).build(); + .setType(SnapshotProtos.SnapshotDescription.Type.FLUSH).build(); // take the snapshot async admin.takeSnapshotAsync( http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java index 305e7d4..4b684e3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestRestoreSnapshotHelper.java @@ -31,12 +31,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.monitoring.MonitoredTask; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock; import org.apache.hadoop.hbase.util.FSTableDescriptors; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java index c49595e..bd9d5dc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.coprocessor.MasterObserver; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.TestTableName; import org.junit.After; http://git-wip-us.apache.org/repos/asf/hbase/blob/37dd8ff7/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java index cd87993..038ee8c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotDescriptionUtils.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription; +import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; @@ -69,7 +69,7 @@ public class TestSnapshotDescriptionUtils { private static final Log LOG = LogFactory.getLog(TestSnapshotDescriptionUtils.class); @Test - public void testValidateMissingTableName() { + public void testValidateMissingTableName() throws IOException { Configuration conf = new Configuration(false); try { SnapshotDescriptionUtils.validate(SnapshotDescription.newBuilder().setName("fail").build(),
