HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota
table (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e0c0e950
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e0c0e950
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e0c0e950
Branch: refs/heads/HBASE-16961
Commit: e0c0e950a72dc77e1071e5d6a32816679a3ebd0e
Parents: 3a5087e
Author: tedyu
Authored: Sat Dec 3 14:30:48 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 25 18:20:14 2017 -0400
--
.../hadoop/hbase/quotas/QuotaTableUtil.java | 13 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 30 +
.../hadoop/hbase/quotas/TestQuotaAdmin.java | 125 ++-
3 files changed, 165 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0c0e950/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index c44090f..8ef4f08 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -53,7 +53,9 @@ import org.apache.hadoop.hbase.util.Strings;
*
* ROW-KEY FAM/QUALDATA
* n. q:s
+ * n. u:du
* t. q:s
+ * t. u:du
* u. q:s
* u. q:s.
* u. q:s.:
@@ -72,6 +74,7 @@ public class QuotaTableUtil {
protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX =
Bytes.toBytes("s.");
+ protected static final byte[] QUOTA_QUALIFIER_DISKUSAGE =
Bytes.toBytes("du");
protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX =
Bytes.toBytes("u.");
protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX =
Bytes.toBytes("t.");
protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX =
Bytes.toBytes("n.");
@@ -330,11 +333,16 @@ public class QuotaTableUtil {
* Quotas protobuf helpers
*/
protected static Quotas quotasFromData(final byte[] data) throws IOException
{
+return quotasFromData(data, 0, data.length);
+ }
+
+ protected static Quotas quotasFromData(
+ final byte[] data, int offset, int length) throws IOException {
int magicLen = ProtobufMagic.lengthOfPBMagic();
-if (!ProtobufMagic.isPBMagicPrefix(data, 0, magicLen)) {
+if (!ProtobufMagic.isPBMagicPrefix(data, offset, magicLen)) {
throw new IOException("Missing pb magic prefix");
}
-return Quotas.parseFrom(new ByteArrayInputStream(data, magicLen,
data.length - magicLen));
+return Quotas.parseFrom(new ByteArrayInputStream(data, offset + magicLen,
length - magicLen));
}
protected static byte[] quotasToData(final Quotas data) throws IOException {
@@ -348,6 +356,7 @@ public class QuotaTableUtil {
boolean hasSettings = false;
hasSettings |= quotas.hasThrottle();
hasSettings |= quotas.hasBypassGlobals();
+hasSettings |= quotas.hasSpace();
return !hasSettings;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0c0e950/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index 5dab2e3..1469268 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.Quota
http://git-wip-us.apache.org/repos/asf/hbase/blob/23194dca/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 01ba8f6..e3c6bfd 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -239,12 +239,20 @@ public final class QuotaProtos {
* THROTTLE = 1;
*/
THROTTLE(1),
+/**
+ * SPACE = 2;
+ */
+SPACE(2),
;
/**
* THROTTLE = 1;
*/
public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
public final int getNumber() {
@@ -262,6 +270,7 @@ public final class QuotaProtos {
public static QuotaType forNumber(int value) {
switch (value) {
case 1: return THROTTLE;
+case 2: return SPACE;
default: return null;
}
}
@@ -311,6 +320,150 @@ public final class QuotaProtos {
// @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
}
+ /**
+ *
+ * Defines what action should be taken when the SpaceQuota is violated
+ *
+ *
+ * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+ */
+ public enum SpaceViolationPolicy
+ implements
org.apache.hadoop.hbase.shaded.com.google.protobuf.ProtocolMessageEnum {
+/**
+ *
+ * Disable the table(s)
+ *
+ *
+ * DISABLE = 1;
+ */
+DISABLE(1),
+/**
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+NO_WRITES_COMPACTIONS(2),
+/**
+ *
+ * No writes or bulk-loads
+ *
+ *
+ * NO_WRITES = 3;
+ */
+NO_WRITES(3),
+/**
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ *
+ * NO_INSERTS = 4;
+ */
+NO_INSERTS(4),
+;
+
+/**
+ *
+ * Disable the table(s)
+ *
+ *
+ * DISABLE = 1;
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ *
+ * No writes or bulk-loads
+ *
+ *
+ * NO_WRITES = 3;
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ *
+ * NO_INSERTS = 4;
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() {
+ return value;
+}
+
+/**
+ * @deprecated Use {@link #forNumber(int)} instead.
+ */
+@java.lang.Deprecated
+public static SpaceViolationPolicy valueOf(int value) {
+ return forNumber(value);
+}
+
+public static SpaceViolationPolicy forNumber(int value) {
+ switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+ }
+}
+
+public static
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap<
+SpaceViolationPolicy> internalValueMap =
+ new
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap()
{
+public SpaceViolationPolicy findValueByNumber(int number) {
+ return SpaceViolationPolicy.forNumber(number);
+}
+ };
+
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(ordinal());
+}
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalAr
HBASE-17568 Better handle stale/missing region size reports
* Expire region reports in the master after a timeout.
* Move regions in violation out of violation when insufficient
region size reports are observed.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d9c22cc2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d9c22cc2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d9c22cc2
Branch: refs/heads/HBASE-16961
Commit: d9c22cc28f020c80fb494bec97b51dd1cb49fe1b
Parents: 986a642
Author: Josh Elser
Authored: Fri Feb 3 16:33:47 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:32:00 2017 -0400
--
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 86 ++-
.../hadoop/hbase/quotas/QuotaObserverChore.java | 53 -
.../hbase/quotas/TestMasterQuotaManager.java| 48 +++-
.../TestQuotaObserverChoreRegionReports.java| 233 +++
5 files changed, 412 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d9c22cc2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index d807b24..ddc58bb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -150,6 +150,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.Updat
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
@@ -1942,8 +1943,9 @@ public class MasterRpcServices extends RSRpcServices
return RegionSpaceUseReportResponse.newBuilder().build();
}
MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+ final long now = EnvironmentEdgeManager.currentTime();
for (RegionSpaceUse report : request.getSpaceUseList()) {
-quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize());
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize(), now);
}
return RegionSpaceUseReportResponse.newBuilder().build();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/d9c22cc2/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index cb614ea..0622dba 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -22,9 +22,12 @@ import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -47,6 +50,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.ThrottleRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Master Quota Manager.
* It is responsible for initialize the quota table on the first-run and
@@ -68,7 +73,7 @@ public class MasterQuotaManager implements
RegionStateListener {
private NamedLock userLocks;
private boolean enabled = false;
private NamespaceAuditor namespaceQuotaManager;
- private ConcurrentHashMap regionSizes;
+ private ConcurrentHashMap
regionSizes;
public MasterQuotaManager(final MasterServices masterServices) {
this.masterServices = masterServices;
@@ -531,21 +536,88 @@ public class MasterQuotaManager implements
RegionStateListener {
}
}
- public void addRegionSize(HRe
http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 1c373ee..23ddd43 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -141,6 +141,22 @@ message SplitTableRegionResponse {
optional uint64 proc_id = 1;
}
+message RegionSpaceUse {
+ optional RegionInfo region = 1; // A region identifier
+ optional uint64 size = 2; // The size in bytes of the region
+}
+
+/**
+ * Reports filesystem usage for regions.
+ */
+message RegionSpaceUseReportRequest {
+ repeated RegionSpaceUse space_use = 1;
+}
+
+message RegionSpaceUseReportResponse {
+
+}
+
service RegionServerStatusService {
/** Called when a region server first starts. */
rpc RegionServerStartup(RegionServerStartupRequest)
@@ -182,4 +198,10 @@ service RegionServerStatusService {
*/
rpc getProcedureResult(GetProcedureResultRequest)
returns(GetProcedureResultResponse);
+
+ /**
+ * Reports Region filesystem space use
+ */
+ rpc ReportRegionSpaceUse(RegionSpaceUseReportRequest)
+returns(RegionSpaceUseReportResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 40c4a71..4ee8f25 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.LockInfo;
import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -95,6 +96,9 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
@@ -1901,4 +1905,19 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(e);
}
}
+
+ @Override
+ public RegionSpaceUseReportResponse reportRegionSpaceUse(RpcController
controller,
+ RegionSpaceUseReportRequest request) throws ServiceException {
+try {
+ master.checkInitialized();
+ MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+ for (RegionSpaceUse report : request.getSpaceUseList()) {
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize());
+ }
+ return RegionSpaceUseReportResponse.newBuilder().build();
+} catch (Exception e) {
+ throw new ServiceException(e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
new file mode 100644
index 000..01540eb
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * c
HBASE-16314 Retry on table snapshot failure during full backup (Vladimir
Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e95cf479
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e95cf479
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e95cf479
Branch: refs/heads/HBASE-16961
Commit: e95cf479c7615ae160a6ba963cc7689f3b440efd
Parents: a3b6f4a
Author: tedyu
Authored: Fri Apr 21 16:15:07 2017 -0700
Committer: tedyu
Committed: Fri Apr 21 16:15:07 2017 -0700
--
.../hbase/backup/BackupRestoreConstants.java| 10 ++
.../backup/impl/FullTableBackupClient.java | 36 ++--
2 files changed, 44 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e95cf479/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
index e46904b..d1ab246 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
@@ -37,6 +37,16 @@ public interface BackupRestoreConstants {
public final static int BACKUP_SYSTEM_TTL_DEFAULT = HConstants.FOREVER;
public final static String BACKUP_ENABLE_KEY = "hbase.backup.enable";
public final static boolean BACKUP_ENABLE_DEFAULT = false;
+
+
+ public static final String BACKUP_MAX_ATTEMPTS_KEY =
"hbase.backup.attempts.max";
+ public static final int DEFAULT_BACKUP_MAX_ATTEMPTS = 10;
+
+ public static final String BACKUP_ATTEMPTS_PAUSE_MS_KEY =
"hbase.backup.attempts.pause.ms";
+ public static final int DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS = 1;
+
+
+
/*
* Drivers option list
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/e95cf479/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
index 77d1184..ee7a841 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/FullTableBackupClient.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.hbase.backup.impl;
+import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_ATTEMPTS_PAUSE_MS_KEY;
+import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.BACKUP_MAX_ATTEMPTS_KEY;
+import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS;
+import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_BACKUP_MAX_ATTEMPTS;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
@@ -148,8 +153,7 @@ public class FullTableBackupClient extends
TableBackupClient {
"snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime())
+ "_"
+ tableName.getNamespaceAsString() + "_" +
tableName.getQualifierAsString();
-admin.snapshot(snapshotName, tableName);
-
+snapshotTable(admin, tableName, snapshotName);
backupInfo.setSnapshotName(tableName, snapshotName);
}
@@ -186,4 +190,32 @@ public class FullTableBackupClient extends
TableBackupClient {
}
+ private void snapshotTable(Admin admin, TableName tableName, String
snapshotName)
+ throws IOException {
+
+int maxAttempts =
+conf.getInt(BACKUP_MAX_ATTEMPTS_KEY, DEFAULT_BACKUP_MAX_ATTEMPTS);
+int pause =
+conf.getInt(BACKUP_ATTEMPTS_PAUSE_MS_KEY,
DEFAULT_BACKUP_ATTEMPTS_PAUSE_MS);
+int attempts = 0;
+
+while (attempts++ < maxAttempts) {
+ try {
+admin.snapshot(snapshotName, tableName);
+return;
+ } catch (IOException ee) {
+LOG.warn("Snapshot attempt " + attempts + " failed for table " +
tableName
++ ", sleeping for " + pause + "ms", ee);
+if (attempts < maxAttempts) {
+ try {
+Thread.sleep(pause);
+ } catch (InterruptedException e) {
+Thread.currentThread().interrupt();
+break;
+ }
+}
+ }
+}
+throw new IOException("Failed to snapshot table "+ tableName);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/91b27eda/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index d7d4db0..e90c934 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10164,6 +10164,1912 @@ public final class RegionServerStatusProtos {
}
+ public interface RegionSpaceUseOrBuilder extends
+ // @@protoc_insertion_point(interface_extends:hbase.pb.RegionSpaceUse)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+boolean hasRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo
getRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
getRegionOrBuilder();
+
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+boolean hasSize();
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+long getSize();
+ }
+ /**
+ * Protobuf type {@code hbase.pb.RegionSpaceUse}
+ */
+ public static final class RegionSpaceUse extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ // @@protoc_insertion_point(message_implements:hbase.pb.RegionSpaceUse)
+ RegionSpaceUseOrBuilder {
+// Use RegionSpaceUse.newBuilder() to construct.
+private
RegionSpaceUse(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private RegionSpaceUse() {
+ size_ = 0L;
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private RegionSpaceUse(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+ int mutable_bitField0_ = 0;
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+case 10: {
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder
subBuilder = null;
+ if (((bitField0_ & 0x0001) == 0x0001)) {
+subBuilder = region_.toBuilder();
+ }
+ region_ =
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
extensionRegistry);
+ if (subBuilder != null) {
+subBuilder.mergeFrom(region_);
+region_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x0001;
+ break;
+}
+case 16: {
+ bitField0_ |= 0x0002;
+ size_ = input.readUInt64();
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index c493b25..943c898 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -22,16 +22,12 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -40,20 +36,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -62,7 +53,6 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
@@ -72,11 +62,8 @@ import com.google.common.collect.Multimap;
@Category(LargeTests.class)
public class TestQuotaObserverChoreWithMiniCluster {
private static final Log LOG =
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
- private static final int SIZE_PER_VALUE = 256;
- private static final String F1 = "f1";
private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private static final AtomicLong COUNTER = new AtomicLong(0);
- private static final long ONE_MEGABYTE = 1024L * 1024L;
private static final long DEFAULT_WAIT_MILLIS = 500;
@Rule
@@ -84,18 +71,19 @@ public class TestQuotaObserverChoreWithMiniCluster {
private HMaster master;
private QuotaObserverChore chore;
- private SpaceQuotaViolationNotifierForTest violationNotifier;
+ private SpaceQuotaSnapshotNotifierForTest snapshotNotifier;
+ private SpaceQuotaHelperForTests helper;
@BeforeClass
public static void setUp() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY,
1000);
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000);
conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
-conf.setClass(SpaceQuotaViolationNotifierFactory.VIOLATION_NOTIFIER_KEY,
-SpaceQuotaViolationNotifierForTest.class,
SpaceQuotaViolationNotifier.class);
+conf.setClass(SpaceQuotaSnapshotNotifierFactory.SNAPSHOT_NOTIFIER_KEY,
+SpaceQuotaSnapshotNotifierForTest.class,
SpaceQuotaSnapshotNotifier.class);
TEST_UTIL.startMiniCluster(1);
}
@@ -131,40 +119,55 @@ public class TestQuotaObserverChoreWithMiniCluster {
}
master = TEST_UTIL.getMiniHBaseCluster().getMaster();
-violationNotifier =
-(SpaceQuotaViolationNotifierForTest)
master.getSpaceQuotaViolationNotifier();
-violationNotifier.clearTableViolations();
+snapshotNotifier =
+(SpaceQuotaSnapshotNotifierForTest)
master.getSpaceQuotaSnapshotNotifier();
+snapshotNotifier.clearSnapshots();
chore = master.getQuotaObserverChore();
+helpe
http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
index 6dbf9b2..99853a5 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/LockServiceProtos.java
@@ -104,6 +104,114 @@ public final class LockServiceProtos {
// @@protoc_insertion_point(enum_scope:hbase.pb.LockType)
}
+ /**
+ * Protobuf enum {@code hbase.pb.ResourceType}
+ */
+ public enum ResourceType
+ implements
org.apache.hadoop.hbase.shaded.com.google.protobuf.ProtocolMessageEnum {
+/**
+ * RESOURCE_TYPE_SERVER = 1;
+ */
+RESOURCE_TYPE_SERVER(1),
+/**
+ * RESOURCE_TYPE_NAMESPACE = 2;
+ */
+RESOURCE_TYPE_NAMESPACE(2),
+/**
+ * RESOURCE_TYPE_TABLE = 3;
+ */
+RESOURCE_TYPE_TABLE(3),
+/**
+ * RESOURCE_TYPE_REGION = 4;
+ */
+RESOURCE_TYPE_REGION(4),
+;
+
+/**
+ * RESOURCE_TYPE_SERVER = 1;
+ */
+public static final int RESOURCE_TYPE_SERVER_VALUE = 1;
+/**
+ * RESOURCE_TYPE_NAMESPACE = 2;
+ */
+public static final int RESOURCE_TYPE_NAMESPACE_VALUE = 2;
+/**
+ * RESOURCE_TYPE_TABLE = 3;
+ */
+public static final int RESOURCE_TYPE_TABLE_VALUE = 3;
+/**
+ * RESOURCE_TYPE_REGION = 4;
+ */
+public static final int RESOURCE_TYPE_REGION_VALUE = 4;
+
+
+public final int getNumber() {
+ return value;
+}
+
+/**
+ * @deprecated Use {@link #forNumber(int)} instead.
+ */
+@java.lang.Deprecated
+public static ResourceType valueOf(int value) {
+ return forNumber(value);
+}
+
+public static ResourceType forNumber(int value) {
+ switch (value) {
+case 1: return RESOURCE_TYPE_SERVER;
+case 2: return RESOURCE_TYPE_NAMESPACE;
+case 3: return RESOURCE_TYPE_TABLE;
+case 4: return RESOURCE_TYPE_REGION;
+default: return null;
+ }
+}
+
+public static
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap<
+ResourceType> internalValueMap =
+ new
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap()
{
+public ResourceType findValueByNumber(int number) {
+ return ResourceType.forNumber(number);
+}
+ };
+
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(ordinal());
+}
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos.getDescriptor().getEnumTypes().get(1);
+}
+
+private static final ResourceType[] VALUES = values();
+
+public static ResourceType valueOf(
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+}
+
+private final int value;
+
+private ResourceType(int value) {
+ this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.ResourceType)
+ }
+
public interface LockRequestOrBuilder extends
// @@protoc_insertion_point(interface_extends:hbase.pb.LockRequest)
org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
@@ -4898,70 +5006,2193 @@ public final class LockServiceProtos {
}
+ public interface WaitingProcedureOrBuilder extends
+ // @@protoc_insertion_point(interface_extends:hbase.pb.WaitingProcedure)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ * required .hbase.pb.LockType lock_type = 1;
+ */
+boolean hasLockType();
+/**
+ * required .hbase.pb.LockType lock_type = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos.LockType
getLockType();
+
+/**
+ * required .hbase.
http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index cc40536..d466e59 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -5778,6 +5778,1284 @@ public final class QuotaProtos {
// @@protoc_insertion_point(class_scope:hbase.pb.SpaceLimitRequest)
}
+ public interface SpaceQuotaStatusOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+// optional .hbase.pb.SpaceViolationPolicy policy = 1;
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+boolean hasPolicy();
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy
getPolicy();
+
+// optional bool in_violation = 2;
+/**
+ * optional bool in_violation = 2;
+ */
+boolean hasInViolation();
+/**
+ * optional bool in_violation = 2;
+ */
+boolean getInViolation();
+ }
+ /**
+ * Protobuf type {@code hbase.pb.SpaceQuotaStatus}
+ *
+ *
+ * Represents the state of a quota on a table. Either the quota is not in
violation
+ * or it is in violatino there is a violation policy which should be in
effect.
+ *
+ */
+ public static final class SpaceQuotaStatus extends
+ com.google.protobuf.GeneratedMessage
+ implements SpaceQuotaStatusOrBuilder {
+// Use SpaceQuotaStatus.newBuilder() to construct.
+private SpaceQuotaStatus(com.google.protobuf.GeneratedMessage.Builder
builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+}
+private SpaceQuotaStatus(boolean noInit) { this.unknownFields =
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SpaceQuotaStatus defaultInstance;
+public static SpaceQuotaStatus getDefaultInstance() {
+ return defaultInstance;
+}
+
+public SpaceQuotaStatus getDefaultInstanceForType() {
+ return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private SpaceQuotaStatus(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+case 8: {
+ int rawValue = input.readEnum();
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy
value =
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy.valueOf(rawValue);
+ if (value == null) {
+unknownFields.mergeVarintField(1, rawValue);
+ } else {
+bitField0_ |= 0x0001;
+policy_ = value;
+ }
+ break;
+}
+case 16: {
+ bitField0_ |= 0x0002;
+ inViolation_ = input.readBool();
+ break;
+}
+ }
+}
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.internal_static_hbase_pb_SpaceQuotaStatus_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+ return
org.apache
http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
new file mode 100644
index 000..6b754b9
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import
org.apache.hadoop.hbase.quotas.policies.BulkLoadVerifyingViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.DisableTableViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.NoInsertsViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.NoWritesCompactionsViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.NoWritesViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * A factory class for instantiating {@link SpaceViolationPolicyEnforcement}
instances.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SpaceViolationPolicyEnforcementFactory {
+
+ private static final SpaceViolationPolicyEnforcementFactory INSTANCE =
+ new SpaceViolationPolicyEnforcementFactory();
+
+ private SpaceViolationPolicyEnforcementFactory() {}
+
+ /**
+ * Returns an instance of this factory.
+ */
+ public static SpaceViolationPolicyEnforcementFactory getInstance() {
+return INSTANCE;
+ }
+
+ /**
+ * Constructs the appropriate {@link SpaceViolationPolicyEnforcement} for
tables that are
+ * in violation of their space quota.
+ */
+ public SpaceViolationPolicyEnforcement create(
+ RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot
snapshot) {
+SpaceViolationPolicyEnforcement enforcement;
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (!status.isInViolation()) {
+ throw new IllegalArgumentException(tableName + " is not in violation.
Snapshot=" + snapshot);
+}
+switch (status.getPolicy()) {
+ case DISABLE:
+enforcement = new DisableTableViolationPolicyEnforcement();
+break;
+ case NO_WRITES_COMPACTIONS:
+enforcement = new NoWritesCompactionsViolationPolicyEnforcement();
+break;
+ case NO_WRITES:
+enforcement = new NoWritesViolationPolicyEnforcement();
+break;
+ case NO_INSERTS:
+enforcement = new NoInsertsViolationPolicyEnforcement();
+break;
+ default:
+throw new IllegalArgumentException("Unhandled SpaceViolationPolicy: "
+ status.getPolicy());
+}
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+ }
+
+ /**
+ * Creates the "default" {@link SpaceViolationPolicyEnforcement} for a table
that isn't in
+ * violation. This is used to have uniform policy checking for tables in and
not quotas.
+ */
+ public SpaceViolationPolicyEnforcement createWithoutViolation(
+ RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot
snapshot) {
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (status.isInViolation()) {
+ throw new IllegalArgumentException(
+ tableName + " is in violation. Logic error. Snapshot=" + snapshot);
+}
+BulkLoadVerifyingViolationPolicyEnforcement enforcement = new
BulkLoadVerifyingViolationPolicyEnforcement();
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java
---
HBASE-17001 Enforce quota violation policies in the RegionServer
The nuts-and-bolts of filesystem quotas. The Master must inform
RegionServers of the violation of a quota by a table. The RegionServer
must apply the violation policy as configured. Need to ensure
that the proper interfaces exist to satisfy all necessary policies.
This required a massive rewrite of the internal tracking by
the general space quota feature. Instead of tracking "violations",
we need to start tracking "usage". This allows us to make the decision
at the RegionServer level as to when the files in a bulk load request
should be accept or rejected which ultimately lets us avoid bulk loads
dramatically exceeding a configured space quota.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/66cf4809
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/66cf4809
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/66cf4809
Branch: refs/heads/HBASE-16961
Commit: 66cf48095cfcfbc04bf34873e909b1e60b24fb5e
Parents: cfd374d
Author: Josh Elser
Authored: Thu Dec 15 13:27:56 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:24:20 2017 -0400
--
.../hbase/quotas/QuotaExceededException.java|4 +
.../hadoop/hbase/quotas/QuotaTableUtil.java | 47 +-
.../hadoop/hbase/quotas/SpaceQuotaSnapshot.java | 192 +++
.../shaded/protobuf/generated/QuotaProtos.java | 1384 +-
.../src/main/protobuf/Quota.proto | 15 +
.../hbase/protobuf/generated/QuotaProtos.java | 1324 -
hbase-protocol/src/main/protobuf/Quota.proto| 15 +
.../org/apache/hadoop/hbase/master/HMaster.java | 29 +-
.../hbase/quotas/ActivePolicyEnforcement.java | 86 ++
.../quotas/NamespaceQuotaSnapshotStore.java | 127 ++
.../quotas/NamespaceQuotaViolationStore.java| 127 --
.../hadoop/hbase/quotas/QuotaObserverChore.java | 344 +++--
.../hadoop/hbase/quotas/QuotaSnapshotStore.java | 96 ++
.../hbase/quotas/QuotaViolationStore.java | 89 --
.../quotas/RegionServerSpaceQuotaManager.java | 179 ++-
.../hbase/quotas/SpaceLimitingException.java| 95 ++
.../hbase/quotas/SpaceQuotaRefresherChore.java | 225 +++
.../quotas/SpaceQuotaSnapshotNotifier.java | 45 +
.../SpaceQuotaSnapshotNotifierFactory.java | 62 +
.../quotas/SpaceQuotaViolationNotifier.java | 54 -
.../SpaceQuotaViolationNotifierFactory.java | 62 -
.../SpaceQuotaViolationNotifierForTest.java | 54 -
...SpaceQuotaViolationPolicyRefresherChore.java | 154 --
.../quotas/SpaceViolationPolicyEnforcement.java | 91 ++
.../SpaceViolationPolicyEnforcementFactory.java | 95 ++
.../hbase/quotas/TableQuotaSnapshotStore.java | 127 ++
.../hbase/quotas/TableQuotaViolationStore.java | 127 --
.../quotas/TableSpaceQuotaSnapshotNotifier.java | 52 +
.../TableSpaceQuotaViolationNotifier.java | 55 -
.../AbstractViolationPolicyEnforcement.java | 118 ++
...LoadVerifyingViolationPolicyEnforcement.java | 50 +
.../DisableTableViolationPolicyEnforcement.java | 80 +
.../NoInsertsViolationPolicyEnforcement.java| 55 +
...esCompactionsViolationPolicyEnforcement.java | 64 +
.../NoWritesViolationPolicyEnforcement.java | 54 +
.../hbase/regionserver/CompactSplitThread.java | 12 +
.../hbase/regionserver/RSRpcServices.java | 92 +-
.../hbase/quotas/SpaceQuotaHelperForTests.java | 228 +++
.../SpaceQuotaSnapshotNotifierForTest.java | 55 +
.../quotas/TestActivePolicyEnforcement.java | 74 +
.../quotas/TestFileSystemUtilizationChore.java |3 +-
.../TestNamespaceQuotaViolationStore.java | 16 +-
.../hbase/quotas/TestQuotaObserverChore.java| 30 +-
.../TestQuotaObserverChoreWithMiniCluster.java | 351 ++---
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 34 +-
.../TestRegionServerSpaceQuotaManager.java | 174 ++-
...SpaceQuotaViolationPolicyRefresherChore.java | 193 ++-
.../hadoop/hbase/quotas/TestSpaceQuotas.java| 452 ++
.../quotas/TestTableQuotaViolationStore.java| 22 +-
.../TestTableSpaceQuotaViolationNotifier.java | 48 +-
.../hbase/quotas/TestTablesWithQuotas.java |8 +-
.../BaseViolationPolicyEnforcement.java | 31 +
...kLoadCheckingViolationPolicyEnforcement.java | 142 ++
...tDisableTableViolationPolicyEnforcement.java | 59 +
...TestNoInsertsViolationPolicyEnforcement.java | 57 +
...esCompactionsViolationPolicyEnforcement.java | 58 +
.../TestNoWritesViolationPolicyEnforcement.java | 57 +
57 files changed, 6492 insertions(+), 1481 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaExceededException.java
-
HBASE-17000 Implement computation of online region sizes and report to the
Master
Includes a trivial implementation of the Master-side collection to
avoid. Only enough to write a test to verify RS collection.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/91b27eda
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/91b27eda
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/91b27eda
Branch: refs/heads/HBASE-16961
Commit: 91b27eda65c438836cd4e3166a9cd683485275a8
Parents: e0c0e95
Author: Josh Elser
Authored: Mon Nov 7 13:46:42 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:24:20 2017 -0400
--
.../generated/RegionServerStatusProtos.java | 2071 +-
.../src/main/protobuf/RegionServerStatus.proto | 22 +
.../hadoop/hbase/master/MasterRpcServices.java | 19 +
.../quotas/FileSystemUtilizationChore.java | 205 ++
.../hadoop/hbase/quotas/MasterQuotaManager.java | 15 +
.../hbase/regionserver/HRegionServer.java | 72 +
.../quotas/TestFileSystemUtilizationChore.java | 357 +++
.../hadoop/hbase/quotas/TestRegionSizeUse.java | 194 ++
.../TestRegionServerRegionSpaceUseReport.java | 99 +
9 files changed, 3032 insertions(+), 22 deletions(-)
--
HBASE-17943 Addendum increases the threshold value of in-memory compaction for
TestWalAndCompactingMemStoreFlush
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9053ec6f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9053ec6f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9053ec6f
Branch: refs/heads/HBASE-16961
Commit: 9053ec6fe6505eba4f14adfdd83329511e4a77f0
Parents: e95cf47
Author: Chia-Ping Tsai
Authored: Sat Apr 22 20:47:55 2017 +0800
Committer: Chia-Ping Tsai
Committed: Sat Apr 22 20:47:55 2017 +0800
--
.../hbase/regionserver/TestWalAndCompactingMemStoreFlush.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/9053ec6f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
index 2c16399..3b2ebe2 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
@@ -36,7 +36,6 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -136,7 +135,7 @@ public class TestWalAndCompactingMemStoreFlush {
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY,
FlushNonSloppyStoresFirstPolicy.class.getName());
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN,
75 * 1024);
-conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
0.25);
+conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
0.5);
// set memstore to do data compaction
conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
String.valueOf(MemoryCompactionPolicy.EAGER));
@@ -771,7 +770,7 @@ public class TestWalAndCompactingMemStoreFlush {
FlushNonSloppyStoresFirstPolicy.class.getName());
conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN,
75 * 1024);
-conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
0.5);
+conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
0.8);
// set memstore to do index compaction with merge
conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
String.valueOf(MemoryCompactionPolicy.BASIC));
HBASE-17864: Implement async snapshot/cloneSnapshot/restoreSnapshot methods
Signed-off-by: Guanghao Zhang
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d39f40e7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d39f40e7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d39f40e7
Branch: refs/heads/HBASE-16961
Commit: d39f40e787ecab54ee597ac4463bbbd2f5e944d9
Parents: 33dadc1
Author: huzheng
Authored: Thu Apr 20 18:59:43 2017 +0800
Committer: Guanghao Zhang
Committed: Fri Apr 21 18:57:43 2017 +0800
--
.../apache/hadoop/hbase/client/AsyncAdmin.java | 88 +++
.../hadoop/hbase/client/AsyncHBaseAdmin.java| 145 +++
.../apache/hadoop/hbase/client/HBaseAdmin.java | 3 +-
.../org/apache/hadoop/hbase/HConstants.java | 4 +
.../hbase/client/TestAsyncSnapshotAdminApi.java | 112 ++
5 files changed, 351 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d39f40e7/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index 5d2955f..b7c60dd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -573,4 +573,92 @@ public interface AsyncAdmin {
* {@link CompletableFuture}.
*/
CompletableFuture> listReplicatedTableCFs();
+
+ /**
+ * Take a snapshot for the given table. If the table is enabled, a
FLUSH-type snapshot will be
+ * taken. If the table is disabled, an offline snapshot is taken. Snapshots
are considered unique
+ * based on the name of the snapshot. Attempts to take a snapshot
with the same name (even
+ * a different type or with different parameters) will fail with a
+ * {@link org.apache.hadoop.hbase.snapshot.SnapshotCreationException}
indicating the duplicate
+ * naming. Snapshot names follow the same naming constraints as tables in
HBase. See
+ * {@link
org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+ * @param snapshotName name of the snapshot to be created
+ * @param tableName name of the table for which snapshot is created
+ */
+ CompletableFuture snapshot(String snapshotName, TableName tableName);
+
+ /**
+ * Create typed snapshot of the table. Snapshots are considered unique based
on the name of the
+ * snapshot. Attempts to take a snapshot with the same name (even a
different type or with
+ * different parameters) will fail with a
+ * {@link org.apache.hadoop.hbase.snapshot.SnapshotCreationException}
indicating the duplicate
+ * naming. Snapshot names follow the same naming constraints as tables in
HBase. See
+ * {@link
org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+ * @param snapshotName name to give the snapshot on the filesystem. Must be
unique from all other
+ * snapshots stored on the cluster
+ * @param tableName name of the table to snapshot
+ * @param type type of snapshot to take
+ */
+ CompletableFuture snapshot(final String snapshotName, final TableName
tableName,
+ SnapshotType type);
+
+ /**
+ * Take a snapshot and wait for the server to complete that snapshot
asynchronously. Only a single
+ * snapshot should be taken at a time for an instance of HBase, or results
may be undefined (you
+ * can tell multiple HBase clusters to snapshot at the same time, but only
one at a time for a
+ * single cluster). Snapshots are considered unique based on the name of
the snapshot.
+ * Attempts to take a snapshot with the same name (even a different type or
with different
+ * parameters) will fail with a {@link
org.apache.hadoop.hbase.snapshot.SnapshotCreationException}
+ * indicating the duplicate naming. Snapshot names follow the same naming
constraints as tables in
+ * HBase. See {@link
org.apache.hadoop.hbase.TableName#isLegalFullyQualifiedTableName(byte[])}.
+ * You should probably use {@link #snapshot(String,
org.apache.hadoop.hbase.TableName)} unless you
+ * are sure about the type of snapshot that you want to take.
+ * @param snapshot snapshot to take
+ */
+ CompletableFuture snapshot(SnapshotDescription snapshot);
+
+ /**
+ * Check the current state of the passed snapshot. There are three possible
states:
+ *
+ * running - returns false
+ * finished - returns true
+ * finished with error - throws the exception that caused the snapshot
to fail
+ *
+ * The cluster only knows about the most recent snapshot. Therefore, if
another snapsh
HBASE-17302 The region flush request disappeared from flushQueue - addendum
(Guangxu Cheng)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/435104af
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/435104af
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/435104af
Branch: refs/heads/HBASE-16961
Commit: 435104af70232076145df4211da297c9235cd58f
Parents: 9053ec6
Author: tedyu
Authored: Sun Apr 23 07:37:34 2017 -0700
Committer: tedyu
Committed: Sun Apr 23 07:37:34 2017 -0700
--
.../org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/435104af/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index aaa9572..2f2a4cf 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -816,8 +816,8 @@ class MemStoreFlusher implements FlushRequester {
return false;
}
FlushRegionEntry other = (FlushRegionEntry) obj;
- if (!this.region.getRegionInfo().getRegionNameAsString().equals(
- other.region.getRegionInfo().getRegionNameAsString())) {
+ if (!Bytes.equals(this.region.getRegionInfo().getRegionName(),
+ other.region.getRegionInfo().getRegionName())) {
return false;
}
return compareTo(other) == 0;
HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23194dca
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23194dca
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23194dca
Branch: refs/heads/HBASE-16961
Commit: 23194dcac8b4e4dbe7598fd51fa89f18b77a59dc
Parents: 2557506
Author: tedyu
Authored: Thu Nov 17 10:19:52 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 25 18:19:48 2017 -0400
--
.../hbase/quotas/QuotaSettingsFactory.java | 47 +
.../apache/hadoop/hbase/quotas/QuotaType.java |1 +
.../hadoop/hbase/quotas/SpaceLimitSettings.java | 166 ++
.../hbase/quotas/SpaceViolationPolicy.java | 44 +
.../hbase/shaded/protobuf/ProtobufUtil.java | 51 +
.../hbase/quotas/TestQuotaSettingsFactory.java | 148 ++
.../hbase/quotas/TestSpaceLimitSettings.java| 119 ++
.../shaded/protobuf/generated/MasterProtos.java | 588 --
.../shaded/protobuf/generated/QuotaProtos.java | 1739 +-
.../src/main/protobuf/Master.proto |2 +
.../src/main/protobuf/Quota.proto | 21 +
.../hbase/protobuf/generated/QuotaProtos.java | 1682 -
hbase-protocol/src/main/protobuf/Quota.proto| 21 +
13 files changed, 4293 insertions(+), 336 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/23194dca/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 3622a32..8512e39 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -27,6 +27,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRe
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
@InterfaceAudience.Public
public class QuotaSettingsFactory {
@@ -89,6 +90,9 @@ public class QuotaSettingsFactory {
if (quotas.getBypassGlobals() == true) {
settings.add(new QuotaGlobalsSettingsBypass(userName, tableName,
namespace, true));
}
+if (quotas.hasSpace()) {
+ settings.add(fromSpace(tableName, namespace, quotas.getSpace()));
+}
return settings;
}
@@ -122,6 +126,18 @@ public class QuotaSettingsFactory {
return settings;
}
+ static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota
protoQuota) {
+if ((null == table && null == namespace) || (null != table && null !=
namespace)) {
+ throw new IllegalArgumentException("Can only construct
SpaceLimitSettings for a table or namespace.");
+}
+if (null != table) {
+ return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
+} else {
+ // namespace must be non-null
+ return SpaceLimitSettings.fromSpaceQuota(namespace, protoQuota);
+}
+ }
+
/* ==
* RPC Throttle
*/
@@ -278,4 +294,35 @@ public class QuotaSettingsFactory {
public static QuotaSettings bypassGlobals(final String userName, final
boolean bypassGlobals) {
return new QuotaGlobalsSettingsBypass(userName, null, null, bypassGlobals);
}
+
+ /* ==
+ * FileSystem Space Settings
+ */
+
+ /**
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given table to the given size in bytes.
+ * When the space usage is exceeded by the table, the provided {@link
SpaceViolationPolicy} is enacted on the table.
+ *
+ * @param tableName The name of the table on which the quota should be
applied.
+ * @param sizeLimit The limit of a table's size in bytes.
+ * @param violationPolicy The action to take when the quota is exceeded.
+ * @return An {@link QuotaSettings} object.
+ */
+ public static QuotaSettings limitTableSpace(final TableName tableName, long
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
+ }
+
+ /**
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given namespace to the given size in bytes.
+ * When the space usage is exceeded by a
HBASE-17952 The new options for PE tool do not work
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/72fac379
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/72fac379
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/72fac379
Branch: refs/heads/HBASE-16961
Commit: 72fac379815d79fdeac1452b36cd12fb5492f627
Parents: 49f707f
Author: zhangduo
Authored: Mon Apr 24 17:00:32 2017 +0800
Committer: zhangduo
Committed: Tue Apr 25 09:34:49 2017 +0800
--
.../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 3 +++
1 file changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/72fac379/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index a3d3254..d0b7319 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -685,6 +685,9 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.columns = that.columns;
this.caching = that.caching;
this.inMemoryCompaction = that.inMemoryCompaction;
+ this.asyncPrefetch = that.asyncPrefetch;
+ this.cacheBlocks = that.cacheBlocks;
+ this.scanReadType = that.scanReadType;
}
public int getCaching() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f592b00/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 000..98236c2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+ private static final Log LOG =
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+ private static final int SIZE_PER_VALUE = 256;
+ private static final String F1 = "f1";
+ private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
+ private static final AtomicLong COUNTER = new AtomicLong(0);
+ private static final long ONE_MEGABYTE = 1024L * 1024L;
+ private static final long DEFAULT_WAIT_MILLIS = 500;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private HMaster master;
+ private QuotaObserverChore chore;
+ private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+Configuration conf = TEST_UTIL.getConfiguration();
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY,
1000);
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+final Connection conn = TEST_UTIL.getConnection();
+
HBASE-16995 Build client Java API and client protobuf messages - addendum fixes
line lengths (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a5087ed
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a5087ed
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a5087ed
Branch: refs/heads/HBASE-16961
Commit: 3a5087ed113c48ddd33a96b47c23bf62b5cceaa3
Parents: 2b9b00b
Author: tedyu
Authored: Mon Nov 21 13:00:27 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 25 18:20:14 2017 -0400
--
.../hbase/quotas/QuotaSettingsFactory.java | 20
.../hadoop/hbase/quotas/SpaceLimitSettings.java | 8
.../hbase/shaded/protobuf/ProtobufUtil.java | 7 ---
3 files changed, 20 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3a5087ed/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 8512e39..7f1c180 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -128,7 +128,8 @@ public class QuotaSettingsFactory {
static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota
protoQuota) {
if ((null == table && null == namespace) || (null != table && null !=
namespace)) {
- throw new IllegalArgumentException("Can only construct
SpaceLimitSettings for a table or namespace.");
+ throw new IllegalArgumentException(
+ "Can only construct SpaceLimitSettings for a table or namespace.");
}
if (null != table) {
return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
@@ -300,29 +301,32 @@ public class QuotaSettingsFactory {
*/
/**
- * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given table to the given size in bytes.
- * When the space usage is exceeded by the table, the provided {@link
SpaceViolationPolicy} is enacted on the table.
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given table
+ * to the given size in bytes. When the space usage is exceeded by the
table, the provided
+ * {@link SpaceViolationPolicy} is enacted on the table.
*
* @param tableName The name of the table on which the quota should be
applied.
* @param sizeLimit The limit of a table's size in bytes.
* @param violationPolicy The action to take when the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
- public static QuotaSettings limitTableSpace(final TableName tableName, long
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+ public static QuotaSettings limitTableSpace(
+ final TableName tableName, long sizeLimit, final SpaceViolationPolicy
violationPolicy) {
return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
}
/**
- * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given namespace to the given size in bytes.
- * When the space usage is exceeded by all tables in the namespace, the
provided {@link SpaceViolationPolicy} is enacted on
- * all tables in the namespace.
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given
+ * namespace to the given size in bytes. When the space usage is exceeded by
all tables in the
+ * namespace, the provided {@link SpaceViolationPolicy} is enacted on all
tables in the namespace.
*
* @param namespace The namespace on which the quota should be applied.
* @param sizeLimit The limit of the namespace's size in bytes.
* @param violationPolicy The action to take when the the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
- public static QuotaSettings limitNamespaceSpace(final String namespace, long
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+ public static QuotaSettings limitNamespaceSpace(
+ final String namespace, long sizeLimit, final SpaceViolationPolicy
violationPolicy) {
return new SpaceLimitSettings(namespace, sizeLimit, violationPolicy);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3a5087ed/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
b/hbase-client/src/main/java/org/apach
HBASE-15143 Procedure v2 - Web UI displaying queues
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/25575064
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/25575064
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/25575064
Branch: refs/heads/HBASE-16961
Commit: 25575064154fe1cc7ff8970e8f15a3cff648f37a
Parents: 1367519
Author: Balazs Meszaros
Authored: Mon Feb 13 13:50:56 2017 -0800
Committer: Michael Stack
Committed: Tue Apr 25 09:39:28 2017 -0700
--
.../org/apache/hadoop/hbase/client/Admin.java |9 +
.../hbase/client/ConnectionImplementation.java | 11 +-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 42 +-
.../client/ShortCircuitMasterConnection.java|6 +
.../hbase/shaded/protobuf/ProtobufUtil.java | 183 +-
.../hadoop/hbase/procedure2/LockInfo.java | 128 +
.../hadoop/hbase/procedure2/LockAndQueue.java | 21 +-
.../hadoop/hbase/procedure2/LockStatus.java |1 +
.../hbase/procedure2/ProcedureScheduler.java|7 +
.../hadoop/hbase/procedure2/ProcedureUtil.java |4 +-
.../procedure2/SimpleProcedureScheduler.java| 10 +-
.../protobuf/generated/LockServiceProtos.java | 2423 +-
.../shaded/protobuf/generated/MasterProtos.java | 2152
.../src/main/protobuf/LockService.proto | 22 +
.../src/main/protobuf/Master.proto | 11 +
.../hbase/tmpl/master/MasterStatusTmpl.jamon|2 +-
.../hbase/coprocessor/MasterObserver.java | 19 +
.../org/apache/hadoop/hbase/master/HMaster.java | 37 +-
.../hbase/master/MasterCoprocessorHost.java | 21 +
.../hadoop/hbase/master/MasterRpcServices.java | 147 +-
.../hadoop/hbase/master/MasterServices.java |9 +-
.../hbase/master/locking/LockProcedure.java |8 +-
.../procedure/MasterProcedureScheduler.java | 119 +-
.../hbase-webapps/master/procedures.jsp | 127 +-
.../resources/hbase-webapps/master/snapshot.jsp |2 +-
.../hbase-webapps/master/snapshotsStats.jsp |2 +-
.../resources/hbase-webapps/master/table.jsp|2 +-
.../hbase-webapps/master/tablesDetailed.jsp |2 +-
.../main/resources/hbase-webapps/master/zk.jsp |2 +-
.../hbase/coprocessor/TestMasterObserver.java | 38 +
.../hbase/master/MockNoopMasterServices.java|9 +-
.../procedure/TestMasterProcedureScheduler.java | 169 +-
.../hadoop/hbase/protobuf/TestProtobufUtil.java | 41 +-
.../hbase/shaded/protobuf/TestProtobufUtil.java | 151 ++
hbase-shell/src/main/ruby/hbase/admin.rb|5 +
hbase-shell/src/main/ruby/shell.rb |3 +-
hbase-shell/src/main/ruby/shell/commands.rb |5 +
.../src/main/ruby/shell/commands/list_locks.rb | 60 +
hbase-shell/src/main/ruby/shell/formatter.rb|9 +-
.../src/test/ruby/shell/list_locks_test.rb | 152 ++
40 files changed, 5409 insertions(+), 762 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index f2fc9a5..3e767d2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -45,6 +45,7 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.replication.TableCFs;
import org.apache.hadoop.hbase.client.security.SecurityCapability;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
+import org.apache.hadoop.hbase.procedure2.LockInfo;
import org.apache.hadoop.hbase.quotas.QuotaFilter;
import org.apache.hadoop.hbase.quotas.QuotaRetriever;
import org.apache.hadoop.hbase.quotas.QuotaSettings;
@@ -1250,6 +1251,14 @@ public interface Admin extends Abortable, Closeable {
throws IOException;
/**
+ * List locks.
+ * @return lock list
+ * @throws IOException if a remote or network exception occurs
+ */
+ LockInfo[] listLocks()
+ throws IOException;
+
+ /**
* Roll the log writer. I.e. for filesystem based write ahead logs, start
writing to a new file.
*
* Note that the actual rolling of the log writer is asynchronous and may
not be complete when
http://git-wip-us.apache.org/repos/asf/hbase/blob/25575064/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase
HBASE-16995 Build client Java API and client protobuf messages - addendum fixes
white spaces (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2b9b00b8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2b9b00b8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2b9b00b8
Branch: refs/heads/HBASE-16961
Commit: 2b9b00b835bc163302b68152fbb1feace4896b9b
Parents: 23194dc
Author: tedyu
Authored: Thu Nov 17 10:42:18 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 25 18:19:48 2017 -0400
--
.../hbase/quotas/TestQuotaSettingsFactory.java| 2 +-
.../shaded/protobuf/generated/MasterProtos.java | 2 +-
.../shaded/protobuf/generated/QuotaProtos.java| 18 +-
.../hbase/protobuf/generated/QuotaProtos.java | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b9b00b8/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
index 17015d6..e0012a7 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
@@ -44,7 +44,7 @@ import org.junit.experimental.categories.Category;
*/
@Category(SmallTests.class)
public class TestQuotaSettingsFactory {
-
+
@Test
public void testAllQuotasAddedToList() {
final SpaceQuota spaceQuota = SpaceQuota.newBuilder()
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b9b00b8/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
index da6c65e..42ab969 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
@@ -64879,7 +64879,7 @@ public final class MasterProtos {
* optional .hbase.pb.SpaceLimitRequest space_limit = 8;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
getSpaceLimitFieldBuilder() {
if (spaceLimitBuilder_ == null) {
spaceLimitBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b9b00b8/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index e3c6bfd..0ab2576 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.gene
http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index e90c934..c70b736 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10429,7 +10429,7 @@ public final class RegionServerStatusProtos {
return memoizedHashCode;
}
int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
+ hash = (19 * hash) + getDescriptor().hashCode();
if (hasRegion()) {
hash = (37 * hash) + REGION_FIELD_NUMBER;
hash = (53 * hash) + getRegion().hashCode();
@@ -10824,7 +10824,7 @@ public final class RegionServerStatusProtos {
* optional .hbase.pb.RegionInfo region = 1;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo,
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo,
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>
getRegionFieldBuilder() {
if (regionBuilder_ == null) {
regionBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -10940,7 +10940,7 @@ public final class RegionServerStatusProtos {
/**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
-
java.util.List
+
java.util.List
getSpaceUseList();
/**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
@@ -10953,7 +10953,7 @@ public final class RegionServerStatusProtos {
/**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
-java.util.List
+java.util.List
getSpaceUseOrBuilderList();
/**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
@@ -11056,7 +11056,7 @@ public final class RegionServerStatusProtos {
/**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
-public java.util.List
+public java.util.List
getSpaceUseOrBuilderList() {
return spaceUse_;
}
@@ -11142,7 +11142,7 @@ public final class RegionServerStatusProtos {
return memoizedHashCode;
}
int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
+ hash = (19 * hash) + getDescriptor().hashCode();
if (getSpaceUseCount() > 0) {
hash = (37 * hash) + SPACE_USE_FIELD_NUMBER;
hash = (53 * hash) + getSpaceUseList().hashCode();
@@ -11368,7 +11368,7 @@ public final class RegionServerStatusProtos {
spaceUseBuilder_ = null;
spaceUse_ = other.spaceUse_;
bitField0_ = (bitField0_ & ~0x0001);
- spaceUseBuilder_ =
+ spaceUseBuilder_ =
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders
?
getSpaceUseFieldBuilder() : null;
} else {
@@ -11604,7 +11604,7 @@ public final class RegionServerStatusProtos {
/**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
- public java.util.List
+ public java.util.List
getSpaceUseOrBuilderList() {
if (spaceUseBuilder_ != null) {
return spaceUseBuilder_.getMessageOrBuilderList();
@@ -11630,12 +11630,12 @@ public final class RegionServerStatusProtos {
/**
* repeated .hbase.pb.RegionSpaceUse space_use = 1;
*/
- public
java.util.List
+ public
java.util.List
getSpaceUseBuilderList() {
return getSpaceUseFieldBuilder().getBuilderList();
}
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.RepeatedFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse,
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse,
org.apache.hadoop.hbase.shaded
http://git-wip-us.apache.org/repos/asf/hbase/blob/66cf4809/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 8b127d9..973ac8c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -37,9 +37,8 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.quotas.QuotaViolationStore.ViolationState;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import com.google.common.annotations.VisibleForTesting;
@@ -54,51 +53,51 @@ import com.google.common.collect.Multimap;
@InterfaceAudience.Private
public class QuotaObserverChore extends ScheduledChore {
private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
- static final String VIOLATION_OBSERVER_CHORE_PERIOD_KEY =
- "hbase.master.quotas.violation.observer.chore.period";
- static final int VIOLATION_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; //
5 minutes in millis
+ static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
+ "hbase.master.quotas.observer.chore.period";
+ static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
- static final String VIOLATION_OBSERVER_CHORE_DELAY_KEY =
- "hbase.master.quotas.violation.observer.chore.delay";
- static final long VIOLATION_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
+ "hbase.master.quotas.observer.chore.delay";
+ static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
- static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_KEY =
- "hbase.master.quotas.violation.observer.chore.timeunit";
- static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_DEFAULT =
TimeUnit.MILLISECONDS.name();
+ static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
+ "hbase.master.quotas.observer.chore.timeunit";
+ static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_DEFAULT =
TimeUnit.MILLISECONDS.name();
- static final String VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_KEY =
- "hbase.master.quotas.violation.observer.report.percent";
- static final double VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
+ static final String QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY =
+ "hbase.master.quotas.observer.report.percent";
+ static final double QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
private final Connection conn;
private final Configuration conf;
private final MasterQuotaManager quotaManager;
/*
- * Callback that changes in quota violation are passed to.
+ * Callback that changes in quota snapshots are passed to.
*/
- private final SpaceQuotaViolationNotifier violationNotifier;
+ private final SpaceQuotaSnapshotNotifier snapshotNotifier;
/*
- * Preserves the state of quota violations for tables and namespaces
+ * Preserves the state of quota snapshots for tables and namespaces
*/
- private final Map tableQuotaViolationStates;
- private final Map namespaceQuotaViolationStates;
+ private final Map tableQuotaSnapshots;
+ private final Map namespaceQuotaSnapshots;
/*
- * Encapsulates logic for moving tables/namespaces into or out of quota
violation
+ * Encapsulates logic for tracking the state of a table/namespace WRT space
quotas
*/
- private QuotaViolationStore tableViolationStore;
- private QuotaViolationStore namespaceViolationStore;
+ private QuotaSnapshotStore tableSnapshotStore;
+ private QuotaSnapshotStore namespaceSnapshotStore;
public QuotaObserverChore(HMaster master) {
this(
master.getConnection(), master.getConfiguration(),
-master.getSpaceQuotaViolationNotifier(),
master.getMasterQuotaManager(),
+master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
master);
}
QuotaObserverChore(
- Connection conn, Configuration conf, SpaceQuotaViolationNotifier
violationNotifier,
+ Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier
snapshotNotifier,
MasterQuotaManager quotaManager, Stoppable stopper) {
super(
QuotaObserverChore.class.getSimpleName(), stopper
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfd374d0/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
new file mode 100644
index 000..4a7000c
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Objects;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * Test case for {@link TableSpaceQuotaViolationNotifier}.
+ */
+@Category(SmallTests.class)
+public class TestTableSpaceQuotaViolationNotifier {
+
+ private TableSpaceQuotaViolationNotifier notifier;
+ private Connection conn;
+
+ @Before
+ public void setup() throws Exception {
+notifier = new TableSpaceQuotaViolationNotifier();
+conn = mock(Connection.class);
+notifier.initialize(conn);
+ }
+
+ @Test
+ public void testToViolation() throws Exception {
+final TableName tn = TableName.valueOf("inviolation");
+final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Put expectedPut = new Put(Bytes.toBytes("t." +
tn.getNameAsString()));
+final SpaceQuota protoQuota = SpaceQuota.newBuilder()
+.setViolationPolicy(ProtobufUtil.toProtoViolationPolicy(policy))
+.build();
+expectedPut.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"),
protoQuota.toByteArray());
+
+notifier.transitionTableToViolation(tn, policy);
+
+verify(quotaTable).put(argThat(new SingleCellPutMatcher(expectedPut)));
+ }
+
+ @Test
+ public void testToObservance() throws Exception {
+final TableName tn = TableName.valueOf("notinviolation");
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Delete expectedDelete = new Delete(Bytes.toBytes("t." +
tn.getNameAsString()));
+expectedDelete.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"));
+
+notifier.transitionTableToObservance(tn);
+
+verify(quotaTable).delete(argThat(new
SingleCellDeleteMatcher(expectedDelete)));
+ }
+
+ /**
+ * Parameterized for Puts.
+ */
+ private static class SingleCellPutMatcher extends
SingleCellMutationMatcher {
+private SingleCellPutMatcher(Put expected) {
+ super(expected);
+}
+ }
+
+ /**
+ * Parameterized for Deletes.
+ */
+ private static class SingleCellDeleteMatcher extends
SingleCellMutationMatcher {
+private SingleCellDeleteMatcher(Delete expected) {
+ super(expected);
+}
+ }
+
+ /**
+ * Quick hack to verify a Mutation with one column.
+ */
+ private static class SingleCellMutationMatcher extends ArgumentMatcher
{
+private final Mutation expected;
+
+private SingleCellMutationMatcher(Mutation expected) {
+ this.
HBASE-16999 Implement master and regionserver synchronization of quota state
* Implement the RegionServer reading violation from the quota table
* Implement the Master reporting violations to the quota table
* RegionServers need to track its enforced policies
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cfd374d0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cfd374d0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cfd374d0
Branch: refs/heads/HBASE-16961
Commit: cfd374d04f008e2e2890f800692b176c143f32bb
Parents: 5f592b0
Author: Josh Elser
Authored: Fri Nov 18 15:38:19 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:24:20 2017 -0400
--
.../hadoop/hbase/quotas/QuotaTableUtil.java | 92 -
.../org/apache/hadoop/hbase/master/HMaster.java | 35 +++-
.../hadoop/hbase/quotas/QuotaObserverChore.java | 5 +-
.../hbase/quotas/RegionServerQuotaManager.java | 200 ---
.../quotas/RegionServerRpcQuotaManager.java | 200 +++
.../quotas/RegionServerSpaceQuotaManager.java | 169
.../quotas/SpaceQuotaViolationNotifier.java | 16 +-
.../SpaceQuotaViolationNotifierFactory.java | 62 ++
.../SpaceQuotaViolationNotifierForTest.java | 4 +
...SpaceQuotaViolationPolicyRefresherChore.java | 154 ++
.../TableSpaceQuotaViolationNotifier.java | 55 +
.../hbase/regionserver/HRegionServer.java | 21 +-
.../hbase/regionserver/RSRpcServices.java | 7 +-
.../regionserver/RegionServerServices.java | 12 +-
.../hadoop/hbase/MockRegionServerServices.java | 10 +-
.../hadoop/hbase/master/MockRegionServer.java | 10 +-
.../TestQuotaObserverChoreWithMiniCluster.java | 2 +
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 47 +
.../hadoop/hbase/quotas/TestQuotaThrottle.java | 4 +-
.../TestRegionServerSpaceQuotaManager.java | 127
...SpaceQuotaViolationPolicyRefresherChore.java | 131
.../TestTableSpaceQuotaViolationNotifier.java | 144 +
22 files changed, 1281 insertions(+), 226 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cfd374d0/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 8ef4f08..b5eac48 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -24,16 +24,20 @@ import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
@@ -44,7 +48,12 @@ import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
+import
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Strings;
@@ -53,9 +62,8 @@ import org.apache.hadoop.hbase.util.Strings;
*
* ROW-KEY FAM/QUALDATA
* n. q:s
- * n. u:du
* t. q:s
- * t. u:du
+ * t. u:v
* u. q:s
* u.
http://git-wip-us.apache.org/repos/asf/hbase/blob/23194dca/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index 05894b9..1925828 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -217,12 +217,20 @@ public final class QuotaProtos {
* THROTTLE = 1;
*/
THROTTLE(0, 1),
+/**
+ * SPACE = 2;
+ */
+SPACE(1, 2),
;
/**
* THROTTLE = 1;
*/
public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
public final int getNumber() { return value; }
@@ -230,6 +238,7 @@ public final class QuotaProtos {
public static QuotaType valueOf(int value) {
switch (value) {
case 1: return THROTTLE;
+case 2: return SPACE;
default: return null;
}
}
@@ -281,6 +290,142 @@ public final class QuotaProtos {
// @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
}
+ /**
+ * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+ *
+ *
+ * Defines what action should be taken when the SpaceQuota is violated
+ *
+ */
+ public enum SpaceViolationPolicy
+ implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * DISABLE = 1;
+ *
+ *
+ * Disable the table(s)
+ *
+ */
+DISABLE(0, 1),
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ */
+NO_WRITES_COMPACTIONS(1, 2),
+/**
+ * NO_WRITES = 3;
+ *
+ *
+ * No writes or bulk-loads
+ *
+ */
+NO_WRITES(2, 3),
+/**
+ * NO_INSERTS = 4;
+ *
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ */
+NO_INSERTS(3, 4),
+;
+
+/**
+ * DISABLE = 1;
+ *
+ *
+ * Disable the table(s)
+ *
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ * NO_WRITES = 3;
+ *
+ *
+ * No writes or bulk-loads
+ *
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ * NO_INSERTS = 4;
+ *
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() { return value; }
+
+public static SpaceViolationPolicy valueOf(int value) {
+ switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+ }
+}
+
+public static
com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static
com.google.protobuf.Internal.EnumLiteMap
+internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap()
{
+public SpaceViolationPolicy findValueByNumber(int number) {
+ return SpaceViolationPolicy.valueOf(number);
+}
+ };
+
+public final com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+}
+public final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+}
+
+private final int index;
+private final int value;
+
+private SpaceViolationPolicy(int index, int value) {
+ this.index = index;
+ this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.SpaceViolationPolicy)
+ }
+
public interface TimedQuotaOrBuilder
extends com.google.protobuf.MessageOrBuilder {
@@ -3315,6 +3460,20 @@ public final class QuotaProtos {
HBASE-17025 Add shell commands for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3a35c425
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3a35c425
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3a35c425
Branch: refs/heads/HBASE-16961
Commit: 3a35c425c6fae27a736fb7d25631b27025de3732
Parents: 67f7c8c
Author: Josh Elser
Authored: Wed Jan 11 11:55:29 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:24:20 2017 -0400
--
hbase-shell/src/main/ruby/hbase/quotas.rb | 62 -
hbase-shell/src/main/ruby/hbase_constants.rb| 1 +
.../src/main/ruby/shell/commands/set_quota.rb | 45 +-
.../hadoop/hbase/client/AbstractTestShell.java | 1 +
hbase-shell/src/test/ruby/hbase/quotas_test.rb | 137 +++
hbase-shell/src/test/ruby/tests_runner.rb | 1 +
6 files changed, 242 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3a35c425/hbase-shell/src/main/ruby/hbase/quotas.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb
b/hbase-shell/src/main/ruby/hbase/quotas.rb
index bf2dc63..d99fe72 100644
--- a/hbase-shell/src/main/ruby/hbase/quotas.rb
+++ b/hbase-shell/src/main/ruby/hbase/quotas.rb
@@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType
java_import org.apache.hadoop.hbase.quotas.QuotaFilter
java_import org.apache.hadoop.hbase.quotas.QuotaRetriever
java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory
+java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy
module HBaseQuotasConstants
+ # RPC Quota constants
GLOBAL_BYPASS = 'GLOBAL_BYPASS'
THROTTLE_TYPE = 'THROTTLE_TYPE'
THROTTLE = 'THROTTLE'
REQUEST = 'REQUEST'
WRITE = 'WRITE'
READ = 'READ'
+ # Space quota constants
+ SPACE = 'SPACE'
+ NO_INSERTS = 'NO_INSERTS'
+ NO_WRITES = 'NO_WRITES'
+ NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'
+ DISABLE = 'DISABLE'
end
module Hbase
@@ -107,6 +115,54 @@ module Hbase
@admin.setQuota(settings)
end
+def limit_space(args)
+ raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil?
and args.kind_of?(Hash))
+ # Let the user provide a raw number
+ if args[LIMIT].is_a?(Numeric)
+limit = args[LIMIT]
+ else
+# Parse a string a 1K, 2G, etc.
+limit = _parse_size(args[LIMIT])
+ end
+ # Extract the policy, failing if something bogus was provided
+ policy = SpaceViolationPolicy.valueOf(args[POLICY])
+ # Create a table or namespace quota
+ if args.key?(TABLE)
+if args.key?(NAMESPACE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+settings =
QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)),
limit, policy)
+ elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+settings =
QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy)
+ else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+ end
+ # Apply the quota
+ @admin.setQuota(settings)
+end
+
+def remove_space_limit(args)
+ raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil?
and args.kind_of?(Hash))
+ if args.key?(TABLE)
+if args.key?(NAMESPACE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+table = TableName.valueOf(args.delete(TABLE))
+settings = QuotaSettingsFactory.removeTableSpaceLimit(table)
+ elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+settings =
QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE))
+ else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+ end
+ @admin.setQuota(settings)
+end
+
def set_global_bypass(bypass, args)
raise(ArgumentError, "Arguments should be a Hash") unless
args.kind_of?(Hash)
@@ -171,7 +227,7 @@ module Hbase
return _size_from_str(match[1].to_i, match[2])
end
else
-raise "Invalid size limit syntax"
+raise(ArgumentError, "Invalid size limit syntax")
end
end
@@ -188,7 +244,7 @@ module Hbase
end
if limit <= 0
- raise "Invalid throttle limit, must be greater then 0"
+ raise(ArgumentError, "Invalid throttle limit, must be greater then
0")
end
case match[3]
@@ -200,7 +256,7 @@ modul
HBASE-16998 Implement Master-side analysis of region space reports
Adds a new Chore to the Master that analyzes the reports that are
sent by RegionServers. The Master must then, for all tables with
quotas, determine the tables that are violating quotas and move
those tables into violation. Similarly, tables no longer violating
the quota can be moved out of violation.
The Chore is the "stateful" bit, managing which tables are and
are not in violation. Everything else is just performing
computation and informing the Chore on the updated state.
Added InterfaceAudience annotations and clean up the QuotaObserverChore
constructor. Cleaned up some javadoc and QuotaObserverChore. Reuse
the QuotaViolationStore impl objects.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f592b00
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f592b00
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f592b00
Branch: refs/heads/HBASE-16961
Commit: 5f592b000572597b17601cc12286f38b4719b4d7
Parents: cf7dd95
Author: Josh Elser
Authored: Tue Nov 8 18:55:12 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:24:20 2017 -0400
--
.../hadoop/hbase/quotas/QuotaRetriever.java | 27 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 20 +
.../hadoop/hbase/quotas/MasterQuotaManager.java | 1 +
.../quotas/NamespaceQuotaViolationStore.java| 127
.../hadoop/hbase/quotas/QuotaObserverChore.java | 618 +++
.../hbase/quotas/QuotaViolationStore.java | 89 +++
.../quotas/SpaceQuotaViolationNotifier.java | 44 ++
.../SpaceQuotaViolationNotifierForTest.java | 50 ++
.../hbase/quotas/TableQuotaViolationStore.java | 127
.../TestNamespaceQuotaViolationStore.java | 156 +
.../hbase/quotas/TestQuotaObserverChore.java| 106
.../TestQuotaObserverChoreWithMiniCluster.java | 596 ++
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 4 -
.../quotas/TestTableQuotaViolationStore.java| 151 +
.../hbase/quotas/TestTablesWithQuotas.java | 198 ++
15 files changed, 2305 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f592b00/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
index 0f7baa5..4482693 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Objects;
import java.util.Queue;
import org.apache.commons.logging.Log;
@@ -54,11 +55,23 @@ public class QuotaRetriever implements Closeable,
Iterable {
private Connection connection;
private Table table;
- private QuotaRetriever() {
+ /**
+ * Should QutoaRetriever manage the state of the connection, or leave it be.
+ */
+ private boolean isManagedConnection = false;
+
+ QuotaRetriever() {
}
void init(final Configuration conf, final Scan scan) throws IOException {
-this.connection = ConnectionFactory.createConnection(conf);
+// Set this before creating the connection and passing it down to make sure
+// it's cleaned up if we fail to construct the Scanner.
+this.isManagedConnection = true;
+init(ConnectionFactory.createConnection(conf), scan);
+ }
+
+ void init(final Connection conn, final Scan scan) throws IOException {
+this.connection = Objects.requireNonNull(conn);
this.table = this.connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME);
try {
scanner = table.getScanner(scan);
@@ -77,10 +90,14 @@ public class QuotaRetriever implements Closeable,
Iterable {
this.table.close();
this.table = null;
}
-if (this.connection != null) {
- this.connection.close();
- this.connection = null;
+// Null out the connection on close() even if we didn't explicitly close it
+// to maintain typical semantics.
+if (isManagedConnection) {
+ if (this.connection != null) {
+this.connection.close();
+ }
}
+this.connection = null;
}
public QuotaSettings next() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f592b00/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/m
HBASE-17447 Implement a MasterObserver for automatically deleting space quotas
When a table or namespace is deleted, it would be nice to automatically
delete the quota on said table/NS. It's possible that not all people
would want this functionality so we can leave it up to the user to
configure this Observer.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb08814a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb08814a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb08814a
Branch: refs/heads/HBASE-16961
Commit: cb08814a409c5b69f7f5e74c0f0aab2cc0d77890
Parents: d09d967
Author: Josh Elser
Authored: Thu Mar 16 18:54:01 2017 -0400
Committer: Josh Elser
Committed: Tue Apr 25 18:37:38 2017 -0400
--
.../hbase/quotas/MasterSpaceQuotaObserver.java | 85 ++
.../quotas/TestMasterSpaceQuotaObserver.java| 169 +++
src/main/asciidoc/_chapters/ops_mgt.adoc| 17 ++
3 files changed, 271 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb08814a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
new file mode 100644
index 000..a3abf32
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+
+/**
+ * An observer to automatically delete space quotas when a table/namespace
+ * are deleted.
+ */
+@InterfaceAudience.Private
+public class MasterSpaceQuotaObserver implements MasterObserver {
+ private CoprocessorEnvironment cpEnv;
+ private Configuration conf;
+ private boolean quotasEnabled = false;
+
+ @Override
+ public void start(CoprocessorEnvironment ctx) throws IOException {
+this.cpEnv = ctx;
+this.conf = cpEnv.getConfiguration();
+this.quotasEnabled = QuotaUtil.isQuotaEnabled(conf);
+ }
+
+ @Override
+ public void postDeleteTable(
+ ObserverContext ctx, TableName tableName)
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+ return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName);
+if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings =
QuotaSettingsFactory.removeTableSpaceLimit(tableName);
+ try (Admin admin = conn.getAdmin()) {
+admin.setQuota(settings);
+ }
+}
+ }
+
+ @Override
+ public void postDeleteNamespace(
+ ObserverContext ctx, String namespace)
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+ return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(),
namespace);
+if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings =
QuotaSettingsFactory
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-protocol-shaded/src/main/protobuf/Master.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto
b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index ada9b35..0d0fb93 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -941,7 +941,11 @@ service MasterService {
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
returns(RemoveDrainFromRegionServersResponse);
- /** Fetches the Master's view of space quotas */
+ /** Fetches the Master's view of space utilization */
rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
returns(GetSpaceQuotaRegionSizesResponse);
+
+ /** Fetches the Master's view of quotas */
+ rpc GetQuotaStates(GetQuotaStatesRequest)
+returns(GetQuotaStatesResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 2d7e5f5..1a6d5ed 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
message RegionSizes {
optional TableName table_name = 1;
optional uint64 size = 2;
+
}
repeated RegionSizes sizes = 1;
}
@@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
}
repeated TableViolationPolicy violation_policies = 1;
}
+
+message GetQuotaStatesRequest {
+}
+
+message GetQuotaStatesResponse {
+ message TableQuotaSnapshot {
+optional TableName table_name = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+ }
+ message NamespaceQuotaSnapshot {
+optional string namespace = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+ }
+ repeated TableQuotaSnapshot table_snapshots = 1;
+ repeated NamespaceQuotaSnapshot ns_snapshots = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 128dc18..64960ac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -911,7 +911,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
// Create the quota snapshot notifier
spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
- this.quotaObserverChore = new QuotaObserverChore(this);
+ this.quotaObserverChore = new QuotaObserverChore(this,
getMasterMetrics());
// Start the chore to read the region FS space reports and act on them
getChoreService().scheduleChore(quotaObserverChore);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index ddc58bb..faffb3f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -62,7 +62,9 @@ import
org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.LockInfo;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
import org.apache.hadoop.hbase.quotas.QuotaUtil;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -113,8 +115,12 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
+i
HBASE-17602 Reduce some quota chore periods/delays
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/986a642a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/986a642a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/986a642a
Branch: refs/heads/HBASE-16961
Commit: 986a642a2d5cf23d52f80f031103abe60ab4af13
Parents: 359e36c
Author: Josh Elser
Authored: Tue Feb 7 11:21:08 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:32:00 2017 -0400
--
.../java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java | 4 ++--
.../org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/986a642a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index b9f4592..7f894e4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -55,11 +55,11 @@ public class QuotaObserverChore extends ScheduledChore {
private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
"hbase.master.quotas.observer.chore.period";
- static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
+ static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1
minutes in millis
static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
"hbase.master.quotas.observer.chore.delay";
- static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15
seconds in millis
static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
"hbase.master.quotas.observer.chore.timeunit";
http://git-wip-us.apache.org/repos/asf/hbase/blob/986a642a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
index e1a2693..8587e79 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
@@ -44,11 +44,11 @@ public class SpaceQuotaRefresherChore extends
ScheduledChore {
static final String POLICY_REFRESHER_CHORE_PERIOD_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.period";
- static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
+ static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1
minute in millis
static final String POLICY_REFRESHER_CHORE_DELAY_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.delay";
- static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15
seconds in millis
static final String POLICY_REFRESHER_CHORE_TIMEUNIT_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.timeunit";
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index d56def5..4577bcf 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -13024,6 +13024,3031 @@ public final class QuotaProtos {
}
+ public interface GetQuotaStatesRequestOrBuilder extends
+ //
@@protoc_insertion_point(interface_extends:hbase.pb.GetQuotaStatesRequest)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+ }
+ /**
+ * Protobuf type {@code hbase.pb.GetQuotaStatesRequest}
+ */
+ public static final class GetQuotaStatesRequest extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ //
@@protoc_insertion_point(message_implements:hbase.pb.GetQuotaStatesRequest)
+ GetQuotaStatesRequestOrBuilder {
+// Use GetQuotaStatesRequest.newBuilder() to construct.
+private
GetQuotaStatesRequest(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private GetQuotaStatesRequest() {
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private GetQuotaStatesRequest(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_descriptor;
+}
+
+protected
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+internalGetFieldAccessorTable() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.class,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.Builder.class);
+}
+
+private byte memoizedIsInitialized = -1;
+public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized == 1) return true;
+ if (isInitialized == 0) return false;
+
+ memoizedIsInitialized = 1;
+ return true;
+}
+
+public void
writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream
output)
+throws java.io.IOException {
+ unknownFields.writeTo(output);
+}
+
+public int getSerializedSize() {
+ int size = memoizedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ size += unknownFields.getSerializedSize();
+ memoizedSize = size;
+ return size;
+}
+
+private static final long serialVersionUID = 0L;
+@java.lang.Override
+public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+
HBASE-17428 Implement informational RPCs for space quotas
Create some RPCs that can expose the in-memory state that the
RegionServers and Master hold to drive the space quota "state machine".
Then, create some hbase shell commands to interact with those.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/609333c9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/609333c9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/609333c9
Branch: refs/heads/HBASE-16961
Commit: 609333c9fcd9dae7a244cfcbc714d8614a1242b4
Parents: a714675
Author: Josh Elser
Authored: Tue Feb 21 15:36:39 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:32:00 2017 -0400
--
.../hbase/client/ConnectionImplementation.java |9 +
.../hadoop/hbase/client/QuotaStatusCalls.java | 125 +
.../client/ShortCircuitMasterConnection.java|7 +
.../hadoop/hbase/quotas/QuotaTableUtil.java | 77 +
.../hbase/shaded/protobuf/RequestConverter.java | 33 +
.../shaded/protobuf/generated/AdminProtos.java | 394 +-
.../shaded/protobuf/generated/MasterProtos.java | 92 +-
.../shaded/protobuf/generated/QuotaProtos.java | 5986 +-
.../generated/RegionServerStatusProtos.java | 28 +-
.../src/main/protobuf/Admin.proto |9 +
.../src/main/protobuf/Master.proto |4 +
.../src/main/protobuf/Quota.proto | 35 +
.../hbase/protobuf/generated/QuotaProtos.java |6 +-
.../hadoop/hbase/master/MasterRpcServices.java | 60 +
.../hbase/quotas/ActivePolicyEnforcement.java |8 +
.../hbase/regionserver/RSRpcServices.java | 57 +
.../hadoop/hbase/master/MockRegionServer.java | 18 +
.../hbase/quotas/TestQuotaStatusRPCs.java | 192 +
hbase-shell/src/main/ruby/hbase/quotas.rb | 16 +
hbase-shell/src/main/ruby/shell.rb |3 +
.../ruby/shell/commands/list_quota_snapshots.rb | 59 +
.../shell/commands/list_quota_table_sizes.rb| 47 +
.../shell/commands/list_quota_violations.rb | 48 +
hbase-shell/src/test/ruby/hbase/quotas_test.rb | 24 -
.../test/ruby/hbase/quotas_test_no_cluster.rb | 69 +
25 files changed, 7086 insertions(+), 320 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 6859cb3..3022ae4 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -92,6 +92,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.DisableReplicationPeerRequest;
@@ -1738,6 +1740,13 @@ class ConnectionImplementation implements
ClusterConnection, Closeable {
ListReplicationPeersRequest request) throws ServiceException {
return stub.listReplicationPeers(controller, request);
}
+
+ @Override
+ public GetSpaceQuotaRegionSizesResponse getSpaceQuotaRegionSizes(
+ RpcController controller, GetSpaceQuotaRegionSizesRequest request)
+ throws ServiceException {
+return stub.getSpaceQuotaRegionSizes(controller, request);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
new file mode 100644
index 000..f0f385d
--- /dev/null
+++
b/hbase-client/src/main/java/
HBASE-17003 Documentation updates for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/546908ed
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/546908ed
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/546908ed
Branch: refs/heads/HBASE-16961
Commit: 546908ed4b741129b8bc245c67aade1aaee29527
Parents: cb91790
Author: Josh Elser
Authored: Thu Mar 16 16:21:14 2017 -0400
Committer: Josh Elser
Committed: Tue Apr 25 18:37:38 2017 -0400
--
src/main/asciidoc/_chapters/ops_mgt.adoc | 64 ++-
1 file changed, 63 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/546908ed/src/main/asciidoc/_chapters/ops_mgt.adoc
--
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc
b/src/main/asciidoc/_chapters/ops_mgt.adoc
index e4c077f..f9009f3 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1705,7 +1705,7 @@ handling multiple workloads:
[[quota]]
=== Quotas
-HBASE-11598 introduces quotas, which allow you to throttle requests based on
+HBASE-11598 introduces RPC quotas, which allow you to throttle requests based
on
the following limits:
. <>
@@ -1885,6 +1885,68 @@ at the same time and that fewer scans can be executed at
the same time. A value
`0.9` will give more queue/handlers to scans, so the number of scans executed
will
increase and the number of gets will decrease.
+[[space-quotas]]
+=== Space Quotas
+
+link:https://issues.apache.org/jira/browse/HBASE-16961[HBASE-16961] introduces
a new type of
+quotas for HBase to leverage: filesystem quotas. These "space" quotas limit
the amount of space
+on the filesystem that HBase namespaces and tables can consume. If a user,
malicious or ignorant,
+has the ability to write data into HBase, with enough time, that user can
effectively crash HBase
+(or worse HDFS) by consuming all available space. When there is no filesystem
space available,
+HBase crashes because it can no longer create/sync data to the write-ahead log.
+
+This feature allows a for a limit to be set on the size of a table or
namespace. When a space quota is set
+on a namespace, the quota's limit applies to the sum of usage of all tables in
that namespace.
+When a table with a quota exists in a namespace with a quota, the table quota
takes priority
+over the namespace quota. This allows for a scenario where a large limit can
be placed on
+a collection of tables, but a single table in that collection can have a
fine-grained limit set.
+
+The existing `set_quota` and `list_quota` HBase shell commands can be used to
interact with
+space quotas. Space quotas are quotas with a `TYPE` of `SPACE` and have
`LIMIT` and `POLICY`
+attributes. The `LIMIT` is a string that refers to the amount of space on the
filesystem
+that the quota subject (e.g. the table or namespace) may consume. For example,
valid values
+of `LIMIT` are `'10G'`, `'2T'`, or `'256M'`. The `POLICY` refers to the action
that HBase will
+take when the quota subject's usage exceeds the `LIMIT`. The following are
valid `POLICY` values.
+
+* `NO_INSERTS` - No new data may be written (e.g. `Put`, `Increment`,
`Append`).
+* `NO_WRITES` - Same as `NO_INSERTS` but `Deletes` are also disallowed.
+* `NO_WRITES_COMPACTIONS` - Same as `NO_WRITES` but compactions are also
disallowed.
+* `DISABLE` - The table(s) are disabled, preventing all read/write access.
+
+.Setting simple space quotas
+
+# Sets a quota on the table 't1' with a limit of 1GB, disallowing
Puts/Increments/Appends when the table exceeds 1GB
+hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY =>
NO_INSERTS
+
+# Sets a quota on the namespace 'ns1' with a limit of 50TB, disallowing
Puts/Increments/Appends/Deletes
+hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY =>
NO_WRITES
+
+# Sets a quota on the table 't3' with a limit of 2TB, disallowing any writes
and compactions when the table exceeds 2TB.
+hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY =>
NO_WRITES_COMPACTIONS
+
+# Sets a quota on the table 't2' with a limit of 50GB, disabling the table
when it exceeds 50GB
+hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY =>
DISABLE
+
+
+Consider the following scenario to set up quotas on a namespace, overriding
the quota on tables in that namespace
+
+.Table and Namespace space quotas
+
+hbase> create_namespace 'ns1'
+hbase> create 'ns1:t1'
+hbase> create 'ns1:t2'
+hbase> create 'ns1:t3'
+hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '100T', POLICY =>
NO_INSERTS
+hbase> set_quota TYPE => SPACE, TABLE => 'ns1:t2', LIMIT => '200G', POLICY =>
NO_WRITES
+h
http://git-wip-us.apache.org/repos/asf/hbase/blob/609333c9/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index a4c6095..d56def5 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
getSpaceFieldBuilder() {
if (spaceBuilder_ == null) {
spaceBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6077,7 +6077,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota quota = 1;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
getQuotaFieldBuilder() {
if (quotaBuilder_ == null) {
quotaBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6351,7 +6351,7 @@ public final class QuotaProtos {
return memoizedHashCode;
}
int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
+ hash = (19 * hash) + getDescriptor().hashCode();
if (hasPolicy()) {
hash = (37 * hash) + POLICY_FIELD_NUMBER;
hash = (53 * hash) + policy_;
@@ -6978,7 +6978,7 @@ public final class QuotaProtos {
return memoizedHashCode;
}
int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
+ hash = (19 * hash) + getDescriptor().hashCode();
if (hasStatus()) {
hash = (37 * hash) + STATUS_FIELD_NUMBER;
hash = (53 * hash) + getStatus().hashCode();
@@ -7351,7 +7351,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuotaStatus status = 1;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatusOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatusOrBuilder>
getStatusFieldBuilder() {
if (statusBuilder_ == null) {
statusBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -7476,163 +7476,5829 @@ public final class QuotaProtos {
}
- private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-internal_static_hbase_pb_TimedQuota_descriptor;
- private static final
-
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
- internal_static_hbase_pb_TimedQuota_fieldAccessorTable;
- private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-internal_static_hbase_pb_Throttle_descriptor;
- private static final
-
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
- internal_static_hbase_pb_Throttle_fieldAccessorTable;
- private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-
HBASE-17794 Swap "violation" for "snapshot" where appropriate
A couple of variables and comments in which violation is incorrectly
used to describe what the code is doing. This was a hold over from early
implementation -- need to scrub these out for clarity.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d09d967c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d09d967c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d09d967c
Branch: refs/heads/HBASE-16961
Commit: d09d967c7788cf088c98d967502197e3a6c108a7
Parents: 546908e
Author: Josh Elser
Authored: Thu Mar 16 19:26:14 2017 -0400
Committer: Josh Elser
Committed: Tue Apr 25 18:37:38 2017 -0400
--
.../java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java| 4 ++--
hbase-protocol-shaded/src/main/protobuf/Quota.proto| 2 +-
.../org/apache/hadoop/hbase/quotas/QuotaObserverChore.java | 6 +++---
.../apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java| 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d09d967c/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index ad59517..c008702 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -228,7 +228,7 @@ public class QuotaTableUtil {
}
/**
- * Creates a {@link Scan} which returns only quota violations from the quota
table.
+ * Creates a {@link Scan} which returns only quota snapshots from the quota
table.
*/
public static Scan makeQuotaSnapshotScan() {
Scan s = new Scan();
@@ -246,7 +246,7 @@ public class QuotaTableUtil {
* will throw an {@link IllegalArgumentException}.
*
* @param result A row from the quota table.
- * @param snapshots A map of violations to add the result of this method
into.
+ * @param snapshots A map of snapshots to add the result of this method into.
*/
public static void extractQuotaSnapshot(
Result result, Map snapshots) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/d09d967c/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 1a6d5ed..364c58b 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -98,7 +98,7 @@ message SpaceLimitRequest {
}
// Represents the state of a quota on a table. Either the quota is not in
violation
-// or it is in violatino there is a violation policy which should be in effect.
+// or it is in violation there is a violation policy which should be in effect.
message SpaceQuotaStatus {
optional SpaceViolationPolicy policy = 1;
optional bool in_violation = 2;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d09d967c/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 94c5c87..254f2a1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -532,9 +532,9 @@ public class QuotaObserverChore extends ScheduledChore {
}
/**
- * Stores the quota violation state for the given table.
+ * Stores the quota state for the given table.
*/
- void setTableQuotaViolation(TableName table, SpaceQuotaSnapshot snapshot) {
+ void setTableQuotaSnapshot(TableName table, SpaceQuotaSnapshot snapshot) {
this.tableQuotaSnapshots.put(table, snapshot);
}
@@ -552,7 +552,7 @@ public class QuotaObserverChore extends ScheduledChore {
}
/**
- * Stores the quota violation state for the given namespace.
+ * Stores the quota state for the given namespace.
*/
void setNamespaceQuotaSnapshot(String namespace, SpaceQuotaSnapshot
snapshot) {
this.namespaceQuotaSnapshots.put(namespace, snapshot);
http://git-wip-us.apache.org/repos/asf/hbase/blob/d09d967c/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java
---
HBASE-17259 API to remove space quotas on a table/namespace
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/67f7c8c9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/67f7c8c9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/67f7c8c9
Branch: refs/heads/HBASE-16961
Commit: 67f7c8c945d6d49d742f04758b312fa2aff2573b
Parents: 66cf480
Author: Josh Elser
Authored: Wed Jan 11 12:47:06 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:24:20 2017 -0400
--
.../hbase/quotas/QuotaSettingsFactory.java | 22 +++
.../hadoop/hbase/quotas/QuotaTableUtil.java | 6 +-
.../hadoop/hbase/quotas/SpaceLimitSettings.java | 44 -
.../hbase/quotas/TestQuotaSettingsFactory.java | 20 +++
.../shaded/protobuf/generated/QuotaProtos.java | 157 +++---
.../src/main/protobuf/Quota.proto | 1 +
.../hbase/protobuf/generated/QuotaProtos.java | 159 ---
hbase-protocol/src/main/protobuf/Quota.proto| 1 +
.../hadoop/hbase/quotas/MasterQuotaManager.java | 9 +-
.../hadoop/hbase/quotas/TestQuotaAdmin.java | 49 +-
10 files changed, 423 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/67f7c8c9/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 7f1c180..184277d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -316,6 +316,17 @@ public class QuotaSettingsFactory {
}
/**
+ * Creates a {@link QuotaSettings} object to remove the FileSystem space
quota for the given
+ * table.
+ *
+ * @param tableName The name of the table to remove the quota for.
+ * @return A {@link QuotaSettings} object.
+ */
+ public static QuotaSettings removeTableSpaceLimit(TableName tableName) {
+return new SpaceLimitSettings(tableName, true);
+ }
+
+ /**
* Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given
* namespace to the given size in bytes. When the space usage is exceeded by
all tables in the
* namespace, the provided {@link SpaceViolationPolicy} is enacted on all
tables in the namespace.
@@ -329,4 +340,15 @@ public class QuotaSettingsFactory {
final String namespace, long sizeLimit, final SpaceViolationPolicy
violationPolicy) {
return new SpaceLimitSettings(namespace, sizeLimit, violationPolicy);
}
+
+ /**
+ * Creates a {@link QuotaSettings} object to remove the FileSystem space
quota for the given
+* namespace.
+ *
+ * @param namespace The namespace to remove the quota on.
+ * @return A {@link QuotaSettings} object.
+ */
+ public static QuotaSettings removeNamespaceSpaceLimit(String namespace) {
+return new SpaceLimitSettings(namespace, true);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/67f7c8c9/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 66535b2..ce4cd04 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -422,7 +422,11 @@ public class QuotaTableUtil {
boolean hasSettings = false;
hasSettings |= quotas.hasThrottle();
hasSettings |= quotas.hasBypassGlobals();
-hasSettings |= quotas.hasSpace();
+// Only when there is a space quota, make sure there's actually both
fields provided
+// Otherwise, it's a noop.
+if (quotas.hasSpace()) {
+ hasSettings |= (quotas.getSpace().hasSoftLimit() &&
quotas.getSpace().hasViolationPolicy());
+}
return !hasSettings;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/67f7c8c9/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
index e54882e..8ff7623 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/h
HBASE-17002 JMX metrics and some UI additions for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cb91790d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cb91790d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cb91790d
Branch: refs/heads/HBASE-16961
Commit: cb91790d2ade376c0ac895fd5b8598b7b1f76b45
Parents: d9c22cc
Author: Josh Elser
Authored: Wed Feb 15 14:24:57 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 25 18:37:38 2017 -0400
--
.../hbase/client/ConnectionImplementation.java |8 +
.../hadoop/hbase/client/QuotaStatusCalls.java | 39 +-
.../client/ShortCircuitMasterConnection.java|8 +
.../hadoop/hbase/quotas/QuotaTableUtil.java | 41 +
.../hbase/shaded/protobuf/RequestConverter.java | 11 +
.../hbase/master/MetricsMasterQuotaSource.java | 75 +
.../master/MetricsMasterQuotaSourceFactory.java | 26 +
.../hbase/master/MetricsMasterWrapper.java | 13 +
.../MetricsRegionServerQuotaSource.java | 54 +
.../MetricsMasterQuotaSourceFactoryImpl.java| 36 +
.../master/MetricsMasterQuotaSourceImpl.java| 129 +
...hadoop.hbase.master.MetricsMasterQuotaSource | 18 +
...hbase.master.MetricsMasterQuotaSourceFactory | 18 +
.../shaded/protobuf/generated/MasterProtos.java | 93 +-
.../shaded/protobuf/generated/QuotaProtos.java | 3099 +-
.../src/main/protobuf/Master.proto |6 +-
.../src/main/protobuf/Quota.proto | 17 +
.../org/apache/hadoop/hbase/master/HMaster.java |2 +-
.../hadoop/hbase/master/MasterRpcServices.java | 38 +
.../hadoop/hbase/master/MetricsMaster.java | 42 +
.../hbase/master/MetricsMasterWrapperImpl.java | 42 +-
.../hadoop/hbase/quotas/QuotaObserverChore.java | 92 +-
.../resources/hbase-webapps/master/table.jsp| 59 +
.../hbase/master/TestMasterMetricsWrapper.java | 17 +
.../hbase/quotas/TestQuotaStatusRPCs.java | 83 +
25 files changed, 4032 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 3022ae4..9476ce7 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -92,6 +92,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
@@ -1747,6 +1749,12 @@ class ConnectionImplementation implements
ClusterConnection, Closeable {
throws ServiceException {
return stub.getSpaceQuotaRegionSizes(controller, request);
}
+
+ @Override
+ public GetQuotaStatesResponse getQuotaStates(
+ RpcController controller, GetQuotaStatesRequest request) throws
ServiceException {
+return stub.getQuotaStates(controller, request);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cb91790d/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
index f0f385d..af36d1e 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.Re
55 matches
Mail list logo
