Repository: hbase Updated Branches: refs/heads/branch-1 3339bac56 -> 46bf3c327 refs/heads/branch-1.2 72a5e8da7 -> 4ee6b3a3b refs/heads/branch-1.3 a623f4503 -> 65651b500 refs/heads/master 7e4e8dc3e -> 9e967e5c1
HBASE-15113 Procedure v2 - Speedup eviction of sys operation results Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/46bf3c32 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/46bf3c32 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/46bf3c32 Branch: refs/heads/branch-1 Commit: 46bf3c32750fd8e03271dd5d8d99918149b928f7 Parents: c60da69 Author: Matteo Bertozzi <[email protected]> Authored: Tue Mar 8 10:46:54 2016 -0800 Committer: Matteo Bertozzi <[email protected]> Committed: Tue Mar 8 11:08:05 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/ProcedureInfo.java | 2 +- .../hadoop/hbase/procedure2/Procedure.java | 13 ++++++++++ .../hbase/procedure2/ProcedureExecutor.java | 26 ++++++++++++-------- .../procedure/CreateNamespaceProcedure.java | 17 ++++++++++--- .../master/procedure/CreateTableProcedure.java | 7 ++++++ .../master/procedure/ServerCrashProcedure.java | 7 ++++++ 6 files changed, 57 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/46bf3c32/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java index a24db64..bcbe41f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java @@ -203,7 +203,7 @@ public class ProcedureInfo implements Cloneable { @InterfaceAudience.Private public boolean hasClientAckTime() { - return clientAckTime > 0; + return clientAckTime != -1; } @InterfaceAudience.Private http://git-wip-us.apache.org/repos/asf/hbase/blob/46bf3c32/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java index 64f817a..aff2b15 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java @@ -190,6 +190,19 @@ public abstract class Procedure<TEnvironment> implements Comparable<Procedure> { return false; } + /** + * By default, the executor will keep the procedure result around util + * the eviction TTL is expired. The client can cut down the waiting time + * by requesting that the result is removed from the executor. + * In case of system started procedure, we can force the executor to auto-ack. + * @param env the environment passed to the ProcedureExecutor + * @return true if the executor should wait the client ack for the result. + * Defaults to return true. + */ + protected boolean shouldWaitClientAck(final TEnvironment env) { + return true; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/hbase/blob/46bf3c32/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java ---------------------------------------------------------------------- diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index f45f2f9..37bcb00 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -165,22 +165,23 @@ public class ProcedureExecutor<TEnvironment> { final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL); final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL); - long now = EnvironmentEdgeManager.currentTime(); - Iterator<Map.Entry<Long, ProcedureInfo>> it = completed.entrySet().iterator(); + final long now = EnvironmentEdgeManager.currentTime(); + final Iterator<Map.Entry<Long, ProcedureInfo>> it = completed.entrySet().iterator(); + final boolean isDebugEnabled = LOG.isDebugEnabled(); while (it.hasNext() && store.isRunning()) { - Map.Entry<Long, ProcedureInfo> entry = it.next(); - ProcedureInfo result = entry.getValue(); + final Map.Entry<Long, ProcedureInfo> entry = it.next(); + final ProcedureInfo procInfo = entry.getValue(); // TODO: Select TTL based on Procedure type - if ((result.hasClientAckTime() && (now - result.getClientAckTime()) >= evictAckTtl) || - (now - result.getLastUpdate()) >= evictTtl) { - if (LOG.isDebugEnabled()) { - LOG.debug("Evict completed procedure " + entry.getKey()); + if ((procInfo.hasClientAckTime() && (now - procInfo.getClientAckTime()) >= evictAckTtl) || + (now - procInfo.getLastUpdate()) >= evictTtl) { + if (isDebugEnabled) { + LOG.debug("Evict completed procedure: " + procInfo); } store.delete(entry.getKey()); it.remove(); - NonceKey nonceKey = result.getNonceKey(); + NonceKey nonceKey = procInfo.getNonceKey(); if (nonceKey != null) { nonceKeysToProcIdsMap.remove(nonceKey); } @@ -1272,7 +1273,12 @@ public class ProcedureExecutor<TEnvironment> { } // update the executor internal state maps - completed.put(proc.getProcId(), Procedure.createProcedureInfo(proc, proc.getNonceKey())); + ProcedureInfo procInfo = Procedure.createProcedureInfo(proc, proc.getNonceKey()); + if (!proc.shouldWaitClientAck(getEnvironment())) { + procInfo.setClientAckTime(0); + } + + completed.put(procInfo.getProcId(), procInfo); rollbackStack.remove(proc.getProcId()); procedures.remove(proc.getProcId()); http://git-wip-us.apache.org/repos/asf/hbase/blob/46bf3c32/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index da64a6c..1dd39d9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -194,16 +194,18 @@ public class CreateNamespaceProcedure sb.append(")"); } + private boolean isBootstrapNamespace() { + return nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE) || + nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE); + } + @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.getMasterServices().isInitialized()) { // Namespace manager might not be ready if master is not fully initialized, // return false to reject user namespace creation; return true for default // and system namespace creation (this is part of master initialization). - boolean isBootstrapNs = nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE) || - nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE); - - if (!isBootstrapNs && env.waitInitialized(this)) { + if (!isBootstrapNamespace() && env.waitInitialized(this)) { return false; } } @@ -364,4 +366,11 @@ public class CreateNamespaceProcedure } return traceEnabled; } + + @Override + protected boolean shouldWaitClientAck(MasterProcedureEnv env) { + // hbase and default namespaces are created on bootstrap internally by the system + // the client does not know about this procedures. + return !isBootstrapNamespace(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/46bf3c32/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index cdb5d61..6d4955c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -471,4 +471,11 @@ public class CreateTableProcedure final TableName tableName) throws IOException { env.getMasterServices().getTableDescriptors().get(tableName); } + + @Override + protected boolean shouldWaitClientAck(MasterProcedureEnv env) { + // system tables are created on bootstrap internally by the system + // the client does not know about this procedures. + return !getTableName().isSystemTable(); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/46bf3c32/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java index d402b38..0e35ddb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java @@ -804,4 +804,11 @@ implements ServerProcedureInterface { protected boolean isYieldBeforeExecuteFromState(MasterProcedureEnv env, ServerCrashState state) { return true; } + + @Override + protected boolean shouldWaitClientAck(MasterProcedureEnv env) { + // The operation is triggered internally on the server + // the client does not know about this procedure. + return false; + } }
