[1/3] phoenix git commit: PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors [Forced Update!]

2018-04-22 Thread pboado 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-cdh5.14 213a00161 -> b21867b83 (forced update)


PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1bfe303f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1bfe303f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1bfe303f

Branch: refs/heads/4.x-cdh5.14
Commit: 1bfe303f6a861feea0d23e64bb1ec93d5db69a1f
Parents: 9948036
Author: Vincent Poon 
Authored: Fri Apr 20 21:35:06 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:09 2018 +0100

--
 .../org/apache/phoenix/hbase/index/Indexer.java | 16 ++-
 .../hbase/index/builder/IndexBuildManager.java  |  4 
 .../hbase/index/covered/IndexMetaData.java  | 12 ++-
 .../hbase/index/write/IndexCommitter.java   |  2 +-
 .../phoenix/hbase/index/write/IndexWriter.java  | 21 ++--
 .../write/ParallelWriterIndexCommitter.java |  2 +-
 .../hbase/index/write/RecoveryIndexWriter.java  |  4 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  2 +-
 .../index/PhoenixTransactionalIndexer.java  |  9 +++--
 .../index/covered/LocalTableStateTest.java  | 18 -
 .../hbase/index/write/TestIndexWriter.java  |  7 +++
 .../index/write/TestParalleIndexWriter.java |  4 +++-
 .../write/TestParalleWriterIndexCommitter.java  |  3 ++-
 13 files changed, 74 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/1bfe303f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index c7dbff2..1ef09fe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -91,6 +91,7 @@ import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 import com.google.common.collect.Lists;
@@ -132,8 +133,13 @@ public class Indexer extends BaseRegionObserver {
   // Hack to get around not being able to save any state between
   // coprocessor calls. TODO: remove after HBASE-18127 when available
   private static class BatchMutateContext {
+  public final int clientVersion;
   public Collection> indexUpdates = 
Collections.emptyList();
   public List rowLocks = 
Lists.newArrayListWithExpectedSize(QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
+
+  public BatchMutateContext(int clientVersion) {
+  this.clientVersion = clientVersion;
+  }
   }
   
   private ThreadLocal batchMutateContext =
@@ -399,7 +405,7 @@ public class Indexer extends BaseRegionObserver {
* Exclusively lock all rows so we get a consistent read
* while determining the index updates
*/
-  BatchMutateContext context = new BatchMutateContext();
+  BatchMutateContext context = new 
BatchMutateContext(this.builder.getIndexMetaData(miniBatchOp).getClientVersion());
   setBatchMutateContext(c, context);
   Durability durability = Durability.SKIP_WAL;
   boolean copyMutations = false;
@@ -627,10 +633,10 @@ public class Indexer extends BaseRegionObserver {
  }
  }
   if(!remoteUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(remoteUpdates, false);
+  writer.writeAndKillYourselfOnFailure(remoteUpdates, false, 
context.clientVersion);
   }
   if(!localUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(localUpdates, true);
+  writer.writeAndKillYourselfOnFailure(localUpdates, true, 
context.clientVersion);
   }
 
   long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
@@ -700,7 +706,7 @@ public class Indexer extends BaseRegionObserver {
 // do the usual writer stuff, killing the server again, if we can't 
manage to make the index
 // writes succeed again
 try {
-writer.writeAndKillYourselfOnFailure(updates, true);
+writer.writeAndKillYourselfOnFailure(updates, true, 
ScanUtil.UNKNOWN_CLIENT_VERSION);
 } catch (IOException e) {
 LOG.error("During WAL replay of outstanding index updates, "
 + "Exception is thrown

[3/3] phoenix git commit: Changes for CDH 5.14.x

2018-04-22 Thread pboado 
Changes for CDH 5.14.x


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b21867b8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b21867b8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b21867b8

Branch: refs/heads/4.x-cdh5.14
Commit: b21867b837b4487aa4f4d948d0acabf2d040d6d8
Parents: 8bba239
Author: Pedro Boado 
Authored: Sat Mar 10 17:54:04 2018 +
Committer: Pedro Boado 
Committed: Sun Apr 22 11:14:08 2018 +0100

--
 phoenix-assembly/pom.xml|  2 +-
 phoenix-client/pom.xml  |  2 +-
 phoenix-core/pom.xml|  2 +-
 .../hadoop/hbase/ipc/PhoenixRpcScheduler.java   | 34 ++--
 phoenix-flume/pom.xml   |  2 +-
 phoenix-hive/pom.xml|  2 +-
 phoenix-kafka/pom.xml   |  2 +-
 phoenix-load-balancer/pom.xml   |  2 +-
 phoenix-parcel/pom.xml  |  2 +-
 phoenix-pherf/pom.xml   |  2 +-
 phoenix-pig/pom.xml |  2 +-
 phoenix-queryserver-client/pom.xml  |  2 +-
 phoenix-queryserver/pom.xml |  2 +-
 phoenix-server/pom.xml  |  2 +-
 phoenix-spark/pom.xml   |  2 +-
 phoenix-tracing-webapp/pom.xml  |  2 +-
 pom.xml |  4 +--
 17 files changed, 49 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/b21867b8/phoenix-assembly/pom.xml
--
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index 55a9a6e..862ec23 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -27,7 +27,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.14.0-SNAPSHOT
   
   phoenix-assembly
   Phoenix Assembly

http://git-wip-us.apache.org/repos/asf/phoenix/blob/b21867b8/phoenix-client/pom.xml
--
diff --git a/phoenix-client/pom.xml b/phoenix-client/pom.xml
index 2454de6..470dd05 100644
--- a/phoenix-client/pom.xml
+++ b/phoenix-client/pom.xml
@@ -27,7 +27,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.14.0-SNAPSHOT
   
   phoenix-client
   Phoenix Client

http://git-wip-us.apache.org/repos/asf/phoenix/blob/b21867b8/phoenix-core/pom.xml
--
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index e1f8e2a..f8afaa1 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -4,7 +4,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.14.0-SNAPSHOT
   
   phoenix-core
   Phoenix Core

http://git-wip-us.apache.org/repos/asf/phoenix/blob/b21867b8/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
 
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
index 4fdddf5..d1f05f8 100644
--- 
a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
+++ 
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
@@ -124,6 +124,36 @@ public class PhoenixRpcScheduler extends RpcScheduler {
 public void setMetadataExecutorForTesting(RpcExecutor executor) {
 this.metadataCallExecutor = executor;
 }
-
-
+
+@Override
+public int getReadQueueLength() {
+return delegate.getReadQueueLength();
+}
+
+@Override
+public int getWriteQueueLength() {
+return delegate.getWriteQueueLength();
+}
+
+@Override
+public int getScanQueueLength() {
+return delegate.getScanQueueLength();
+}
+
+@Override
+public int getActiveReadRpcHandlerCount() {
+return delegate.getActiveReadRpcHandlerCount();
+}
+
+@Override
+public int getActiveWriteRpcHandlerCount() {
+return delegate.getActiveWriteRpcHandlerCount();
+}
+
+@Override
+public int getActiveScanRpcHandlerCount() {
+return delegate.getActiveScanRpcHandlerCount();
+}
+
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/b21867b8/phoenix-flume/pom.xml
--
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index d61a9aa..4b8a0ff 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -26,7 +26,7 @@

[2/3] phoenix git commit: PHOENIX-4601 Perform server-side retries if client version < 4.14

2018-04-22 Thread pboado 
PHOENIX-4601 Perform server-side retries if client version < 4.14


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8bba2397
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8bba2397
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8bba2397

Branch: refs/heads/4.x-cdh5.14
Commit: 8bba23976be4b5d4f0139660831532ac4ca23f9e
Parents: 1bfe303
Author: Vincent Poon 
Authored: Fri Apr 20 22:22:10 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:22 2018 +0100

--
 .../hbase/index/write/IndexWriterUtils.java | 36 
 .../write/ParallelWriterIndexCommitter.java | 12 +--
 .../TrackingParallelWriterIndexCommitter.java   | 13 ---
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  1 +
 4 files changed, 48 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8bba2397/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index 76d6800..0d3004f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -21,6 +21,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -58,15 +59,19 @@ public class IndexWriterUtils {
public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = 
"phoenix.index.writes.threads.max";
public static final String HTABLE_KEEP_ALIVE_KEY = 
"hbase.htable.threads.keepalivetime";
 
+   @Deprecated
public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = 
"phoenix.index.writes.rpc.retries.number";
-/**
- * Retry server-server index write rpc only once, and let the client retry 
the data write
- * instead to avoid typing up the handler
- */
-   // note in HBase 2+, numTries = numRetries + 1
-   // in prior versions, numTries = numRetries
-   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 1;
+   /**
+* Based on the logic in HBase's AsyncProcess, a default of 11 retries with 
a pause of 100ms
+* approximates 48 sec total retry time (factoring in backoffs).  The total 
time should be less
+* than HBase's rpc timeout (default of 60 sec) or else the client will 
retry before receiving
+* the response
+*/
+   @Deprecated
+   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
+   @Deprecated
public static final String INDEX_WRITER_RPC_PAUSE = 
"phoenix.index.writes.rpc.pause";
+   @Deprecated
public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
 
   private IndexWriterUtils() {
@@ -76,12 +81,29 @@ public class IndexWriterUtils {
 public static HTableFactory 
getDefaultDelegateHTableFactory(CoprocessorEnvironment env) {
 // create a simple delegate factory, setup the way we need
 Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+return ServerUtil.getDelegateHTableFactory(env, conf);
+}
+
+private static void setHTableThreads(Configuration conf) {
 // set the number of threads allowed per table.
 int htableThreads =
 
conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
 IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
 LOG.trace("Creating HTableFactory with " + htableThreads + " threads 
for each HTable.");
 IndexManagementUtil.setIfNotSet(conf, HTABLE_THREAD_KEY, 
htableThreads);
+}
+
+/**
+ * Retry server-server index write rpc only once, and let the client retry 
the data write
+ * instead to avoid tying up the handler
+ */
+public static HTableFactory 
getNoRetriesHTableFactory(CoprocessorEnvironment env) {
+Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+// note in HBase 2+, numTries = numRetries + 1
+// in prior versions, numTries = numRetries
+conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
 return ServerUtil.getDelegateHTableFactory(env, conf);
 }

[2/3] phoenix git commit: PHOENIX-4601 Perform server-side retries if client version < 4.14

2018-04-22 Thread pboado 
PHOENIX-4601 Perform server-side retries if client version < 4.14


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8bba2397
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8bba2397
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8bba2397

Branch: refs/heads/4.x-cdh5.13
Commit: 8bba23976be4b5d4f0139660831532ac4ca23f9e
Parents: 1bfe303
Author: Vincent Poon 
Authored: Fri Apr 20 22:22:10 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:22 2018 +0100

--
 .../hbase/index/write/IndexWriterUtils.java | 36 
 .../write/ParallelWriterIndexCommitter.java | 12 +--
 .../TrackingParallelWriterIndexCommitter.java   | 13 ---
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  1 +
 4 files changed, 48 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8bba2397/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index 76d6800..0d3004f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -21,6 +21,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -58,15 +59,19 @@ public class IndexWriterUtils {
public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = 
"phoenix.index.writes.threads.max";
public static final String HTABLE_KEEP_ALIVE_KEY = 
"hbase.htable.threads.keepalivetime";
 
+   @Deprecated
public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = 
"phoenix.index.writes.rpc.retries.number";
-/**
- * Retry server-server index write rpc only once, and let the client retry 
the data write
- * instead to avoid typing up the handler
- */
-   // note in HBase 2+, numTries = numRetries + 1
-   // in prior versions, numTries = numRetries
-   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 1;
+   /**
+* Based on the logic in HBase's AsyncProcess, a default of 11 retries with 
a pause of 100ms
+* approximates 48 sec total retry time (factoring in backoffs).  The total 
time should be less
+* than HBase's rpc timeout (default of 60 sec) or else the client will 
retry before receiving
+* the response
+*/
+   @Deprecated
+   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
+   @Deprecated
public static final String INDEX_WRITER_RPC_PAUSE = 
"phoenix.index.writes.rpc.pause";
+   @Deprecated
public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
 
   private IndexWriterUtils() {
@@ -76,12 +81,29 @@ public class IndexWriterUtils {
 public static HTableFactory 
getDefaultDelegateHTableFactory(CoprocessorEnvironment env) {
 // create a simple delegate factory, setup the way we need
 Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+return ServerUtil.getDelegateHTableFactory(env, conf);
+}
+
+private static void setHTableThreads(Configuration conf) {
 // set the number of threads allowed per table.
 int htableThreads =
 
conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
 IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
 LOG.trace("Creating HTableFactory with " + htableThreads + " threads 
for each HTable.");
 IndexManagementUtil.setIfNotSet(conf, HTABLE_THREAD_KEY, 
htableThreads);
+}
+
+/**
+ * Retry server-server index write rpc only once, and let the client retry 
the data write
+ * instead to avoid tying up the handler
+ */
+public static HTableFactory 
getNoRetriesHTableFactory(CoprocessorEnvironment env) {
+Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+// note in HBase 2+, numTries = numRetries + 1
+// in prior versions, numTries = numRetries
+conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
 return ServerUtil.getDelegateHTableFactory(env, conf);
 }

[1/3] phoenix git commit: PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors [Forced Update!]

2018-04-22 Thread pboado 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-cdh5.13 1dd0830ed -> d396e455a (forced update)


PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1bfe303f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1bfe303f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1bfe303f

Branch: refs/heads/4.x-cdh5.13
Commit: 1bfe303f6a861feea0d23e64bb1ec93d5db69a1f
Parents: 9948036
Author: Vincent Poon 
Authored: Fri Apr 20 21:35:06 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:09 2018 +0100

--
 .../org/apache/phoenix/hbase/index/Indexer.java | 16 ++-
 .../hbase/index/builder/IndexBuildManager.java  |  4 
 .../hbase/index/covered/IndexMetaData.java  | 12 ++-
 .../hbase/index/write/IndexCommitter.java   |  2 +-
 .../phoenix/hbase/index/write/IndexWriter.java  | 21 ++--
 .../write/ParallelWriterIndexCommitter.java |  2 +-
 .../hbase/index/write/RecoveryIndexWriter.java  |  4 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  2 +-
 .../index/PhoenixTransactionalIndexer.java  |  9 +++--
 .../index/covered/LocalTableStateTest.java  | 18 -
 .../hbase/index/write/TestIndexWriter.java  |  7 +++
 .../index/write/TestParalleIndexWriter.java |  4 +++-
 .../write/TestParalleWriterIndexCommitter.java  |  3 ++-
 13 files changed, 74 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/1bfe303f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index c7dbff2..1ef09fe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -91,6 +91,7 @@ import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 import com.google.common.collect.Lists;
@@ -132,8 +133,13 @@ public class Indexer extends BaseRegionObserver {
   // Hack to get around not being able to save any state between
   // coprocessor calls. TODO: remove after HBASE-18127 when available
   private static class BatchMutateContext {
+  public final int clientVersion;
   public Collection> indexUpdates = 
Collections.emptyList();
   public List rowLocks = 
Lists.newArrayListWithExpectedSize(QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
+
+  public BatchMutateContext(int clientVersion) {
+  this.clientVersion = clientVersion;
+  }
   }
   
   private ThreadLocal batchMutateContext =
@@ -399,7 +405,7 @@ public class Indexer extends BaseRegionObserver {
* Exclusively lock all rows so we get a consistent read
* while determining the index updates
*/
-  BatchMutateContext context = new BatchMutateContext();
+  BatchMutateContext context = new 
BatchMutateContext(this.builder.getIndexMetaData(miniBatchOp).getClientVersion());
   setBatchMutateContext(c, context);
   Durability durability = Durability.SKIP_WAL;
   boolean copyMutations = false;
@@ -627,10 +633,10 @@ public class Indexer extends BaseRegionObserver {
  }
  }
   if(!remoteUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(remoteUpdates, false);
+  writer.writeAndKillYourselfOnFailure(remoteUpdates, false, 
context.clientVersion);
   }
   if(!localUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(localUpdates, true);
+  writer.writeAndKillYourselfOnFailure(localUpdates, true, 
context.clientVersion);
   }
 
   long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
@@ -700,7 +706,7 @@ public class Indexer extends BaseRegionObserver {
 // do the usual writer stuff, killing the server again, if we can't 
manage to make the index
 // writes succeed again
 try {
-writer.writeAndKillYourselfOnFailure(updates, true);
+writer.writeAndKillYourselfOnFailure(updates, true, 
ScanUtil.UNKNOWN_CLIENT_VERSION);
 } catch (IOException e) {
 LOG.error("During WAL replay of outstanding index updates, "
 + "Exception is thrown

[3/3] phoenix git commit: Changes for CDH 5.13.x

2018-04-22 Thread pboado 
Changes for CDH 5.13.x


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d396e455
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d396e455
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d396e455

Branch: refs/heads/4.x-cdh5.13
Commit: d396e455a522123db9559afccc490711e94ec3aa
Parents: 8bba239
Author: Pedro Boado 
Authored: Sat Mar 10 17:54:04 2018 +
Committer: Pedro Boado 
Committed: Sun Apr 22 11:12:38 2018 +0100

--
 phoenix-assembly/pom.xml|  2 +-
 phoenix-client/pom.xml  |  2 +-
 phoenix-core/pom.xml|  2 +-
 .../hadoop/hbase/ipc/PhoenixRpcScheduler.java   | 34 ++--
 phoenix-flume/pom.xml   |  2 +-
 phoenix-hive/pom.xml|  2 +-
 phoenix-kafka/pom.xml   |  2 +-
 phoenix-load-balancer/pom.xml   |  2 +-
 phoenix-parcel/pom.xml  |  2 +-
 phoenix-pherf/pom.xml   |  2 +-
 phoenix-pig/pom.xml |  2 +-
 phoenix-queryserver-client/pom.xml  |  2 +-
 phoenix-queryserver/pom.xml |  2 +-
 phoenix-server/pom.xml  |  2 +-
 phoenix-spark/pom.xml   |  2 +-
 phoenix-tracing-webapp/pom.xml  |  2 +-
 pom.xml |  4 +--
 17 files changed, 49 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d396e455/phoenix-assembly/pom.xml
--
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index 55a9a6e..f0cd238 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -27,7 +27,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.13.2-SNAPSHOT
   
   phoenix-assembly
   Phoenix Assembly

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d396e455/phoenix-client/pom.xml
--
diff --git a/phoenix-client/pom.xml b/phoenix-client/pom.xml
index 2454de6..b4da311 100644
--- a/phoenix-client/pom.xml
+++ b/phoenix-client/pom.xml
@@ -27,7 +27,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.13.2-SNAPSHOT
   
   phoenix-client
   Phoenix Client

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d396e455/phoenix-core/pom.xml
--
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index e1f8e2a..19ddeb5 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -4,7 +4,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.13.2-SNAPSHOT
   
   phoenix-core
   Phoenix Core

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d396e455/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
 
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
index 4fdddf5..d1f05f8 100644
--- 
a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
+++ 
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
@@ -124,6 +124,36 @@ public class PhoenixRpcScheduler extends RpcScheduler {
 public void setMetadataExecutorForTesting(RpcExecutor executor) {
 this.metadataCallExecutor = executor;
 }
-
-
+
+@Override
+public int getReadQueueLength() {
+return delegate.getReadQueueLength();
+}
+
+@Override
+public int getWriteQueueLength() {
+return delegate.getWriteQueueLength();
+}
+
+@Override
+public int getScanQueueLength() {
+return delegate.getScanQueueLength();
+}
+
+@Override
+public int getActiveReadRpcHandlerCount() {
+return delegate.getActiveReadRpcHandlerCount();
+}
+
+@Override
+public int getActiveWriteRpcHandlerCount() {
+return delegate.getActiveWriteRpcHandlerCount();
+}
+
+@Override
+public int getActiveScanRpcHandlerCount() {
+return delegate.getActiveScanRpcHandlerCount();
+}
+
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d396e455/phoenix-flume/pom.xml
--
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index d61a9aa..bcc037c 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -26,7 +26,7 @@

[1/3] phoenix git commit: PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors [Forced Update!]

2018-04-22 Thread pboado 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-cdh5.12 3fc3c5f3f -> c402afa2f (forced update)


PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1bfe303f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1bfe303f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1bfe303f

Branch: refs/heads/4.x-cdh5.12
Commit: 1bfe303f6a861feea0d23e64bb1ec93d5db69a1f
Parents: 9948036
Author: Vincent Poon 
Authored: Fri Apr 20 21:35:06 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:09 2018 +0100

--
 .../org/apache/phoenix/hbase/index/Indexer.java | 16 ++-
 .../hbase/index/builder/IndexBuildManager.java  |  4 
 .../hbase/index/covered/IndexMetaData.java  | 12 ++-
 .../hbase/index/write/IndexCommitter.java   |  2 +-
 .../phoenix/hbase/index/write/IndexWriter.java  | 21 ++--
 .../write/ParallelWriterIndexCommitter.java |  2 +-
 .../hbase/index/write/RecoveryIndexWriter.java  |  4 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  2 +-
 .../index/PhoenixTransactionalIndexer.java  |  9 +++--
 .../index/covered/LocalTableStateTest.java  | 18 -
 .../hbase/index/write/TestIndexWriter.java  |  7 +++
 .../index/write/TestParalleIndexWriter.java |  4 +++-
 .../write/TestParalleWriterIndexCommitter.java  |  3 ++-
 13 files changed, 74 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/1bfe303f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index c7dbff2..1ef09fe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -91,6 +91,7 @@ import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 import com.google.common.collect.Lists;
@@ -132,8 +133,13 @@ public class Indexer extends BaseRegionObserver {
   // Hack to get around not being able to save any state between
   // coprocessor calls. TODO: remove after HBASE-18127 when available
   private static class BatchMutateContext {
+  public final int clientVersion;
   public Collection> indexUpdates = 
Collections.emptyList();
   public List rowLocks = 
Lists.newArrayListWithExpectedSize(QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
+
+  public BatchMutateContext(int clientVersion) {
+  this.clientVersion = clientVersion;
+  }
   }
   
   private ThreadLocal batchMutateContext =
@@ -399,7 +405,7 @@ public class Indexer extends BaseRegionObserver {
* Exclusively lock all rows so we get a consistent read
* while determining the index updates
*/
-  BatchMutateContext context = new BatchMutateContext();
+  BatchMutateContext context = new 
BatchMutateContext(this.builder.getIndexMetaData(miniBatchOp).getClientVersion());
   setBatchMutateContext(c, context);
   Durability durability = Durability.SKIP_WAL;
   boolean copyMutations = false;
@@ -627,10 +633,10 @@ public class Indexer extends BaseRegionObserver {
  }
  }
   if(!remoteUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(remoteUpdates, false);
+  writer.writeAndKillYourselfOnFailure(remoteUpdates, false, 
context.clientVersion);
   }
   if(!localUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(localUpdates, true);
+  writer.writeAndKillYourselfOnFailure(localUpdates, true, 
context.clientVersion);
   }
 
   long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
@@ -700,7 +706,7 @@ public class Indexer extends BaseRegionObserver {
 // do the usual writer stuff, killing the server again, if we can't 
manage to make the index
 // writes succeed again
 try {
-writer.writeAndKillYourselfOnFailure(updates, true);
+writer.writeAndKillYourselfOnFailure(updates, true, 
ScanUtil.UNKNOWN_CLIENT_VERSION);
 } catch (IOException e) {
 LOG.error("During WAL replay of outstanding index updates, "
 + "Exception is thrown

[3/3] phoenix git commit: Changes for CDH 5.12.x

2018-04-22 Thread pboado 
Changes for CDH 5.12.x


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c402afa2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c402afa2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c402afa2

Branch: refs/heads/4.x-cdh5.12
Commit: c402afa2fca34ae6a07f066eea32a2eb2ac9c992
Parents: 8bba239
Author: Pedro Boado 
Authored: Sat Mar 10 17:54:04 2018 +
Committer: Pedro Boado 
Committed: Sun Apr 22 11:11:04 2018 +0100

--
 phoenix-assembly/pom.xml|  2 +-
 phoenix-client/pom.xml  |  2 +-
 phoenix-core/pom.xml|  2 +-
 .../hadoop/hbase/ipc/PhoenixRpcScheduler.java   | 34 ++--
 phoenix-flume/pom.xml   |  2 +-
 phoenix-hive/pom.xml|  2 +-
 phoenix-kafka/pom.xml   |  2 +-
 phoenix-load-balancer/pom.xml   |  2 +-
 phoenix-parcel/pom.xml  |  2 +-
 phoenix-pherf/pom.xml   |  2 +-
 phoenix-pig/pom.xml |  2 +-
 phoenix-queryserver-client/pom.xml  |  2 +-
 phoenix-queryserver/pom.xml |  2 +-
 phoenix-server/pom.xml  |  2 +-
 phoenix-spark/pom.xml   |  2 +-
 phoenix-tracing-webapp/pom.xml  |  2 +-
 pom.xml |  4 +--
 17 files changed, 49 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/c402afa2/phoenix-assembly/pom.xml
--
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index 55a9a6e..14225ee 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -27,7 +27,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.12.2-SNAPSHOT
   
   phoenix-assembly
   Phoenix Assembly

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c402afa2/phoenix-client/pom.xml
--
diff --git a/phoenix-client/pom.xml b/phoenix-client/pom.xml
index 2454de6..e211008 100644
--- a/phoenix-client/pom.xml
+++ b/phoenix-client/pom.xml
@@ -27,7 +27,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.12.2-SNAPSHOT
   
   phoenix-client
   Phoenix Client

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c402afa2/phoenix-core/pom.xml
--
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index e1f8e2a..2d837a2 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -4,7 +4,7 @@
   
 org.apache.phoenix
 phoenix
-4.14.0-cdh5.11.2-SNAPSHOT
+4.14.0-cdh5.12.2-SNAPSHOT
   
   phoenix-core
   Phoenix Core

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c402afa2/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
 
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
index 4fdddf5..d1f05f8 100644
--- 
a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
+++ 
b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java
@@ -124,6 +124,36 @@ public class PhoenixRpcScheduler extends RpcScheduler {
 public void setMetadataExecutorForTesting(RpcExecutor executor) {
 this.metadataCallExecutor = executor;
 }
-
-
+
+@Override
+public int getReadQueueLength() {
+return delegate.getReadQueueLength();
+}
+
+@Override
+public int getWriteQueueLength() {
+return delegate.getWriteQueueLength();
+}
+
+@Override
+public int getScanQueueLength() {
+return delegate.getScanQueueLength();
+}
+
+@Override
+public int getActiveReadRpcHandlerCount() {
+return delegate.getActiveReadRpcHandlerCount();
+}
+
+@Override
+public int getActiveWriteRpcHandlerCount() {
+return delegate.getActiveWriteRpcHandlerCount();
+}
+
+@Override
+public int getActiveScanRpcHandlerCount() {
+return delegate.getActiveScanRpcHandlerCount();
+}
+
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/c402afa2/phoenix-flume/pom.xml
--
diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml
index d61a9aa..8a78010 100644
--- a/phoenix-flume/pom.xml
+++ b/phoenix-flume/pom.xml
@@ -26,7 +26,7 @@

[2/3] phoenix git commit: PHOENIX-4601 Perform server-side retries if client version < 4.14

2018-04-22 Thread pboado 
PHOENIX-4601 Perform server-side retries if client version < 4.14


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8bba2397
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8bba2397
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8bba2397

Branch: refs/heads/4.x-cdh5.12
Commit: 8bba23976be4b5d4f0139660831532ac4ca23f9e
Parents: 1bfe303
Author: Vincent Poon 
Authored: Fri Apr 20 22:22:10 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:22 2018 +0100

--
 .../hbase/index/write/IndexWriterUtils.java | 36 
 .../write/ParallelWriterIndexCommitter.java | 12 +--
 .../TrackingParallelWriterIndexCommitter.java   | 13 ---
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  1 +
 4 files changed, 48 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8bba2397/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index 76d6800..0d3004f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -21,6 +21,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -58,15 +59,19 @@ public class IndexWriterUtils {
public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = 
"phoenix.index.writes.threads.max";
public static final String HTABLE_KEEP_ALIVE_KEY = 
"hbase.htable.threads.keepalivetime";
 
+   @Deprecated
public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = 
"phoenix.index.writes.rpc.retries.number";
-/**
- * Retry server-server index write rpc only once, and let the client retry 
the data write
- * instead to avoid typing up the handler
- */
-   // note in HBase 2+, numTries = numRetries + 1
-   // in prior versions, numTries = numRetries
-   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 1;
+   /**
+* Based on the logic in HBase's AsyncProcess, a default of 11 retries with 
a pause of 100ms
+* approximates 48 sec total retry time (factoring in backoffs).  The total 
time should be less
+* than HBase's rpc timeout (default of 60 sec) or else the client will 
retry before receiving
+* the response
+*/
+   @Deprecated
+   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
+   @Deprecated
public static final String INDEX_WRITER_RPC_PAUSE = 
"phoenix.index.writes.rpc.pause";
+   @Deprecated
public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
 
   private IndexWriterUtils() {
@@ -76,12 +81,29 @@ public class IndexWriterUtils {
 public static HTableFactory 
getDefaultDelegateHTableFactory(CoprocessorEnvironment env) {
 // create a simple delegate factory, setup the way we need
 Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+return ServerUtil.getDelegateHTableFactory(env, conf);
+}
+
+private static void setHTableThreads(Configuration conf) {
 // set the number of threads allowed per table.
 int htableThreads =
 
conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
 IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
 LOG.trace("Creating HTableFactory with " + htableThreads + " threads 
for each HTable.");
 IndexManagementUtil.setIfNotSet(conf, HTABLE_THREAD_KEY, 
htableThreads);
+}
+
+/**
+ * Retry server-server index write rpc only once, and let the client retry 
the data write
+ * instead to avoid tying up the handler
+ */
+public static HTableFactory 
getNoRetriesHTableFactory(CoprocessorEnvironment env) {
+Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+// note in HBase 2+, numTries = numRetries + 1
+// in prior versions, numTries = numRetries
+conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
 return ServerUtil.getDelegateHTableFactory(env, conf);
 }

[2/2] phoenix git commit: PHOENIX-4601 Perform server-side retries if client version < 4.14

2018-04-22 Thread pboado 
PHOENIX-4601 Perform server-side retries if client version < 4.14


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8bba2397
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8bba2397
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8bba2397

Branch: refs/heads/4.x-cdh5.11
Commit: 8bba23976be4b5d4f0139660831532ac4ca23f9e
Parents: 1bfe303
Author: Vincent Poon 
Authored: Fri Apr 20 22:22:10 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:22 2018 +0100

--
 .../hbase/index/write/IndexWriterUtils.java | 36 
 .../write/ParallelWriterIndexCommitter.java | 12 +--
 .../TrackingParallelWriterIndexCommitter.java   | 13 ---
 .../phoenix/jdbc/PhoenixDatabaseMetaData.java   |  1 +
 4 files changed, 48 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8bba2397/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index 76d6800..0d3004f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -21,6 +21,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.phoenix.hbase.index.table.HTableFactory;
 import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
 import org.apache.phoenix.util.PropertiesUtil;
@@ -58,15 +59,19 @@ public class IndexWriterUtils {
public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = 
"phoenix.index.writes.threads.max";
public static final String HTABLE_KEEP_ALIVE_KEY = 
"hbase.htable.threads.keepalivetime";
 
+   @Deprecated
public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = 
"phoenix.index.writes.rpc.retries.number";
-/**
- * Retry server-server index write rpc only once, and let the client retry 
the data write
- * instead to avoid typing up the handler
- */
-   // note in HBase 2+, numTries = numRetries + 1
-   // in prior versions, numTries = numRetries
-   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 1;
+   /**
+* Based on the logic in HBase's AsyncProcess, a default of 11 retries with 
a pause of 100ms
+* approximates 48 sec total retry time (factoring in backoffs).  The total 
time should be less
+* than HBase's rpc timeout (default of 60 sec) or else the client will 
retry before receiving
+* the response
+*/
+   @Deprecated
+   public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
+   @Deprecated
public static final String INDEX_WRITER_RPC_PAUSE = 
"phoenix.index.writes.rpc.pause";
+   @Deprecated
public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
 
   private IndexWriterUtils() {
@@ -76,12 +81,29 @@ public class IndexWriterUtils {
 public static HTableFactory 
getDefaultDelegateHTableFactory(CoprocessorEnvironment env) {
 // create a simple delegate factory, setup the way we need
 Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+return ServerUtil.getDelegateHTableFactory(env, conf);
+}
+
+private static void setHTableThreads(Configuration conf) {
 // set the number of threads allowed per table.
 int htableThreads =
 
conf.getInt(IndexWriterUtils.INDEX_WRITER_PER_TABLE_THREADS_CONF_KEY,
 IndexWriterUtils.DEFAULT_NUM_PER_TABLE_THREADS);
 LOG.trace("Creating HTableFactory with " + htableThreads + " threads 
for each HTable.");
 IndexManagementUtil.setIfNotSet(conf, HTABLE_THREAD_KEY, 
htableThreads);
+}
+
+/**
+ * Retry server-server index write rpc only once, and let the client retry 
the data write
+ * instead to avoid tying up the handler
+ */
+public static HTableFactory 
getNoRetriesHTableFactory(CoprocessorEnvironment env) {
+Configuration conf = 
PropertiesUtil.cloneConfig(env.getConfiguration());
+setHTableThreads(conf);
+// note in HBase 2+, numTries = numRetries + 1
+// in prior versions, numTries = numRetries
+conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
 return ServerUtil.getDelegateHTableFactory(env, conf);
 }

[1/2] phoenix git commit: PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors

2018-04-22 Thread pboado 
Repository: phoenix
Updated Branches:
  refs/heads/4.x-cdh5.11 9948036b6 -> 8bba23976


PHOENIX-4613 Thread clientVersion through to IndexCommitter implementors


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1bfe303f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1bfe303f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1bfe303f

Branch: refs/heads/4.x-cdh5.11
Commit: 1bfe303f6a861feea0d23e64bb1ec93d5db69a1f
Parents: 9948036
Author: Vincent Poon 
Authored: Fri Apr 20 21:35:06 2018 +0100
Committer: Pedro Boado 
Committed: Sun Apr 22 11:07:09 2018 +0100

--
 .../org/apache/phoenix/hbase/index/Indexer.java | 16 ++-
 .../hbase/index/builder/IndexBuildManager.java  |  4 
 .../hbase/index/covered/IndexMetaData.java  | 12 ++-
 .../hbase/index/write/IndexCommitter.java   |  2 +-
 .../phoenix/hbase/index/write/IndexWriter.java  | 21 ++--
 .../write/ParallelWriterIndexCommitter.java |  2 +-
 .../hbase/index/write/RecoveryIndexWriter.java  |  4 ++--
 .../TrackingParallelWriterIndexCommitter.java   |  2 +-
 .../index/PhoenixTransactionalIndexer.java  |  9 +++--
 .../index/covered/LocalTableStateTest.java  | 18 -
 .../hbase/index/write/TestIndexWriter.java  |  7 +++
 .../index/write/TestParalleIndexWriter.java |  4 +++-
 .../write/TestParalleWriterIndexCommitter.java  |  3 ++-
 13 files changed, 74 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/phoenix/blob/1bfe303f/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java 
b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index c7dbff2..1ef09fe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -91,6 +91,7 @@ import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ScanUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 import com.google.common.collect.Lists;
@@ -132,8 +133,13 @@ public class Indexer extends BaseRegionObserver {
   // Hack to get around not being able to save any state between
   // coprocessor calls. TODO: remove after HBASE-18127 when available
   private static class BatchMutateContext {
+  public final int clientVersion;
   public Collection> indexUpdates = 
Collections.emptyList();
   public List rowLocks = 
Lists.newArrayListWithExpectedSize(QueryServicesOptions.DEFAULT_MUTATE_BATCH_SIZE);
+
+  public BatchMutateContext(int clientVersion) {
+  this.clientVersion = clientVersion;
+  }
   }
   
   private ThreadLocal batchMutateContext =
@@ -399,7 +405,7 @@ public class Indexer extends BaseRegionObserver {
* Exclusively lock all rows so we get a consistent read
* while determining the index updates
*/
-  BatchMutateContext context = new BatchMutateContext();
+  BatchMutateContext context = new 
BatchMutateContext(this.builder.getIndexMetaData(miniBatchOp).getClientVersion());
   setBatchMutateContext(c, context);
   Durability durability = Durability.SKIP_WAL;
   boolean copyMutations = false;
@@ -627,10 +633,10 @@ public class Indexer extends BaseRegionObserver {
  }
  }
   if(!remoteUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(remoteUpdates, false);
+  writer.writeAndKillYourselfOnFailure(remoteUpdates, false, 
context.clientVersion);
   }
   if(!localUpdates.isEmpty()) {
-  writer.writeAndKillYourselfOnFailure(localUpdates, true);
+  writer.writeAndKillYourselfOnFailure(localUpdates, true, 
context.clientVersion);
   }
 
   long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
@@ -700,7 +706,7 @@ public class Indexer extends BaseRegionObserver {
 // do the usual writer stuff, killing the server again, if we can't 
manage to make the index
 // writes succeed again
 try {
-writer.writeAndKillYourselfOnFailure(updates, true);
+writer.writeAndKillYourselfOnFailure(updates, true, 
ScanUtil.UNKNOWN_CLIENT_VERSION);
 } catch (IOException e) {
 LOG.error("During WAL replay of outstanding index updates, "
 + "Exception is thrown instead of killing

Build failed in Jenkins: Phoenix Compile Compatibility with HBase #614

2018-04-22 Thread Apache Jenkins Server 
See 


--
[...truncated 37.15 KB...]
  symbol:   class HBaseRpcController
  location: class 
org.apache.hadoop.hbase.ipc.controller.ClientRpcControllerFactory
[ERROR] 
:[52,9]
 cannot find symbol
  symbol:   class HBaseRpcController
  location: class 
org.apache.hadoop.hbase.ipc.controller.ClientRpcControllerFactory
[ERROR] 
:[180,14]
 cannot find symbol
  symbol: class MetricRegistry
[ERROR] 
:[179,7]
 method does not override or implement a method from a supertype
[ERROR] 
:[454,78]
 cannot find symbol
  symbol: class HBaseRpcController
[ERROR] 
:[432,17]
 cannot find symbol
  symbol: class HBaseRpcController
[ERROR] Failed to execute goal 
org.apache.maven.plugins:maven-compiler-plugin:3.0:compile (default-compile) on 
project phoenix-core: Compilation failure: Compilation failure: 
[ERROR] 
:[34,39]
 cannot find symbol
[ERROR]   symbol:   class MetricRegistry
[ERROR]   location: package org.apache.hadoop.hbase.metrics
[ERROR] 
:[144,16]
 cannot find symbol
[ERROR]   symbol:   class MetricRegistry
[ERROR]   location: class 
org.apache.phoenix.coprocessor.PhoenixMetaDataCoprocessorHost.PhoenixMetaDataControllerEnvironment
[ERROR] 
:[24,35]
 cannot find symbol
[ERROR]   symbol:   class DelegatingHBaseRpcController
[ERROR]   location: package org.apache.hadoop.hbase.ipc
[ERROR] 
:[25,35]
 cannot find symbol
[ERROR]   symbol:   class HBaseRpcController
[ERROR]   location: package org.apache.hadoop.hbase.ipc
[ERROR] 
:[37,37]
 cannot find symbol
[ERROR]   symbol: class DelegatingHBaseRpcController
[ERROR] 
:[56,38]
 cannot find symbol
[ERROR]   symbol:   class HBaseRpcController
[ERROR]   location: class 
org.apache.hadoop.hbase.ipc.controller.MetadataRpcController
[ERROR] 
:[26,35]
 cannot find symbol
[ERROR]   symbol:   class HBaseRpcController
[ERROR]   location: package org.apache.hadoop.hbase.ipc
[ERROR] 
:[40,12]
 cannot find symbol
[ERROR]   symbol:   class HBaseRpcController
[ERROR]   location: class 
org.apache.hadoop.hbase.ipc.controller.InterRegionServerMetadataRpcControllerFactory
[ERROR] 
:[46,12]
 cannot find symbol
[ERROR]   symbol:   class HBaseRpcController
[ERROR]   location: class 
org.apache.hadoop.hbase.ipc.controller.InterRegionServerMetadataRpcControllerFactory
[ERROR]