[phoenix] branch 4.14-HBase-1.3 updated: PHOENIX-5478 IndexTool mapper task should not timeout (addendum for start/close region operation)

2019-10-30 Thread kadir 
This is an automated email from the ASF dual-hosted git repository.

kadir pushed a commit to branch 4.14-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git


The following commit(s) were added to refs/heads/4.14-HBase-1.3 by this push:
 new 1a497ed  PHOENIX-5478 IndexTool mapper task should not timeout 
(addendum for start/close region operation)
1a497ed is described below

commit 1a497ed84b5b81687eeaae642137665c7215
Author: Kadir 
AuthorDate: Wed Oct 30 21:32:22 2019 -0700

PHOENIX-5478 IndexTool mapper task should not timeout (addendum for 
start/close region operation)
---
 .../phoenix/coprocessor/UngroupedAggregateRegionObserver.java   | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index e31c5dc..31e 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -1147,6 +1147,7 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 @Override
 public boolean next(List results) throws IOException {
 int rowCount = 0;
+region.startRegionOperation();
 try {
 byte[] uuidValue = ServerCacheClient.generateId();
 synchronized (innerScanner) {
@@ -1196,9 +1197,7 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 LOGGER.error("IOException during rebuilding: " + 
Throwables.getStackTraceAsString(e));
 throw e;
 } finally {
-if (!hasMore) {
-region.closeRegionOperation();
-}
+region.closeRegionOperation();
 }
 byte[] rowCountBytes = 
PLong.INSTANCE.toBytes(Long.valueOf(rowCount));
 final KeyValue aggKeyValue = 
KeyValueUtil.newKeyValue(UNGROUPED_AGG_ROW_KEY, SINGLE_COLUMN_FAMILY,
@@ -1216,7 +1215,6 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 private RegionScanner rebuildIndices(final RegionScanner innerScanner, 
final Region region, final Scan scan,
  final Configuration config) throws 
IOException {
 
-region.startRegionOperation();
 RegionScanner scanner = new IndexRebuildRegionScanner(innerScanner, 
region, scan, config);
 return scanner;
 }

[phoenix] branch 4.14-HBase-1.3 updated: PHOENIX-5478 IndexTool mapper task should not timeout (addendum for setting config params)

2019-10-25 Thread kadir 
This is an automated email from the ASF dual-hosted git repository.

kadir pushed a commit to branch 4.14-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git


The following commit(s) were added to refs/heads/4.14-HBase-1.3 by this push:
 new 5d23c74  PHOENIX-5478 IndexTool mapper task should not timeout 
(addendum for setting config params)
5d23c74 is described below

commit 5d23c74b52d8a66526f4f19257e26a47bec3a017
Author: Kadir 
AuthorDate: Fri Oct 25 08:49:50 2019 -0700

PHOENIX-5478 IndexTool mapper task should not timeout (addendum for setting 
config params)
---
 .../apache/phoenix/mapreduce/index/IndexTool.java  | 22 ++
 .../apache/phoenix/query/QueryServicesOptions.java |  2 +-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java 
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index cb67374..dd56093 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -481,6 +481,28 @@ public class IndexTool extends Configured implements Tool {
 
 private Job configureJobForServerBuildIndex()
 throws Exception {
+long indexRebuildQueryTimeoutMs =
+
configuration.getLong(QueryServices.INDEX_REBUILD_QUERY_TIMEOUT_ATTRIB,
+
QueryServicesOptions.DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT);
+long indexRebuildRPCTimeoutMs =
+
configuration.getLong(QueryServices.INDEX_REBUILD_RPC_TIMEOUT_ATTRIB,
+
QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_TIMEOUT);
+long indexRebuildClientScannerTimeOutMs =
+
configuration.getLong(QueryServices.INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT_ATTRIB,
+
QueryServicesOptions.DEFAULT_INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT);
+int indexRebuildRpcRetriesCounter =
+
configuration.getInt(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER,
+
QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_RETRIES_COUNTER);
+// Set various phoenix and hbase level timeouts and rpc retries
+configuration.set(QueryServices.THREAD_TIMEOUT_MS_ATTRIB,
+Long.toString(indexRebuildQueryTimeoutMs));
+configuration.set(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
+Long.toString(indexRebuildClientScannerTimeOutMs));
+configuration.set(HConstants.HBASE_RPC_TIMEOUT_KEY,
+Long.toString(indexRebuildRPCTimeoutMs));
+configuration.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+Long.toString(indexRebuildRpcRetriesCounter));
+configuration.set("mapreduce.task.timeout", 
Long.toString(indexRebuildQueryTimeoutMs));
 
 PhoenixConfigurationUtil.setIndexToolDataTableName(configuration, 
qDataTable);
 PhoenixConfigurationUtil.setIndexToolIndexTableName(configuration, 
qIndexTable);
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java 
b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index afb7153..68763db 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -342,7 +342,7 @@ public class QueryServicesOptions {
 
 public static final long 
DEFAULT_GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS = 7*24*60*60*1000; /* 7 
days */
 public static final boolean DEFAULT_INDEX_REGION_OBSERVER_ENABLED = true;
-public static final long DEFAULT_INDEX_REBUILD_PAGE_SIZE_IN_ROWS = 
1024*1024;
+public static final long DEFAULT_INDEX_REBUILD_PAGE_SIZE_IN_ROWS = 16*1024;
 
 public static final boolean DEFAULT_PROPERTY_POLICY_PROVIDER_ENABLED = 
true;

[phoenix] branch 4.14-HBase-1.3 updated: PHOENIX-5478 IndexTool mapper task should not timeout

2019-10-24 Thread kadir 
This is an automated email from the ASF dual-hosted git repository.

kadir pushed a commit to branch 4.14-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git


The following commit(s) were added to refs/heads/4.14-HBase-1.3 by this push:
 new 12b70b6  PHOENIX-5478 IndexTool mapper task should not timeout
12b70b6 is described below

commit 12b70b6f8f7611d9047cd73a3eea8e087b203b5c
Author: Kadir 
AuthorDate: Wed Oct 23 22:38:13 2019 -0700

PHOENIX-5478 IndexTool mapper task should not timeout
---
 .../org/apache/phoenix/end2end/IndexToolIT.java|   2 +-
 .../coprocessor/BaseScannerRegionObserver.java |   2 +
 .../UngroupedAggregateRegionObserver.java  | 220 +++--
 .../apache/phoenix/index/GlobalIndexChecker.java   |   4 -
 .../PhoenixServerBuildIndexInputFormat.java|   2 +
 .../apache/phoenix/mapreduce/index/IndexTool.java  |  12 --
 .../org/apache/phoenix/query/QueryServices.java|   2 +
 .../apache/phoenix/query/QueryServicesOptions.java |   3 +-
 8 files changed, 130 insertions(+), 117 deletions(-)

diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index 9cc2393..6fc01bd 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -81,7 +81,6 @@ import com.google.common.collect.Maps;
 @RunWith(Parameterized.class)
 @Category(NeedsOwnMiniClusterTest.class)
 public class IndexToolIT extends ParallelStatsEnabledIT {
-
 private final boolean localIndex;
 private final boolean transactional;
 private final boolean directApi;
@@ -117,6 +116,7 @@ public class IndexToolIT extends ParallelStatsEnabledIT {
 Map serverProps = Maps.newHashMapWithExpectedSize(2);
 serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
 QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
+serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, 
Long.toString(8));
 Map clientProps = Maps.newHashMapWithExpectedSize(2);
 clientProps.put(QueryServices.TRANSACTIONS_ENABLED, 
Boolean.TRUE.toString());
 clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, 
Boolean.TRUE.toString());
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index b73615f..cb4d0af 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -75,6 +75,8 @@ abstract public class BaseScannerRegionObserver extends 
BaseRegionObserver {
 public static final String GROUP_BY_LIMIT = "_GroupByLimit";
 public static final String LOCAL_INDEX = "_LocalIndex";
 public static final String LOCAL_INDEX_BUILD = "_LocalIndexBuild";
+// The number of index rows to be rebuild in one RPC call
+public static final String INDEX_REBUILD_PAGING = "_IndexRebuildPaging";
 /* 
 * Attribute to denote that the index maintainer has been serialized using 
its proto-buf presentation.
 * Needed for backward compatibility purposes. TODO: get rid of this in 
next major release.
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 0166206..3cae671 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -21,6 +21,7 @@ import static 
org.apache.phoenix.query.QueryConstants.AGG_TIMESTAMP;
 import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
 import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
 import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
+import static 
org.apache.phoenix.query.QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS;
 import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
 import static 
org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_BYTES_ATTRIB;
 import static 
org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
@@ -1034,116 +1035,137 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 throw new RuntimeException(e);
 }
 }
-
-private RegionScanner rebuildIndices(final RegionScanner innerScanner, 
final Region region, final Scan scan,
-Configuration config) throws IOException {
-byte[] indexMetaData = 
scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD);
-boolean useProto =