PHOENIX-3525 Cap automatic index rebuilding to inactive timestamp

Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ed180c4d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ed180c4d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ed180c4d

Branch: refs/heads/4.x-HBase-1.1
Commit: ed180c4dcb06e50d092840dacb7de875cca56812
Parents: 82ab8b9
Author: James Taylor <[email protected]>
Authored: Mon Aug 7 19:06:46 2017 -0700
Committer: James Taylor <[email protected]>
Committed: Mon Aug 7 23:05:01 2017 -0700

----------------------------------------------------------------------
 .../end2end/index/MutableIndexFailureIT.java    |   1 +
 .../end2end/index/PartialIndexRebuilderIT.java  |   1 +
 .../coprocessor/MetaDataEndpointImpl.java       |  18 ++++
 .../coprocessor/MetaDataRegionObserver.java     | 105 ++++++++++++-------
 .../org/apache/phoenix/query/QueryServices.java |   5 +
 .../phoenix/query/QueryServicesOptions.java     |  13 +--
 .../java/org/apache/phoenix/util/IndexUtil.java |   5 -
 .../java/org/apache/phoenix/util/TestUtil.java  |   3 +-
 8 files changed, 101 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 855bd75..2b2c0d7 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -138,6 +138,7 @@ public class MutableIndexFailureIT extends BaseTest {
         
serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, 
"30000"); // give up rebuilding after 30 seconds
         // need to override rpc retries otherwise test doesn't pass
         serverProps.put(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, 
Long.toString(1));
+        
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB,
 Long.toString(1000));
         Map<String, String> clientProps = 
Collections.singletonMap(QueryServices.TRANSACTIONS_ENABLED, 
Boolean.TRUE.toString());
         NUM_SLAVES_BASE = 4;
         setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), 
new ReadOnlyProps(clientProps.entrySet().iterator()));

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
index ef9ae1b..cc5f138 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
@@ -51,6 +51,7 @@ public class PartialIndexRebuilderIT extends 
BaseUniqueNamesOwnClusterIT {
         serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, 
Boolean.TRUE.toString());
         
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, 
"1000");
         
serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, 
"30000"); // give up rebuilding after 30 seconds
+        
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB,
 Long.toString(1000));
         setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), 
ReadOnlyProps.EMPTY_PROPS);
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 93b9d2e..85a6603 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -3519,6 +3519,24 @@ public class MetaDataEndpointImpl extends 
MetaDataProtocol implements Coprocesso
                     Cell newDisableTimeStampCell = 
newKVs.get(disableTimeStampKVIndex);
                     long newDisableTimeStamp = (Long) 
PLong.INSTANCE.toObject(newDisableTimeStampCell.getValueArray(),
                             newDisableTimeStampCell.getValueOffset(), 
newDisableTimeStampCell.getValueLength());
+                    // We never set the INDEX_DISABLE_TIMESTAMP to a positive 
value when we're setting the state to ACTIVE.
+                    // Instead, we're passing in what we expect the 
INDEX_DISABLE_TIMESTAMP to be currently. If it's
+                    // changed, it means that a data table row failed to write 
while we were partially rebuilding it
+                    // and we must rerun it.
+                    if (newState == PIndexState.ACTIVE && newDisableTimeStamp 
> 0) {
+                        // Don't allow setting to ACTIVE if the 
INDEX_DISABLE_TIMESTAMP doesn't match
+                        // what we expect.
+                        if (newDisableTimeStamp != Math.abs(curTimeStampVal)) {
+                            
builder.setReturnCode(MetaDataProtos.MutationCode.UNALLOWED_TABLE_MUTATION);
+                            
builder.setMutationTime(EnvironmentEdgeManager.currentTimeMillis());
+                            done.run(builder.build());
+                            return;
+                       }
+                        // Reset INDEX_DISABLE_TIMESTAMP_BYTES to zero as 
we're good to go.
+                        newKVs.set(disableTimeStampKVIndex, 
+                                CellUtil.createCell(key, TABLE_FAMILY_BYTES, 
INDEX_DISABLE_TIMESTAMP_BYTES, 
+                                        timeStamp, 
KeyValue.Type.Put.getCode(), PLong.INSTANCE.toBytes(0L)));
+                    }
                     // We use the sign of the INDEX_DISABLE_TIMESTAMP to 
differentiate the keep-index-active (negative)
                     // from block-writes-to-data-table case. In either case, 
we want to keep the oldest timestamp to
                     // drive the partial index rebuild rather than update it 
with each attempt to update the index

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
index 9b68cd4..e6a2cc3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.CoprocessorEnvironment;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.client.HTableInterface;
@@ -53,6 +54,7 @@ import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.apache.phoenix.cache.GlobalCache;
@@ -205,6 +207,7 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
         private long rebuildIndexBatchSize = HConstants.LATEST_TIMESTAMP;
         private long configuredBatches = 10;
         private long indexDisableTimestampThreshold;
+        private final ReadOnlyProps props;
 
         public BuildIndexScheduleTask(RegionCoprocessorEnvironment env) {
             this.env = env;
@@ -216,6 +219,7 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
             this.indexDisableTimestampThreshold =
                     
configuration.getLong(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD,
                         
QueryServicesOptions.DEFAULT_INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD);
+            this.props = new ReadOnlyProps(env.getConfiguration().iterator());
         }
 
         @Override
@@ -247,7 +251,7 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                 scan.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                     PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
 
-                Map<PTable, List<PTable>> dataTableToIndexesMap = null;
+                Map<PTable, List<Pair<PTable,Long>>> dataTableToIndexesMap = 
null;
                 boolean hasMore = false;
                 List<Cell> results = new ArrayList<Cell>();
                 scanner = this.env.getRegion().getScanner(scan);
@@ -260,20 +264,23 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                     Result r = Result.create(results);
                     byte[] disabledTimeStamp = 
r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
-                    byte[] indexState = 
r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
-                            PhoenixDatabaseMetaData.INDEX_STATE_BYTES);
+                    Cell indexStateCell = 
r.getColumnLatestCell(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, 
PhoenixDatabaseMetaData.INDEX_STATE_BYTES);
 
                     if (disabledTimeStamp == null || disabledTimeStamp.length 
== 0) {
                         continue;
                     }
 
+                    long indexDisableTimestamp =
+                            
PLong.INSTANCE.getCodec().decodeLong(disabledTimeStamp, 0,
+                                SortOrder.ASC);
                     byte[] dataTable = 
r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                         PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES);
-                    if ((dataTable == null || dataTable.length == 0) || 
(indexState == null || indexState.length == 0)) {
+                    if ((dataTable == null || dataTable.length == 0) || 
indexStateCell == null) {
                         // data table name can't be empty
                         continue;
                     }
 
+                    byte[] indexState = CellUtil.cloneValue(indexStateCell);
                     byte[][] rowKeyMetaData = new byte[3][];
                     SchemaUtil.getVarChars(r.getRow(), 3, rowKeyMetaData);
                     byte[] schemaName = 
rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
@@ -305,12 +312,7 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                                 + indexPTable.getName() + " are online.");
                         continue;
                     }
-                    long indexDisableTimestamp =
-                            
PLong.INSTANCE.getCodec().decodeLong(disabledTimeStamp, 0,
-                                SortOrder.ASC);
-                    PIndexState state = 
PIndexState.fromSerializedValue(indexState[0]);
-                    if (indexDisableTimestamp > 0 && System.currentTimeMillis()
-                            - indexDisableTimestamp > 
indexDisableTimestampThreshold) {
+                    if (System.currentTimeMillis() - 
Math.abs(indexDisableTimestamp) > indexDisableTimestampThreshold) {
                         /*
                          * It has been too long since the index has been 
disabled and any future
                          * attempts to reenable it likely will fail. So we are 
going to mark the
@@ -334,8 +336,28 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                     // cannot transition directly from DISABLED -> ACTIVE
                     if 
(Bytes.compareTo(PIndexState.DISABLE.getSerializedBytes(), indexState) == 0) {
                         IndexUtil.updateIndexState(conn, indexTableFullName, 
PIndexState.INACTIVE, null);
+                        continue; // Must wait until clients start to do index 
maintenance again
                     }
-                    List<PTable> indexesToPartiallyRebuild = 
dataTableToIndexesMap.get(dataPTable);
+                    long currentTime = System.currentTimeMillis();
+                    long forwardOverlapDurationMs = 
env.getConfiguration().getLong(
+                            
QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, 
+                                    
QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME);
+                    // Wait until no failures have occurred in at least 
forwardOverlapDurationMs
+                    if (indexStateCell.getTimestamp() + 
forwardOverlapDurationMs > currentTime) {
+                        continue; // Haven't waited long enough yet
+                    }
+                    Long upperBoundOfRebuild = HConstants.LATEST_TIMESTAMP;
+                    if 
(Bytes.compareTo(PIndexState.INACTIVE.getSerializedBytes(), indexState) == 0) {
+                        upperBoundOfRebuild = indexStateCell.getTimestamp() + 
forwardOverlapDurationMs;
+                    } else if 
(Bytes.compareTo(PIndexState.ACTIVE.getSerializedBytes(), indexState) == 0) {
+                        // Since the index state cell is updated every time 
the INDEX_DISABLED_TIMESTAMP
+                        // changes, we know the upper bound.
+                        upperBoundOfRebuild = indexStateCell.getTimestamp() + 
1;
+                    } else {
+                        LOG.warn("Unexpected index state of " + 
indexTableFullName + ":" + Bytes.toStringBinary(indexState));
+                        continue; // Ignore as this is an unexpected state
+                    }
+                    List<Pair<PTable,Long>> indexesToPartiallyRebuild = 
dataTableToIndexesMap.get(dataPTable);
                     if (indexesToPartiallyRebuild == null) {
                         indexesToPartiallyRebuild = 
Lists.newArrayListWithExpectedSize(dataPTable.getIndexes().size());
                         dataTableToIndexesMap.put(dataPTable, 
indexesToPartiallyRebuild);
@@ -343,24 +365,26 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                     LOG.debug("We have found " + indexPTable.getIndexState() + 
" Index:" + indexPTable.getName()
                             + " on data table:" + dataPTable.getName() + " 
which failed to be updated at "
                             + indexPTable.getIndexDisableTimestamp());
-                    indexesToPartiallyRebuild.add(indexPTable);
+                    indexesToPartiallyRebuild.add(new 
Pair<PTable,Long>(indexPTable,upperBoundOfRebuild));
                 } while (hasMore);
 
                                if (dataTableToIndexesMap != null) {
-                                       long overlapTime = 
env.getConfiguration().getLong(
-                                                       
QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB,
-                                                       
QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME);
-                                       for (Map.Entry<PTable, List<PTable>> 
entry : dataTableToIndexesMap.entrySet()) {
+                                       long backwardOverlapDurationMs = 
env.getConfiguration().getLong(
+                                                       
QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME_ATTRIB,
+                                                       
env.getConfiguration().getLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB,
 
+                                                               
QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME));
+                                       for (Map.Entry<PTable, 
List<Pair<PTable,Long>>> entry : dataTableToIndexesMap.entrySet()) {
                                                PTable dataPTable = 
entry.getKey();
-                                               List<PTable> 
indexesToPartiallyRebuild = entry.getValue();
-                                               ReadOnlyProps props = new 
ReadOnlyProps(env.getConfiguration().iterator());
+                                               List<Pair<PTable,Long>> pairs = 
entry.getValue();
+                        List<PTable> indexesToPartiallyRebuild = 
Lists.newArrayListWithExpectedSize(pairs.size());
                                                try (HTableInterface metaTable 
= env.getTable(
                                                                
SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, 
props))) {
                                                        long 
earliestDisableTimestamp = Long.MAX_VALUE;
+                            long latestUpperBoundTimestamp = Long.MIN_VALUE;
                                                        List<IndexMaintainer> 
maintainers = Lists
-                                                                       
.newArrayListWithExpectedSize(indexesToPartiallyRebuild.size());
+                                                                       
.newArrayListWithExpectedSize(pairs.size());
                                                        int 
signOfDisableTimeStamp = 0;
-                                                       for (PTable index : 
indexesToPartiallyRebuild) {
+                                                       for (Pair<PTable,Long> 
pair : pairs) {
                                                    // We need a way of 
differentiating the block writes to data table case from
                                                    // the leave index active 
case. In either case, we need to know the time stamp
                                                    // at which writes started 
failing so we can rebuild from that point. If we
@@ -368,6 +392,8 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                                                    // then writes to the data 
table will be blocked (this is client side logic
                                                    // and we can't change this 
in a minor release). So we use the sign of the
                                                    // time stamp to 
differentiate.
+                                                           PTable index = 
pair.getFirst();
+                                                           Long 
upperBoundTimestamp = pair.getSecond();
                                                                long 
disabledTimeStampVal = index.getIndexDisableTimestamp();
                                                                if 
(disabledTimeStampVal != 0) {
                                     if (signOfDisableTimeStamp != 0 && 
signOfDisableTimeStamp != Long.signum(disabledTimeStampVal)) {
@@ -379,16 +405,20 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                                                                                
earliestDisableTimestamp = disabledTimeStampVal;
                                                                        }
 
+                                                                       
indexesToPartiallyRebuild.add(index);
                                                                        
maintainers.add(index.getIndexMaintainer(dataPTable, conn));
                                                                }
+                                                               if 
(upperBoundTimestamp > latestUpperBoundTimestamp) {
+                                                                   
latestUpperBoundTimestamp = upperBoundTimestamp;
+                                                               }
                                                        }
                                                        // No indexes are 
disabled, so skip this table
                                                        if 
(earliestDisableTimestamp == Long.MAX_VALUE) {
                                                                continue;
                                                        }
-                                                       long timeStamp = 
Math.max(0, earliestDisableTimestamp - overlapTime);
+                                                       long scanBeginTime = 
Math.max(0, earliestDisableTimestamp - backwardOverlapDurationMs);
                                                        LOG.info("Starting to 
build " + dataPTable + " indexes " + indexesToPartiallyRebuild
-                                                                       + " 
from timestamp=" + timeStamp);
+                                                                       + " 
from timestamp=" + scanBeginTime + " until " + latestUpperBoundTimestamp);
                                                        
                                                        TableRef tableRef = new 
TableRef(null, dataPTable, HConstants.LATEST_TIMESTAMP, false);
                                                        // TODO Need to set 
high timeout
@@ -398,11 +428,10 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                                                        Scan dataTableScan = 
IndexManagementUtil.newLocalStateScan(plan.getContext().getScan(),
                                                                        
maintainers);
 
-                                                       long scanEndTime = 
getTimestampForBatch(timeStamp,
-                                                                       
batchExecutedPerTableMap.get(dataPTable.getName()));
+                                                       long scanEndTime = 
Math.min(latestUpperBoundTimestamp,
+                                                               
getTimestampForBatch(scanBeginTime,batchExecutedPerTableMap.get(dataPTable.getName())));
                                                        // We can't allow 
partial results
-                                                       
dataTableScan.setAllowPartialResults(false);
-                                                       
dataTableScan.setTimeRange(timeStamp, scanEndTime);
+                                                       
dataTableScan.setTimeRange(scanBeginTime, scanEndTime);
                                                        
dataTableScan.setCacheBlocks(false);
                                                        
dataTableScan.setAttribute(BaseScannerRegionObserver.REBUILD_INDEXES, 
TRUE_BYTES);
 
@@ -418,7 +447,7 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                                     + (scanEndTime == 
HConstants.LATEST_TIMESTAMP ? "LATEST_TIMESTAMP" : scanEndTime));
                                                        MutationState 
mutationState = plan.execute();
                                                        long rowCount = 
mutationState.getUpdateCount();
-                            if (scanEndTime == HConstants.LATEST_TIMESTAMP) {
+                            if (scanEndTime == latestUpperBoundTimestamp) {
                                 LOG.info("Rebuild completed for all 
inactive/disabled indexes in data table:"
                                         + dataPTable.getName());
                             }
@@ -427,13 +456,17 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                                                                String 
indexTableFullName = SchemaUtil.getTableName(
                                                                                
indexPTable.getSchemaName().getString(),
                                                                                
indexPTable.getTableName().getString());
-                                                               if (scanEndTime 
== HConstants.LATEST_TIMESTAMP) {
-                                                                       
IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.ACTIVE, 0l);
+                                                               if (scanEndTime 
== latestUpperBoundTimestamp) {
+                                                                   // We 
compare the absolute value of the index disable timestamp. We don't want to
+                                                                   // pass a 
negative value because that means an additional index write failed.
+                                                                       
IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.ACTIVE, 
Math.abs(indexPTable.getIndexDisableTimestamp()));
                                                                        
batchExecutedPerTableMap.remove(dataPTable.getName());
                                     LOG.info("Making Index:" + 
indexPTable.getTableName() + " active after rebuilding");
                                                                } else {
                                                                    // Maintain 
sign of INDEX_DISABLE_TIMESTAMP (see comment above)
-                                                                       
updateDisableTimestamp(conn, indexTableFullName, env, scanEndTime * 
signOfDisableTimeStamp, metaTable);
+                                                                       if 
(!updateDisableTimestamp(conn, indexTableFullName, scanEndTime * 
signOfDisableTimeStamp, metaTable, indexPTable.getIndexDisableTimestamp())) {
+                                                                           
LOG.warn("The index disabled timestamp for " + indexTableFullName + " was 
updated outside of rebuilder. Will reattempt rebuild next iteration.");
+                                                                       }
                                                                        Long 
noOfBatches = batchExecutedPerTableMap.get(dataPTable.getName());
                                                                        if 
(noOfBatches == null) {
                                                                                
noOfBatches = 0l;
@@ -445,8 +478,7 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
                                                                                
        dataPTable.getTableName().getString());
                                                                        new 
MetaDataClient(conn).updateCache(indexPTable.getSchemaName().getString(),
                                                                                
        indexPTable.getTableName().getString());
-                                                                       
LOG.info(
-                                                                               
        "During Round-robin build: Successfully updated index disabled 
timestamp  for "
+                                                                       
LOG.info("During Round-robin build: Successfully updated index disabled 
timestamp  for "
                                                                                
                        + indexTableFullName + " to " + scanEndTime);
                                                                }
                                                        }
@@ -490,16 +522,15 @@ public class MetaDataRegionObserver extends 
BaseRegionObserver {
         }
     }
     
-       private static void updateDisableTimestamp(PhoenixConnection conn, 
String indexTableName,
-                       RegionCoprocessorEnvironment env, long 
disabledTimestamp, HTableInterface metaTable) throws IOException {
+       private static boolean updateDisableTimestamp(PhoenixConnection conn, 
String indexTableName,
+               long disabledTimestamp, HTableInterface metaTable, long 
expectedDisabledTimestamp) throws IOException {
                byte[] indexTableKey = 
SchemaUtil.getTableKeyFromFullName(indexTableName);
                Put put = new Put(indexTableKey);
                put.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, 
PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES,
                                PLong.INSTANCE.toBytes(disabledTimestamp));
-               metaTable.checkAndPut(indexTableKey, 
PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
-                               
PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, CompareOp.NOT_EQUAL, 
PLong.INSTANCE.toBytes(0),
+               return metaTable.checkAndPut(indexTableKey, 
PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+                               
PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, CompareOp.EQUAL, 
PLong.INSTANCE.toBytes(expectedDisabledTimestamp),
                                put);
-
        }
 
     private static synchronized void 
initRebuildIndexConnectionProps(Configuration config) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java 
b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index e13a527..cbd756e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -151,8 +151,13 @@ public interface QueryServices extends SQLCloseable {
     public static final String INDEX_FAILURE_THROW_EXCEPTION_ATTRIB = 
"phoenix.index.failure.throw.exception";
 
     // Index will be partially re-built from index disable time stamp - 
following overlap time
+    @Deprecated
     public static final String 
INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB =
         "phoenix.index.failure.handling.rebuild.overlap.time";
+    public static final String 
INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME_ATTRIB =
+            "phoenix.index.failure.handling.rebuild.overlap.backward.time";
+    public static final String 
INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB =
+            "phoenix.index.failure.handling.rebuild.overlap.forward.time";
     public static final String INDEX_PRIOIRTY_ATTRIB = 
"phoenix.index.rpc.priority";
     public static final String METADATA_PRIOIRTY_ATTRIB = 
"phoenix.metadata.rpc.priority";
     public static final String ALLOW_LOCAL_INDEX_ATTRIB = 
"phoenix.index.allowLocalIndex";

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java 
b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index c6593da..ec5c95b 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -79,15 +79,15 @@ import static 
org.apache.phoenix.query.QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB
 import static 
org.apache.phoenix.query.QueryServices.STATS_USE_CURRENT_TIME_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.THREAD_POOL_SIZE_ATTRIB;
 import static org.apache.phoenix.query.QueryServices.THREAD_TIMEOUT_MS_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.TRANSACTIONS_ENABLED;
-import static 
org.apache.phoenix.query.QueryServices.UPLOAD_BINARY_DATA_TYPE_ENCODING;
-import static 
org.apache.phoenix.query.QueryServices.USE_BYTE_BASED_REGEX_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.USE_INDEXES_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.TRACING_BATCH_SIZE;
 import static org.apache.phoenix.query.QueryServices.TRACING_ENABLED;
 import static 
org.apache.phoenix.query.QueryServices.TRACING_STATS_TABLE_NAME_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.TRACING_BATCH_SIZE;
 import static org.apache.phoenix.query.QueryServices.TRACING_THREAD_POOL_SIZE;
 import static org.apache.phoenix.query.QueryServices.TRACING_TRACE_BUFFER_SIZE;
+import static org.apache.phoenix.query.QueryServices.TRANSACTIONS_ENABLED;
+import static 
org.apache.phoenix.query.QueryServices.UPLOAD_BINARY_DATA_TYPE_ENCODING;
+import static 
org.apache.phoenix.query.QueryServices.USE_BYTE_BASED_REGEX_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.USE_INDEXES_ATTRIB;
 import static 
org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION;
 
 import java.util.HashSet;
@@ -182,7 +182,8 @@ public class QueryServicesOptions {
     public static final boolean DEFAULT_INDEX_FAILURE_DISABLE_INDEX = true; 
     public static final boolean DEFAULT_INDEX_FAILURE_THROW_EXCEPTION = true; 
     public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL = 
60000; // 60 secs
-    public static final long 
DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME = 1; // 1 ms
+    public static final long 
DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME = 1; // 1 ms
+    public static final long 
DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME = 60000 * 3; // 3 
mins
     // 30 min rpc timeout * 5 tries, with 2100ms total pause time between 
retries
     public static final long DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT = (5 * 30000 
* 60) + 2100;
     public static final long DEFAULT_INDEX_REBUILD_RPC_TIMEOUT = 30000 * 60; 
// 30 mins

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java 
b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 18e543c..75a7e55 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -114,7 +114,6 @@ import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.transaction.PhoenixTransactionContext;
 
-import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 
 public class IndexUtil {
@@ -790,10 +789,6 @@ public class IndexUtil {
 
     public static void updateIndexState(PhoenixConnection conn, String 
indexTableName,
                PIndexState newState, Long indexDisableTimestamp) throws 
SQLException {
-        if (newState == PIndexState.ACTIVE) {
-            Preconditions.checkArgument(indexDisableTimestamp == 0,
-                "Index disable timestamp has to be 0 when marking an index as 
active");
-        }
        byte[] indexTableKey = 
SchemaUtil.getTableKeyFromFullName(indexTableName);
        String schemaName = 
SchemaUtil.getSchemaNameFromFullName(indexTableName);
        String indexName = SchemaUtil.getTableNameFromFullName(indexTableName);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java 
b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 266f4da..1716928 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -870,8 +870,7 @@ public class TestUtil {
                     + "AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS 
NULL AND " + PhoenixDatabaseMetaData.COLUMN_NAME + " IS NULL"
                     + " AND " + PhoenixDatabaseMetaData.INDEX_STATE + " = '" + 
expectedIndexState.getSerializedValue() + "'";
             ResultSet rs = conn.createStatement().executeQuery(query);
-            assertTrue(rs.next());
-            if (expectedIndexState == PIndexState.ACTIVE) {
+            if (rs.next() && expectedIndexState == PIndexState.ACTIVE) {
                 if (rs.getLong(1) == 0 && !rs.wasNull()) {
                     isActive = true;
                     break;

Reply via email to