PHOENIX-3525 Cap automatic index rebuilding to inactive timestamp
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ed180c4d Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ed180c4d Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ed180c4d Branch: refs/heads/4.x-HBase-1.1 Commit: ed180c4dcb06e50d092840dacb7de875cca56812 Parents: 82ab8b9 Author: James Taylor <[email protected]> Authored: Mon Aug 7 19:06:46 2017 -0700 Committer: James Taylor <[email protected]> Committed: Mon Aug 7 23:05:01 2017 -0700 ---------------------------------------------------------------------- .../end2end/index/MutableIndexFailureIT.java | 1 + .../end2end/index/PartialIndexRebuilderIT.java | 1 + .../coprocessor/MetaDataEndpointImpl.java | 18 ++++ .../coprocessor/MetaDataRegionObserver.java | 105 ++++++++++++------- .../org/apache/phoenix/query/QueryServices.java | 5 + .../phoenix/query/QueryServicesOptions.java | 13 +-- .../java/org/apache/phoenix/util/IndexUtil.java | 5 - .../java/org/apache/phoenix/util/TestUtil.java | 3 +- 8 files changed, 101 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java index 855bd75..2b2c0d7 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java @@ -138,6 +138,7 @@ public class MutableIndexFailureIT extends BaseTest { serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "30000"); // give up rebuilding after 30 seconds // need to override rpc retries otherwise test doesn't pass serverProps.put(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, Long.toString(1)); + serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(1000)); Map<String, String> clientProps = Collections.singletonMap(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); NUM_SLAVES_BASE = 4; setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java index ef9ae1b..cc5f138 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java @@ -51,6 +51,7 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT { serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.TRUE.toString()); serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, "1000"); serverProps.put(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, "30000"); // give up rebuilding after 30 seconds + serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, Long.toString(1000)); setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), ReadOnlyProps.EMPTY_PROPS); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 93b9d2e..85a6603 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -3519,6 +3519,24 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso Cell newDisableTimeStampCell = newKVs.get(disableTimeStampKVIndex); long newDisableTimeStamp = (Long) PLong.INSTANCE.toObject(newDisableTimeStampCell.getValueArray(), newDisableTimeStampCell.getValueOffset(), newDisableTimeStampCell.getValueLength()); + // We never set the INDEX_DISABLE_TIMESTAMP to a positive value when we're setting the state to ACTIVE. + // Instead, we're passing in what we expect the INDEX_DISABLE_TIMESTAMP to be currently. If it's + // changed, it means that a data table row failed to write while we were partially rebuilding it + // and we must rerun it. + if (newState == PIndexState.ACTIVE && newDisableTimeStamp > 0) { + // Don't allow setting to ACTIVE if the INDEX_DISABLE_TIMESTAMP doesn't match + // what we expect. + if (newDisableTimeStamp != Math.abs(curTimeStampVal)) { + builder.setReturnCode(MetaDataProtos.MutationCode.UNALLOWED_TABLE_MUTATION); + builder.setMutationTime(EnvironmentEdgeManager.currentTimeMillis()); + done.run(builder.build()); + return; + } + // Reset INDEX_DISABLE_TIMESTAMP_BYTES to zero as we're good to go. + newKVs.set(disableTimeStampKVIndex, + CellUtil.createCell(key, TABLE_FAMILY_BYTES, INDEX_DISABLE_TIMESTAMP_BYTES, + timeStamp, KeyValue.Type.Put.getCode(), PLong.INSTANCE.toBytes(0L))); + } // We use the sign of the INDEX_DISABLE_TIMESTAMP to differentiate the keep-index-active (negative) // from block-writes-to-data-table case. In either case, we want to keep the oldest timestamp to // drive the partial index rebuild rather than update it with each attempt to update the index http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index 9b68cd4..e6a2cc3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -38,6 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.HTableInterface; @@ -53,6 +54,7 @@ import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.phoenix.cache.GlobalCache; @@ -205,6 +207,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver { private long rebuildIndexBatchSize = HConstants.LATEST_TIMESTAMP; private long configuredBatches = 10; private long indexDisableTimestampThreshold; + private final ReadOnlyProps props; public BuildIndexScheduleTask(RegionCoprocessorEnvironment env) { this.env = env; @@ -216,6 +219,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver { this.indexDisableTimestampThreshold = configuration.getLong(QueryServices.INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD, QueryServicesOptions.DEFAULT_INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD); + this.props = new ReadOnlyProps(env.getConfiguration().iterator()); } @Override @@ -247,7 +251,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver { scan.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES); - Map<PTable, List<PTable>> dataTableToIndexesMap = null; + Map<PTable, List<Pair<PTable,Long>>> dataTableToIndexesMap = null; boolean hasMore = false; List<Cell> results = new ArrayList<Cell>(); scanner = this.env.getRegion().getScanner(scan); @@ -260,20 +264,23 @@ public class MetaDataRegionObserver extends BaseRegionObserver { Result r = Result.create(results); byte[] disabledTimeStamp = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES); - byte[] indexState = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, - PhoenixDatabaseMetaData.INDEX_STATE_BYTES); + Cell indexStateCell = r.getColumnLatestCell(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES); if (disabledTimeStamp == null || disabledTimeStamp.length == 0) { continue; } + long indexDisableTimestamp = + PLong.INSTANCE.getCodec().decodeLong(disabledTimeStamp, 0, + SortOrder.ASC); byte[] dataTable = r.getValue(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES); - if ((dataTable == null || dataTable.length == 0) || (indexState == null || indexState.length == 0)) { + if ((dataTable == null || dataTable.length == 0) || indexStateCell == null) { // data table name can't be empty continue; } + byte[] indexState = CellUtil.cloneValue(indexStateCell); byte[][] rowKeyMetaData = new byte[3][]; SchemaUtil.getVarChars(r.getRow(), 3, rowKeyMetaData); byte[] schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX]; @@ -305,12 +312,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver { + indexPTable.getName() + " are online."); continue; } - long indexDisableTimestamp = - PLong.INSTANCE.getCodec().decodeLong(disabledTimeStamp, 0, - SortOrder.ASC); - PIndexState state = PIndexState.fromSerializedValue(indexState[0]); - if (indexDisableTimestamp > 0 && System.currentTimeMillis() - - indexDisableTimestamp > indexDisableTimestampThreshold) { + if (System.currentTimeMillis() - Math.abs(indexDisableTimestamp) > indexDisableTimestampThreshold) { /* * It has been too long since the index has been disabled and any future * attempts to reenable it likely will fail. So we are going to mark the @@ -334,8 +336,28 @@ public class MetaDataRegionObserver extends BaseRegionObserver { // cannot transition directly from DISABLED -> ACTIVE if (Bytes.compareTo(PIndexState.DISABLE.getSerializedBytes(), indexState) == 0) { IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.INACTIVE, null); + continue; // Must wait until clients start to do index maintenance again } - List<PTable> indexesToPartiallyRebuild = dataTableToIndexesMap.get(dataPTable); + long currentTime = System.currentTimeMillis(); + long forwardOverlapDurationMs = env.getConfiguration().getLong( + QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB, + QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME); + // Wait until no failures have occurred in at least forwardOverlapDurationMs + if (indexStateCell.getTimestamp() + forwardOverlapDurationMs > currentTime) { + continue; // Haven't waited long enough yet + } + Long upperBoundOfRebuild = HConstants.LATEST_TIMESTAMP; + if (Bytes.compareTo(PIndexState.INACTIVE.getSerializedBytes(), indexState) == 0) { + upperBoundOfRebuild = indexStateCell.getTimestamp() + forwardOverlapDurationMs; + } else if (Bytes.compareTo(PIndexState.ACTIVE.getSerializedBytes(), indexState) == 0) { + // Since the index state cell is updated every time the INDEX_DISABLED_TIMESTAMP + // changes, we know the upper bound. + upperBoundOfRebuild = indexStateCell.getTimestamp() + 1; + } else { + LOG.warn("Unexpected index state of " + indexTableFullName + ":" + Bytes.toStringBinary(indexState)); + continue; // Ignore as this is an unexpected state + } + List<Pair<PTable,Long>> indexesToPartiallyRebuild = dataTableToIndexesMap.get(dataPTable); if (indexesToPartiallyRebuild == null) { indexesToPartiallyRebuild = Lists.newArrayListWithExpectedSize(dataPTable.getIndexes().size()); dataTableToIndexesMap.put(dataPTable, indexesToPartiallyRebuild); @@ -343,24 +365,26 @@ public class MetaDataRegionObserver extends BaseRegionObserver { LOG.debug("We have found " + indexPTable.getIndexState() + " Index:" + indexPTable.getName() + " on data table:" + dataPTable.getName() + " which failed to be updated at " + indexPTable.getIndexDisableTimestamp()); - indexesToPartiallyRebuild.add(indexPTable); + indexesToPartiallyRebuild.add(new Pair<PTable,Long>(indexPTable,upperBoundOfRebuild)); } while (hasMore); if (dataTableToIndexesMap != null) { - long overlapTime = env.getConfiguration().getLong( - QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, - QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME); - for (Map.Entry<PTable, List<PTable>> entry : dataTableToIndexesMap.entrySet()) { + long backwardOverlapDurationMs = env.getConfiguration().getLong( + QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME_ATTRIB, + env.getConfiguration().getLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, + QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME)); + for (Map.Entry<PTable, List<Pair<PTable,Long>>> entry : dataTableToIndexesMap.entrySet()) { PTable dataPTable = entry.getKey(); - List<PTable> indexesToPartiallyRebuild = entry.getValue(); - ReadOnlyProps props = new ReadOnlyProps(env.getConfiguration().iterator()); + List<Pair<PTable,Long>> pairs = entry.getValue(); + List<PTable> indexesToPartiallyRebuild = Lists.newArrayListWithExpectedSize(pairs.size()); try (HTableInterface metaTable = env.getTable( SchemaUtil.getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, props))) { long earliestDisableTimestamp = Long.MAX_VALUE; + long latestUpperBoundTimestamp = Long.MIN_VALUE; List<IndexMaintainer> maintainers = Lists - .newArrayListWithExpectedSize(indexesToPartiallyRebuild.size()); + .newArrayListWithExpectedSize(pairs.size()); int signOfDisableTimeStamp = 0; - for (PTable index : indexesToPartiallyRebuild) { + for (Pair<PTable,Long> pair : pairs) { // We need a way of differentiating the block writes to data table case from // the leave index active case. In either case, we need to know the time stamp // at which writes started failing so we can rebuild from that point. If we @@ -368,6 +392,8 @@ public class MetaDataRegionObserver extends BaseRegionObserver { // then writes to the data table will be blocked (this is client side logic // and we can't change this in a minor release). So we use the sign of the // time stamp to differentiate. + PTable index = pair.getFirst(); + Long upperBoundTimestamp = pair.getSecond(); long disabledTimeStampVal = index.getIndexDisableTimestamp(); if (disabledTimeStampVal != 0) { if (signOfDisableTimeStamp != 0 && signOfDisableTimeStamp != Long.signum(disabledTimeStampVal)) { @@ -379,16 +405,20 @@ public class MetaDataRegionObserver extends BaseRegionObserver { earliestDisableTimestamp = disabledTimeStampVal; } + indexesToPartiallyRebuild.add(index); maintainers.add(index.getIndexMaintainer(dataPTable, conn)); } + if (upperBoundTimestamp > latestUpperBoundTimestamp) { + latestUpperBoundTimestamp = upperBoundTimestamp; + } } // No indexes are disabled, so skip this table if (earliestDisableTimestamp == Long.MAX_VALUE) { continue; } - long timeStamp = Math.max(0, earliestDisableTimestamp - overlapTime); + long scanBeginTime = Math.max(0, earliestDisableTimestamp - backwardOverlapDurationMs); LOG.info("Starting to build " + dataPTable + " indexes " + indexesToPartiallyRebuild - + " from timestamp=" + timeStamp); + + " from timestamp=" + scanBeginTime + " until " + latestUpperBoundTimestamp); TableRef tableRef = new TableRef(null, dataPTable, HConstants.LATEST_TIMESTAMP, false); // TODO Need to set high timeout @@ -398,11 +428,10 @@ public class MetaDataRegionObserver extends BaseRegionObserver { Scan dataTableScan = IndexManagementUtil.newLocalStateScan(plan.getContext().getScan(), maintainers); - long scanEndTime = getTimestampForBatch(timeStamp, - batchExecutedPerTableMap.get(dataPTable.getName())); + long scanEndTime = Math.min(latestUpperBoundTimestamp, + getTimestampForBatch(scanBeginTime,batchExecutedPerTableMap.get(dataPTable.getName()))); // We can't allow partial results - dataTableScan.setAllowPartialResults(false); - dataTableScan.setTimeRange(timeStamp, scanEndTime); + dataTableScan.setTimeRange(scanBeginTime, scanEndTime); dataTableScan.setCacheBlocks(false); dataTableScan.setAttribute(BaseScannerRegionObserver.REBUILD_INDEXES, TRUE_BYTES); @@ -418,7 +447,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver { + (scanEndTime == HConstants.LATEST_TIMESTAMP ? "LATEST_TIMESTAMP" : scanEndTime)); MutationState mutationState = plan.execute(); long rowCount = mutationState.getUpdateCount(); - if (scanEndTime == HConstants.LATEST_TIMESTAMP) { + if (scanEndTime == latestUpperBoundTimestamp) { LOG.info("Rebuild completed for all inactive/disabled indexes in data table:" + dataPTable.getName()); } @@ -427,13 +456,17 @@ public class MetaDataRegionObserver extends BaseRegionObserver { String indexTableFullName = SchemaUtil.getTableName( indexPTable.getSchemaName().getString(), indexPTable.getTableName().getString()); - if (scanEndTime == HConstants.LATEST_TIMESTAMP) { - IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.ACTIVE, 0l); + if (scanEndTime == latestUpperBoundTimestamp) { + // We compare the absolute value of the index disable timestamp. We don't want to + // pass a negative value because that means an additional index write failed. + IndexUtil.updateIndexState(conn, indexTableFullName, PIndexState.ACTIVE, Math.abs(indexPTable.getIndexDisableTimestamp())); batchExecutedPerTableMap.remove(dataPTable.getName()); LOG.info("Making Index:" + indexPTable.getTableName() + " active after rebuilding"); } else { // Maintain sign of INDEX_DISABLE_TIMESTAMP (see comment above) - updateDisableTimestamp(conn, indexTableFullName, env, scanEndTime * signOfDisableTimeStamp, metaTable); + if (!updateDisableTimestamp(conn, indexTableFullName, scanEndTime * signOfDisableTimeStamp, metaTable, indexPTable.getIndexDisableTimestamp())) { + LOG.warn("The index disabled timestamp for " + indexTableFullName + " was updated outside of rebuilder. Will reattempt rebuild next iteration."); + } Long noOfBatches = batchExecutedPerTableMap.get(dataPTable.getName()); if (noOfBatches == null) { noOfBatches = 0l; @@ -445,8 +478,7 @@ public class MetaDataRegionObserver extends BaseRegionObserver { dataPTable.getTableName().getString()); new MetaDataClient(conn).updateCache(indexPTable.getSchemaName().getString(), indexPTable.getTableName().getString()); - LOG.info( - "During Round-robin build: Successfully updated index disabled timestamp for " + LOG.info("During Round-robin build: Successfully updated index disabled timestamp for " + indexTableFullName + " to " + scanEndTime); } } @@ -490,16 +522,15 @@ public class MetaDataRegionObserver extends BaseRegionObserver { } } - private static void updateDisableTimestamp(PhoenixConnection conn, String indexTableName, - RegionCoprocessorEnvironment env, long disabledTimestamp, HTableInterface metaTable) throws IOException { + private static boolean updateDisableTimestamp(PhoenixConnection conn, String indexTableName, + long disabledTimestamp, HTableInterface metaTable, long expectedDisabledTimestamp) throws IOException { byte[] indexTableKey = SchemaUtil.getTableKeyFromFullName(indexTableName); Put put = new Put(indexTableKey); put.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, PLong.INSTANCE.toBytes(disabledTimestamp)); - metaTable.checkAndPut(indexTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, - PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, CompareOp.NOT_EQUAL, PLong.INSTANCE.toBytes(0), + return metaTable.checkAndPut(indexTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES, CompareOp.EQUAL, PLong.INSTANCE.toBytes(expectedDisabledTimestamp), put); - } private static synchronized void initRebuildIndexConnectionProps(Configuration config) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index e13a527..cbd756e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -151,8 +151,13 @@ public interface QueryServices extends SQLCloseable { public static final String INDEX_FAILURE_THROW_EXCEPTION_ATTRIB = "phoenix.index.failure.throw.exception"; // Index will be partially re-built from index disable time stamp - following overlap time + @Deprecated public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB = "phoenix.index.failure.handling.rebuild.overlap.time"; + public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME_ATTRIB = + "phoenix.index.failure.handling.rebuild.overlap.backward.time"; + public static final String INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME_ATTRIB = + "phoenix.index.failure.handling.rebuild.overlap.forward.time"; public static final String INDEX_PRIOIRTY_ATTRIB = "phoenix.index.rpc.priority"; public static final String METADATA_PRIOIRTY_ATTRIB = "phoenix.metadata.rpc.priority"; public static final String ALLOW_LOCAL_INDEX_ATTRIB = "phoenix.index.allowLocalIndex"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index c6593da..ec5c95b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -79,15 +79,15 @@ import static org.apache.phoenix.query.QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB import static org.apache.phoenix.query.QueryServices.STATS_USE_CURRENT_TIME_ATTRIB; import static org.apache.phoenix.query.QueryServices.THREAD_POOL_SIZE_ATTRIB; import static org.apache.phoenix.query.QueryServices.THREAD_TIMEOUT_MS_ATTRIB; -import static org.apache.phoenix.query.QueryServices.TRANSACTIONS_ENABLED; -import static org.apache.phoenix.query.QueryServices.UPLOAD_BINARY_DATA_TYPE_ENCODING; -import static org.apache.phoenix.query.QueryServices.USE_BYTE_BASED_REGEX_ATTRIB; -import static org.apache.phoenix.query.QueryServices.USE_INDEXES_ATTRIB; +import static org.apache.phoenix.query.QueryServices.TRACING_BATCH_SIZE; import static org.apache.phoenix.query.QueryServices.TRACING_ENABLED; import static org.apache.phoenix.query.QueryServices.TRACING_STATS_TABLE_NAME_ATTRIB; -import static org.apache.phoenix.query.QueryServices.TRACING_BATCH_SIZE; import static org.apache.phoenix.query.QueryServices.TRACING_THREAD_POOL_SIZE; import static org.apache.phoenix.query.QueryServices.TRACING_TRACE_BUFFER_SIZE; +import static org.apache.phoenix.query.QueryServices.TRANSACTIONS_ENABLED; +import static org.apache.phoenix.query.QueryServices.UPLOAD_BINARY_DATA_TYPE_ENCODING; +import static org.apache.phoenix.query.QueryServices.USE_BYTE_BASED_REGEX_ATTRIB; +import static org.apache.phoenix.query.QueryServices.USE_INDEXES_ATTRIB; import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION; import java.util.HashSet; @@ -182,7 +182,8 @@ public class QueryServicesOptions { public static final boolean DEFAULT_INDEX_FAILURE_DISABLE_INDEX = true; public static final boolean DEFAULT_INDEX_FAILURE_THROW_EXCEPTION = true; public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL = 60000; // 60 secs - public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME = 1; // 1 ms + public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_BACKWARD_TIME = 1; // 1 ms + public static final long DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_FORWARD_TIME = 60000 * 3; // 3 mins // 30 min rpc timeout * 5 tries, with 2100ms total pause time between retries public static final long DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT = (5 * 30000 * 60) + 2100; public static final long DEFAULT_INDEX_REBUILD_RPC_TIMEOUT = 30000 * 60; // 30 mins http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index 18e543c..75a7e55 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -114,7 +114,6 @@ import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.transaction.PhoenixTransactionContext; -import com.google.common.base.Preconditions; import com.google.common.collect.Lists; public class IndexUtil { @@ -790,10 +789,6 @@ public class IndexUtil { public static void updateIndexState(PhoenixConnection conn, String indexTableName, PIndexState newState, Long indexDisableTimestamp) throws SQLException { - if (newState == PIndexState.ACTIVE) { - Preconditions.checkArgument(indexDisableTimestamp == 0, - "Index disable timestamp has to be 0 when marking an index as active"); - } byte[] indexTableKey = SchemaUtil.getTableKeyFromFullName(indexTableName); String schemaName = SchemaUtil.getSchemaNameFromFullName(indexTableName); String indexName = SchemaUtil.getTableNameFromFullName(indexTableName); http://git-wip-us.apache.org/repos/asf/phoenix/blob/ed180c4d/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index 266f4da..1716928 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -870,8 +870,7 @@ public class TestUtil { + "AND " + PhoenixDatabaseMetaData.COLUMN_FAMILY + " IS NULL AND " + PhoenixDatabaseMetaData.COLUMN_NAME + " IS NULL" + " AND " + PhoenixDatabaseMetaData.INDEX_STATE + " = '" + expectedIndexState.getSerializedValue() + "'"; ResultSet rs = conn.createStatement().executeQuery(query); - assertTrue(rs.next()); - if (expectedIndexState == PIndexState.ACTIVE) { + if (rs.next() && expectedIndexState == PIndexState.ACTIVE) { if (rs.getLong(1) == 0 && !rs.wasNull()) { isActive = true; break;
