This is an automated email from the ASF dual-hosted git repository.
lokiore pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 96c05a47cb PHOENIX-7996 Correctly maintain immutable global indexes on
the server side for partial upserts (#2611)
96c05a47cb is described below
commit 96c05a47cbd32cbe81a8b324aa202131b57d3e91
Author: Lokesh Khurana <[email protected]>
AuthorDate: Thu Sep 17 10:41:00 2026 -0700
PHOENIX-7996 Correctly maintain immutable global indexes on the server side
for partial upserts (#2611)
Co-authored-by: lokesh-khurana <[email protected]>
Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]>
---
.../org/apache/phoenix/compile/DeleteCompiler.java | 19 +-
.../org/apache/phoenix/compile/UpsertCompiler.java | 7 +-
.../org/apache/phoenix/execute/MutationState.java | 4 +-
.../org/apache/phoenix/index/IndexMaintainer.java | 8 +-
.../phoenix/index/IndexMetaDataCacheClient.java | 9 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 23 +-
.../phoenix/hbase/index/IndexRegionObserver.java | 40 ++
.../java/org/apache/phoenix/end2end/DeleteIT.java | 10 +-
.../org/apache/phoenix/end2end/IndexToolIT.java | 9 +-
.../end2end/index/BaseImmutableIndexIT.java | 454 +++++++++++++++++++++
.../apache/phoenix/end2end/index/BaseIndexIT.java | 17 +-
.../end2end/index/BaseIndexWithRegionMovesIT.java | 14 +-
.../end2end/index/GlobalIndexCheckerIT.java | 71 ++++
.../phoenix/monitoring/BasePhoenixMetricsIT.java | 2 +
.../monitoring/PhoenixTableLevelMetricsIT.java | 2 +
.../org/apache/phoenix/rpc/PhoenixClientRpcIT.java | 9 +-
16 files changed, 659 insertions(+), 39 deletions(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 3d1e9ffdcd..08f9803eb6 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -18,8 +18,6 @@
package org.apache.phoenix.compile;
import static
org.apache.phoenix.execute.MutationState.RowTimestampColInfo.NULL_ROWTIMESTAMP_INFO;
-import static
org.apache.phoenix.query.QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB;
-import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED;
import static org.apache.phoenix.util.NumberUtil.add;
import edu.umd.cs.findbugs.annotations.NonNull;
@@ -214,7 +212,7 @@ public class DeleteCompiler {
// The data table is always the last one in the list if it's
// not chosen as the best of the possible plans.
dataTable = otherTableRefs.get(otherTableRefs.size() - 1).getTable();
- if (!isMaintainedOnClient(table, connection)) {
+ if (!isMaintainedOnClient(table, dataTable, connection)) {
// dataTable is a projected table and may not include all the
indexed columns and so we
// need to get
// the actual data table
@@ -259,7 +257,7 @@ public class DeleteCompiler {
// row timestamp column, then the
// row key will already have its value.
// Check for otherTableRefs being empty required when deleting
directly from the index
- if (otherTableRefs.isEmpty() || isMaintainedOnClient(table,
connection)) {
+ if (otherTableRefs.isEmpty() || isMaintainedOnClient(table, dataTable,
connection)) {
mutations.put(rowKeyPtr,
new RowMutationState(PRow.DELETE_MARKER, 0,
statement.getConnection().getStatementExecutionCounter(),
NULL_ROWTIMESTAMP_INFO,
@@ -385,7 +383,7 @@ public class DeleteCompiler {
for (PTable index : table.getIndexes()) {
if (
!index.getIndexState().isDisabled()
- && isMaintainedOnClient(index, statement.getConnection())
+ && isMaintainedOnClient(index, table, statement.getConnection())
) {
nonDisabledIndexes.add(index);
}
@@ -569,7 +567,7 @@ public class DeleteCompiler {
// mutations are generated on the client side. Indexed columns are needed
to identify index rows
// to be deleted
for (PTable index : table.getIndexes()) {
- if (isMaintainedOnClient(index, connection)) {
+ if (isMaintainedOnClient(index, table, connection)) {
IndexMaintainer maintainer = index.getIndexMaintainer(table,
connection);
// Go through maintainer as it handles functional indexes correctly
for (Pair<String, String> columnInfo :
maintainer.getIndexedColumnInfo()) {
@@ -1135,15 +1133,16 @@ public class DeleteCompiler {
}
}
- private static boolean isMaintainedOnClient(PTable table, PhoenixConnection
connection) {
+ private static boolean isMaintainedOnClient(PTable table, PTable dataTable,
+ PhoenixConnection connection) {
if (CDCUtil.isCDCIndex(table)) {
return false;
}
+ // The server-side-maintenance flag is read against the data table so a
ROW_TIMESTAMP data
+ // table keeps its indexes client-maintained (see IndexUtil).
if (
!table.isTransactional() && table.getIndexType() != IndexType.LOCAL
- && connection.getQueryServices().getConfiguration().getBoolean(
- SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
- DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED)
+ && IndexUtil.isServerSideImmutableIndexMaintenanceEnabled(dataTable,
connection)
) {
return false;
}
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 106f5071e3..b1cb9255ad 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -17,8 +17,6 @@
*/
package org.apache.phoenix.compile;
-import static
org.apache.phoenix.query.QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB;
-import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED;
import static
org.apache.phoenix.thirdparty.com.google.common.base.Preconditions.checkArgument;
import static
org.apache.phoenix.thirdparty.com.google.common.collect.Lists.newArrayListWithCapacity;
@@ -626,9 +624,8 @@ public class UpsertCompiler {
// region space managed by region servers. So we bail out on executing
on server side.
// Disable running upsert select on server side if a table has global
mutable secondary
// indexes on it
- boolean serverSideImmutableIndexes =
connection.getQueryServices().getConfiguration()
- .getBoolean(SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
- DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED);
+ boolean serverSideImmutableIndexes =
+ IndexUtil.isServerSideImmutableIndexMaintenanceEnabled(table,
connection);
boolean hasGlobalServerSideIndexes = SchemaUtil.hasGlobalIndex(table)
&& (serverSideImmutableIndexes || !table.isImmutableRows());
boolean hasWhereSubquery = select.getWhere() != null &&
select.getWhere().hasSubquery();
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
index b0bfac0b04..4931620a07 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -1927,7 +1927,9 @@ public class MutationState implements SQLCloseable {
}
PTable logicalTable = tableInfo.getPTable();
if (
- !this.serverSideImmutableIndexes &&
tableInfo.getOrigTableRef().getTable().isImmutableRows()
+ !IndexUtil.isServerSideImmutableIndexMaintenanceEnabled(
+ tableInfo.getOrigTableRef().getTable(),
this.serverSideImmutableIndexes)
+ && tableInfo.getOrigTableRef().getTable().isImmutableRows()
&& (this.indexRegionObserverEnabledAllTables
|| IndexUtil.isGlobalIndexCheckerEnabled(connection,
tableInfo.getHTableName()))
) {
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index 70b83d1795..533ddadb10 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -17,8 +17,6 @@
*/
package org.apache.phoenix.index;
-import static
org.apache.phoenix.query.QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB;
-import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED;
import static
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -198,11 +196,9 @@ public class IndexMaintainer implements Writable,
Iterable<ColumnReference> {
return Iterators.filter(indexes, new Predicate<PTable>() {
@Override
public boolean apply(PTable index) {
- return sendIndexMaintainer(index) && ((index.getIndexType() ==
IndexType.GLOBAL
+ return sendIndexMaintainer(index) && ((IndexUtil.isGlobalIndex(index)
&& (dataTable.getImmutableStorageScheme() !=
index.getImmutableStorageScheme()
- || connection.getQueryServices().getConfiguration().getBoolean(
- SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
- DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED)))
+ ||
IndexUtil.isServerSideImmutableIndexMaintenanceEnabled(dataTable, connection)))
|| index.getIndexType() == IndexType.LOCAL ||
CDCUtil.isCDCIndex(index));
}
});
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
index 53a833bbe0..7b1e83bc0d 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java
@@ -40,6 +40,7 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.IndexUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.ScanUtil;
@@ -145,9 +146,11 @@ public class IndexMetaDataCacheClient {
QueryServicesOptions.DEFAULT_INDEX_USE_SERVER_METADATA)
&& props.getBoolean(QueryServices.INDEX_REGION_OBSERVER_ENABLED_ATTRIB,
QueryServicesOptions.DEFAULT_INDEX_REGION_OBSERVER_ENABLED);
- boolean serverSideImmutableIndexes =
- props.getBoolean(SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
- DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED);
+ // Keep ROW_TIMESTAMP tables client-maintained so the send-metadata
decision matches the
+ // client-vs-server maintenance decision made for the same table
elsewhere.
+ boolean serverSideImmutableIndexes =
IndexUtil.isServerSideImmutableIndexMaintenanceEnabled(
+ table, props.getBoolean(SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
+ DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED));
boolean useServerCacheRpc =
useIndexMetadataCache(connection, mutations,
indexMetaDataPtr.getLength() + txState.length)
&& sendIndexMaintainers;
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/IndexUtil.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 5262437af8..830c2d49c6 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -761,6 +761,25 @@ public class IndexUtil {
}
}
+ /**
+ * Whether server-side maintenance of an immutable data table's global
indexes applies. A table
+ * with a ROW_TIMESTAMP column stays client-maintained even when the flag is
set: server-side
+ * maintenance stamps every cell with the server batch timestamp, which
would overwrite the
+ * user-supplied ROW_TIMESTAMP value and silently drop rows on ROW_TIMESTAMP
range scans.
+ */
+ public static boolean isServerSideImmutableIndexMaintenanceEnabled(PTable
dataTable,
+ boolean configured) {
+ return configured && dataTable.getRowTimestampColPos() == -1;
+ }
+
+ public static boolean isServerSideImmutableIndexMaintenanceEnabled(PTable
dataTable,
+ PhoenixConnection connection) {
+ return isServerSideImmutableIndexMaintenanceEnabled(dataTable,
+ connection.getQueryServices().getConfiguration().getBoolean(
+ QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
+ QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED));
+ }
+
public static List<PTable> getClientMaintainedIndexes(PTable table,
boolean serverSideImmutableIndex) {
Iterator<PTable> indexIterator = // Only maintain tables with immutable
rows through this
@@ -768,7 +787,9 @@ public class IndexUtil {
(table.isTransactional() &&
table.getTransactionProvider().getTransactionProvider()
.isUnsupported(Feature.MAINTAIN_LOCAL_INDEX_ON_SERVER))
? IndexMaintainer.maintainedIndexes(table.getIndexes().iterator())
- : (table.isImmutableRows() && !serverSideImmutableIndex ||
table.isTransactional())
+ : (table.isImmutableRows()
+ && !isServerSideImmutableIndexMaintenanceEnabled(table,
serverSideImmutableIndex)
+ || table.isTransactional())
// If the data table has a different storage scheme than index
table, don't maintain
// this on the client. For example, if the index is single cell but
the data table is
// one_cell, and there is a partial update on the data table, index
can't be built
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
index b1145c721b..5125c783e4 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/hbase/index/IndexRegionObserver.java
@@ -1189,6 +1189,44 @@ public class IndexRegionObserver implements
RegionCoprocessor, RegionObserver {
return false;
}
+ /**
+ * Determines whether any data table mutation in the batch omits an on-disk
column that a global
+ * index materializes (an indexed, covered, or index-WHERE column). For an
immutable table the
+ * region-side build skips the current-row read-back, so such a partial
upsert would drop or
+ * misbuild that column in the index while it survives in the data table.
Columns are resolved to
+ * their on-disk qualifiers per the data table storage scheme, so a
single-cell table (whose
+ * upserts rewrite the whole cell) is never treated as partial. Both covered
and uncovered global
+ * maintainers are considered; transform and local index maintainers are
skipped.
+ */
+ private boolean isPartialGlobalIndexMutation(PhoenixIndexMetaData
indexMetaData,
+ MiniBatchOperationInProgress<Mutation> miniBatchOp) {
+ Set<ColumnReference> columns = new HashSet<ColumnReference>();
+ for (IndexMaintainer indexMaintainer :
indexMetaData.getIndexMaintainers()) {
+ if (indexMaintainer instanceof TransformMaintainer ||
indexMaintainer.isLocalIndex()) {
+ continue;
+ }
+ columns.addAll(indexMaintainer.getAllColumnsForDataTable());
+ }
+ if (columns.isEmpty()) {
+ return false;
+ }
+ for (int i = 0; i < miniBatchOp.size(); i++) {
+ if (isAtomicOperationComplete(miniBatchOp.getOperationStatus(i))) {
+ continue;
+ }
+ Mutation m = miniBatchOp.getOperation(i);
+ if (!this.builder.isEnabled(m)) {
+ continue;
+ }
+ for (ColumnReference column : columns) {
+ if (m.get(column.getFamily(), column.getQualifier()).isEmpty()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
/**
* Retrieve the data row state either from memory or disk. The rows are
locked by the caller.
* <p>
@@ -1985,6 +2023,8 @@ public class IndexRegionObserver implements
RegionCoprocessor, RegionObserver {
|| context.hasStrictConditionalTTL()
|| !context.immutableRows && context.hasUncoveredIndex
&& isPartialUncoveredIndexMutation(indexMetaData, miniBatchOp)
+ || context.immutableRows && (context.hasGlobalIndex ||
context.hasUncoveredIndex)
+ && isPartialGlobalIndexMutation(indexMetaData, miniBatchOp)
) {
getCurrentRowStates(c, context, batchTimestamp);
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
index c2a31842f9..cb7ffbfd27 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DeleteIT.java
@@ -60,6 +60,7 @@ import org.apache.phoenix.parse.DeleteStatement;
import org.apache.phoenix.parse.SQLParser;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.junit.Test;
@@ -505,7 +506,14 @@ public class DeleteIT extends ParallelStatsDisabledIT {
psDelete.setString(3, "CC");
psDelete.setDate(4, date);
String explainPlan = QueryUtil.getExplainPlan(psDelete.executeQuery());
- if (addNonPKIndex) {
+ // A non-PK index needs the row's non-PK values to maintain its entries.
When immutable
+ // indexes are maintained client side the client must read the row
first, so the delete is
+ // not a single-row plan. When they are maintained server side the
region server reads and
+ // maintains the index, so the point delete stays a single-row plan.
+ boolean serverSideImmutableIndexes =
con.unwrap(PhoenixConnection.class).getQueryServices()
+
.getConfiguration().getBoolean(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
+ QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED);
+ if (addNonPKIndex && !serverSideImmutableIndexes) {
assertNotEquals("DELETE SINGLE ROW", explainPlan);
} else {
assertEquals("DELETE SINGLE ROW", explainPlan);
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index 23f083f096..cd65cc45b8 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -888,8 +888,13 @@ public class IndexToolIT extends BaseTest {
0, IndexTool.IndexVerifyType.ONLY, "-fi");
CounterGroup mrJobCounters = getMRJobCounters(indexTool);
- // Extra index rows should be detected
- if (mutable) {
+ boolean serverSideImmutableIndexes =
conn.unwrap(PhoenixConnection.class).getQueryServices()
+
.getConfiguration().getBoolean(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
+ QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED);
+ // Extra index rows should be detected. Immutable indexes maintained
server-side adopt the
+ // same two-phase verified write protocol as mutable indexes, so the
directly-written extra
+ // rows are counted as UNVERIFIED rather than VERIFIED.
+ if (mutable || serverSideImmutableIndexes) {
assertEquals(3, mrJobCounters
.findCounter(BEFORE_REPAIR_EXTRA_UNVERIFIED_INDEX_ROW_COUNT.name()).getValue());
} else {
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java
index aaeeea6d07..dbf8487952 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseImmutableIndexIT.java
@@ -761,4 +761,458 @@ public abstract class BaseImmutableIndexIT extends
BaseTest {
conn.commit();
}
}
+
+ //
---------------------------------------------------------------------------------------------
+ // Partial-upsert and delete coverage for server-side immutable index
maintenance
+ // (SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB). These run under both
storage schemes (the
+ // columnEncoded parameter selects ONE_CELL_PER_COLUMN vs
SINGLE_CELL_ARRAY_WITH_OFFSETS) and
+ // under both flag states -- ServerSideImmutableIndexIT forces the flag on,
+ // ClientSideImmutableIndexIT
+ // forces it off -- so every assertion below must hold whether the index is
maintained on the
+ // server or the client. Only non-transactional global indexes are affected
by the flag, so each
+ // test bails out for local and transactional parameterizations.
+ //
---------------------------------------------------------------------------------------------
+
+ /**
+ * Reads every row of a query into "col1|col2|..." strings (a null column
renders as the literal
+ * "null"), so an index-served result can be compared for exact agreement
with the data table
+ * under either storage scheme.
+ */
+ private static List<String> readRows(Connection conn, String sql) throws
SQLException {
+ List<String> rows = new ArrayList<>();
+ try (ResultSet rs = conn.createStatement().executeQuery(sql)) {
+ int cols = rs.getMetaData().getColumnCount();
+ while (rs.next()) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 1; i <= cols; i++) {
+ if (i > 1) {
+ sb.append('|');
+ }
+ sb.append(rs.getString(i));
+ }
+ rows.add(sb.toString());
+ }
+ }
+ return rows;
+ }
+
+ /**
+ * A partial upsert on an immutable table with a covered global index must
leave the index
+ * consistent with the data table under BOTH storage schemes. Under
ONE_CELL_PER_COLUMN the
+ * omitted covered column keeps its earlier value; under
SINGLE_CELL_ARRAY_WITH_OFFSETS the whole
+ * row is a single cell so the omitted column is overwritten to null. Either
way the index the
+ * server maintains must read back exactly what the data table returns. The
SINGLE_CELL branch
+ * (columnEncoded=true) is the load-bearing case the ONE_CELL-only
GlobalIndexCheckerIT test
+ * skips.
+ */
+ @Test
+ public void testPartialUpsertForImmutableCoveredIndex() throws Exception {
+ if (localIndex || transactionProvider != null) {
+ return;
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ conn.createStatement()
+ .execute("CREATE TABLE " + tableName
+ + " (id VARCHAR NOT NULL PRIMARY KEY, val1 VARCHAR, val2 VARCHAR,
val3 VARCHAR) "
+ + tableDDLOptions);
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES ('a', 'ab', 'abc',
'abcd')");
+ conn.commit();
+ conn.createStatement()
+ .execute("CREATE INDEX " + indexName + " ON " + tableName + " (val1)
INCLUDE (val2, val3)");
+
+ String idxSql = "SELECT /*+ INDEX(" + tableName + " " + indexName
+ + ") */ id, val1, val2, val3 " + "FROM " + tableName + " WHERE val1 =
'ab'";
+ String dataSql =
+ "SELECT /*+ NO_INDEX */ id, val1, val2, val3 FROM " + tableName + "
WHERE val1 = 'ab'";
+ // Full upsert: the index is actually used (guards against a vacuous
data-scan agreement) and
+ // returns the complete row.
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + idxSql)),
+ tableName, indexName);
+ assertEquals("[a|ab|abc|abcd]", readRows(conn, idxSql).toString());
+ assertEquals(readRows(conn, dataSql), readRows(conn, idxSql));
+
+ // Partial upsert re-supplies val1 (the index key) and val2 but omits
the covered column val3.
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " (id, val1, val2) VALUES ('a',
'ab', 'abcc')");
+ conn.commit();
+ // The server-maintained index must still agree with the data table,
whatever the storage
+ // scheme did to the omitted column.
+ assertEquals("index must agree with data after partial upsert",
readRows(conn, dataSql),
+ readRows(conn, idxSql));
+ }
+ }
+
+ /**
+ * A partial upsert that omits the indexed column of an UNCOVERED global
index must leave the
+ * index path resolving exactly the rows the data table does. Under
ONE_CELL_PER_COLUMN the
+ * indexed column keeps its earlier value so both find the row under 'ab';
under
+ * SINGLE_CELL_ARRAY_WITH_OFFSETS the single-cell overwrite nulls the
indexed column so both find
+ * nothing. The scan path agrees under both maintenance modes because it
self-heals unverified
+ * index rows at read time. The COUNT (aggregate) path does not self-heal,
so it only agrees when
+ * the index is maintained on the server: server-side maintenance reads the
current row back and
+ * rewrites a verified entry for the retained key, whereas client-side
maintenance cannot rebuild
+ * the omitted index key and leaves the COUNT path undercounting -- the
pre-existing gap this PR's
+ * server-side path closes.
+ */
+ @Test
+ public void testPartialUpsertForImmutableUncoveredIndex() throws Exception {
+ if (localIndex || transactionProvider != null) {
+ return;
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ conn.createStatement().execute("CREATE TABLE " + tableName
+ + " (id VARCHAR NOT NULL PRIMARY KEY, val1 VARCHAR, val2 VARCHAR) " +
tableDDLOptions);
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " (id, val1, val2) VALUES ('a',
'ab', 'abc')");
+ conn.commit();
+ conn.createStatement()
+ .execute("CREATE UNCOVERED INDEX " + indexName + " ON " + tableName +
" (val1)");
+
+ String idxSql = "SELECT /*+ INDEX(" + tableName + " " + indexName + ")
*/ id FROM "
+ + tableName + " WHERE val1 = 'ab'";
+ String dataSql = "SELECT /*+ NO_INDEX */ id FROM " + tableName + " WHERE
val1 = 'ab'";
+ String idxCount = "SELECT /*+ INDEX(" + tableName + " " + indexName + ")
*/ COUNT(*) FROM "
+ + tableName + " WHERE val1 = 'ab'";
+ String dataCount = "SELECT /*+ NO_INDEX */ COUNT(*) FROM " + tableName +
" WHERE val1 = 'ab'";
+ // Full upsert: the uncovered index is used and resolves the row.
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + idxSql)),
+ tableName, indexName);
+ assertEquals("[a]", readRows(conn, idxSql).toString());
+
+ // Partial upsert omits the indexed column val1.
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " (id, val2) VALUES ('a',
'xyz')");
+ conn.commit();
+ // The scan path self-heals unverified index rows against the data
table, so it agrees under
+ // both maintenance modes.
+ assertEquals("uncovered index scan must agree with data after partial
upsert",
+ readRows(conn, dataSql), readRows(conn, idxSql));
+ if (serverSideIndex) {
+ // The COUNT (aggregate) path does not self-heal. Only server-side
maintenance rebuilds a
+ // verified uncovered-index entry from the read-back row, so only then
does the COUNT path
+ // agree with the data table; the client-side path is the pre-existing
gap this PR closes.
+ assertEquals("uncovered index COUNT must agree with data after partial
upsert",
+ readRows(conn, dataCount), readRows(conn, idxCount));
+ }
+ }
+ }
+
+ /**
+ * A partial upsert on a table with more than one covered global index must
leave every maintained
+ * index in agreement with the data table for the covered column it reads
back.
+ */
+ @Test
+ public void testPartialUpsertForImmutableMultipleIndexes() throws Exception {
+ if (localIndex || transactionProvider != null) {
+ return;
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName1 = "IND_" + generateUniqueName();
+ String indexName2 = "IND_" + generateUniqueName();
+ conn.createStatement()
+ .execute("CREATE TABLE " + tableName
+ + " (id VARCHAR NOT NULL PRIMARY KEY, val1 VARCHAR, val2 VARCHAR,
val3 VARCHAR) "
+ + tableDDLOptions);
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES ('a', 'ab', 'abc',
'abcd')");
+ conn.commit();
+ conn.createStatement()
+ .execute("CREATE INDEX " + indexName1 + " ON " + tableName + " (val1)
INCLUDE (val3)");
+ conn.createStatement()
+ .execute("CREATE INDEX " + indexName2 + " ON " + tableName + " (val2)
INCLUDE (val3)");
+ // Partial upsert re-supplies val1 and val2 (both index keys) and omits
the covered column
+ // val3.
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " (id, val1, val2) VALUES ('a',
'ab', 'abc')");
+ conn.commit();
+
+ String viaIdx1 = "SELECT /*+ INDEX(" + tableName + " " + indexName1 + ")
*/ val3 FROM "
+ + tableName + " WHERE val1 = 'ab'";
+ String viaIdx2 = "SELECT /*+ INDEX(" + tableName + " " + indexName2 + ")
*/ val3 FROM "
+ + tableName + " WHERE val2 = 'abc'";
+ String data = "SELECT /*+ NO_INDEX */ val3 FROM " + tableName + " WHERE
id = 'a'";
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + viaIdx1)),
+ tableName, indexName1);
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + viaIdx2)),
+ tableName, indexName2);
+ assertEquals("index1 must agree with data", readRows(conn, data),
readRows(conn, viaIdx1));
+ assertEquals("index2 must agree with data", readRows(conn, data),
readRows(conn, viaIdx2));
+ }
+ }
+
+ /**
+ * A partial upsert that touches only one column family of a
multi-column-family immutable table
+ * must read back a covered column that lives in an untouched family. Family
b is never written by
+ * the partial upsert, so its cell survives under BOTH storage schemes and
val3 must remain 'abcd'
+ * on the data table -- and the server-maintained index must read back the
same value.
+ */
+ @Test
+ public void testPartialUpsertForImmutableMultipleColumnFamilies() throws
Exception {
+ if (localIndex || transactionProvider != null) {
+ return;
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ conn.createStatement()
+ .execute("CREATE TABLE " + tableName
+ + " (id VARCHAR NOT NULL PRIMARY KEY, a.val1 VARCHAR, a.val2
VARCHAR, b.val3 VARCHAR) "
+ + tableDDLOptions);
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES ('a', 'ab', 'abc',
'abcd')");
+ conn.commit();
+ conn.createStatement()
+ .execute("CREATE INDEX " + indexName + " ON " + tableName + " (val1)
INCLUDE (val2, val3)");
+ // Partial upsert touches only family a (val1, val2); family b's val3 is
untouched.
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " (id, val1, val2) VALUES ('a',
'ab', 'abcc')");
+ conn.commit();
+
+ String idxSql = "SELECT /*+ INDEX(" + tableName + " " + indexName
+ + ") */ val1, val2, val3 FROM " + tableName + " WHERE val1 = 'ab'";
+ String dataSql =
+ "SELECT /*+ NO_INDEX */ val1, val2, val3 FROM " + tableName + " WHERE
id = 'a'";
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + idxSql)),
+ tableName, indexName);
+ List<String> dataRows = readRows(conn, dataSql);
+ List<String> idxRows = readRows(conn, idxSql);
+ if (tableDDLOptions.contains("ONE_CELL_PER_COLUMN")) {
+ // ONE_CELL_PER_COLUMN keeps the untouched family's cell, so val3 must
survive as 'abcd'.
+ // This
+ // anchors the cross-family read-back so an all-null agreement cannot
pass vacuously.
+ // (SINGLE_CELL_ARRAY_WITH_OFFSETS does not retain the omitted column
on a partial upsert;
+ // the
+ // agreement check below still verifies the index tracks the data
table under that scheme.)
+ assertEquals("[ab|abcc|abcd]", dataRows.toString());
+ }
+ // The server-maintained index must read back identically to the data
table across families.
+ assertEquals("index must agree with data across column families",
dataRows, idxRows);
+ }
+ }
+
+ /**
+ * A partial upsert that omits the WHERE-clause column of a partial index
(CREATE INDEX ... WHERE)
+ * must leave the partial index selecting exactly the rows the data-table
predicate does. Under
+ * ONE_CELL_PER_COLUMN the WHERE column keeps its earlier value so the row
stays indexed; under
+ * SINGLE_CELL_ARRAY_WITH_OFFSETS the single-cell overwrite nulls it so
neither the index nor the
+ * data path matches val2 = 'keep'. Either way the index path and the data
path must agree.
+ */
+ @Test
+ public void testPartialUpsertForImmutablePartialIndex() throws Exception {
+ if (localIndex || transactionProvider != null) {
+ return;
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ conn.createStatement()
+ .execute("CREATE TABLE " + tableName
+ + " (id VARCHAR NOT NULL PRIMARY KEY, val1 VARCHAR, val2 VARCHAR,
val3 VARCHAR) "
+ + tableDDLOptions);
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES ('a', 'ab', 'keep',
'abcd')");
+ conn.commit();
+ // Only rows with val2 = 'keep' are indexed; val2 is the WHERE-clause
column.
+ conn.createStatement().execute("CREATE INDEX " + indexName + " ON " +
tableName
+ + " (val1) INCLUDE (val3) WHERE val2 = 'keep'");
+ // Partial upsert omits val2 (the partial-index predicate column) and
updates val3.
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " (id, val1, val3) VALUES ('a',
'ab', 'updated')");
+ conn.commit();
+
+ // The query predicate implies the partial-index predicate, so the index
is applicable; force
+ // it and compare against the data table for the same predicate.
+ String idxSql = "SELECT /*+ INDEX(" + tableName + " " + indexName + ")
*/ val3 FROM "
+ + tableName + " WHERE val1 = 'ab' AND val2 = 'keep'";
+ String dataSql =
+ "SELECT /*+ NO_INDEX */ val3 FROM " + tableName + " WHERE val1 = 'ab'
AND val2 = 'keep'";
+ assertEquals("partial index must agree with data after partial upsert",
+ readRows(conn, dataSql), readRows(conn, idxSql));
+ }
+ }
+
+ /**
+ * A partial upsert on an immutable table carrying BOTH a
matching-storage-scheme index (inherits
+ * the base scheme) and a mismatched-storage-scheme index (explicit
SINGLE_CELL_ARRAY_WITH_OFFSETS
+ * on a ONE_CELL_PER_COLUMN base) must leave every index in agreement with
the data table. The
+ * mismatched index is always maintained on the server because its storage
scheme differs from the
+ * data table's (IndexMaintainer), independent of the
server-side-immutable-index flag; the
+ * matching index is maintained on the server only when the flag is on. So
with the flag off the
+ * same upsert batch carries one server-maintained and one client-maintained
index, exercising the
+ * single per-batch immutableRows classification -- both indexes must still
read back the retained
+ * covered column. The mismatched pairing is only legal in the upgrade
direction (a SINGLE_CELL
+ * base rejects a ONE_CELL index with
INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE), so the mismatched
+ * index is created only under the ONE_CELL_PER_COLUMN base; under the
SINGLE_CELL base only the
+ * matching index runs.
+ */
+ @Test
+ public void testPartialUpsertForImmutableMixedStorageSchemeIndexes() throws
Exception {
+ if (localIndex || transactionProvider != null) {
+ return;
+ }
+ boolean oneCellBase = tableDDLOptions.contains("ONE_CELL_PER_COLUMN");
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(false);
+ String tableName = "TBL_" + generateUniqueName();
+ String matchingIndex = "IND_" + generateUniqueName();
+ String mismatchedIndex = "IND_" + generateUniqueName();
+ conn.createStatement()
+ .execute("CREATE TABLE " + tableName
+ + " (id VARCHAR NOT NULL PRIMARY KEY, val1 VARCHAR, val2 VARCHAR,
val3 VARCHAR) "
+ + tableDDLOptions);
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES ('a', 'ab', 'abc',
'abcd')");
+ conn.commit();
+ // Matching-scheme index: inherits the base table's storage scheme.
+ conn.createStatement()
+ .execute("CREATE INDEX " + matchingIndex + " ON " + tableName + "
(val1) INCLUDE (val3)");
+ // Mismatched-scheme index: explicit SINGLE_CELL on a ONE_CELL base (the
only legal mismatch
+ // direction). Skipped under a SINGLE_CELL base, where a ONE_CELL index
would be rejected.
+ if (oneCellBase) {
+ conn.createStatement()
+ .execute("CREATE INDEX " + mismatchedIndex + " ON " + tableName
+ + " (val2) INCLUDE (val3)
IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, "
+ + "COLUMN_ENCODED_BYTES=2");
+ }
+ // Partial upsert re-supplies both index keys (val1, val2) and omits the
covered column val3.
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " (id, val1, val2) VALUES ('a',
'ab', 'abc')");
+ conn.commit();
+
+ String viaMatching = "SELECT /*+ INDEX(" + tableName + " " +
matchingIndex
+ + ") */ id, val3 FROM " + tableName + " WHERE val1 = 'ab'";
+ String data = "SELECT /*+ NO_INDEX */ id, val3 FROM " + tableName + "
WHERE id = 'a'";
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + viaMatching)),
+ tableName, matchingIndex);
+ List<String> dataRows = readRows(conn, data);
+ if (oneCellBase) {
+ // ONE_CELL base keeps the omitted covered column, so val3 must
survive as 'abcd' (id
+ // anchors
+ // the row so an all-null agreement cannot pass vacuously).
+ assertEquals("[a|abcd]", dataRows.toString());
+ }
+ assertEquals("matching-scheme index must agree with data", dataRows,
+ readRows(conn, viaMatching));
+ if (oneCellBase) {
+ String viaMismatched = "SELECT /*+ INDEX(" + tableName + " " +
mismatchedIndex + ") */ id, "
+ + "val3 FROM " + tableName + " WHERE val2 = 'abc'";
+ assertExplainPlan(false,
+
QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN " +
viaMismatched)),
+ tableName, mismatchedIndex);
+ // The mismatched index is always server-maintained (schemes differ);
it must read back the
+ // same retained covered column as the data table, even when the
matching index in the same
+ // batch is maintained on the client (flag off).
+ assertEquals("mismatched-scheme index must agree with data", dataRows,
+ readRows(conn, viaMismatched));
+
+ // Aggregate (COUNT) path does not self-heal against the data table
the way the point scan
+ // above does, so it is a strong guard that the SINGLE_CELL index
actually retained the
+ // omitted covered column. The single-cell family is rewritten
wholesale on the partial
+ // upsert, so without the always-on server read-back val3 would be
lost and this COUNT would
+ // drop to 0. It must equal the data-table COUNT in BOTH flag modes
because a mismatched
+ // storage scheme keeps the index server-maintained regardless of the
flag.
+ String countByCovered = "COUNT(*) FROM " + tableName + " WHERE val3 =
'abcd'";
+ String dataCount = "SELECT /*+ NO_INDEX */ " + countByCovered;
+ String mismatchedCount =
+ "SELECT /*+ INDEX(" + tableName + " " + mismatchedIndex + ") */ " +
countByCovered;
+ // Filtering on a covered (non-key) column is a full index scan, not a
range scan, so assert
+ // the index is used by name rather than via the range-scan-only
assertExplainPlan helper.
+ String mismatchedCountPlan = QueryUtil
+ .getExplainPlan(conn.createStatement().executeQuery("EXPLAIN " +
mismatchedCount));
+ assertTrue(
+ "COUNT must be served from the mismatched-scheme index; plan was: "
+ mismatchedCountPlan,
+ mismatchedCountPlan.contains(mismatchedIndex));
+ assertEquals("mismatched-scheme index COUNT must agree with data",
+ readRows(conn, dataCount), readRows(conn, mismatchedCount));
+ }
+ }
+ }
+
+ /**
+ * Deleting from an immutable table that has a ROW_TIMESTAMP column and a
secondary index. The
+ * ROW_TIMESTAMP carve-out must keep index maintenance on the client even
when the server-side
+ * flag is on, so the region server does not re-stamp the cells with a
server clock; otherwise a
+ * ROW_TIMESTAMP range scan would silently drop the surviving rows.
+ */
+ @Test
+ public void testDeleteFromImmutableRowTimestampTableWithIndex() throws
Exception {
+ if (localIndex || transactionProvider != null) {
+ return;
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ conn.setAutoCommit(true);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ conn.createStatement().execute(
+ "CREATE TABLE " + tableName + " (k1 BIGINT NOT NULL, k2 VARCHAR NOT
NULL, val VARCHAR "
+ + "CONSTRAINT pk PRIMARY KEY (k1 ROW_TIMESTAMP, k2)) " +
tableDDLOptions);
+ conn.createStatement().execute("CREATE INDEX " + indexName + " ON " +
tableName + " (val)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES
(100, 'a', 'v1')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES
(200, 'b', 'v2')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES
(300, 'c', 'v3')");
+
+ conn.createStatement().execute("DELETE FROM " + tableName + " WHERE k1 =
200 AND k2 = 'b'");
+
+ // Data and index counts must agree after the delete.
+ try (ResultSet rs =
+ conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*)
FROM " + tableName)) {
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ }
+ String countViaIndex = "SELECT COUNT(*) FROM " + tableName + " WHERE val
IS NOT NULL";
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + countViaIndex)),
+ tableName, indexName);
+ try (ResultSet rs = conn.createStatement().executeQuery(countViaIndex)) {
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ }
+ // Each surviving row keeps its ROW_TIMESTAMP-derived cell timestamp: a
scan bounded to a
+ // surviving row's exact ROW_TIMESTAMP band must still return it. This
is the ROW_TIMESTAMP
+ // carve-out under test -- maintenance stays on the client so cells are
not re-stamped with
+ // the
+ // server clock; a server-clock re-stamp would move the cell outside its
ROW_TIMESTAMP band
+ // and
+ // this bounded scan would return 0. The band deliberately excludes the
deleted k1=200: a
+ // DELETE
+ // marker is stamped at wall-clock time, outside any ROW_TIMESTAMP band,
so a bounded scan
+ // cannot observe the delete -- an orthogonal ROW_TIMESTAMP semantics
unrelated to this flag.
+ for (long k1 : new long[] { 100L, 300L }) {
+ try (
+ ResultSet rs = conn.createStatement().executeQuery("SELECT /*+
NO_INDEX */ COUNT(*) FROM "
+ + tableName + " WHERE k1 >= " + k1 + " AND k1 <= " + k1)) {
+ assertTrue(rs.next());
+ assertEquals("surviving ROW_TIMESTAMP row " + k1 + " must remain
reachable in its band",
+ 1, rs.getInt(1));
+ }
+ }
+ // Index read resolves a surviving row to its original ROW_TIMESTAMP key.
+ String viaIndex = "SELECT k1 FROM " + tableName + " WHERE val = 'v3'";
+ assertExplainPlan(false,
+ QueryUtil.getExplainPlan(conn.createStatement().executeQuery("EXPLAIN
" + viaIndex)),
+ tableName, indexName);
+ try (ResultSet rs = conn.createStatement().executeQuery(viaIndex)) {
+ assertTrue(rs.next());
+ assertEquals(300L, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ }
+ }
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java
index 4dca35db98..9c70f18d6b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java
@@ -69,6 +69,7 @@ import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableImpl;
@@ -271,15 +272,21 @@ public abstract class BaseIndexIT extends
ParallelStatsDisabledIT {
Iterator<Pair<byte[], List<Cell>>> iterator =
PhoenixRuntime.getUncommittedDataIterator(conn);
if (iterator.hasNext()) {
byte[] tableName = iterator.next().getFirst(); // skip data table
mutations
- PTable table =
conn.unwrap(PhoenixConnection.class).getTable(Bytes.toString(tableName));
- boolean clientSideUpdate =
- (!localIndex || (transactional &&
table.getTransactionProvider().getTransactionProvider()
- .isUnsupported(Feature.MAINTAIN_LOCAL_INDEX_ON_SERVER))) &&
(!mutable || transactional);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(Bytes.toString(tableName));
+ // Immutable indexes are maintained server side unless client-side
maintenance is enabled.
+ boolean serverSideImmutableIndexes =
pconn.getQueryServices().getConfiguration().getBoolean(
+ QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
+ QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED);
+ boolean clientSideUpdate = (!localIndex || (transactional &&
table.getTransactionProvider()
+
.getTransactionProvider().isUnsupported(Feature.MAINTAIN_LOCAL_INDEX_ON_SERVER)))
+ && ((!mutable && !serverSideImmutableIndexes) || transactional);
if (!clientSideUpdate) {
assertTrue(table.getType() == PTableType.TABLE); // should be data
table
}
boolean hasIndexData = iterator.hasNext();
- // global immutable and global transactional tables are processed client
side
+ // global transactional (and immutable when maintained client side)
tables are processed
+ // client side
assertEquals(clientSideUpdate, hasIndexData);
}
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java
index 329cb4df55..afff9ebeeb 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java
@@ -69,6 +69,7 @@ import org.apache.phoenix.parse.TableName;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableImpl;
@@ -300,14 +301,19 @@ public abstract class BaseIndexWithRegionMovesIT extends
ParallelStatsDisabledWi
if (iterator.hasNext()) {
byte[] tableName = iterator.next().getFirst(); // skip data table
mutations
PTable table = PhoenixRuntime.getTable(conn, Bytes.toString(tableName));
- boolean clientSideUpdate =
- (!localIndex || (transactional &&
table.getTransactionProvider().getTransactionProvider()
- .isUnsupported(Feature.MAINTAIN_LOCAL_INDEX_ON_SERVER))) &&
(!mutable || transactional);
+ // Immutable indexes are maintained server side unless client-side
maintenance is enabled.
+ boolean serverSideImmutableIndexes =
conn.unwrap(PhoenixConnection.class).getQueryServices()
+
.getConfiguration().getBoolean(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
+ QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED);
+ boolean clientSideUpdate = (!localIndex || (transactional &&
table.getTransactionProvider()
+
.getTransactionProvider().isUnsupported(Feature.MAINTAIN_LOCAL_INDEX_ON_SERVER)))
+ && ((!mutable && !serverSideImmutableIndexes) || transactional);
if (!clientSideUpdate) {
assertTrue(table.getType() == PTableType.TABLE); // should be data
table
}
boolean hasIndexData = iterator.hasNext();
- // global immutable and global transactional tables are processed client
side
+ // global transactional (and immutable when maintained client side)
tables are processed
+ // client side
assertEquals(clientSideUpdate, hasIndexData);
}
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
index 8d7d4c546a..927612f737 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexCheckerIT.java
@@ -55,10 +55,12 @@ import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.phoenix.end2end.IndexToolIT;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.hbase.index.IndexRegionObserver;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixResultSet;
import org.apache.phoenix.mapreduce.index.IndexTool;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -725,6 +727,7 @@ public class GlobalIndexCheckerIT extends BaseTest {
conn.createStatement().execute("upsert into " + dataTableName + " (id,
val1, val2, val3) "
+ "values ('a', 'ab', 'abcc', null)");
conn.commit();
+ waitForEventualConsistency();
String selectSql = "SELECT * from " + dataTableName + " WHERE val1 =
'ab'";
// Verify that we will read from the index table
assertExplainPlan(conn, selectSql, dataTableName, indexTableName);
@@ -753,6 +756,74 @@ public class GlobalIndexCheckerIT extends BaseTest {
}
}
+ /**
+ * Same partial-upsert scenario as testPartialRowUpdateForImmutable but for
an UNCOVERED global
+ * index. A partial upsert that omits the indexed column (val1) must not
create a spurious
+ * NULL-keyed index entry: for an immutable table val1 keeps its earlier
value, so the index must
+ * still resolve the row under its original key and never under IS NULL.
Verified via the index
+ * path (explain plan asserts the index table is used).
+ */
+ @Test
+ public void testPartialRowUpdateForImmutableUncovered() throws Exception {
+ if (async || encoded) {
+ // No need to run this test more than once
+ return;
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String dataTableName = generateUniqueName();
+ conn.createStatement()
+ .execute("create table " + dataTableName
+ + " (id varchar(10) not null primary key, val1 varchar(10), val2
varchar(10))"
+ + " IMMUTABLE_ROWS=true, IMMUTABLE_STORAGE_SCHEME="
+ + PTableImpl.ImmutableStorageScheme.ONE_CELL_PER_COLUMN);
+ // full upsert that sets the indexed column
+ conn.createStatement()
+ .execute("upsert into " + dataTableName + " (id, val1, val2) values
('a', 'ab', 'abc')");
+ conn.commit();
+ String indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE UNCOVERED INDEX " +
indexTableName + " on "
+ + dataTableName + " (val1)" + this.indexDDLOptions);
+ // PARTIAL upsert that omits the indexed column val1. For immutable rows
val1 stays 'ab'.
+ conn.createStatement()
+ .execute("upsert into " + dataTableName + " (id, val2) values ('a',
'xyz')");
+ conn.commit();
+ waitForEventualConsistency();
+
+ // (a) index path must still resolve the row under its original key 'ab'
+ String selectAb = "SELECT id from " + dataTableName + " WHERE val1 =
'ab'";
+ assertExplainPlan(conn, selectAb, dataTableName, indexTableName);
+ try (ResultSet rs = conn.createStatement().executeQuery(selectAb)) {
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertFalse(rs.next());
+ }
+
+ // (b) count via the index must be exactly 1. The scan path (a)/(c)
self-heals unverified
+ // index rows at read time, but the COUNT aggregate does not: only
server-side maintenance
+ // reads the current row back and rewrites a verified entry for the
retained key, so the
+ // COUNT path agrees only when server-side immutable-index maintenance
is enabled.
+ boolean serverSideImmutableIndexes =
conn.unwrap(PhoenixConnection.class).getQueryServices()
+
.getConfiguration().getBoolean(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
+ QueryServicesOptions.DEFAULT_SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED);
+ if (serverSideImmutableIndexes) {
+ String countAb = "SELECT COUNT(*) from " + dataTableName + " WHERE
val1 = 'ab'";
+ assertExplainPlan(conn, countAb, dataTableName, indexTableName);
+ try (ResultSet rs = conn.createStatement().executeQuery(countAb)) {
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ }
+ }
+
+ // (c) a partial build lacking the read-back would emit a spurious
NULL-keyed index entry
+ // pointing at id='a'; the index path for IS NULL must therefore return
nothing
+ String selectNull = "SELECT id from " + dataTableName + " WHERE val1 IS
NULL";
+ assertExplainPlan(conn, selectNull, dataTableName, indexTableName);
+ try (ResultSet rs = conn.createStatement().executeQuery(selectNull)) {
+ assertFalse(rs.next());
+ }
+ }
+ }
+
@Test
public void testFailPreIndexRowUpdate() throws Exception {
String dataTableName = generateUniqueName();
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
index fe059f8451..e768d096b3 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
@@ -67,6 +67,8 @@ public abstract class BasePhoenixMetricsIT extends BaseTest {
// disable renewing leases as this will force spooling to happen.
props.put(QueryServices.RENEW_LEASE_ENABLED, String.valueOf(false));
props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false));
+ // Keep immutable-index maintenance client-side so mutation metrics
account for index mutations.
+ props.put(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
Boolean.FALSE.toString());
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
// need the non-test driver for some tests that check number of
hconnections, etc.
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
index 4ca910964a..c8424ca00b 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
@@ -164,6 +164,8 @@ public class PhoenixTableLevelMetricsIT extends BaseTest {
conf.set(QueryServices.TABLE_LEVEL_METRICS_ENABLED, String.valueOf(true));
conf.set(QueryServices.METRIC_PUBLISHER_ENABLED, String.valueOf(true));
conf.set(QueryServices.COLLECT_REQUEST_LEVEL_METRICS,
String.valueOf(true));
+ // Keep immutable-index maintenance client-side so index-usage mutation
metrics are recorded.
+ conf.set(QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
String.valueOf(false));
InstanceResolver.clearSingletons();
// Override to get required config for static fields loaded that require
HBase config
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
index ba4c06457b..fd0827a4a2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/PhoenixClientRpcIT.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.ipc.CallRunner;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -58,8 +59,14 @@ public class PhoenixClientRpcIT extends BaseTest {
Collections.singletonMap(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
TestPhoenixIndexRpcSchedulerFactory.class.getName());
NUM_SLAVES_BASE = 2;
+ // Keep immutable-index maintenance client-side: this test verifies that
client-originated
+ // index writes do not use the region-server index RPC queue. Under
server-side maintenance
+ // that invariant no longer holds (the write is routed through the index
handler pool only
+ // when the index region is not colocated with the data region), so pin
the flag off here.
+ Map<String, String> clientProps = Collections.singletonMap(
+ QueryServices.SERVER_SIDE_IMMUTABLE_INDEXES_ENABLED_ATTRIB,
Boolean.FALSE.toString());
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
- ReadOnlyProps.EMPTY_PROPS);
+ new ReadOnlyProps(clientProps.entrySet().iterator()));
}
@AfterClass
