This is an automated email from the ASF dual-hosted git repository.
shahrs87 pushed a commit to branch PHOENIX-6883-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/PHOENIX-6883-feature by this
push:
new 01a28f1f0c PHOENIX-7166 : Enable feature flags introduced by metadata
caching redesign (#1859)
01a28f1f0c is described below
commit 01a28f1f0cccbf791d24dca0d890fdc03dd9d2ea
Author: palash <[email protected]>
AuthorDate: Fri Apr 19 10:20:42 2024 -0700
PHOENIX-7166 : Enable feature flags introduced by metadata caching redesign
(#1859)
---
.../phoenix/compile/CreateIndexCompiler.java | 4 +-
.../org/apache/phoenix/compile/FromCompiler.java | 9 +++-
.../apache/phoenix/compile/StatementContext.java | 2 +-
.../org/apache/phoenix/compile/UpsertCompiler.java | 2 +-
.../org/apache/phoenix/jdbc/PhoenixStatement.java | 22 ++++++--
.../phoenix/query/ConnectionQueryServicesImpl.java | 4 +-
.../apache/phoenix/query/QueryServicesOptions.java | 18 ++++++-
.../org/apache/phoenix/schema/MetaDataClient.java | 20 +++++--
.../java/org/apache/phoenix/schema/TableRef.java | 2 +-
.../phoenix/util/ValidateLastDDLTimestampUtil.java | 14 ++++-
.../apache/phoenix/schema/transform/Transform.java | 6 +++
.../phoenix/cache/ServerMetadataCacheIT.java | 61 ++++++++++++++++++++--
.../phoenix/end2end/AlterTableWithViewsIT.java | 19 +++----
.../apache/phoenix/end2end/AppendOnlySchemaIT.java | 2 +-
.../phoenix/end2end/BackwardCompatibilityIT.java | 1 -
.../apache/phoenix/end2end/BasePermissionsIT.java | 51 +-----------------
.../org/apache/phoenix/end2end/ConnectionIT.java | 3 ++
.../apache/phoenix/end2end/ConnectionUtilIT.java | 3 ++
.../org/apache/phoenix/end2end/CreateTableIT.java | 3 +-
.../phoenix/end2end/LogicalTableNameBaseIT.java | 10 ++--
.../end2end/LogicalTableNameExtendedIT.java | 21 +++++---
.../MigrateSystemTablesToSystemNamespaceIT.java | 10 ++++
.../PartialResultServerConfigurationIT.java | 2 +
.../SystemTablesCreationOnConnectionIT.java | 10 ++++
.../UpdateCacheAcrossDifferentClientsIT.java | 17 +++---
.../java/org/apache/phoenix/end2end/UpgradeIT.java | 4 +-
.../end2end/index/IndexAsyncThresholdIT.java | 4 +-
.../end2end/index/MutableIndexFailureIT.java | 3 ++
.../phoenix/iterate/ScannerLeaseRenewalIT.java | 2 +
.../jdbc/HighAvailabilityTestingUtility.java | 7 +++
.../jdbc/LoggingSingleConnectionLimiterIT.java | 2 +-
.../phoenix/jdbc/ParallelPhoenixConnectionIT.java | 1 +
.../monitoring/PhoenixTableLevelMetricsIT.java | 53 ++++++++++++-------
.../phoenix/query/MaxConcurrentConnectionsIT.java | 8 ++-
.../apache/phoenix/query/MetaDataCachingIT.java | 19 +++++--
.../rpc/UpdateCacheConnectionLevelPropIT.java | 10 ++++
.../java/org/apache/phoenix/rpc/UpdateCacheIT.java | 29 ++++++++--
.../apache/phoenix/compile/WhereOptimizerTest.java | 4 +-
.../jdbc/ParallelPhoenixConnectionFailureTest.java | 7 +--
.../phoenix/query/BaseConnectionlessQueryTest.java | 5 +-
.../java/org/apache/phoenix/query/BaseTest.java | 3 ++
41 files changed, 335 insertions(+), 142 deletions(-)
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java
index 35d193de5f..534ce6ebaf 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java
@@ -224,7 +224,9 @@ public class CreateIndexCompiler {
}
public MutationPlan compile(final CreateIndexStatement create) throws
SQLException {
final PhoenixConnection connection = statement.getConnection();
- final ColumnResolver resolver = FromCompiler.getResolver(create,
connection, create.getUdfParseNodes());
+ final ColumnResolver resolver
+ = FromCompiler.getResolverForCreateIndex(
+ create, connection, create.getUdfParseNodes());
Scan scan = new Scan();
final StatementContext context = new StatementContext(statement,
resolver, scan, new SequenceManager(statement));
verifyIndexWhere(create.getWhere(), context,
create.getTable().getName());
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/FromCompiler.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index 15b23438b7..9d3a6b8952 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -295,9 +295,14 @@ public class FromCompiler {
}
}
- public static ColumnResolver getResolver(SingleTableStatement statement,
PhoenixConnection connection, Map<String, UDFParseNode> udfParseNodes)
+ public static ColumnResolver
getResolverForCreateIndex(SingleTableStatement statement,
+ PhoenixConnection connection, Map<String,
UDFParseNode> udfParseNodes)
throws SQLException {
- SingleTableColumnResolver visitor = new
SingleTableColumnResolver(connection, statement.getTable(), true, 0,
udfParseNodes);
+ // use alwaysHitServer=true to ensure client's cache is up-to-date
even when client is
+ // validating last_ddl_timestamps and UCF = never.
+ SingleTableColumnResolver visitor
+ = new SingleTableColumnResolver(connection,
statement.getTable(), true, 0,
+ udfParseNodes, true, null);
return visitor;
}
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
index a7abbd5dd5..58b768452c 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/StatementContext.java
@@ -285,7 +285,7 @@ public class StatementContext {
}
public long getCurrentTime() throws SQLException {
- long ts = this.getCurrentTable().getCurrentTime();
+ long ts = this.getCurrentTable().getTimeStamp();
// if the table is transactional then it is only resolved once per
query, so we can't use the table timestamp
if (this.getCurrentTable().getTable().getType() != PTableType.SUBQUERY
&& this.getCurrentTable().getTable().getType() !=
PTableType.PROJECTED
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 890b99a90c..c996516628 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -636,7 +636,7 @@ public class UpsertCompiler {
// as max TS, so that the query can safely restarted and still
work of a snapshot
// (so it won't see its own data in case of concurrent splits)
// see PHOENIX-4849
- long serverTime =
selectResolver.getTables().get(0).getCurrentTime();
+ long serverTime =
selectResolver.getTables().get(0).getTimeStamp();
if (serverTime == QueryConstants.UNSET_TIMESTAMP) {
// if this is the first time this table is resolved the
ref's current time might not be defined, yet
// in that case force an RPC to get the server time
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 897de9350a..405fa80cf1 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -358,6 +358,7 @@ public class PhoenixStatement implements
PhoenixMonitoredStatement, SQLCloseable
String tableName = null;
clearResultSet();
PhoenixResultSet rs = null;
+ QueryPlan plan = null;
try {
PhoenixConnection conn = getConnection();
conn.checkOpen();
@@ -367,9 +368,7 @@ public class PhoenixStatement implements
PhoenixMonitoredStatement, SQLCloseable
&& stmt.getOperation() !=
Operation.UPGRADE) {
throw new UpgradeRequiredException();
}
- QueryPlan
- plan =
- stmt.compilePlan(PhoenixStatement.this,
+ plan = stmt.compilePlan(PhoenixStatement.this,
Sequence.ValueOp.VALIDATE_SEQUENCE);
// Send mutations to hbase, so they are
visible to subsequent reads.
// Use original plan for data table so that
data and immutable indexes will be sent
@@ -437,13 +436,26 @@ public class PhoenixStatement implements
PhoenixMonitoredStatement, SQLCloseable
//Force update cache and retry if meta not found
error occurs
catch (MetaDataEntityNotFoundException e) {
if (doRetryOnMetaNotFoundError &&
e.getTableName() != null) {
+ String sName = e.getSchemaName();
+ String tName = e.getTableName();
+ // when the query plan uses the local
index PTable,
+ // the TNFE can still be for the base table
+ if (plan != null && plan.getTableRef() !=
null) {
+ PTable queryPlanTable =
plan.getTableRef().getTable();
+ if (queryPlanTable != null
+ &&
queryPlanTable.getIndexType()
+ == IndexType.LOCAL) {
+ sName =
queryPlanTable.getSchemaName().getString();
+ tName =
queryPlanTable.getTableName().getString();
+ }
+ }
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("Reloading table {} data
from server",
- e.getTableName());
+ tName);
}
if (new MetaDataClient(connection)
.updateCache(connection.getTenantId(),
- e.getSchemaName(),
e.getTableName(), true)
+ sName, tName, true)
.wasUpdated()) {
updateMetrics = false;
//TODO we can log retry count and
error for debugging in LOG table
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 0ff374003a..384cda9325 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -75,6 +75,7 @@ import static
org.apache.phoenix.monitoring.MetricType.NUM_SYSTEM_TABLE_RPC_SUCC
import static
org.apache.phoenix.monitoring.MetricType.TIME_SPENT_IN_SYSTEM_TABLE_RPC_CALLS;
import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY;
import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
+import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED;
import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_ENABLED;
import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE;
import static
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS;
@@ -6328,7 +6329,8 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
public void
invalidateServerMetadataCache(List<InvalidateServerMetadataCacheRequest>
requests)
throws Throwable {
boolean invalidateCacheEnabled =
- config.getBoolean(PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED,
false);
+ config.getBoolean(PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED,
+ DEFAULT_PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED);
if (!invalidateCacheEnabled) {
LOGGER.info("Skip invalidating server metadata cache since conf
property"
+ " phoenix.metadata.invalidate.cache.enabled is set to
false");
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 1f55135c93..5be43c3a91 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -119,6 +119,7 @@ import static
org.apache.phoenix.query.QueryServices.WAL_EDIT_CODEC_ATTRIB;
import java.util.Map.Entry;
import
org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+import org.apache.phoenix.schema.ConnectionProperty;
import org.apache.phoenix.schema.PIndexState;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Coprocessor;
@@ -370,10 +371,23 @@ public class QueryServicesOptions {
//Security defaults
public static final boolean DEFAULT_PHOENIX_ACLS_ENABLED = false;
- //default update cache frequency
- public static final long DEFAULT_UPDATE_CACHE_FREQUENCY = 0;
public static final int DEFAULT_SMALL_SCAN_THRESHOLD = 100;
+
+ /**
+ * Metadata caching configs, see
https://issues.apache.org/jira/browse/PHOENIX-6883.
+ * Disable the boolean flags and set UCF=always to disable the caching
re-design.
+ *
+ * Disable caching re-design if you use Online Data Format Change since
the cutover logic
+ * is currently incompatible and clients may not learn about the physical
table change.
+ * See https://issues.apache.org/jira/browse/PHOENIX-7284.
+ *
+ * Disable caching re-design if your clients will not have ADMIN perms to
call region server
+ * RPC. See https://issues.apache.org/jira/browse/HBASE-28508
+ */
+ public static final long DEFAULT_UPDATE_CACHE_FREQUENCY
+ = (long)
ConnectionProperty.UPDATE_CACHE_FREQUENCY.getValue("ALWAYS");
public static final boolean DEFAULT_LAST_DDL_TIMESTAMP_VALIDATION_ENABLED
= false;
+ public static final boolean
DEFAULT_PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED = false;
public static final String
DEFAULT_UPDATE_CACHE_FREQUENCY_FOR_PENDING_DISABLED_INDEX
=
Long.toString(0L);
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index d4e2fd7552..0e58a3ad07 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -200,6 +200,7 @@ import org.apache.phoenix.schema.stats.GuidePostsInfo;
import org.apache.phoenix.schema.transform.TransformClient;
import org.apache.phoenix.util.ClientUtil;
import org.apache.phoenix.util.TaskMetaDataServiceCallBack;
+import org.apache.phoenix.util.ValidateLastDDLTimestampUtil;
import org.apache.phoenix.util.ViewUtil;
import org.apache.phoenix.util.JacksonUtil;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -771,6 +772,7 @@ public class MetaDataClient {
// What if the table is created with UPDATE_CACHE_FREQUENCY
explicitly set to ALWAYS?
// i.e. explicitly set to 0. We should ideally be checking for
something like
// hasUpdateCacheFrequency().
+
//always fetch an Index in PENDING_DISABLE state to retrieve
server timestamp
//QueryOptimizer needs that to decide whether the index can be used
if (PIndexState.PENDING_DISABLE.equals(table.getIndexState())) {
@@ -1516,14 +1518,17 @@ public class MetaDataClient {
Set<String> acquiredColumnMutexSet =
Sets.newHashSetWithExpectedSize(3);
String physicalSchemaName = null;
String physicalTableName = null;
+ PTable dataTable = null;
try {
- ColumnResolver resolver = FromCompiler.getResolver(statement,
connection, statement.getUdfParseNodes());
+ ColumnResolver resolver
+ = FromCompiler.getResolverForCreateIndex(
+ statement, connection,
statement.getUdfParseNodes());
tableRef = resolver.getTables().get(0);
Date asyncCreatedDate = null;
if (statement.isAsync()) {
- asyncCreatedDate = new Date(tableRef.getTimeStamp());
+ asyncCreatedDate = new Date(tableRef.getCurrentTime());
}
- PTable dataTable = tableRef.getTable();
+ dataTable = tableRef.getTable();
boolean isTenantConnection = connection.getTenantId() != null;
if (isTenantConnection) {
if (dataTable.getType() != PTableType.VIEW) {
@@ -1776,7 +1781,14 @@ public class MetaDataClient {
return buildIndexAtTimeStamp(table, statement.getTable());
}
- return buildIndex(table, tableRef);
+ MutationState state = buildIndex(table, tableRef);
+ // If client is validating LAST_DDL_TIMESTAMPS, parent's
last_ddl_timestamp changed
+ // so remove it from client's cache. It will be refreshed when table
is accessed next time.
+ if
(ValidateLastDDLTimestampUtil.getValidateLastDdlTimestampEnabled(connection)) {
+ connection.removeTable(connection.getTenantId(),
dataTable.getName().getString(),
+ null, dataTable.getTimeStamp());
+ }
+ return state;
}
/**
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/TableRef.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/TableRef.java
index 71a839eb38..64f13acf6b 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -91,7 +91,7 @@ public class TableRef {
this.table = table;
// if UPDATE_CACHE_FREQUENCY is set, always let the server set
timestamps
this.upperBoundTimeStamp = table.getUpdateCacheFrequency()!=0 ?
QueryConstants.UNSET_TIMESTAMP : upperBoundTimeStamp;
- this.currentTime = this.upperBoundTimeStamp;
+ this.currentTime = upperBoundTimeStamp;
this.lowerBoundTimeStamp = lowerBoundTimeStamp;
this.hasDynamicCols = hasDynamicCols;
this.hinted = hinted;
diff --git
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
index bc1ae34f6a..8e87fbcafe 100644
---
a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
+++
b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ValidateLastDDLTimestampUtil.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Admin;
@@ -76,6 +77,17 @@ public class ValidateLastDDLTimestampUtil {
QueryServicesOptions.DEFAULT_LAST_DDL_TIMESTAMP_VALIDATION_ENABLED);
}
+ /**
+ * Get whether last ddl timestamp validation is enabled in the
Configuration
+ * @param config
+ * @return true if it is enabled, false otherwise
+ */
+ public static boolean getValidateLastDdlTimestampEnabled(Configuration
config) {
+ return config.getBoolean(
+ QueryServices.LAST_DDL_TIMESTAMP_VALIDATION_ENABLED,
+
QueryServicesOptions.DEFAULT_LAST_DDL_TIMESTAMP_VALIDATION_ENABLED);
+ }
+
/**
* Verifies that table metadata for given tables is up-to-date in client
cache with server.
* A random live region server is picked for invoking the RPC to validate
LastDDLTimestamp.
@@ -196,7 +208,7 @@ public class ValidateLastDDLTimestampUtil {
byte[] tenantIDBytes = key.getTenantId() == null
? HConstants.EMPTY_BYTE_ARRAY
: key.getTenantId().getBytes();
- byte[] schemaBytes = schemaName == null
+ byte[] schemaBytes = (schemaName == null || schemaName.isEmpty())
? HConstants.EMPTY_BYTE_ARRAY
: key.getSchemaName().getBytes();
builder.setTenantId(ByteStringer.wrap(tenantIDBytes));
diff --git
a/phoenix-core-server/src/main/java/org/apache/phoenix/schema/transform/Transform.java
b/phoenix-core-server/src/main/java/org/apache/phoenix/schema/transform/Transform.java
index 53f95537cf..b216c9d7e3 100644
---
a/phoenix-core-server/src/main/java/org/apache/phoenix/schema/transform/Transform.java
+++
b/phoenix-core-server/src/main/java/org/apache/phoenix/schema/transform/Transform.java
@@ -121,6 +121,12 @@ public class Transform extends TransformClient {
).execute();
}
+ /**
+ * Disable caching re-design if you use Online Data Format Change since
the cutover logic
+ * is currently incompatible and clients may not learn about the physical
table change.
+ * See https://issues.apache.org/jira/browse/PHOENIX-6883 and
+ * https://issues.apache.org/jira/browse/PHOENIX-7284.
+ */
public static void doCutover(PhoenixConnection connection,
SystemTransformRecord systemTransformRecord) throws Exception{
String tenantId = systemTransformRecord.getTenantId();
String schema = systemTransformRecord.getSchemaName();
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/cache/ServerMetadataCacheIT.java
similarity index 97%
rename from
phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheIT.java
rename to
phoenix-core/src/it/java/org/apache/phoenix/cache/ServerMetadataCacheIT.java
index 80e86eb832..2135cc20aa 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/cache/ServerMetadataCacheIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/cache/ServerMetadataCacheIT.java
@@ -851,6 +851,30 @@ public class ServerMetadataCacheIT extends
ParallelStatsDisabledIT {
}
}
+ /**
+ * Test that a client does not see TableNotFoundException when trying to
validate
+ * LAST_DDL_TIMESTAMP for a view and its parent after the table was
altered and removed from
+ * the client's cache.
+ */
+ @Test
+ public void testQueryViewAfterParentRemovedFromCache() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String url = QueryUtil.getConnectionUrl(props, config);
+ ConnectionQueryServices cqs = driver.getConnectionQueryServices(url,
props);
+ String tableName = generateUniqueName();
+ String viewName = generateUniqueName();
+ try (Connection conn = cqs.connect(url, props)) {
+ createTable(conn, tableName);
+ createView(conn, tableName, viewName);
+ query(conn, viewName);
+ // this removes the parent table from the client cache
+ alterTableDropColumn(conn, tableName, "v2");
+ query(conn, viewName);
+ } catch (TableNotFoundException e) {
+ fail("TableNotFoundException should not be encountered by
client.");
+ }
+ }
+
/**
* Test query on index with stale last ddl timestamp.
* Client-1 creates a table and an index on it. Client-2 queries table to
populate its cache.
@@ -1323,6 +1347,30 @@ public class ServerMetadataCacheIT extends
ParallelStatsDisabledIT {
}
}
+ /**
+ * Test that a client can not create an index on a column after another
client dropped the column.
+ */
+ @Test
+ public void testClientCannotCreateIndexOnDroppedColumn() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String url1 = QueryUtil.getConnectionUrl(props, config, "client1");
+ String url2 = QueryUtil.getConnectionUrl(props, config, "client2");
+ String tableName = generateUniqueName();
+ String indexName = generateUniqueName();
+ ConnectionQueryServices spyCqs1 =
Mockito.spy(driver.getConnectionQueryServices(url1, props));
+ ConnectionQueryServices spyCqs2 =
Mockito.spy(driver.getConnectionQueryServices(url2, props));
+
+ try (Connection conn1 = spyCqs1.connect(url1, props);
+ Connection conn2 = spyCqs2.connect(url2, props)) {
+ createTable(conn1, tableName);
+ alterTableDropColumn(conn2, tableName, "v2");
+ createIndex(conn1, tableName, indexName, "v2");
+ fail("Client should not be able to create index on dropped
column.");
+ }
+ catch (ColumnNotFoundException expected) {
+ }
+ }
+
/**
* Test that upserts into a view whose parent was dropped throws a
TableNotFoundException.
*/
@@ -1606,14 +1654,14 @@ public class ServerMetadataCacheIT extends
ParallelStatsDisabledIT {
map = viewPTable2.getAncestorLastDDLTimestampMap();
assertEquals(basePTable2.getLastDDLTimestamp(),
map.get(basePTable2.getKey()));
assertEquals(2, viewPTable2.getIndexes().size());
- for (PTable indexT : viewPTable2.getIndexes()) {
+ for (PTable indexOfView : viewPTable2.getIndexes()) {
// inherited index
- if (indexT.getTableName().getString().equals(index2)) {
- map = indexT.getAncestorLastDDLTimestampMap();
+ if (indexOfView.getTableName().getString().equals(index2)) {
+ map = indexOfView.getAncestorLastDDLTimestampMap();
assertEquals(basePTable2.getLastDDLTimestamp(),
map.get(basePTable2.getKey()));
} else {
// view index
- map = indexT.getAncestorLastDDLTimestampMap();
+ map = indexOfView.getAncestorLastDDLTimestampMap();
assertEquals(basePTable2.getLastDDLTimestamp(),
map.get(basePTable2.getKey()));
assertEquals(viewPTable2.getLastDDLTimestamp(),
map.get(viewPTable2.getKey()));
}
@@ -1724,7 +1772,10 @@ public class ServerMetadataCacheIT extends
ParallelStatsDisabledIT {
// create index, getTable RPCs for base table
createIndex(conn1, tableName, indexName, "v2");
- numTableRPCs += 3; // one rpc each for getting current time,
create index, alter index state after building
+ // getting current time,
+ // create index(compile+execute),
+ // alter index state after building(compile+execute)
+ numTableRPCs += 5;
assertNumGetTableRPC(spyCqs1, tableName, numTableRPCs);
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
index 12106b2753..f02f358cd4 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
@@ -21,6 +21,7 @@ import static
org.apache.phoenix.exception.SQLExceptionCode.CANNOT_MUTATE_TABLE;
import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -1108,7 +1109,8 @@ public class AlterTableWithViewsIT extends
SplitSystemCatalogIT {
String viewIndex2 = generateUniqueName();
String fullNameViewIndex1 =
SchemaUtil.getTableName(viewSchemaName, viewIndex1);
String fullNameViewIndex2 =
SchemaUtil.getTableName(viewSchemaName, viewIndex2);
-
+ List<String> fullViewIndexNames =
Arrays.asList(fullNameViewIndex1, fullNameViewIndex2);
+
conn.setAutoCommit(false);
viewConn.setAutoCommit(false);
String ddlFormat =
@@ -1149,10 +1151,8 @@ public class AlterTableWithViewsIT extends
SplitSystemCatalogIT {
byte[] viewIndexPhysicalTable =
viewIndex.getPhysicalName().getBytes();
assertNotNull("Can't find view index", viewIndex);
assertEquals("Unexpected number of indexes ", 2,
view.getIndexes().size());
- assertEquals("Unexpected index ", fullNameViewIndex1 ,
view.getIndexes().get(0).getName()
- .getString());
- assertEquals("Unexpected index ", fullNameViewIndex2 ,
view.getIndexes().get(1).getName()
- .getString());
+ assertTrue("Expected index not found ",
fullViewIndexNames.contains(view.getIndexes().get(0).getName().getString()));
+ assertTrue("Expected index not found ",
fullViewIndexNames.contains(view.getIndexes().get(1).getName().getString()));
assertEquals("Unexpected salt buckets", view.getBucketNum(),
view.getIndexes().get(0).getBucketNum());
assertEquals("Unexpected salt buckets", view.getBucketNum(),
@@ -1181,14 +1181,15 @@ public class AlterTableWithViewsIT extends
SplitSystemCatalogIT {
}
pconn = viewConn.unwrap(PhoenixConnection.class);
- view = pconn.getTable(new PTableKey(tenantId, viewOfTable ));
+ view = pconn.getTableNoCache(viewOfTable);
+ assertEquals("Unexpected number of indexes ", 1,
view.getIndexes().size());
+ assertEquals("Unexpected index ", fullNameViewIndex2 ,
view.getIndexes().get(0).getName().getString());
+ assertNotEquals("Dropped index should not be in view metadata ",
fullNameViewIndex1 , view.getIndexes().get(0).getName().getString());
try {
- viewIndex = pconn.getTable(new PTableKey(tenantId,
fullNameViewIndex1 ));
+ viewIndex = pconn.getTableNoCache(fullNameViewIndex1);
fail("View index should have been dropped");
} catch (TableNotFoundException e) {
}
- assertEquals("Unexpected number of indexes ", 1,
view.getIndexes().size());
- assertEquals("Unexpected index ", fullNameViewIndex2 ,
view.getIndexes().get(0).getName().getString());
// verify that the physical index view table is *not* dropped
conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(viewIndexPhysicalTable);
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
index 3d25c1a932..dba8c65045 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
@@ -305,7 +305,7 @@ public class AppendOnlySchemaIT extends
ParallelStatsDisabledIT {
"create table IF NOT EXISTS " + tableName + " ( id char(1)
NOT NULL,"
+ " col1 integer NOT NULL,"
+ " CONSTRAINT NAME_PK PRIMARY KEY (id, col1))"
- + " APPEND_ONLY_SCHEMA = true");
+ + " APPEND_ONLY_SCHEMA = true,
UPDATE_CACHE_FREQUENCY=always");
fail("UPDATE_CACHE_FREQUENCY attribute must not be set to
ALWAYS if APPEND_ONLY_SCHEMA is true");
} catch (SQLException e) {
assertEquals(SQLExceptionCode.UPDATE_CACHE_FREQUENCY_INVALID.getErrorCode(),
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
index 9b258cead2..6348497119 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
@@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.phoenix.cache.ServerMetadataCache;
import org.apache.phoenix.coprocessor.SystemCatalogRegionObserver;
import org.apache.phoenix.coprocessor.TaskMetaDataEndpoint;
import
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.MavenCoordinates;
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
index d7aea5a252..397c0b4992 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
@@ -16,7 +16,6 @@
*/
package org.apache.phoenix.end2end;
-import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.phoenix.thirdparty.com.google.common.base.Joiner;
import org.apache.phoenix.thirdparty.com.google.common.base.Throwables;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
@@ -146,7 +145,6 @@ public abstract class BasePermissionsIT extends BaseTest {
private static final int NUM_RECORDS = 5;
boolean isNamespaceMapped;
- boolean isReadAccessEnabledForListDecomRs;
private String schemaName;
private String tableName;
@@ -161,7 +159,6 @@ public abstract class BasePermissionsIT extends BaseTest {
BasePermissionsIT(final boolean isNamespaceMapped) throws Exception {
this.isNamespaceMapped = isNamespaceMapped;
this.tableName = generateUniqueName();
- isReadAccessEnabledForListDecomRs =
isReadAccessEnabledForListDecomRs();
}
static void initCluster(boolean isNamespaceMapped) throws Exception {
@@ -170,7 +167,6 @@ public abstract class BasePermissionsIT extends BaseTest {
static void initCluster(boolean isNamespaceMapped, boolean
useCustomAccessController) throws Exception {
if (null != testUtil) {
- ServerMetadataCacheTestImpl.resetCache();
testUtil.shutdownMiniCluster();
testUtil = null;
}
@@ -182,47 +178,11 @@ public abstract class BasePermissionsIT extends BaseTest {
configureNamespacesOnServer(config, isNamespaceMapped);
configureStatsConfigurations(config);
config.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true);
- setPhoenixRegionServerEndpoint(config);
+
testUtil.startMiniCluster(1);
superUser1 = User.createUserForTesting(config, SUPER_USER, new
String[0]);
superUser2 = User.createUserForTesting(config, "superUser2", new
String[0]);
- /**
- * CQSI initialization needs to make an Admin API call to fetch a list
of live region servers.
- * Permissions were relaxed for that API call in HBASE-28391.
- * Disable metadata caching re-design on server if API call needs
ADMIN access.
- */
- if (!isReadAccessEnabledForListDecomRs()) {
-
config.setLong(QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, 0L);
-
config.setBoolean(QueryServices.LAST_DDL_TIMESTAMP_VALIDATION_ENABLED, false);
-
config.setBoolean(QueryServices.PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED,
false);
- }
- }
-
- // See https://issues.apache.org/jira/browse/HBASE-28391
- private static boolean isReadAccessEnabledForListDecomRs() {
- // true for 2.4.18+, 2.5.8+
- String hbaseVersion = VersionInfo.getVersion();
- String[] versionArr = hbaseVersion.split("\\.");
- int majorVersion = Integer.parseInt(versionArr[0]);
- int minorVersion = Integer.parseInt(versionArr[1]);
- int patchVersion = Integer.parseInt(versionArr[2].split("-hadoop")[0]);
- if (majorVersion > 2) {
- return true;
- }
- if (majorVersion < 2) {
- return false;
- }
- if (minorVersion > 5) {
- return true;
- }
- if (minorVersion < 4) {
- return false;
- }
- if (minorVersion == 4) {
- return patchVersion >= 18;
- }
- return patchVersion >= 8;
}
@Before
@@ -380,15 +340,6 @@ public abstract class BasePermissionsIT extends BaseTest {
props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
}
props.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
Boolean.toString(isNamespaceMapped));
- /**
- * CQSI initialization needs to make an Admin API call to fetch a list
of live region servers.
- * Permissions were relaxed for that API call in HBASE-28391.
- * Disable metadata caching re-design on client side if API call needs
ADMIN access.
- */
- if (!isReadAccessEnabledForListDecomRs) {
- props.put(QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB,
Long.toString(0L));
- props.put(QueryServices.LAST_DDL_TIMESTAMP_VALIDATION_ENABLED,
Boolean.toString(false));
- }
return props;
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionIT.java
index 243b3c730a..0eba0b8f51 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionIT.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assume.assumeTrue;
import java.sql.Connection;
+import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.query.ConfigurationFactory;
import org.apache.phoenix.util.InstanceResolver;
@@ -58,6 +60,7 @@ public class ConnectionIT {
conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase-test");
hbaseTestUtil.startMiniCluster();
Class.forName(PhoenixDriver.class.getName());
+ DriverManager.registerDriver(new PhoenixTestDriver());
InstanceResolver.clearSingletons();
// Make sure the ConnectionInfo doesn't try to pull a default
Configuration
InstanceResolver.getSingleton(ConfigurationFactory.class, new
ConfigurationFactory() {
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionUtilIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionUtilIT.java
index 1937d052d6..d8454a8ff5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionUtilIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ConnectionUtilIT.java
@@ -22,6 +22,7 @@ import static
org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster;
import static org.junit.Assert.assertEquals;
import java.sql.Connection;
+import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
@@ -30,6 +31,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -49,6 +51,7 @@ public class ConnectionUtilIT {
conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase-test");
hbaseTestUtil.startMiniCluster();
Class.forName(PhoenixDriver.class.getName());
+ DriverManager.registerDriver(new PhoenixTestDriver());
}
@Test
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index d7473e5baf..ceaa1402bb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -906,7 +906,8 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
String createTableString = "CREATE TABLE " + tableName + " (k VARCHAR
PRIMARY KEY, "
+ "v1 VARCHAR, v2 VARCHAR)";
- verifyUCFValueInSysCat(tableName, createTableString, props, 0L);
+ verifyUCFValueInSysCat(tableName, createTableString, props,
+ QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY);
}
@Test
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameBaseIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameBaseIT.java
index 7039a85873..bae0589b68 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameBaseIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameBaseIT.java
@@ -31,6 +31,7 @@ import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -530,13 +531,16 @@ public abstract class LogicalTableNameBaseIT extends
BaseTest {
}
public static void renameAndDropPhysicalTable(Connection conn, String
tenantId, String schema, String tableName, String physicalName, boolean
isNamespaceEnabled) throws Exception {
+ // if client is validating last_ddl_timestamp, this change in physical
table name should be visible to the client
+ // UPDATE LAST_DDL_TIMESTAMP of the table and clear the server
metadata cache on region servers
+ long lastDDLTimestamp = EnvironmentEdgeManager.currentTimeMillis();
String
changeName = String.format(
- "UPSERT INTO SYSTEM.CATALOG (TENANT_ID, TABLE_SCHEM,
TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, PHYSICAL_TABLE_NAME) VALUES (%s, %s,
'%s', NULL, NULL, '%s')",
- tenantId, schema==null ? null : ("'" + schema + "'"),
tableName, physicalName);
+ "UPSERT INTO SYSTEM.CATALOG (TENANT_ID, TABLE_SCHEM,
TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, PHYSICAL_TABLE_NAME,
LAST_DDL_TIMESTAMP) VALUES (%s, %s, '%s', NULL, NULL, '%s', %d)",
+ tenantId, schema==null ? null : ("'" + schema + "'"),
tableName, physicalName, lastDDLTimestamp);
conn.createStatement().execute(changeName);
conn.commit();
-
+ ServerMetadataCacheTestImpl.resetCache();
String fullTableName = SchemaUtil.getTableName(schema, tableName);
if (isNamespaceEnabled && !(Strings.isNullOrEmpty(schema) ||
NULL_STRING.equals(schema))) {
fullTableName = schema + NAMESPACE_SEPARATOR + tableName;
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java
index 496d2bd262..b99f66d1ca 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LogicalTableNameExtendedIT.java
@@ -209,21 +209,26 @@ public class LogicalTableNameExtendedIT extends
LogicalTableNameBaseIT {
createIndexOnTable(conn, tableName, indexName2);
populateTable(conn, tableName, 1, 2);
- // Test hint
String tableSelect = "SELECT V1,V2,V3 FROM " + tableName;
ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " +
tableSelect);
- assertEquals(true,
QueryUtil.getExplainPlan(rs).contains(indexName));
+ String plan = QueryUtil.getExplainPlan(rs);
+ // plan should use one of the indexes
+ assertEquals(true, plan.contains(indexName) ||
plan.contains(indexName2));
+ // Test hint for the other index
+ String hintedIndex =
QueryUtil.getExplainPlan(rs).contains(indexName)
+ ? indexName2
+ : indexName;
try (Admin admin =
conn.unwrap(PhoenixConnection.class).getQueryServices()
.getAdmin()) {
- String snapshotName = new
StringBuilder(indexName2).append("-Snapshot").toString();
- admin.snapshot(snapshotName, TableName.valueOf(indexName2));
- String newName = "NEW_" + indexName2;
+ String snapshotName = new
StringBuilder(hintedIndex).append("-Snapshot").toString();
+ admin.snapshot(snapshotName, TableName.valueOf(hintedIndex));
+ String newName = "NEW_" + hintedIndex;
admin.cloneSnapshot(snapshotName, TableName.valueOf(newName));
- renameAndDropPhysicalTable(conn, "NULL", null, indexName2,
newName, true);
+ renameAndDropPhysicalTable(conn, "NULL", null, hintedIndex,
newName, true);
}
- String indexSelect = "SELECT /*+ INDEX(" + tableName + " " +
indexName2 + ")*/ V1,V2,V3 FROM " + tableName;
+ String indexSelect = "SELECT /*+ INDEX(" + tableName + " " +
hintedIndex + ")*/ V1,V2,V3 FROM " + tableName;
rs = conn.createStatement().executeQuery("EXPLAIN " + indexSelect);
- assertEquals(true,
QueryUtil.getExplainPlan(rs).contains(indexName2));
+ assertEquals(true,
QueryUtil.getExplainPlan(rs).contains(hintedIndex));
rs = conn.createStatement().executeQuery(indexSelect);
assertEquals(true, rs.next());
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index 29fc6c526a..918f7a1928 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -16,6 +16,7 @@
*/
package org.apache.phoenix.end2end;
+import static
org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -41,10 +42,12 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.phoenix.coprocessor.PhoenixRegionServerEndpoint;
import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
import org.apache.phoenix.exception.UpgradeInProgressException;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.ConnectionQueryServicesImpl;
@@ -54,6 +57,7 @@ import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.After;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -85,9 +89,15 @@ public class MigrateSystemTablesToSystemNamespaceIT extends
BaseTest {
final UserGroupInformation user4 =
UserGroupInformation.createUserForTesting("user4", new String[0]);
+ @BeforeClass
+ public static synchronized void registerTestDriver() throws SQLException {
+ DriverManager.registerDriver(new PhoenixTestDriver());
+ }
+
public final void doSetup(boolean systemMappingEnabled) throws Exception {
testUtil = new HBaseTestingUtility();
Configuration conf = testUtil.getConfiguration();
+ conf.set(REGIONSERVER_COPROCESSOR_CONF_KEY,
PhoenixRegionServerEndpointTestImpl.class.getName());
enableNamespacesOnServer(conf, systemMappingEnabled);
configureRandomHMasterPort(conf);
testUtil.startMiniCluster(1);
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PartialResultServerConfigurationIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PartialResultServerConfigurationIT.java
index b636f56fa3..2cc4629cd7 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PartialResultServerConfigurationIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PartialResultServerConfigurationIT.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.query.ConnectionQueryServices.Feature;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -77,6 +78,7 @@ public class PartialResultServerConfigurationIT {
url = PhoenixRuntime.JDBC_PROTOCOL +
PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum;
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
+ DriverManager.registerDriver(new PhoenixTestDriver());
}
@AfterClass
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
index 8680325ed9..1ff648b805 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablesCreationOnConnectionIT.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.end2end;
+import static
org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES;
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_MUTEX_HBASE_TABLE_NAME;
@@ -51,6 +52,8 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.ipc.ServerRpcController;
import org.apache.hadoop.hbase.ipc.controller.ServerToServerRpcController;
+import org.apache.phoenix.cache.ServerMetadataCacheImpl;
+import org.apache.phoenix.coprocessor.PhoenixRegionServerEndpoint;
import org.apache.phoenix.coprocessorclient.MetaDataProtocol;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.UpgradeRequiredException;
@@ -69,6 +72,7 @@ import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.UpgradeUtil;
import org.junit.After;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -174,6 +178,10 @@ public class SystemTablesCreationOnConnectionIT {
}
}
+ @BeforeClass
+ public static synchronized void registerTestDriver() throws SQLException {
+ DriverManager.registerDriver(new PhoenixTestDriver());
+ }
@Before
public void resetVariables() {
setOldTimestampToInduceUpgrade = false;
@@ -192,6 +200,7 @@ public class SystemTablesCreationOnConnectionIT {
refCountLeaked =
BaseTest.isAnyStoreRefCountLeaked(testUtil.getAdmin());
}
testUtil.shutdownMiniCluster();
+ ServerMetadataCacheTestImpl.resetCache();
testUtil = null;
assertFalse("refCount leaked", refCountLeaked);
}
@@ -661,6 +670,7 @@ public class SystemTablesCreationOnConnectionIT {
conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED,
isNamespaceMappingEnabled);
// Avoid multiple clusters trying to bind to the master's info port
(16010)
conf.setInt(HConstants.MASTER_INFO_PORT, -1);
+ conf.set(REGIONSERVER_COPROCESSOR_CONF_KEY,
PhoenixRegionServerEndpointTestImpl.class.getName());
testUtil.startMiniCluster(1);
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpdateCacheAcrossDifferentClientsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpdateCacheAcrossDifferentClientsIT.java
index 23606d8684..76f5a701eb 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpdateCacheAcrossDifferentClientsIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpdateCacheAcrossDifferentClientsIT.java
@@ -38,6 +38,7 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
@@ -104,17 +105,19 @@ public class UpdateCacheAcrossDifferentClientsIT extends
BaseTest {
} catch (TableNotFoundException e) {
//Expected
}
- rs = conn2.createStatement().executeQuery("select * from
"+tableName);
try {
+ rs = conn2.createStatement().executeQuery("select * from
"+tableName);
rs.next();
fail("Should throw
org.apache.hadoop.hbase.TableNotFoundException since the latest metadata " +
"wasn't fetched");
- } catch (PhoenixIOException ex) {
- boolean foundHBaseTableNotFound = false;
- for(Throwable throwable : Throwables.getCausalChain(ex)) {
-
if(org.apache.hadoop.hbase.TableNotFoundException.class.equals(throwable.getClass()))
{
- foundHBaseTableNotFound = true;
- break;
+ } catch (SQLException ex) {
+ boolean foundHBaseTableNotFound = (ex instanceof
TableNotFoundException);
+ if (!foundHBaseTableNotFound) {
+ for(Throwable throwable : Throwables.getCausalChain(ex)) {
+
if(org.apache.hadoop.hbase.TableNotFoundException.class.equals(throwable.getClass()))
{
+ foundHBaseTableNotFound = true;
+ break;
+ }
}
}
assertTrue("Should throw
org.apache.hadoop.hbase.TableNotFoundException since the latest" +
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
index c59726adfb..2c5478fa7f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
@@ -68,7 +68,6 @@ import
org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
import
org.apache.phoenix.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
@@ -731,13 +730,14 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
}
private void nullDDLTimestamps(Connection conn) throws SQLException {
+ //ignore system tables since that can interfere with other tests.
String pkCols = TENANT_ID + ", " + TABLE_SCHEM +
", " + TABLE_NAME + ", " + COLUMN_NAME + ", " + COLUMN_FAMILY;
String upsertSql =
"UPSERT INTO " + SYSTEM_CATALOG_NAME + " (" + pkCols + ", " +
LAST_DDL_TIMESTAMP + ")" + " " +
"SELECT " + pkCols + ", NULL FROM " + SYSTEM_CATALOG_NAME + "
" +
- "WHERE " + TABLE_TYPE + " IS NOT NULL";
+ "WHERE " + TABLE_TYPE + " " + " != '" +
PTableType.SYSTEM.getSerializedValue() + "'";
conn.createStatement().execute(upsertSql);
conn.commit();
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java
index aaf58d2e65..4732ebd711 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexAsyncThresholdIT.java
@@ -162,9 +162,7 @@ public class IndexAsyncThresholdIT extends BaseTest {
exception = (SQLException) e;
}
connection.commit();
- PTableKey key = new PTableKey(null, this.tableName);
- PMetaData metaCache =
connection.unwrap(PhoenixConnection.class).getMetaDataCache();
- List<PTable> indexes =
metaCache.getTableRef(key).getTable().getIndexes();
+ List<PTable> indexes =
connection.unwrap(PhoenixConnection.class).getTable(this.tableName).getIndexes();
if (!overThreshold) {
if (this.mode == Mode.ASYNC) {
assertEquals(PIndexState.BUILDING,
indexes.get(0).getIndexState());
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 79a82094d8..9b7b277779 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.end2end.index;
+import static
org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -54,6 +55,7 @@ import org.apache.phoenix.compile.ExplainPlanAttributes;
import org.apache.phoenix.coprocessor.MetaDataRegionObserver;
import
org.apache.phoenix.coprocessor.MetaDataRegionObserver.BuildIndexScheduleTask;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
+import org.apache.phoenix.end2end.PhoenixRegionServerEndpointTestImpl;
import org.apache.phoenix.execute.CommitException;
import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
@@ -176,6 +178,7 @@ public class MutableIndexFailureIT extends BaseTest {
* because we want to control it's execution ourselves
*/
serverProps.put(QueryServices.INDEX_REBUILD_TASK_INITIAL_DELAY,
Long.toString(Long.MAX_VALUE));
+ serverProps.put(REGIONSERVER_COPROCESSOR_CONF_KEY,
PhoenixRegionServerEndpointTestImpl.class.getName());
return serverProps;
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java
index 94700e04cf..42876eae60 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/iterate/ScannerLeaseRenewalIT.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.query.ConnectionQueryServices.Feature;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
@@ -87,6 +88,7 @@ public class ScannerLeaseRenewalIT {
// use round robin iterator
driverProps.put(FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.toString(false));
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
+ DriverManager.registerDriver(new PhoenixTestDriver());
try (PhoenixConnection phxConn = DriverManager.getConnection(url,
driverProps).unwrap(PhoenixConnection.class)) {
// run test methods only if we are at the hbase version that
supports lease renewal.
Assume.assumeTrue(phxConn.getQueryServices().supportsFeature(Feature.RENEW_LEASE));
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
index bc67e4272a..526b83e292 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/HighAvailabilityTestingUtility.java
@@ -18,7 +18,9 @@
package org.apache.phoenix.jdbc;
import org.apache.hadoop.hbase.StartMiniClusterOption;
+import org.apache.phoenix.end2end.PhoenixRegionServerEndpointTestImpl;
import org.apache.phoenix.end2end.ServerMetadataCacheTestImpl;
+import org.apache.phoenix.query.QueryServicesOptions;
import
org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.phoenix.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.commons.lang3.RandomUtils;
@@ -53,6 +55,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.hadoop.hbase.HConstants.*;
+import static
org.apache.hadoop.hbase.coprocessor.CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY;
import static org.apache.hadoop.hbase.ipc.RpcClient.*;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.test.GenericTestUtils.waitFor;
@@ -536,6 +539,10 @@ public class HighAvailabilityTestingUtility {
// Hadoop cluster settings to avoid failing tests
conf.setInt(DFS_REPLICATION_KEY, 1); // we only need one replica
for testing
+
+ // Phoenix Region Server Endpoint needed for metadata caching
+ conf.set(REGIONSERVER_COPROCESSOR_CONF_KEY,
+ PhoenixRegionServerEndpointTestImpl.class.getName());
}
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingSingleConnectionLimiterIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingSingleConnectionLimiterIT.java
index fb0ad2871d..f5b1330def 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingSingleConnectionLimiterIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/LoggingSingleConnectionLimiterIT.java
@@ -88,7 +88,7 @@ public class LoggingSingleConnectionLimiterIT extends
LoggingConnectionLimiterIT
String zkQuorum = "localhost:" +
hBaseTestingUtility.getZkCluster().getClientPort();
url = PhoenixRuntime.JDBC_PROTOCOL +
PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum;
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
-
+ DriverManager.registerDriver(new PhoenixTestDriver());
String profileName = "setup";
final String urlWithPrinc = url +
PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + profileName
+ PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
index 7ecacd2261..5772da8f9e 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionIT.java
@@ -97,6 +97,7 @@ public class ParallelPhoenixConnectionIT {
public static void setUpBeforeClass() throws Exception {
CLUSTERS.start();
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
+ DriverManager.registerDriver(new PhoenixTestDriver());
GLOBAL_PROPERTIES.setProperty(AUTO_COMMIT_ATTRIB, "true");
GLOBAL_PROPERTIES.setProperty(QueryServices.COLLECT_REQUEST_LEVEL_METRICS,
String.valueOf(true));
GLOBAL_PROPERTIES.setProperty(QueryServices.LOG_LEVEL,
LogLevel.DEBUG.name()); //Need logging for query metrics
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
index 20c06c3611..4c5d495f5a 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/PhoenixTableLevelMetricsIT.java
@@ -38,6 +38,7 @@ import org.apache.phoenix.query.ConfigurationFactory;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.query.QueryServicesTestImpl;
import org.apache.phoenix.util.EnvironmentEdge;
import org.apache.phoenix.util.EnvironmentEdgeManager;
@@ -46,6 +47,7 @@ import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.DelayedOrFailingRegionServer;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.ValidateLastDDLTimestampUtil;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -54,6 +56,8 @@ import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import javax.annotation.concurrent.GuardedBy;
+
import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
import static
org.apache.phoenix.exception.SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY;
import static
org.apache.phoenix.exception.SQLExceptionCode.GET_TABLE_REGIONS_FAIL;
@@ -133,6 +137,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -153,6 +158,7 @@ public class PhoenixTableLevelMetricsIT extends BaseTest {
@BeforeClass public static void doSetup() throws Exception {
final Configuration conf = HBaseConfiguration.create();
+ setUpConfigForMiniCluster(conf);
conf.set(QueryServices.TABLE_LEVEL_METRICS_ENABLED,
String.valueOf(true));
conf.set(QueryServices.METRIC_PUBLISHER_ENABLED, String.valueOf(true));
conf.set(QueryServices.COLLECT_REQUEST_LEVEL_METRICS,
String.valueOf(true));
@@ -171,7 +177,7 @@ public class PhoenixTableLevelMetricsIT extends BaseTest {
return copy;
}
});
- hbaseTestUtil = new HBaseTestingUtility();
+ hbaseTestUtil = new HBaseTestingUtility(conf);
hbaseTestUtil.startMiniCluster(1, 1, null, null,
DelayedOrFailingRegionServer.class);
// establish url and quorum. Need to use PhoenixDriver and not
PhoenixTestDriver
String zkQuorum = "localhost:" +
hbaseTestUtil.getZkCluster().getClientPort();
@@ -1276,14 +1282,21 @@ public class PhoenixTableLevelMetricsIT extends
BaseTest {
assertTrue(metricExists);
metricExists = false;
//assert BaseTable is not being queried
- for (PhoenixTableMetric metric :
getPhoenixTableClientMetrics().get(dataTable)) {
- if (metric.getMetricType().equals(SELECT_SQL_COUNTER)) {
- metricExists = true;
- assertMetricValue(metric, SELECT_SQL_COUNTER, 0,
CompareOp.EQ);
- break;
+ //if client is validating last_ddl_timestamps with ucf=never,
+ //there will be no metrics for base table (like getTable RPC
times/counts).
+ if (ValidateLastDDLTimestampUtil
+
.getValidateLastDdlTimestampEnabled(conn.unwrap(PhoenixConnection.class))) {
+
assertFalse(getPhoenixTableClientMetrics().containsKey(dataTable));
+ } else {
+ for (PhoenixTableMetric metric :
getPhoenixTableClientMetrics().get(dataTable)) {
+ if (metric.getMetricType().equals(SELECT_SQL_COUNTER))
{
+ metricExists = true;
+ assertMetricValue(metric, SELECT_SQL_COUNTER, 0,
CompareOp.EQ);
+ break;
+ }
}
+ assertTrue(metricExists);
}
- assertTrue(metricExists);
}
}
}
@@ -1586,11 +1599,16 @@ public class PhoenixTableLevelMetricsIT extends
BaseTest {
* Custom driver to return a custom QueryServices object
*/
public static class PhoenixMetricsTestingDriver extends PhoenixTestDriver {
- private ConnectionQueryServices cqs;
+ @GuardedBy("this")
+ private final Map<ConnectionInfo, ConnectionQueryServices>
+ connectionQueryServicesMap = new HashMap<>();
+
+ private final QueryServices qsti;
private ReadOnlyProps overrideProps;
public PhoenixMetricsTestingDriver(ReadOnlyProps props) {
overrideProps = props;
+ qsti = new QueryServicesTestImpl(getDefaultProps(), overrideProps);
}
@Override public boolean acceptsURL(String url) {
@@ -1598,17 +1616,16 @@ public class PhoenixTableLevelMetricsIT extends
BaseTest {
}
@Override public synchronized ConnectionQueryServices
getConnectionQueryServices(String url,
- Properties info) throws SQLException {
- if (cqs == null) {
- QueryServicesTestImpl qsti =
- new QueryServicesTestImpl(getDefaultProps(),
overrideProps);
- cqs =
- new PhoenixMetricsTestingQueryServices(
- qsti,
- ConnectionInfo.create(url, qsti.getProps(),
info), info);
- cqs.init(url, info);
+
Properties info) throws SQLException {
+ ConnectionInfo connInfo = ConnectionInfo.create(url, null, info);
+ ConnectionQueryServices connectionQueryServices =
connectionQueryServicesMap.get(connInfo);
+ if (connectionQueryServices != null) {
+ return connectionQueryServices;
}
- return cqs;
+ connectionQueryServices = new
PhoenixMetricsTestingQueryServices(qsti, connInfo, info);
+ connectionQueryServices.init(url, info);
+ connectionQueryServicesMap.put(connInfo, connectionQueryServices);
+ return connectionQueryServices;
}
}
}
\ No newline at end of file
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java
index 92bf8aefaa..dacd07ee19 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/query/MaxConcurrentConnectionsIT.java
@@ -18,11 +18,14 @@
package org.apache.phoenix.query;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.end2end.ServerMetadataCacheTestImpl;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
import org.apache.phoenix.util.DelayedRegionServer;
import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.AfterClass;
@@ -57,7 +60,9 @@ public class MaxConcurrentConnectionsIT extends BaseTest {
@BeforeClass
public static void setUp() throws Exception {
- hbaseTestUtil = new HBaseTestingUtility();
+ final Configuration conf = HBaseConfiguration.create();
+ setUpConfigForMiniCluster(conf);
+ hbaseTestUtil = new HBaseTestingUtility(conf);
hbaseTestUtil.startMiniCluster(1,1,null,null,DelayedRegionServer.class);
// establish url and quorum. Need to use PhoenixDriver and not
PhoenixTestDriver
@@ -65,6 +70,7 @@ public class MaxConcurrentConnectionsIT extends BaseTest {
url = PhoenixRuntime.JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR +
zkQuorum +
JDBC_PROTOCOL_SEPARATOR + "uniqueConn=A";
DriverManager.registerDriver(PhoenixDriver.INSTANCE);
+ DriverManager.registerDriver(new PhoenixTestDriver());
}
private String getUniqueUrl() {
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java
index 390df78b0f..06cc176fa0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java
@@ -25,6 +25,7 @@ import org.apache.phoenix.schema.*;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.RunUntilFailure;
+import org.apache.phoenix.util.ValidateLastDDLTimestampUtil;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -51,6 +52,9 @@ public class MetaDataCachingIT extends BaseTest {
private static final Logger LOGGER =
LoggerFactory.getLogger(MetaDataCachingIT.class);
private final Random RAND = new Random(11);
+ private boolean isLastDDLTimestampValidationEnabled
+ =
ValidateLastDDLTimestampUtil.getValidateLastDdlTimestampEnabled(config);
+
@BeforeClass
public static synchronized void doSetup() throws Exception {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
@@ -143,7 +147,7 @@ public class MetaDataCachingIT extends BaseTest {
@Test
public void testGlobalClientCacheMetrics() throws Exception {
int numThreads = 5;
- int numTables = 1;
+ int numTables = 1; // test with only 1 table because we pick tables
randomly in the workload
int numMaxDML = 2;
GlobalClientMetrics.GLOBAL_CLIENT_METADATA_CACHE_MISS_COUNTER.getMetric().reset();
@@ -152,12 +156,19 @@ public class MetaDataCachingIT extends BaseTest {
simulateWorkload("testGlobalClientCacheMetrics", numTables,
numThreads, numMaxDML);
// only 1 miss when the table is created
+ int numExpectedMisses = 1;
+ if (isLastDDLTimestampValidationEnabled) {
+ // if we are validating last_ddl_timestamps,
+ // region server will see 2 more misses when trying to update its
cache
+ numExpectedMisses += 2;
+ }
+
assertEquals("Incorrect number of client metadata cache misses",
- 1,
GlobalClientMetrics.GLOBAL_CLIENT_METADATA_CACHE_MISS_COUNTER.getMetric().getValue());
+ numExpectedMisses,
GlobalClientMetrics.GLOBAL_CLIENT_METADATA_CACHE_MISS_COUNTER.getMetric().getValue());
// (2 hits per upsert + 1 hit per select) per thread
- assertEquals("Incorrect number of client metadata cache hits",
- 3*numMaxDML*numThreads,
GlobalClientMetrics.GLOBAL_CLIENT_METADATA_CACHE_HIT_COUNTER.getMetric().getValue());
+ assertTrue("number of total metadata cache hits (server+client) should
be more than or equal to client cache hits",
+ 3*numMaxDML*numThreads <=
GlobalClientMetrics.GLOBAL_CLIENT_METADATA_CACHE_HIT_COUNTER.getMetric().getValue());
}
/*
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheConnectionLevelPropIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheConnectionLevelPropIT.java
index b1f063f57b..59459e81fe 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheConnectionLevelPropIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheConnectionLevelPropIT.java
@@ -22,11 +22,13 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.util.SchemaUtil;
import static org.apache.phoenix.util.TestUtil.DEFAULT_SCHEMA_NAME;
+import org.apache.phoenix.util.ValidateLastDDLTimestampUtil;
import org.junit.AfterClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -61,6 +63,8 @@ public class UpdateCacheConnectionLevelPropIT extends
ParallelStatsDisabledIT {
private static Connection conn1;
private static Connection conn2;
private static ConnectionQueryServices spyForConn2;
+ private boolean isLastDDLTimestampValidationEnabled
+ =
ValidateLastDDLTimestampUtil.getValidateLastDdlTimestampEnabled(config);
@AfterClass
public static synchronized void freeResources() {
@@ -135,6 +139,12 @@ public class UpdateCacheConnectionLevelPropIT extends
ParallelStatsDisabledIT {
// both connection and table level properties are not set
int numExecutions = 2;
int numExpectedGetTableCalls = 4; // 2 for SELECTs, and 2 for UPSERTs
+
+ // there will be no getTable calls if we are validating
last_ddl_timestamps
+ // and schema has not changed.
+ if (isLastDDLTimestampValidationEnabled) {
+ numExpectedGetTableCalls = 0;
+ }
setUpTableAndConnections(fullTableName, null, null);
verifyExpectedGetTableCalls(fullTableName, numExecutions,
numExpectedGetTableCalls);
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index d26536fca0..02e90c124d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -48,6 +48,7 @@ import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.MetaDataClient;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.types.PVarchar;
@@ -56,6 +57,7 @@ import org.apache.phoenix.transaction.TransactionFactory;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
+import org.apache.phoenix.util.ValidateLastDDLTimestampUtil;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
@@ -69,6 +71,8 @@ import org.slf4j.LoggerFactory;
@Category(ParallelStatsDisabledTest.class)
public class UpdateCacheIT extends ParallelStatsDisabledIT {
+ private boolean isLastDDLTimestampValidationEnabled
+ =
ValidateLastDDLTimestampUtil.getValidateLastDdlTimestampEnabled(config);
private static final Logger LOGGER =
LoggerFactory.getLogger(UpdateCacheIT.class);
@@ -87,7 +91,13 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
String fullTableName = INDEX_DATA_SCHEMA +
QueryConstants.NAME_SEPARATOR + tableName;
Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute("create table " + fullTableName +
TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true,TRANSACTION_PROVIDER='" +
provider + "'");
- helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
+ int[] expectedRPCs = new int[] {1, 3};
+ if (isLastDDLTimestampValidationEnabled) {
+ // when validating last_ddl_timestamps, no getTable RPCs will
be performed
+ // since schema has not changed.
+ expectedRPCs = new int[] {0, 0};
+ }
+ helpTestUpdateCache(fullTableName, expectedRPCs, false);
}
}
@@ -97,7 +107,13 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
String fullTableName = INDEX_DATA_SCHEMA +
QueryConstants.NAME_SEPARATOR + tableName;
Connection conn = DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute("create table " + fullTableName +
TestUtil.TEST_TABLE_SCHEMA);
- helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
+ int[] expectedRPCs = new int[] {1, 3};
+ if (isLastDDLTimestampValidationEnabled) {
+ // when validating last_ddl_timestamps, no getTable RPCs will be
performed
+ // since schema has not changed.
+ expectedRPCs = new int[] {0, 0};
+ }
+ helpTestUpdateCache(fullTableName, expectedRPCs, false);
}
@Test
@@ -217,7 +233,14 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT
{
}
// The indexes should have got the UPDATE_CACHE_FREQUENCY value of
their base table
- helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
+ int numRPCUpsert = 0;
+ int numRPCSelect = 0;
+ if (isLastDDLTimestampValidationEnabled) {
+ // we created indexes on the table which will bump the
last_ddl_timestamp of the table
+ // hence we will do 1 getTable RPC for the upsert
+ numRPCUpsert = 1;
+ }
+ helpTestUpdateCache(fullTableName, new int[] {numRPCUpsert,
numRPCSelect}, false);
helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR
+ localIndex,
new int[] {0}, true);
helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR
+ globalIndex,
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
index d2b8004df7..d80134ef63 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereOptimizerTest.java
@@ -3544,8 +3544,8 @@ public class WhereOptimizerTest extends
BaseConnectionlessQueryTest {
PTable viewIndexPTable =
tenantConn.unwrap(PhoenixConnection.class).getTable(globalViewIndexName);
// PK of view index [_INDEX_ID, tenant_id, KV, PK]
byte[] startRow =
ByteUtil.concat(PLong.INSTANCE.toBytes(viewIndexPTable.getViewIndexId()),
- PChar.INSTANCE.toBytes(tenantId),
- PChar.INSTANCE.toBytes("KV"));
+
PChar.INSTANCE.toBytes(tenantId),
+
PChar.INSTANCE.toBytes("KV"));
assertArrayEquals(startRow, scan.getStartRow());
}
}
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFailureTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFailureTest.java
index c64252d530..51d89728bf 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFailureTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/ParallelPhoenixConnectionFailureTest.java
@@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.Test;
import org.mockito.Mockito;
@@ -51,9 +52,9 @@ public class ParallelPhoenixConnectionFailureTest extends
BaseTest {
@Test
public void testExecuteQueryChainFailure() throws SQLException {
HBaseTestingUtility hbaseTestingUtility = new HBaseTestingUtility();
-
- PhoenixConnection conn1 = (PhoenixConnection)
DriverManager.getConnection(url);
- PhoenixConnection conn2 = (PhoenixConnection)
DriverManager.getConnection(url);
+ Properties props = new Properties();
+ PhoenixConnection conn1 = (PhoenixConnection)
DriverManager.getConnection(url, props);
+ PhoenixConnection conn2 = (PhoenixConnection)
DriverManager.getConnection(url, props);
PhoenixConnection connSpy1 = Mockito.spy(conn1);
PhoenixConnection connSpy2 = Mockito.spy(conn2);
AtomicInteger numStatementsCreatedOnConn1 = new AtomicInteger();
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java
index f8350e4940..6a9eb7b8a3 100644
---
a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java
+++
b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java
@@ -35,6 +35,7 @@ import static org.junit.Assert.fail;
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.HConstants;
@@ -49,6 +50,7 @@ import org.apache.phoenix.schema.ColumnRef;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -86,7 +88,8 @@ public class BaseConnectionlessQueryTest extends BaseTest {
// only load the test driver if we are testing locally - for
integration tests, we want to
// test on a wider scale
if (PhoenixEmbeddedDriver.isTestUrl(url)) {
- driver = initDriver(ReadOnlyProps.EMPTY_PROPS);
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ driver = initDriver(new ReadOnlyProps(props));
assertTrue(DriverManager.getDriver(url) == driver);
driver.connect(url, PropertiesUtil.deepCopy(TEST_PROPERTIES));
}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 18892db735..1405ad6355 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -661,6 +661,9 @@ public abstract class BaseTest {
if (value == null) {
value = PhoenixRegionServerEndpointTestImpl.class.getName();
}
+ else {
+ value = value + "," +
PhoenixRegionServerEndpointTestImpl.class.getName();
+ }
conf.set(REGIONSERVER_COPROCESSOR_CONF_KEY, value);
}
private static PhoenixTestDriver newTestDriver(ReadOnlyProps props) throws
Exception {
