This is an automated email from the ASF dual-hosted git repository.
skadam pushed a commit to branch 4.x-HBase-1.5
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-HBase-1.5 by this push:
new 36c642b PHOENIX-5713: Incorrectly handled view and view indexes
with/without … (#704)
36c642b is described below
commit 36c642b1f46088f8031a4f7ba5dee9e4ee3427e5
Author: Swaroopa Kadam <[email protected]>
AuthorDate: Tue Feb 4 15:42:36 2020 -0800
PHOENIX-5713: Incorrectly handled view and view indexes with/without …
(#704)
* PHOENIX-5713: Incorrectly handled view and view indexes with/without
namespace in IndexScrutinyMapper#getTtl()
---
.../NonParameterizedIndexScrutinyToolIT.java | 56 ++++++++-
.../mapreduce/index/IndexScrutinyMapper.java | 46 ++++---
.../phoenix/index/IndexScrutinyMapperTest.java | 132 +++++++++++++++++++++
3 files changed, 209 insertions(+), 25 deletions(-)
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
index cd73af5..93a723d 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NonParameterizedIndexScrutinyToolIT.java
@@ -23,6 +23,7 @@ import
org.apache.phoenix.mapreduce.index.IndexScrutinyMapperForTest;
import org.apache.phoenix.mapreduce.index.IndexScrutinyTool;
import org.apache.phoenix.util.EnvironmentEdgeManager;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.SchemaUtil;
import org.junit.Test;
import java.sql.Connection;
@@ -144,32 +145,75 @@ public class NonParameterizedIndexScrutinyToolIT extends
IndexScrutinyToolBaseIT
@Test
public void testScrutinyOnRowsNearExpiry() throws Exception {
+ String schema = generateUniqueName();
String dataTableName = generateUniqueName();
String indexTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schema,
dataTableName);
String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY
KEY, NAME VARCHAR, "
+ "ZIP INTEGER) TTL="+TEST_TABLE_TTL;
String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP)";
String upsertData = "UPSERT INTO %s VALUES (?, ?, ?)";
- int initialDelta = -5000; //insert row with 5 secs before current
timestamp
IndexScrutinyMapperForTest.ScrutinyTestClock
- testClock = new
IndexScrutinyMapperForTest.ScrutinyTestClock(initialDelta);
+ testClock = new
IndexScrutinyMapperForTest.ScrutinyTestClock(0);
try (Connection conn =
DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
- conn.createStatement().execute(String.format(dataTableDDL,
dataTableName));
+ conn.createStatement().execute(String.format(dataTableDDL,
dataTableFullName));
conn.createStatement().execute(String.format(indexTableDDL,
indexTableName,
- dataTableName));
+ dataTableFullName));
// insert two rows
PreparedStatement
upsertDataStmt =
conn.prepareStatement(String.format(upsertData,
- dataTableName));
+ dataTableFullName));
EnvironmentEdgeManager.injectEdge(testClock);
upsertRow(upsertDataStmt, 1, "name-1", 98051);
upsertRow(upsertDataStmt, 2, "name-2", 98052);
conn.commit();
- List<Job> completedJobs = runScrutiny(null, dataTableName,
indexTableName);
+ List<Job> completedJobs = runScrutiny(schema, dataTableName,
indexTableName);
+ Job job = completedJobs.get(0);
+ assertTrue(job.isSuccessful());
+ Counters counters = job.getCounters();
+ assertEquals(2, getCounterValue(counters, EXPIRED_ROW_COUNT));
+ assertEquals(0, getCounterValue(counters, VALID_ROW_COUNT));
+ assertEquals(0, getCounterValue(counters, INVALID_ROW_COUNT));
+ }
+ }
+
+ @Test
+ public void testScrutinyOnRowsNearExpiry_viewIndex() throws Exception {
+ String schemaName = "S"+generateUniqueName();
+ String dataTableName = "T"+generateUniqueName();
+ String dataTableFullName =
SchemaUtil.getTableName(schemaName,dataTableName);
+ String viewIndexName = "VI"+generateUniqueName();
+ String viewName = "V"+generateUniqueName();
+ String viewFullName = SchemaUtil.getTableName(schemaName,viewName);
+ String dataTableDDL = "CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY
KEY, NAME VARCHAR, "
+ + "ZIP INTEGER) TTL="+TEST_TABLE_TTL;
+ String viewDDL = "CREATE VIEW %s AS SELECT * FROM %s";
+ String indexTableDDL = "CREATE INDEX %s ON %s (NAME) INCLUDE (ZIP)";
+ String upsertData = "UPSERT INTO %s VALUES (?, ?, ?)";
+ IndexScrutinyMapperForTest.ScrutinyTestClock
+ testClock = new
IndexScrutinyMapperForTest.ScrutinyTestClock(0);
+
+ try (Connection conn =
+ DriverManager.getConnection(getUrl(),
PropertiesUtil.deepCopy(TEST_PROPERTIES))) {
+ conn.createStatement().execute(String.format(dataTableDDL,
dataTableFullName));
+ conn.createStatement().execute(String.format(viewDDL,
viewFullName, dataTableFullName));
+ conn.createStatement().execute(String.format(indexTableDDL,
viewIndexName,
+ viewFullName));
+ // insert two rows
+ PreparedStatement
+ upsertDataStmt =
conn.prepareStatement(String.format(upsertData,
+ viewFullName));
+
+ EnvironmentEdgeManager.injectEdge(testClock);
+ upsertRow(upsertDataStmt, 1, "name-1", 98051);
+ upsertRow(upsertDataStmt, 2, "name-2", 98052);
+ conn.commit();
+
+ List<Job> completedJobs = runScrutiny(schemaName, viewName,
viewIndexName);
Job job = completedJobs.get(0);
assertTrue(job.isSuccessful());
Counters counters = job.getCounters();
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
index 9099f05..2359b4a 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
@@ -33,6 +33,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -50,6 +51,8 @@ import
org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import org.apache.phoenix.parse.HintNode.Hint;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.ColumnInfo;
@@ -297,31 +300,36 @@ public class IndexScrutinyMapper extends
Mapper<NullWritable, PhoenixIndexDBWrit
}
private long getTableTtl() throws SQLException, IOException {
- PTable psourceTable = PhoenixRuntime.getTable(connection,
qSourceTable);
- if (psourceTable.getType() == PTableType.INDEX
- && psourceTable.getIndexType() == PTable.IndexType.LOCAL) {
+ PTable pSourceTable = PhoenixRuntime.getTable(connection,
qSourceTable);
+ if (pSourceTable.getType() == PTableType.INDEX
+ && pSourceTable.getIndexType() == PTable.IndexType.LOCAL) {
return Integer.MAX_VALUE;
}
- String schema = psourceTable.getSchemaName().toString();
- String table = getSourceTableName(psourceTable);
- Admin admin =
connection.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
- String fullTableName = SchemaUtil.getQualifiedTableName(schema, table);
- HTableDescriptor tableDesc =
admin.getTableDescriptor(TableName.valueOf(fullTableName));
- return
tableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(psourceTable)).getTimeToLive();
+ ConnectionQueryServices
+ cqsi =
connection.unwrap(PhoenixConnection.class).getQueryServices();
+ Admin admin = cqsi.getAdmin();
+ String physicalTable = getSourceTableName(pSourceTable,
+ SchemaUtil.isNamespaceMappingEnabled(null, cqsi.getProps()));
+ HTableDescriptor tableDesc =
admin.getTableDescriptor(TableName.valueOf(physicalTable));
+ return
tableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(pSourceTable)).getTimeToLive();
}
- private String getSourceTableName(PTable psourceTable) {
- String sourcePhysicalName = psourceTable.getPhysicalName().getString();
- String table;
- if (psourceTable.getType() == PTableType.VIEW) {
- table = sourcePhysicalName;
- } else if (MetaDataUtil.isViewIndex(sourcePhysicalName)) {
- table =
SchemaUtil.getParentTableNameFromIndexTable(sourcePhysicalName,
- MetaDataUtil.VIEW_INDEX_TABLE_PREFIX);
+ @VisibleForTesting
+ public static String getSourceTableName(PTable pSourceTable, boolean
isNamespaceEnabled) {
+ String sourcePhysicalName = pSourceTable.getPhysicalName().getString();
+ String physicalTable, table, schema;
+ if (pSourceTable.getType() == PTableType.VIEW
+ || MetaDataUtil.isViewIndex(sourcePhysicalName)) {
+ // in case of view and view index ptable, getPhysicalName()
returns hbase tables
+ // i.e. without _IDX_ and with _IDX_ respectively
+ physicalTable = sourcePhysicalName;
} else {
- table = psourceTable.getTableName().toString();
+ table = pSourceTable.getTableName().toString();
+ schema = pSourceTable.getSchemaName().toString();
+ physicalTable = SchemaUtil
+ .getPhysicalHBaseTableName(schema, table,
isNamespaceEnabled).toString();
}
- return table;
+ return physicalTable;
}
protected Map<String, Pair<Long, List<Object>>>
buildTargetStatement(PreparedStatement targetStatement)
diff --git
a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexScrutinyMapperTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexScrutinyMapperTest.java
new file mode 100644
index 0000000..863593b
--- /dev/null
+++
b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexScrutinyMapperTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.index;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.mapreduce.index.IndexScrutinyMapper;
+import org.apache.phoenix.query.BaseConnectionlessQueryTest;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+@RunWith(Parameterized.class)
+public class IndexScrutinyMapperTest extends BaseConnectionlessQueryTest {
+ String schema, tableName, indexName;
+ boolean isNamespaceEnabled;
+ PTable inputTable;
+
+ @Before
+ public void setup() {
+ schema = "S_" + generateUniqueName();
+ tableName = "T_" + generateUniqueName();
+ indexName = "I_" + generateUniqueName();
+ inputTable = Mockito.mock(PTable.class);
+
+ }
+
+ @Parameterized.Parameters(name
="IndexUpgradeToolTest_isNamespaceEnabled={0}")
+ public static synchronized Collection<Boolean> data() {
+ return Arrays.asList( false, true);
+ }
+
+ public IndexScrutinyMapperTest(boolean isNamespaceEnabled) {
+ this.isNamespaceEnabled = isNamespaceEnabled;
+ }
+ @Test
+ public void testGetSourceTableName_table() {
+ String fullTableName = SchemaUtil.getQualifiedTableName(schema,
tableName);
+ PName sourcePhysicalName =
SchemaUtil.getPhysicalHBaseTableName(schema, tableName,
+ isNamespaceEnabled);
+ String expectedName =
SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
+ isNamespaceEnabled).toString();
+ //setup
+ Mockito.when(inputTable.getType()).thenReturn(PTableType.TABLE);
+
Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
+
Mockito.when(inputTable.getTableName()).thenReturn(PNameFactory.newName(tableName));
+
Mockito.when(inputTable.getSchemaName()).thenReturn(PNameFactory.newName(schema));
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(inputTable,
isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+
+ @Test
+ public void testGetSourceTableName_view() {
+ String fullTableName = SchemaUtil.getQualifiedTableName(schema,
tableName);
+ PName sourcePhysicalName =
SchemaUtil.getPhysicalHBaseTableName(schema, tableName,
+ isNamespaceEnabled);
+ String expectedName =
SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
+ isNamespaceEnabled).toString();
+ //setup
+ Mockito.when(inputTable.getType()).thenReturn(PTableType.VIEW);
+
Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(inputTable,
isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+
+ @Test
+ public void testGetSourceTableName_index() {
+ String fullTableName = SchemaUtil.getQualifiedTableName(schema,
indexName);
+ PName sourcePhysicalName =
SchemaUtil.getPhysicalHBaseTableName(schema, indexName,
+ isNamespaceEnabled);
+ String expectedName =
SchemaUtil.getPhysicalTableName(Bytes.toBytes(fullTableName),
+ isNamespaceEnabled).toString();
+
+ //setup
+ Mockito.when(inputTable.getType()).thenReturn(PTableType.INDEX);
+
Mockito.when(inputTable.getPhysicalName()).thenReturn(sourcePhysicalName);
+
Mockito.when(inputTable.getTableName()).thenReturn(PNameFactory.newName(indexName));
+
Mockito.when(inputTable.getSchemaName()).thenReturn(PNameFactory.newName(schema));
+
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(inputTable,
isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+
+ @Test
+ public void testGetSourceTableName_viewIndex() {
+ PName physicalTableName = SchemaUtil.getPhysicalHBaseTableName(schema,
tableName,
+ isNamespaceEnabled);
+ String expectedName =
MetaDataUtil.getViewIndexPhysicalName(physicalTableName.getString());
+ PName physicalIndexTableName = PNameFactory
+
.newName(MetaDataUtil.getViewIndexPhysicalName(physicalTableName.getString()));
+
+ PTable pSourceTable = Mockito.mock(PTable.class);
+ //setup
+
Mockito.when(pSourceTable.getPhysicalName()).thenReturn(physicalIndexTableName);
+ //test
+ String output = IndexScrutinyMapper.getSourceTableName(pSourceTable,
isNamespaceEnabled);
+ //assert
+ Assert.assertEquals(expectedName, output);
+ }
+}
