This is an automated email from the ASF dual-hosted git repository. gjacoby pushed a commit to branch 4.x in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push: new 5e4044f PHOENIX-5875 - Optional logging for IndexTool verification 5e4044f is described below commit 5e4044fdf11f7d695dd2cd06110f485752a96ba3 Author: Geoffrey Jacoby <gjac...@apache.org> AuthorDate: Mon May 18 17:07:05 2020 -0700 PHOENIX-5875 - Optional logging for IndexTool verification --- .../end2end/IndexToolForNonTxGlobalIndexIT.java | 149 +++++++++++++++++++-- .../org/apache/phoenix/end2end/IndexToolIT.java | 59 +++++--- .../index/IndexVerificationOutputRepositoryIT.java | 67 +++++++++ .../coprocessor/BaseScannerRegionObserver.java | 2 + .../coprocessor/IndexRebuildRegionScanner.java | 27 +++- .../PhoenixServerBuildIndexInputFormat.java | 3 + .../apache/phoenix/mapreduce/index/IndexTool.java | 97 ++++++++++++++ .../index/IndexVerificationOutputRepository.java | 74 +++++++--- .../mapreduce/util/PhoenixConfigurationUtil.java | 15 +++ .../org/apache/phoenix/index/IndexToolTest.java | 106 +++++++++++++-- 10 files changed, 537 insertions(+), 62 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java index 16813ee..aca88b3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolForNonTxGlobalIndexIT.java @@ -23,6 +23,9 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -34,6 +37,8 @@ import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.mapreduce.index.IndexTool; import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository; +import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow; +import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; @@ -44,6 +49,7 @@ import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; +import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Rule; @@ -89,7 +95,9 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REB import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT; import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.CURRENT_SCN_VALUE; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) @@ -136,6 +144,17 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT new ReadOnlyProps(clientProps.entrySet().iterator())); } + @After + public void cleanup() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + deleteAllRows(conn, + TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES)); + deleteAllRows(conn, + TableName.valueOf(IndexVerificationResultRepository.RESULT_TABLE_NAME)); + } + } + @Test public void testWithSetNull() throws Exception { // This tests the cases where a column having a null value is overwritten with a not null value and vice versa; @@ -166,11 +185,11 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT IndexTool indexTool = IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, new String[0]); assertEquals(NROWS, indexTool.getJob().getCounters().findCounter(INPUT_RECORDS).getValue()); + assertTrue("Index rebuild failed!", indexTool.getJob().isSuccessful()); + TestUtil.assertIndexState(conn, indexTableFullName, PIndexState.ACTIVE, null); long actualRowCount = IndexScrutiny .scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(NROWS, actualRowCount); - actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); - assertEquals(NROWS, actualRowCount); IndexToolIT.setEveryNthRowWithNull(NROWS, 5, stmt); conn.commit(); actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); @@ -192,7 +211,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_MISSING_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue()); assertEquals(0, indexTool.getJob().getCounters().findCounter(BEFORE_REBUILD_BEYOND_MAXLOOKBACK_MISSING_INDEX_ROW_COUNT).getValue()); - IndexToolIT.dropIndexToolTables(conn); } } @@ -259,7 +277,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT .getTable(indexToolOutputTable.getName()); Result r = hIndexToolTable.getScanner(scan).next(); assertTrue(r == null); - IndexToolIT.dropIndexToolTables(conn); } } @@ -376,7 +393,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT assertEquals(0, indexTool.getJob().getCounters().findCounter(AFTER_REBUILD_BEYOND_MAXLOOKBACK_INVALID_INDEX_ROW_COUNT).getValue()); actualRowCount = IndexScrutiny.scrutinizeIndex(conn, dataTableFullName, indexTableFullName); assertEquals(2 * NROWS, actualRowCount); - IndexToolIT.dropIndexToolTables(conn); } } @@ -416,7 +432,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, viewName, indexTableName, null, 0, IndexTool.IndexVerifyType.BOTH); assertEquals(0, IndexToolIT.MutationCountingRegionObserver.getMutationCount()); - IndexToolIT.dropIndexToolTables(conn); } } @@ -457,7 +472,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT Assert.fail("Fail to parsing the error message from IndexToolOutputTable"); } IndexRegionObserver.setIgnoreIndexRebuildForTesting(false); - IndexToolIT.dropIndexToolTables(conn); } } @@ -501,7 +515,6 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT null, 0, IndexTool.IndexVerifyType.AFTER); IndexToolIT.runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, null, 0, IndexTool.IndexVerifyType.ONLY); - IndexToolIT.dropIndexToolTables(conn); } } @@ -566,9 +579,127 @@ public class IndexToolForNonTxGlobalIndexIT extends BaseUniqueNamesOwnClusterIT Assert.assertFalse(it.isValidLastVerifyTime(10L)); Assert.assertFalse(it.isValidLastVerifyTime(EnvironmentEdgeManager.currentTimeMillis() - 1000L)); Assert.assertTrue(it.isValidLastVerifyTime(scn)); + } + } - IndexToolIT.dropIndexToolTables(conn); + @Test + public void testDisableOutputLogging() throws Exception { + if (!mutable || useSnapshot) { + return; + } + + String schemaName = generateUniqueName(); + String dataTableName = generateUniqueName(); + String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName); + String indexTableName = generateUniqueName(); + String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + + try(Connection conn = DriverManager.getConnection(getUrl(), props)) { + String stmString1 = + "CREATE TABLE " + dataTableFullName + + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) " + + tableDDLOptions; + conn.createStatement().execute(stmString1); + String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName); + PreparedStatement stmt1 = conn.prepareStatement(upsertQuery); + + // insert two rows + IndexToolIT.upsertRow(stmt1, 1); + IndexToolIT.upsertRow(stmt1, 2); + conn.commit(); + + //create ASYNC + String stmtString2 = + String.format( + "CREATE INDEX %s ON %s (LPAD(UPPER(NAME, 'en_US'),8,'x')||'_xyz') ASYNC ", + indexTableName, dataTableFullName); + conn.createStatement().execute(stmtString2); + conn.commit(); + + // run the index MR job as ONLY so the index doesn't get rebuilt. Should be 2 missing + //rows. We pass in --disable-logging BEFORE to silence the output logging to + // PHOENIX_INDEX_TOOL, since ONLY logs BEFORE the (non-existent in this case) + // rebuild + assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.ONLY, + IndexTool.IndexDisableLoggingType.BEFORE, null, schemaName, dataTableName, indexTableName, + indexTableFullName, 0); + + //now check that disabling logging AFTER leaves only the BEFORE logs on a BOTH run + assertDisableLogging(conn, 2, IndexTool.IndexVerifyType.BOTH, + IndexTool.IndexDisableLoggingType.AFTER, + IndexVerificationOutputRepository.PHASE_BEFORE_VALUE, schemaName, + dataTableName, indexTableName, + indexTableFullName, 0); + + deleteAllRows(conn, + TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME)); + deleteAllRows(conn, TableName.valueOf(indexTableFullName)); + + //now check that disabling logging BEFORE creates only the AFTER logs on a BOTH run + //the index tool run fails validation at the end because we suppressed the BEFORE logs + //which prevented the rebuild from working properly, but that's ok for this test. + assertDisableLogging(conn, 2, IndexTool.IndexVerifyType.BOTH, + IndexTool.IndexDisableLoggingType.BEFORE, + IndexVerificationOutputRepository.PHASE_AFTER_VALUE, schemaName, + dataTableName, indexTableName, + indexTableFullName, -1); + + deleteAllRows(conn, TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME)); + deleteAllRows(conn, TableName.valueOf(indexTableFullName)); + + //now check that disabling logging BOTH creates no logs on a BOTH run + assertDisableLogging(conn, 0, IndexTool.IndexVerifyType.BOTH, + IndexTool.IndexDisableLoggingType.BOTH, + IndexVerificationOutputRepository.PHASE_AFTER_VALUE, schemaName, + dataTableName, indexTableName, + indexTableFullName, -1); + + } + } + + public void deleteAllRows(Connection conn, TableName tableName) throws SQLException, + IOException { + Scan scan = new Scan(); + HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices(). + getAdmin(); + HConnection hbaseConn = admin.getConnection(); + HTableInterface table = hbaseConn.getTable(tableName); + try (ResultScanner scanner = table.getScanner(scan)) { + for (Result r : scanner) { + Delete del = new Delete(r.getRow()); + table.delete(del); + } + } + } + + private void assertDisableLogging(Connection conn, int expectedRows, + IndexTool.IndexVerifyType verifyType, + IndexTool.IndexDisableLoggingType disableLoggingType, + byte[] expectedPhase, + String schemaName, String dataTableName, + String indexTableName, String indexTableFullName, + int expectedStatus) throws Exception { + IndexTool tool = IndexToolIT.runIndexTool(true, false, schemaName, dataTableName, + indexTableName, + null, + expectedStatus, verifyType, "-et", + Long.toString(EnvironmentEdgeManager.currentTimeMillis()),"-dl", disableLoggingType.toString()); + assertNotNull(tool); + assertNotNull(tool.getEndTime()); + byte[] indexTableFullNameBytes = Bytes.toBytes(indexTableFullName); + + IndexVerificationOutputRepository outputRepository = + new IndexVerificationOutputRepository(indexTableFullNameBytes, conn); + List<IndexVerificationOutputRow> rows = + outputRepository.getOutputRows(tool.getEndTime(), + indexTableFullNameBytes); + assertEquals(expectedRows, rows.size()); + if (expectedRows > 0) { + assertArrayEquals(expectedPhase, rows.get(0).getPhaseValue()); } + TestUtil.dumpTable(conn, + TableName.valueOf(IndexVerificationOutputRepository.OUTPUT_TABLE_NAME)); } private void deleteOneRowFromResultTable(Connection conn, Long scn, String indexTable) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java index a2bd788..5310618 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java @@ -23,11 +23,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -40,11 +42,9 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.mapreduce.Job; -import org.apache.phoenix.coprocessor.IndexRebuildRegionScanner; -import org.apache.phoenix.hbase.index.IndexRegionObserver; import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository; +import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow; import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.mapreduce.index.IndexTool; @@ -53,24 +53,23 @@ import org.apache.phoenix.mapreduce.index.PhoenixServerBuildIndexMapper; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature; import org.apache.phoenix.transaction.TransactionFactory; -import org.apache.phoenix.util.IndexScrutiny; +import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; @@ -106,8 +105,10 @@ import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.BEF import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.REBUILT_INDEX_ROW_COUNT; import static org.apache.phoenix.mapreduce.index.PhoenixIndexToolJobCounters.SCANNED_DATA_ROW_COUNT; import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -161,7 +162,8 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT { } @Parameters( - name = "transactionProvider={0},mutable={1},localIndex={2},directApi={3},useSnapshot={4}") + name = "transactionProvider={0},mutable={1},localIndex={2},directApi={3}," + + "useSnapshot={4},useTenant={5}") public static synchronized Collection<Object[]> data() { List<Object[]> list = Lists.newArrayListWithExpectedSize(48); boolean[] Booleans = new boolean[] { false, true }; @@ -382,7 +384,10 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT { if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), IndexVerificationOutputRepository.DATA_TABLE_NAME_BYTES, 0, IndexVerificationOutputRepository.DATA_TABLE_NAME_BYTES.length) == 0) { dataTableNameCheck = true; - assertTrue(Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(), + assertTrue("Value was different! Expected: " + Bytes.toString(dataTableFullNameBytes) + + " Actual: " + Bytes.toString(cell.getValueArray(), + cell.getValueOffset(), cell.getValueLength()), + Bytes.compareTo(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(), dataTableFullNameBytes, 0, dataTableFullNameBytes.length) == 0); } else if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), IndexVerificationOutputRepository.INDEX_TABLE_NAME_BYTES, 0, IndexVerificationOutputRepository.INDEX_TABLE_NAME_BYTES.length) == 0) { @@ -404,8 +409,10 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT { scan = new Scan(); scanner = hIndexTable.getScanner(scan); Result result = scanner.next(); - assert(result != null); - verifyIndexTableRowKey(CellUtil.cloneRow(result.rawCells()[0]), indexTableFullName); + if (result != null) { + verifyIndexTableRowKey(CellUtil.cloneRow(result.rawCells()[0]), indexTableFullName); + } + return errorMessageCell; } @@ -630,8 +637,18 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT { } private static List<String> getArgList (boolean directApi, boolean useSnapshot, String schemaName, + String dataTable, String indxTable, String tenantId, + IndexTool.IndexVerifyType verifyType, Long startTime, + Long endTime, Long incrementalVerify) { + return getArgList(directApi, useSnapshot, schemaName, dataTable, indxTable, tenantId, + verifyType, startTime, endTime, IndexTool.IndexDisableLoggingType.NONE, incrementalVerify); + } + + private static List<String> getArgList (boolean directApi, boolean useSnapshot, String schemaName, String dataTable, String indxTable, String tenantId, - IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime, Long incrementalVerify) { + IndexTool.IndexVerifyType verifyType, Long startTime, Long endTime, + IndexTool.IndexDisableLoggingType disableLoggingType, + Long incrementalVerify) { List<String> args = Lists.newArrayList(); if (schemaName != null) { args.add("-s"); @@ -644,7 +661,8 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT { if (directApi) { args.add("-direct"); } - args.add("-v" + verifyType.getValue()); // verify index rows inline + args.add("-v"); + args.add(verifyType.getValue()); // verify index rows inline // Need to run this job in foreground for the test to be deterministic args.add("-runfg"); @@ -664,6 +682,12 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT { args.add("-et"); args.add(String.valueOf(endTime)); } + + if (disableLoggingType != IndexTool.IndexDisableLoggingType.NONE) { + args.add("-dl"); + args.add(disableLoggingType.getValue()); + } + if(incrementalVerify!=null) { args.add("-rv"); args.add(String.valueOf(incrementalVerify)); @@ -689,10 +713,13 @@ public class IndexToolIT extends BaseUniqueNamesOwnClusterIT { } public static String [] getArgValues(boolean directApi, boolean useSnapshot, String schemaName, - String dataTable, String indexTable, String tenantId, - IndexTool.IndexVerifyType verifyType, Long incrementalVerify) { + String dataTable, String indexTable, String tenantId, + IndexTool.IndexVerifyType verifyType, Long startTime, + Long endTime, + IndexTool.IndexDisableLoggingType disableLoggingType, + Long incrementalVerify ) { List<String> args = getArgList(directApi, useSnapshot, schemaName, dataTable, indexTable, - tenantId, verifyType, null, null, incrementalVerify); + tenantId, verifyType, startTime, endTime, disableLoggingType, incrementalVerify); return args.toArray(new String[0]); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java index 1464e80..e772fa7 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -28,6 +29,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.mapreduce.index.IndexTool; import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository; import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow; import org.apache.phoenix.query.ConnectionQueryServices; @@ -40,6 +42,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.Mockito; import java.io.IOException; import java.sql.Connection; @@ -55,6 +58,7 @@ import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputReposito import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.PHASE_BEFORE_VALUE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.when; public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT { @@ -136,6 +140,69 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT } } + @Test + public void testDisableLoggingBefore() throws SQLException, IOException { + IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.BEFORE; + boolean expectedBefore = false; + boolean expectedAfter = true; + verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter); + } + + @Test + public void testDisableLoggingAfter() throws SQLException, IOException { + IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.AFTER; + boolean expectedBefore = true; + boolean expectedAfter = false; + verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter); + } + + @Test + public void testDisableLoggingBoth() throws SQLException, IOException { + IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.BOTH; + boolean expectedBefore = false; + boolean expectedAfter = false; + verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter); + } + + @Test + public void testDisableLoggingNone() throws SQLException, IOException { + IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.NONE; + boolean expectedBefore = true; + boolean expectedAfter = true; + verifyDisableLogging(disableLoggingVerifyType, expectedBefore, expectedAfter); + } + + public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableLoggingVerifyType, boolean expectedBefore, boolean expectedAfter) throws SQLException, IOException { + Table mockOutputTable = Mockito.mock(Table.class); + Table mockIndexTable = Mockito.mock(Table.class); + when(mockIndexTable.getName()).thenReturn(TableName.valueOf("testDisableLoggingIndexName")); + IndexVerificationOutputRepository outputRepository = + new IndexVerificationOutputRepository(mockOutputTable, + mockIndexTable, disableLoggingVerifyType); + byte[] dataRowKey = Bytes.toBytes("dataRowKey"); + byte[] indexRowKey = Bytes.toBytes("indexRowKey"); + long dataRowTs = EnvironmentEdgeManager.currentTimeMillis(); + long indexRowTs = EnvironmentEdgeManager.currentTimeMillis(); + String errorMsg = ""; + byte[] expectedValue = Bytes.toBytes("expectedValue"); + byte[] actualValue = Bytes.toBytes("actualValue"); + long scanMaxTs = EnvironmentEdgeManager.currentTimeMillis(); + byte[] tableName = Bytes.toBytes("testDisableLoggingTableName"); + + outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs + , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, true); + outputRepository.logToIndexToolOutputTable(dataRowKey, indexRowKey, dataRowTs, indexRowTs + , errorMsg, expectedValue, actualValue, scanMaxTs, tableName, false); + int expectedRowsLogged = 0; + if (expectedBefore && expectedAfter) { + expectedRowsLogged = 2; + } else if (expectedBefore || expectedAfter) { + expectedRowsLogged = 1; + } + Mockito.verify(mockOutputTable, Mockito.times(expectedRowsLogged)). + put(Mockito.any(Put.class)); + } + public void verifyOutputRow(IndexVerificationOutputRepository outputRepository, long scanMaxTs, byte[] indexNameBytes, IndexVerificationOutputRow expectedRow) throws IOException { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java index 4897741..4d61295 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java @@ -89,6 +89,8 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { // Index verification type done by the index tool public static final String INDEX_REBUILD_VERIFY_TYPE = "_IndexRebuildVerifyType"; public static final String INDEX_RETRY_VERIFY = "_IndexRetryVerify"; + public static final String INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE = + "_IndexRebuildDisableLoggingVerifyType"; /* * Attribute to denote that the index maintainer has been serialized using its proto-buf presentation. diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java index 76ae453..a5a4ec1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/IndexRebuildRegionScanner.java @@ -39,7 +39,6 @@ import java.util.concurrent.Future; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import com.sun.org.apache.xpath.internal.operations.Bool; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; @@ -104,6 +103,7 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner { private boolean useProto = true; private byte[] indexRowKey; private IndexTool.IndexVerifyType verifyType = IndexTool.IndexVerifyType.NONE; + private IndexTool.IndexDisableLoggingType disableLoggingVerifyType = IndexTool.IndexDisableLoggingType.NONE; private boolean verify = false; private Map<byte[], List<Mutation>> indexKeyToMutationMap; private Map<byte[], Pair<Put, Delete>> dataKeyToMutationMap; @@ -152,11 +152,20 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner { if (verifyType != IndexTool.IndexVerifyType.NONE) { verify = true; viewConstants = IndexUtil.deserializeViewConstantsFromScan(scan); + byte[] disableLoggingValueBytes = + scan.getAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE); + if (disableLoggingValueBytes != null) { + disableLoggingVerifyType = + IndexTool.IndexDisableLoggingType.fromValue(disableLoggingValueBytes); + } verificationOutputRepository = - new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName(), hTableFactory); + new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName() + , hTableFactory, disableLoggingVerifyType); verificationResult = new IndexToolVerificationResult(scan); + new IndexVerificationOutputRepository(indexMaintainer.getIndexTableName() + , hTableFactory, disableLoggingVerifyType); verificationResultRepository = - new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory); + new IndexVerificationResultRepository(indexMaintainer.getIndexTableName(), hTableFactory); indexKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); dataKeyToMutationMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); pool = new WaitForCompletionTaskRunner(ThreadPoolManager.getExecutor( @@ -240,14 +249,20 @@ public class IndexRebuildRegionScanner extends GlobalIndexRegionScanner { innerScanner.close(); if (verify) { try { - verificationResultRepository.logToIndexToolResultTable(verificationResult, + if (verificationResultRepository != null) { + verificationResultRepository.logToIndexToolResultTable(verificationResult, verifyType, region.getRegionInfo().getRegionName(), skipped); + } } finally { this.pool.stop("IndexRebuildRegionScanner is closing"); hTableFactory.shutdown(); indexHTable.close(); - verificationResultRepository.close(); - verificationOutputRepository.close(); + if (verificationResultRepository != null) { + verificationResultRepository.close(); + } + if (verificationOutputRepository != null) { + verificationOutputRepository.close(); + } } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java index 9408369..696659b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java @@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getCurrentScnValue; +import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getDisableLoggingVerifyType; import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getIndexToolDataTableName; import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getIndexToolIndexTableName; import static org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getIndexToolLastVerifyTime; @@ -106,6 +107,8 @@ public class PhoenixServerBuildIndexInputFormat<T extends DBWritable> extends Ph scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_PAGING, TRUE_BYTES); scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_VERIFY_TYPE, getIndexVerifyType(configuration).toBytes()); scan.setAttribute(BaseScannerRegionObserver.INDEX_RETRY_VERIFY, Bytes.toBytes(lastVerifyTimeValue)); + scan.setAttribute(BaseScannerRegionObserver.INDEX_REBUILD_DISABLE_LOGGING_VERIFY_TYPE, + getDisableLoggingVerifyType(configuration).toBytes()); } catch (IOException e) { throw new SQLException(e); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java index 4c0c2d2..3477049 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java @@ -156,6 +156,42 @@ public class IndexTool extends Configured implements Tool { } } + public enum IndexDisableLoggingType { + NONE("NONE"), + BEFORE("BEFORE"), + AFTER("AFTER"), + BOTH("BOTH"); + + private String value; + private byte[] valueBytes; + + IndexDisableLoggingType(String value) { + this.value = value; + this.valueBytes = PVarchar.INSTANCE.toBytes(value); + } + + public String getValue() { + return this.value; + } + + public byte[] toBytes() { + return this.valueBytes; + } + + public static IndexDisableLoggingType fromValue(String value) { + for (IndexDisableLoggingType disableLoggingType: IndexDisableLoggingType.values()) { + if (value.equals(disableLoggingType.getValue())) { + return disableLoggingType; + } + } + throw new IllegalStateException("Invalid value: "+ value + " for " + IndexDisableLoggingType.class); + } + + public static IndexDisableLoggingType fromValue(byte[] value) { + return fromValue(Bytes.toString(value)); + } + } + private static final Logger LOGGER = LoggerFactory.getLogger(IndexTool.class); private String schemaName; @@ -163,6 +199,7 @@ public class IndexTool extends Configured implements Tool { private String indexTable; private boolean isPartialBuild, isForeground; private IndexVerifyType indexVerifyType = IndexVerifyType.NONE; + private IndexDisableLoggingType disableLoggingType = IndexDisableLoggingType.NONE; private String qDataTable; private String qIndexTable; private boolean useSnapshot; @@ -243,6 +280,11 @@ public class IndexTool extends Configured implements Tool { private static final Option RETRY_VERIFY_OPTION = new Option("rv", "retry-verify", true, "Max scan ts of the last rebuild/verify that needs to be retried incrementally"); + private static final Option DISABLE_LOGGING_OPTION = new Option("dl", + "disable-logging", true + , "Disable logging of failed verification rows for BEFORE, " + + "AFTER, or BOTH verify jobs"); + public static final String INDEX_JOB_NAME_TEMPLATE = "PHOENIX_%s.%s_INDX_%s"; public static final String INVALID_TIME_RANGE_EXCEPTION_MESSAGE = "startTime is greater than " @@ -280,6 +322,7 @@ public class IndexTool extends Configured implements Tool { options.addOption(START_TIME_OPTION); options.addOption(END_TIME_OPTION); options.addOption(RETRY_VERIFY_OPTION); + options.addOption(DISABLE_LOGGING_OPTION); return options; } @@ -334,9 +377,52 @@ public class IndexTool extends Configured implements Tool { if (splitIndex && cmdLine.hasOption(PARTIAL_REBUILD_OPTION.getOpt())) { throw new IllegalStateException("Cannot split index for a partial rebuild, as the index table is dropped"); } + if (loggingDisabledMismatchesVerifyOption(cmdLine)){ + throw new IllegalStateException("Can't disable index verification logging when no " + + "index verification or the wrong kind of index verification has been requested. " + + "VerifyType: [" + cmdLine.getOptionValue(VERIFY_OPTION.getOpt()) + "] and " + + "DisableLoggingType: [" + + cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt()) + "]"); + } return cmdLine; } + private boolean loggingDisabledMismatchesVerifyOption(CommandLine cmdLine) { + boolean loggingDisabled = cmdLine.hasOption(DISABLE_LOGGING_OPTION.getOpt()); + if (!loggingDisabled) { + return false; + } + boolean hasVerifyOption = + cmdLine.hasOption(VERIFY_OPTION.getOpt()); + if (!hasVerifyOption) { + return true; + } + String loggingDisableValue = cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt()); + String verifyValue = cmdLine.getOptionValue(VERIFY_OPTION.getOpt()); + IndexDisableLoggingType loggingDisableType = IndexDisableLoggingType.fromValue(loggingDisableValue); + if (loggingDisableType != IndexDisableLoggingType.BEFORE && + loggingDisableType != IndexDisableLoggingType.AFTER && + loggingDisableType != IndexDisableLoggingType.BOTH) { + return true; + } + IndexVerifyType verifyType = IndexVerifyType.fromValue(verifyValue); + //error if we're trying to disable logging when we're not doing any verification + if (verifyType.equals(IndexVerifyType.NONE)){ + return true; + } + //error if we're disabling logging after rebuild but we're not verifying after rebuild + if ((verifyType.equals(IndexVerifyType.BEFORE) || verifyType.equals(IndexVerifyType.ONLY)) + && loggingDisableType.equals(IndexDisableLoggingType.AFTER)) { + return true; + } + //error if we're disabling logging before rebuild but we're not verifying before rebuild + if ((verifyType.equals(IndexVerifyType.AFTER)) + && loggingDisableType.equals(IndexDisableLoggingType.BEFORE)) { + return true; + } + return false; + } + private void printHelpAndExit(String errorMessage, Options options) { System.err.println(errorMessage); printHelpAndExit(options, 1); @@ -356,6 +442,10 @@ public class IndexTool extends Configured implements Tool { public Long getLastVerifyTime() { return lastVerifyTime; } + public IndexTool.IndexDisableLoggingType getDisableLoggingType() { + return disableLoggingType; + } + class JobFactory { Connection connection; Configuration configuration; @@ -607,6 +697,7 @@ public class IndexTool extends Configured implements Tool { PhoenixConfigurationUtil.setIndexToolStartTime(configuration, startTime); } PhoenixConfigurationUtil.setIndexVerifyType(configuration, indexVerifyType); + PhoenixConfigurationUtil.setDisableLoggingVerifyType(configuration, disableLoggingType); String physicalIndexTable = pIndexTable.getPhysicalName().getString(); PhoenixConfigurationUtil.setPhysicalTableName(configuration, physicalIndexTable); @@ -758,6 +849,7 @@ public class IndexTool extends Configured implements Tool { boolean useEndTime = cmdLine.hasOption(END_TIME_OPTION.getOpt()); boolean retryVerify = cmdLine.hasOption(RETRY_VERIFY_OPTION.getOpt()); boolean verify = cmdLine.hasOption(VERIFY_OPTION.getOpt()); + boolean disableLogging = cmdLine.hasOption(DISABLE_LOGGING_OPTION.getOpt()); if (useTenantId) { tenantId = cmdLine.getOptionValue(TENANT_ID_OPTION.getOpt()); @@ -778,6 +870,11 @@ public class IndexTool extends Configured implements Tool { if (verify) { String value = cmdLine.getOptionValue(VERIFY_OPTION.getOpt()); indexVerifyType = IndexVerifyType.fromValue(value); + if (disableLogging) { + disableLoggingType = + IndexDisableLoggingType.fromValue( + cmdLine.getOptionValue(DISABLE_LOGGING_OPTION.getOpt())); + } } schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPTION.getOpt()); dataTable = cmdLine.getOptionValue(DATA_TABLE_OPTION.getOpt()); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java index 7e8ee23..4bc91a8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java @@ -50,6 +50,9 @@ public class IndexVerificationOutputRepository implements AutoCloseable { private Table indexTable; private byte[] indexName; private Table outputTable; + private IndexTool.IndexDisableLoggingType disableLoggingVerifyType = + IndexTool.IndexDisableLoggingType.NONE; + public final static String OUTPUT_TABLE_NAME = "PHOENIX_INDEX_TOOL"; public final static byte[] OUTPUT_TABLE_NAME_BYTES = Bytes.toBytes(OUTPUT_TABLE_NAME); public final static byte[] OUTPUT_TABLE_COLUMN_FAMILY = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; @@ -98,10 +101,20 @@ public class IndexVerificationOutputRepository implements AutoCloseable { indexTable = queryServices.getTable(indexName); } - public IndexVerificationOutputRepository(byte[] indexName, HTableFactory hTableFactory) throws IOException { + @VisibleForTesting + public IndexVerificationOutputRepository(Table outputTable, Table indexTable, + IndexTool.IndexDisableLoggingType disableLoggingVerifyType) throws SQLException { + this.outputTable = outputTable; + this.indexTable = indexTable; + this.disableLoggingVerifyType = disableLoggingVerifyType; + } + + public IndexVerificationOutputRepository(byte[] indexName, HTableFactory hTableFactory, + IndexTool.IndexDisableLoggingType disableLoggingVerifyType) throws IOException { this.indexName = indexName; outputTable = hTableFactory.getTable(new ImmutableBytesPtr(OUTPUT_TABLE_NAME_BYTES)); indexTable = hTableFactory.getTable(new ImmutableBytesPtr(indexName)); + this.disableLoggingVerifyType = disableLoggingVerifyType; } public static byte[] generateOutputTableRowKey(long ts, byte[] indexTableName, byte[] dataRowKey ) { @@ -162,29 +175,48 @@ public class IndexVerificationOutputRepository implements AutoCloseable { String errorMsg, byte[] expectedValue, byte[] actualValue, long scanMaxTs, byte[] tableName, boolean isBeforeRebuild) throws IOException { - byte[] rowKey = generateOutputTableRowKey(scanMaxTs, indexTable.getName().toBytes(), dataRowKey); - Put put = new Put(rowKey); - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_NAME_BYTES, tableName); - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_NAME_BYTES, indexName); - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(dataRowTs))); + if (shouldLogOutput(isBeforeRebuild)) { + byte[] rowKey = generateOutputTableRowKey(scanMaxTs, indexTable.getName().toBytes(), dataRowKey); + Put put = new Put(rowKey); + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_NAME_BYTES, tableName); + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_NAME_BYTES, indexName); + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, DATA_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(dataRowTs))); - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_ROW_KEY_BYTES, indexRowKey); - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(indexRowTs))); - byte[] errorMessageBytes; - if (expectedValue != null) { - errorMessageBytes = getErrorMessageBytes(errorMsg, expectedValue, actualValue); - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, EXPECTED_VALUE_BYTES, expectedValue); - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ACTUAL_VALUE_BYTES, actualValue); - } else { - errorMessageBytes = Bytes.toBytes(errorMsg); + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_ROW_KEY_BYTES, indexRowKey); + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, INDEX_TABLE_TS_BYTES, Bytes.toBytes(Long.toString(indexRowTs))); + byte[] errorMessageBytes; + if (expectedValue != null) { + errorMessageBytes = getErrorMessageBytes(errorMsg, expectedValue, actualValue); + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, EXPECTED_VALUE_BYTES, expectedValue); + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ACTUAL_VALUE_BYTES, actualValue); + } else { + errorMessageBytes = Bytes.toBytes(errorMsg); + } + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ERROR_MESSAGE_BYTES, errorMessageBytes); + if (isBeforeRebuild) { + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_BEFORE_VALUE); + } else { + put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_AFTER_VALUE); + } + outputTable.put(put); + } + } + + public boolean shouldLogOutput(boolean isBeforeRebuild) { + if (disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.BOTH)) { + return false; + } + if (disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.NONE)) { + return true; + } + if (isBeforeRebuild && + (disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.AFTER))) { + return true; } - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, ERROR_MESSAGE_BYTES, errorMessageBytes); - if (isBeforeRebuild) { - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_BEFORE_VALUE); - } else { - put.addColumn(OUTPUT_TABLE_COLUMN_FAMILY, VERIFICATION_PHASE_BYTES, PHASE_AFTER_VALUE); + if (!isBeforeRebuild && disableLoggingVerifyType.equals(IndexTool.IndexDisableLoggingType.BEFORE)) { + return true; } - outputTable.put(put); + return false; } public static byte[] getErrorMessageBytes(String errorMsg, byte[] expectedValue, byte[] actualValue) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java index f575b09..348755a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java @@ -148,6 +148,9 @@ public final class PhoenixConfigurationUtil { public static final String INDEX_VERIFY_TYPE = "phoenix.mr.index.IndexVerifyType"; + public static final String DISABLE_LOGGING_TYPE = "phoenix.mr.index" + + ".IndexDisableLoggingType"; + // Generate splits based on scans from stats, or just from region splits public static final String MAPREDUCE_SPLIT_BY_STATS = "phoenix.mapreduce.split.by.stats"; @@ -604,6 +607,12 @@ public final class PhoenixConfigurationUtil { configuration.set(INDEX_VERIFY_TYPE, verifyType.getValue()); } + public static void setDisableLoggingVerifyType(Configuration configuration, + IndexTool.IndexDisableLoggingType disableLoggingType) { + Preconditions.checkNotNull(configuration); + configuration.set(DISABLE_LOGGING_TYPE, disableLoggingType.getValue()); + } + public static String getScrutinyDataTableName(Configuration configuration) { Preconditions.checkNotNull(configuration); return configuration.get(SCRUTINY_DATA_TABLE_NAME); @@ -735,6 +744,12 @@ public final class PhoenixConfigurationUtil { return IndexTool.IndexVerifyType.fromValue(value); } + public static IndexTool.IndexVerifyType getDisableLoggingVerifyType(Configuration configuration) { + Preconditions.checkNotNull(configuration); + String value = configuration.get(DISABLE_LOGGING_TYPE, IndexTool.IndexVerifyType.NONE.getValue()); + return IndexTool.IndexVerifyType.fromValue(value); + } + public static boolean getSplitByStats(final Configuration configuration) { Preconditions.checkNotNull(configuration); boolean split = configuration.getBoolean(MAPREDUCE_SPLIT_BY_STATS, DEFAULT_SPLIT_BY_STATS); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java index fd5164f..87215d0 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexToolTest.java @@ -34,6 +34,7 @@ import org.mockito.MockitoAnnotations; import static org.apache.phoenix.mapreduce.index.IndexTool.FEATURE_NOT_APPLICABLE; import static org.apache.phoenix.mapreduce.index.IndexTool.INVALID_TIME_RANGE_EXCEPTION_MESSAGE; +import static org.junit.Assert.assertEquals; import static org.apache.phoenix.mapreduce.index.IndexTool.RETRY_VERIFY_NOT_APPLICABLE; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -72,8 +73,8 @@ public class IndexToolTest extends BaseTest { startTime , endTime); CommandLine cmdLine = it.parseOptions(args); it.populateIndexToolAttributes(cmdLine); - Assert.assertEquals(startTime, it.getStartTime()); - Assert.assertEquals(endTime, it.getEndTime()); + assertEquals(startTime, it.getStartTime()); + assertEquals(endTime, it.getEndTime()); } @Test @@ -96,8 +97,8 @@ public class IndexToolTest extends BaseTest { startTime , null); CommandLine cmdLine = it.parseOptions(args); it.populateIndexToolAttributes(cmdLine); - Assert.assertEquals(startTime, it.getStartTime()); - Assert.assertEquals(null, it.getEndTime()); + assertEquals(startTime, it.getStartTime()); + assertEquals(null, it.getEndTime()); } @Test @@ -109,8 +110,8 @@ public class IndexToolTest extends BaseTest { null , endTime); CommandLine cmdLine = it.parseOptions(args); it.populateIndexToolAttributes(cmdLine); - Assert.assertEquals(null, it.getStartTime()); - Assert.assertEquals(endTime, it.getEndTime()); + assertEquals(null, it.getStartTime()); + assertEquals(endTime, it.getEndTime()); } @Test @@ -211,7 +212,7 @@ public class IndexToolTest extends BaseTest { String [] args = IndexToolIT.getArgValues(true, true, schema, dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.NONE, - lastVerifyTime); + lastVerifyTime, null, IndexTool.IndexDisableLoggingType.NONE, lastVerifyTime); when(mockTool.parseOptions(args)).thenCallRealMethod(); CommandLine cmdLine = mockTool.parseOptions(args); @@ -234,9 +235,10 @@ public class IndexToolTest extends BaseTest { when(mockTool.getLastVerifyTime()).thenCallRealMethod(); Long lastVerifyTime = 10L; String [] args = - IndexToolIT.getArgValues(true, true, schema, - dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.AFTER, - lastVerifyTime); + IndexToolIT.getArgValues(true, true, schema, + dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.AFTER, + lastVerifyTime, null, IndexTool.IndexDisableLoggingType.NONE, + lastVerifyTime); when(mockTool.parseOptions(args)).thenCallRealMethod(); CommandLine cmdLine = mockTool.parseOptions(args); @@ -249,4 +251,88 @@ public class IndexToolTest extends BaseTest { exceptionRule.expectMessage(RETRY_VERIFY_NOT_APPLICABLE); mockTool.populateIndexToolAttributes(cmdLine); } + + @Test + public void testCheckVerifyAndDisableLogging_defaultsNone() throws Exception { + Long startTime = 1L; + Long endTime = 10L; + String [] args = + IndexToolIT.getArgValues(true, true, schema, + dataTable, indexTable, tenantId, IndexTool.IndexVerifyType.NONE, + startTime , endTime); + CommandLine cmdLine = it.parseOptions(args); + it.populateIndexToolAttributes(cmdLine); + assertEquals(IndexTool.IndexDisableLoggingType.NONE, it.getDisableLoggingType()); + } + + @Test + public void testDisableLogging_allowsNone() throws Exception { + verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.NONE); + verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.ONLY); + verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.BEFORE); + verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.AFTER); + verifyDisableLogging(IndexTool.IndexDisableLoggingType.NONE, IndexTool.IndexVerifyType.BOTH); + } + + @Test + public void testDisableLogging_allowsBefore() throws Exception { + verifyDisableLogging(IndexTool.IndexDisableLoggingType.BEFORE, IndexTool.IndexVerifyType.BEFORE); + verifyDisableLogging(IndexTool.IndexDisableLoggingType.BEFORE, IndexTool.IndexVerifyType.ONLY); + verifyDisableLogging(IndexTool.IndexDisableLoggingType.BEFORE, IndexTool.IndexVerifyType.BOTH); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BEFORE, + IndexTool.IndexVerifyType.AFTER); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BEFORE, + IndexTool.IndexVerifyType.NONE); + } + + @Test + public void testDisableLogging_allowsAfter() throws Exception { + verifyDisableLogging(IndexTool.IndexDisableLoggingType.AFTER, IndexTool.IndexVerifyType.BOTH); + verifyDisableLogging(IndexTool.IndexDisableLoggingType.AFTER, IndexTool.IndexVerifyType.AFTER); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.AFTER, + IndexTool.IndexVerifyType.NONE); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.AFTER, + IndexTool.IndexVerifyType.BEFORE); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH, + IndexTool.IndexVerifyType.ONLY); + } + + @Test + public void testCheckVerifyAndDisableLogging_allowsBoth() throws Exception { + verifyDisableLogging(IndexTool.IndexDisableLoggingType.BOTH, IndexTool.IndexVerifyType.BOTH); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH, + IndexTool.IndexVerifyType.NONE); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH, + IndexTool.IndexVerifyType.ONLY); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH, + IndexTool.IndexVerifyType.BEFORE); + verifyDisableLoggingException(IndexTool.IndexDisableLoggingType.BOTH, + IndexTool.IndexVerifyType.AFTER); + } + + public void verifyDisableLogging(IndexTool.IndexDisableLoggingType disableType, + IndexTool.IndexVerifyType verifyType) throws Exception { + Long startTime = 1L; + Long endTime = 10L; + String[] args = + IndexToolIT.getArgValues(true, true, schema, + dataTable, indexTable, tenantId, verifyType, + startTime, endTime, disableType, null); + CommandLine cmdLine = it.parseOptions(args); + it.populateIndexToolAttributes(cmdLine); + assertEquals(disableType, it.getDisableLoggingType()); + } + + public void verifyDisableLoggingException(IndexTool.IndexDisableLoggingType disableType, + IndexTool.IndexVerifyType verifyType) { + Long startTime = 1L; + Long endTime = 10L; + String[] args = + IndexToolIT.getArgValues(true, true, schema, + dataTable, indexTable, tenantId, verifyType, + startTime, endTime, disableType, null); + exceptionRule.expect(IllegalStateException.class); + CommandLine cmdLine = it.parseOptions(args); + } + }