This is an automated email from the ASF dual-hosted git repository. skadam pushed a commit to branch 4.x in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push: new c273579 PHOENIX-5874: IndexTool does not set TTL on its log tables correctly (#773) c273579 is described below commit c27357978e8df14f9fa97b61e3a246bd12d9d80c Author: Swaroopa Kadam <swaroopa.kada...@gmail.com> AuthorDate: Fri May 1 10:01:42 2020 -0700 PHOENIX-5874: IndexTool does not set TTL on its log tables correctly (#773) Co-authored-by: s.kadam <s.ka...@apache.org> --- .../index/IndexVerificationOutputRepositoryIT.java | 55 ++++++++++++++++ .../index/IndexVerificationResultRepositoryIT.java | 75 +++++++++++++++++++--- .../index/IndexVerificationOutputRepository.java | 4 +- .../index/IndexVerificationResultRepository.java | 3 +- 4 files changed, 125 insertions(+), 12 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java index 0b67044..1464e80 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java @@ -17,6 +17,10 @@ */ package org.apache.phoenix.end2end.index; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -26,8 +30,14 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository; import org.apache.phoenix.mapreduce.index.IndexVerificationOutputRow; +import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.util.EnvironmentEdgeManager; +import org.apache.phoenix.util.ManualEnvironmentEdge; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.TestUtil; +import org.bouncycastle.util.Strings; +import org.junit.After; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -39,6 +49,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import static org.apache.phoenix.coprocessor.MetaDataProtocol.DEFAULT_LOG_TTL; +import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.OUTPUT_TABLE_NAME_BYTES; import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.PHASE_AFTER_VALUE; import static org.apache.phoenix.mapreduce.index.IndexVerificationOutputRepository.PHASE_BEFORE_VALUE; import static org.junit.Assert.assertEquals; @@ -94,6 +106,36 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT } + @Test + public void testTTLOnOutputTable() throws SQLException, IOException { + String mockString = "mock_value"; + byte[] mockStringBytes = Strings.toByteArray(mockString); + ManualEnvironmentEdge customClock = new ManualEnvironmentEdge(); + customClock.setValue(1); + EnvironmentEdgeManager.injectEdge(customClock); + try (Connection conn = DriverManager.getConnection(getUrl())) { + HTable hTable = new HTable(config, OUTPUT_TABLE_NAME_BYTES); + + IndexVerificationOutputRepository + outputRepository = + new IndexVerificationOutputRepository(mockStringBytes, conn); + + outputRepository.createOutputTable(conn); + TestUtil.assertTableHasTtl(conn, TableName.valueOf(OUTPUT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL); + outputRepository.logToIndexToolOutputTable(mockStringBytes, mockStringBytes, + 1, 2, mockString, mockStringBytes, mockStringBytes, + EnvironmentEdgeManager.currentTimeMillis(), mockStringBytes, true); + + Assert.assertEquals(1, TestUtil.getRowCount(hTable, false)); + + customClock.incrementValue(1000*(DEFAULT_LOG_TTL+5)); + EnvironmentEdgeManager.injectEdge(customClock); + int count = TestUtil.getRowCount(hTable, false); + + Assert.assertEquals(0, count); + } + } + public void verifyOutputRow(IndexVerificationOutputRepository outputRepository, long scanMaxTs, byte[] indexNameBytes, IndexVerificationOutputRow expectedRow) throws IOException { @@ -169,4 +211,17 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT dataTableName + " (val1) include (val2, val3)"); conn.commit(); } + + @After + public void dropOutputTable() throws Exception { + try(Connection conn = DriverManager.getConnection(getUrl())) { + ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices(); + Admin admin = queryServices.getAdmin(); + TableName outputTableName = TableName.valueOf(OUTPUT_TABLE_NAME_BYTES); + if (admin.tableExists(outputTableName)) { + ((HBaseAdmin) admin).disableTable(OUTPUT_TABLE_NAME_BYTES); + ((HBaseAdmin) admin).deleteTable(OUTPUT_TABLE_NAME_BYTES); + } + } + } } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java index 0ffd13a..84df8b9 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java @@ -17,21 +17,36 @@ */ package org.apache.phoenix.end2end.index; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.IndexToolVerificationResult; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.mapreduce.index.IndexTool; import org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository; +import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.util.EnvironmentEdgeManager; +import org.apache.phoenix.util.ManualEnvironmentEdge; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.TestUtil; +import org.bouncycastle.util.Strings; +import org.junit.After; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.SQLException; import java.util.Collections; import java.util.Map; +import static org.apache.phoenix.coprocessor.MetaDataProtocol.DEFAULT_LOG_TTL; +import static org.apache.phoenix.mapreduce.index.IndexVerificationResultRepository.RESULT_TABLE_NAME_BYTES; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -50,24 +65,53 @@ public class IndexVerificationResultRepositoryIT extends ParallelStatsDisabledIT byte[] indexNameBytes = Bytes.toBytes(indexName); try (Connection conn = DriverManager.getConnection(getUrl())) { createTableAndIndex(conn, tableName, indexName); - IndexVerificationResultRepository resultRepository = - new IndexVerificationResultRepository(conn, indexNameBytes); - resultRepository.createResultTable(conn); - byte[] regionOne = Bytes.toBytes("a.1.00000000000000000000"); - byte[] regionTwo = Bytes.toBytes("a.2.00000000000000000000"); long scanMaxTs = EnvironmentEdgeManager.currentTimeMillis(); IndexToolVerificationResult expectedResult = getExpectedResult(scanMaxTs); - resultRepository.logToIndexToolResultTable(expectedResult, IndexTool.IndexVerifyType.BOTH, - regionOne); - resultRepository.logToIndexToolResultTable(expectedResult, IndexTool.IndexVerifyType.BOTH, - regionTwo); + IndexVerificationResultRepository resultRepository = setupResultRepository(conn, indexNameBytes, expectedResult); IndexToolVerificationResult actualResult = resultRepository.getVerificationResult(conn, scanMaxTs); assertVerificationResult(expectedResult, actualResult); + } + } + + @Test + public void testTTLOnResultTable() throws SQLException, IOException { + String mockString = "mock_value"; + byte[] mockStringBytes = Strings.toByteArray(mockString); + ManualEnvironmentEdge customClock = new ManualEnvironmentEdge(); + customClock.setValue(1); + EnvironmentEdgeManager.injectEdge(customClock); + try (Connection conn = DriverManager.getConnection(getUrl())) { + HTable hTable = new HTable(config, RESULT_TABLE_NAME_BYTES); + long scanMaxTs = EnvironmentEdgeManager.currentTimeMillis(); + IndexToolVerificationResult expectedResult = getExpectedResult(scanMaxTs); + setupResultRepository(conn, mockStringBytes,expectedResult); + Assert.assertEquals(2, TestUtil.getRowCount(hTable, false)); + + customClock.incrementValue(1000*(DEFAULT_LOG_TTL+5)); + EnvironmentEdgeManager.injectEdge(customClock); + int count = TestUtil.getRowCount(hTable, false); + + Assert.assertEquals(0, count); } } + private IndexVerificationResultRepository setupResultRepository(Connection conn, byte[] indexNameBytes,IndexToolVerificationResult expectedResult) + throws SQLException, IOException { + IndexVerificationResultRepository resultRepository = + new IndexVerificationResultRepository(conn, indexNameBytes); + resultRepository.createResultTable(conn); + TestUtil.assertTableHasTtl(conn, TableName.valueOf(RESULT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL); + byte[] regionOne = Bytes.toBytes("a.1.00000000000000000000"); + byte[] regionTwo = Bytes.toBytes("a.2.00000000000000000000"); + resultRepository.logToIndexToolResultTable(expectedResult, IndexTool.IndexVerifyType.BOTH, + regionOne); + resultRepository.logToIndexToolResultTable(expectedResult, IndexTool.IndexVerifyType.BOTH, + regionTwo); + return resultRepository; + } + private void assertVerificationResult(IndexToolVerificationResult expectedResult, IndexToolVerificationResult actualResult) { assertEquals(expectedResult.getScanMaxTs(), actualResult.getScanMaxTs()); assertArrayEquals(expectedResult.getStartRow(), actualResult.getStartRow()); @@ -139,4 +183,17 @@ public class IndexVerificationResultRepositoryIT extends ParallelStatsDisabledIT dataTableName + " (val1) include (val2, val3)"); conn.commit(); } + + @After + public void dropResultTable() throws Exception { + try(Connection conn = DriverManager.getConnection(getUrl())) { + ConnectionQueryServices queryServices = conn.unwrap(PhoenixConnection.class).getQueryServices(); + Admin admin = queryServices.getAdmin(); + TableName outputTableName = TableName.valueOf(RESULT_TABLE_NAME_BYTES); + if (admin.tableExists(outputTableName)) { + ((HBaseAdmin) admin).disableTable(RESULT_TABLE_NAME_BYTES); + ((HBaseAdmin) admin).deleteTable(RESULT_TABLE_NAME_BYTES); + } + } + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java index dbe4315..6e97a9d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationOutputRepository.java @@ -147,9 +147,9 @@ public class IndexVerificationOutputRepository implements AutoCloseable { if (!admin.tableExists(outputTableName)) { HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(OUTPUT_TABLE_NAME)); - tableDescriptor.setValue(HColumnDescriptor.TTL, - String.valueOf(MetaDataProtocol.DEFAULT_LOG_TTL)); HColumnDescriptor columnDescriptor = new HColumnDescriptor(OUTPUT_TABLE_COLUMN_FAMILY); + columnDescriptor.setValue(HColumnDescriptor.TTL, + String.valueOf(MetaDataProtocol.DEFAULT_LOG_TTL)); tableDescriptor.addFamily(columnDescriptor); admin.createTable(tableDescriptor); outputTable = admin.getConnection().getTable(outputTableName); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java index 988facc..e52823e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexVerificationResultRepository.java @@ -131,8 +131,9 @@ public class IndexVerificationResultRepository implements AutoCloseable { if (!admin.tableExists(resultTableName)) { HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(RESULT_TABLE_NAME)); - tableDescriptor.setValue(HColumnDescriptor.TTL, String.valueOf(MetaDataProtocol.DEFAULT_LOG_TTL)); HColumnDescriptor columnDescriptor = new HColumnDescriptor(RESULT_TABLE_COLUMN_FAMILY); + columnDescriptor.setValue(HColumnDescriptor.TTL, + String.valueOf(MetaDataProtocol.DEFAULT_LOG_TTL)); tableDescriptor.addFamily(columnDescriptor); admin.createTable(tableDescriptor); setResultTable(admin.getConnection().getTable(resultTableName));