This is an automated email from the ASF dual-hosted git repository. tdsilva pushed a commit to branch 4.14-HBase-1.3 in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit c9e850de15f199d13c3fcb6d1f50a0499efdafa5 Author: Kadir <kozde...@salesforce.com> AuthorDate: Wed Dec 12 17:53:38 2018 -0800 PHOENIX-5025 Tool to clean up orphan views (addendum) --- .../apache/phoenix/end2end/OrphanViewToolIT.java | 25 +++--- .../apache/phoenix/mapreduce/OrphanViewTool.java | 89 +++++++++++++--------- 2 files changed, 71 insertions(+), 43 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java index f9a1785..38d4afc 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java @@ -19,7 +19,6 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; @@ -27,9 +26,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.File; +import java.io.FileReader; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; +import java.io.LineNumberReader; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; @@ -54,6 +53,7 @@ import org.slf4j.LoggerFactory; @RunWith(Parameterized.class) public class OrphanViewToolIT extends ParallelStatsDisabledIT { + private static final String SYSTEM_CHILD_LINK_NAME = SYSTEM_CATALOG_NAME; private static final Logger LOG = LoggerFactory.getLogger(OrphanViewToolIT.class); private final boolean isMultiTenant; @@ -206,9 +206,13 @@ public class OrphanViewToolIT extends ParallelStatsDisabledIT { } private void verifyLineCount(String fileName, long lineCount) throws IOException { - if (Files.lines(Paths.get(fileName)).count() != lineCount) - LOG.debug(Files.lines(Paths.get(fileName)).count() + " != " + lineCount); - assertTrue(Files.lines(Paths.get(fileName)).count() == lineCount); + LineNumberReader reader = new LineNumberReader(new FileReader(fileName)); + while (reader.readLine() != null) { + } + int count = reader.getLineNumber(); + if (count != lineCount) + LOG.debug(count + " != " + lineCount); + assertTrue(count == lineCount); } private void verifyCountQuery(Connection connection, String query, String schemaName, long count) @@ -238,7 +242,6 @@ public class OrphanViewToolIT extends ParallelStatsDisabledIT { } } - private void verifyNoChildLink(Connection connection, String viewSchemaName) throws Exception { // Verify that there there is no link in the system child link table verifyCountQuery(connection, countChildLinksQuery, viewSchemaName, 0); @@ -264,6 +267,7 @@ public class OrphanViewToolIT extends ParallelStatsDisabledIT { schemaName == null ? "IS NULL" : "= '" + schemaName + "'")); connection.commit(); } + @Test public void testDeleteBaseTableRows() throws Exception { String baseTableName = generateUniqueName(); @@ -438,7 +442,8 @@ public class OrphanViewToolIT extends ParallelStatsDisabledIT { } } - public static String[] getArgValues(boolean clean, boolean identify, boolean outputPath, boolean inputPath) { + public static String[] getArgValues(boolean clean, boolean identify, boolean outputPath, boolean inputPath) + throws InterruptedException{ final List<String> args = Lists.newArrayList(); if (outputPath) { args.add("-op"); @@ -454,8 +459,10 @@ public class OrphanViewToolIT extends ParallelStatsDisabledIT { if (identify) { args.add("-i"); } + final long ageMs = 2000; + Thread.sleep(ageMs); args.add("-a"); - args.add("0"); + args.add(Long.toString(ageMs)); return args.toArray(new String[0]); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java index a8a30b6..2e0dd0d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java @@ -21,7 +21,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE; @@ -71,6 +70,7 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.SchemaUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,6 +79,7 @@ import org.slf4j.LoggerFactory; * */ public class OrphanViewTool extends Configured implements Tool { + private static final String SYSTEM_CHILD_LINK_NAME = SYSTEM_CATALOG_NAME; private static final Logger LOG = LoggerFactory.getLogger(OrphanViewTool.class); // Query all the views that are not "MAPPED" views private static final String viewQuery = "SELECT " + @@ -394,26 +395,48 @@ public class OrphanViewTool extends Configured implements Tool { } private void gracefullyDropView(PhoenixConnection phoenixConnection, Configuration configuration, - Key key) throws Exception { - PhoenixConnection tenantConnection; - if (key.getTenantId() != null) { - Properties tenantProps = new Properties(); - tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, key.getTenantId()); - tenantConnection = ConnectionUtil.getInputConnection(configuration, tenantProps). - unwrap(PhoenixConnection.class); - } else { - tenantConnection = phoenixConnection; - } - - MetaDataClient client = new MetaDataClient(tenantConnection); - org.apache.phoenix.parse.TableName pTableName = org.apache.phoenix.parse.TableName - .create(key.getSchemaName(), key.getTableName()); + Key key) throws Exception { + PhoenixConnection tenantConnection = null; + boolean newConn = false; try { - client.dropTable( - new DropTableStatement(pTableName, PTableType.VIEW, false, true, true)); + if (key.getTenantId() != null) { + Properties tenantProps = new Properties(); + tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, key.getTenantId()); + tenantConnection = ConnectionUtil.getInputConnection(configuration, tenantProps). + unwrap(PhoenixConnection.class); + newConn = true; + } else { + tenantConnection = phoenixConnection; + } + String fullViewName = SchemaUtil.getTableName(key.getSchemaName(), key.getTableName()); + String dropTable = String.format("DROP VIEW IF EXISTS %s CASCADE", fullViewName); + try { + tenantConnection.createStatement().execute(dropTable); + tenantConnection.commit(); + } + catch (TableNotFoundException e) { + LOG.info("Ignoring view " + fullViewName + " as it has already been dropped"); + } + } finally { + if (newConn) { + tryClosingConnection(tenantConnection); + } } - catch (TableNotFoundException e) { - LOG.info("Ignoring view " + pTableName + " as it has already been dropped"); + } + + /** + * Try closing a connection if it is not null + * @param connection connection object + * @throws RuntimeException if closing the connection fails + */ + private void tryClosingConnection(Connection connection) { + try { + if (connection != null) { + connection.close(); + } + } catch (SQLException sqlE) { + LOG.error("Failed to close connection: ", sqlE); + throw new RuntimeException("Failed to close connection with exception: ", sqlE); } } @@ -812,17 +835,6 @@ public class OrphanViewTool extends Configured implements Tool { } catch (IllegalStateException e) { printHelpAndExit(e.getMessage(), getOptions()); } - - Properties props = new Properties(); - long scn = System.currentTimeMillis() - ageMs; - props.setProperty("CurrentSCN", Long.toString(scn)); - connection = ConnectionUtil.getInputConnection(configuration); - PhoenixConnection phoenixConnection = connection.unwrap(PhoenixConnection.class); - - if (clean) { - // Take a snapshot of system tables to be modified - createSnapshot(phoenixConnection, scn); - } if (outputPath != null) { // Create files to log orphan views and links for (int i = VIEW; i < ORPHAN_TYPE_COUNT; i++) { @@ -834,7 +846,20 @@ public class OrphanViewTool extends Configured implements Tool { writer[i] = new BufferedWriter(new FileWriter(file)); } } + Properties props = new Properties(); + long scn = System.currentTimeMillis() - ageMs; + props.setProperty("CurrentSCN", Long.toString(scn)); + connection = ConnectionUtil.getInputConnection(configuration, props); + PhoenixConnection phoenixConnection = connection.unwrap(PhoenixConnection.class); identifyOrphanViews(phoenixConnection); + if (clean) { + // Close the connection with SCN + phoenixConnection.close(); + connection = ConnectionUtil.getInputConnection(configuration); + phoenixConnection = connection.unwrap(PhoenixConnection.class); + // Take a snapshot of system tables to be modified + createSnapshot(phoenixConnection, scn); + } for (Map.Entry<Key, View> entry : orphanViewSet.entrySet()) { try { dropOrLogOrphanViews(phoenixConnection, configuration, entry.getKey()); @@ -843,10 +868,6 @@ public class OrphanViewTool extends Configured implements Tool { } }; if (clean) { - // Wait for the view drop tasks in the SYSTEM.TASK table to be processed - long timeInterval = configuration.getLong(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, - QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS); - Thread.sleep(maxViewLevel * timeInterval); // Clean up any remaining orphan view records from system tables for (Map.Entry<Key, View> entry : orphanViewSet.entrySet()) { try {