[
https://issues.apache.org/jira/browse/PHOENIX-5025?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16705429#comment-16705429
]
ASF GitHub Bot commented on PHOENIX-5025:
-----------------------------------------
Github user ChinmaySKulkarni commented on a diff in the pull request:
https://github.com/apache/phoenix/pull/404#discussion_r238033365
--- Diff:
phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java ---
@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
+import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.mapreduce.OrphanViewTool;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@RunWith(Parameterized.class)
+public class OrphanViewToolIT extends ParallelStatsDisabledIT {
+ private static final Logger LOG =
LoggerFactory.getLogger(OrphanViewToolIT.class);
+
+ private final boolean isMultiTenant;
+ private final boolean columnEncoded;
+
+ private static final int fanout = 2;
+
+ private static final String filePath = "/tmp/";
+ private static final String fileName = "/tmp/OrphanView.txt";
+
+ protected static String SCHEMA1 = "SCHEMA1";
+ protected static String SCHEMA2 = "SCHEMA2";
+ protected static String SCHEMA3 = "SCHEMA3";
+ protected static String SCHEMA4 = "SCHEMA4";
+
+ private final String TENANT_SPECIFIC_URL = getUrl() + ';' +
TENANT_ID_ATTRIB + "=tenant";
+
+ public OrphanViewToolIT(boolean isMultiTenant, boolean columnEncoded) {
+ this.isMultiTenant = isMultiTenant;
+ this.columnEncoded = columnEncoded;
+ }
+
+ @Parameters(name="OrphanViewToolIT_multiTenant={0},
columnEncoded={1}") // name is used by failsafe as file name in reports
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true },
+ { true, false }, { true, true } });
+ }
+
+ private String generateDDL(String format) {
+ return generateDDL("", format);
+ }
+
+ private String generateDDL(String options, String format) {
+ StringBuilder optionsBuilder = new StringBuilder(options);
+ if (!columnEncoded) {
+ if (optionsBuilder.length()!=0)
+ optionsBuilder.append(",");
+ optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
+ }
+ if (isMultiTenant) {
+ if (optionsBuilder.length()!=0)
+ optionsBuilder.append(",");
+ optionsBuilder.append("MULTI_TENANT=true");
+ }
+ return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR
NOT NULL, " : "",
+ isMultiTenant ? "TENANT_ID, " : "",
optionsBuilder.toString());
+ }
+
+ private void deleteRows(Connection connection, String systemTableName,
String schemaName) throws SQLException {
+ String delete = "DELETE FROM " + systemTableName +
+ " WHERE " + TABLE_SCHEM + " = '" + schemaName + "'";
+ connection.createStatement().execute(delete);
+ }
+
+ private void deleteAllRows(Connection connection, String
baseTableSchema,
+ String childViewSchemaName,
+ String grandchildViewSchemaName, String
grandGrandChildViewSchemaName) throws SQLException {
+ deleteRows(connection, SYSTEM_CATALOG_NAME, baseTableSchema);
+ deleteRows(connection, SYSTEM_CATALOG_NAME, childViewSchemaName);
+ deleteRows(connection, SYSTEM_CATALOG_NAME,
grandchildViewSchemaName);
+ deleteRows(connection, SYSTEM_CATALOG_NAME,
grandGrandChildViewSchemaName);
+ deleteRows(connection, SYSTEM_CHILD_LINK_NAME, baseTableSchema);
+ deleteRows(connection, SYSTEM_CHILD_LINK_NAME,
childViewSchemaName);
+ deleteRows(connection, SYSTEM_CHILD_LINK_NAME,
grandchildViewSchemaName);
+ deleteRows(connection, SYSTEM_CHILD_LINK_NAME,
grandGrandChildViewSchemaName);
+
+ connection.commit();
+ }
+ private void createBaseTableAndViews(Connection baseTableConnection,
String baseTableFullName,
+ Connection viewConnection, String
childViewSchemaName,
+ String grandchildViewSchemaName,
String grandGrandChildViewSchemaName)
+ throws SQLException {
+ String ddlFormat =
+ "CREATE TABLE IF NOT EXISTS " + baseTableFullName + " ("
+ + " %s PK2 VARCHAR NOT NULL, V1 VARCHAR, V2
VARCHAR "
+ + " CONSTRAINT NAME_PK PRIMARY KEY (%s PK2)" + " )
%s";
+
baseTableConnection.createStatement().execute(generateDDL(ddlFormat));
+ // Create a view tree (i.e., tree of views) with depth of 3
+ for (int i = 0; i < fanout; i++) {
+ String childView =
SchemaUtil.getTableName(childViewSchemaName, generateUniqueName());
+ String childViewDDL = "CREATE VIEW " + childView + " AS SELECT
* FROM " + baseTableFullName;
+ viewConnection.createStatement().execute(childViewDDL);
+ for (int j = 0; j < fanout; j++) {
+ String grandchildView =
SchemaUtil.getTableName(grandchildViewSchemaName, generateUniqueName());
+ String grandchildViewDDL = "CREATE VIEW " + grandchildView
+ " AS SELECT * FROM " + childView;
+
viewConnection.createStatement().execute(grandchildViewDDL);
+ for (int k = 0; k < fanout; k++) {
+ viewConnection.createStatement().execute("CREATE VIEW
" +
+
SchemaUtil.getTableName(grandGrandChildViewSchemaName, generateUniqueName()) +
+ " AS SELECT * FROM " + grandchildView);
+ }
+ }
+ }
+ }
+
+ private long getLineCount() throws IOException {
+ return Files.lines(Paths.get(fileName)).count();
+ }
+
+ @Test
+ public void testCreateTableAndViews() throws Exception {
+ String baseTableName = generateUniqueName();
+ String baseTableFullName = SchemaUtil.getTableName(SCHEMA1,
baseTableName);
+ try (Connection connection = DriverManager.getConnection(getUrl());
+ Connection viewConnection =
+ isMultiTenant ?
DriverManager.getConnection(TENANT_SPECIFIC_URL) : connection) {
+ createBaseTableAndViews(connection, baseTableFullName,
viewConnection, SCHEMA2, SCHEMA3, SCHEMA4);
+ // Run the orphan view tool to drop orphan views but no view
should be dropped
+ runOrphanViewTool(true, false, true, false);
+ assertTrue(getLineCount() == 0);
+ // Verify that the views we have created are still in the
system catalog table
+ String viewQuery = "SELECT COUNT(*) FROM " +
+ SYSTEM_CATALOG_NAME +
+ " WHERE " + TABLE_TYPE + " = '" +
PTableType.VIEW.getSerializedValue() + "'";
+ ResultSet rs =
connection.createStatement().executeQuery(viewQuery);
+ assertTrue(rs.next());
+ assertTrue(rs.getLong(1) == fanout + fanout * fanout + fanout
* fanout * fanout);
+ deleteAllRows(connection, SCHEMA1, SCHEMA2, SCHEMA3, SCHEMA4);
+ }
+ }
+
+ private void verifyNoChildLink(Connection connection, String
viewSchemaName) throws Exception {
+ // Verify that there there is no link in the system child link
table
+ String childLinkQuery = "SELECT COUNT(*) FROM " +
SYSTEM_CHILD_LINK_NAME +
+ " WHERE " + TABLE_SCHEM + " = '" + viewSchemaName + "' AND
" +
+ LINK_TYPE + " = " +
PTable.LinkType.CHILD_TABLE.getSerializedValue();
+ ResultSet rs =
connection.createStatement().executeQuery(childLinkQuery);
+ assertTrue(rs.next());
+ assertTrue(rs.getLong(1) == 0);
+ }
+
+ private void verifyNoViewNoLink(Connection connection, String
viewSchemaName) throws Exception {
+ // Verify that the views and links have been removed from the
system catalog table
+ String catalogQuery = "SELECT COUNT(*) FROM " +
SYSTEM_CATALOG_NAME +
+ " WHERE " + TABLE_SCHEM + " = '" + viewSchemaName + "'";
+ ResultSet rs =
connection.createStatement().executeQuery(catalogQuery);
+ assertTrue(rs.next());
+ long count = rs.getLong(1);
+ assertTrue(rs.getLong(1) == 0);
+ }
+
+ @Test
+ public void testDeleteBaseTableRows() throws Exception {
+ String baseTableName = generateUniqueName();
+ String baseTableFullName = SchemaUtil.getTableName(SCHEMA1,
baseTableName);
+ try (Connection connection = DriverManager.getConnection(getUrl());
+ Connection viewConnection =
+ isMultiTenant ?
DriverManager.getConnection(TENANT_SPECIFIC_URL) : connection) {
+ createBaseTableAndViews(connection, baseTableFullName,
viewConnection, SCHEMA2, SCHEMA2, SCHEMA2);
+ // Delete the base table rows from the system catalog
+ String deleteTableRows = "DELETE FROM " + SYSTEM_CATALOG_NAME +
--- End diff --
A lot of these String queries can be extracted out as static final Strings
and you can use String.format wherever required. Will reduce duplication
throughout unit tests and makes it easier to see what the helper queries are in
one spot.
> Tool to clean up orphan views
> -----------------------------
>
> Key: PHOENIX-5025
> URL: https://issues.apache.org/jira/browse/PHOENIX-5025
> Project: Phoenix
> Issue Type: New Feature
> Reporter: Kadir OZDEMIR
> Assignee: Kadir OZDEMIR
> Priority: Major
>
> A view without its base table is an orphan view. Since views are virtual
> tables and their data is stored in their base tables, they are useless when
> they become orphan. A base table can have child views, grandchild views and
> so on. Due to some reasons/bugs, when a base table was dropped, its views
> were not not properly cleaned up in the past. For example, the drop table
> code did not support cleaning up grandchild views. This has been recently
> fixed by PHOENIX-4764. Although PHOENIX-4764 prevents new orphan views due to
> table drop operations, it does not clean up existing orphan views. It is also
> believed that when the system catalog table was split due to a bug in the
> past, it also contributed to creating orphan views as Phoenix did not support
> splittable system catalog. Therefore, Phoenix needs a tool to clean up orphan
> views.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
