virajjasani commented on code in PR #1955: URL: https://github.com/apache/phoenix/pull/1955#discussion_r1722392016
########## phoenix-core/src/it/java/org/apache/phoenix/end2end/UpdateCacheFreqIT.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.end2end; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compile.ExplainPlan; +import org.apache.phoenix.compile.ExplainPlanAttributes; +import org.apache.phoenix.coprocessor.MetaDataEndpointImpl; +import org.apache.phoenix.coprocessor.generated.MetaDataProtos; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.exception.PhoenixIOException; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixPreparedStatement; +import org.apache.phoenix.protobuf.ProtobufUtil; +import org.apache.phoenix.query.BaseTest; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; +import org.apache.phoenix.util.ClientUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.TestUtil; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +import static org.apache.phoenix.query.QueryServices.DISABLE_VIEW_SUBTREE_VALIDATION; + +@Category(NeedsOwnMiniClusterTest.class) +public class UpdateCacheFreqIT extends BaseTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(UpdateCacheFreqIT.class); + + private static void initCluster() throws Exception { + Map<String, String> props = Maps.newConcurrentMap(); + props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + Integer.toString(60 * 60 * 1000)); + props.put(QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, "ALWAYS"); + props.put(QueryServices.LAST_DDL_TIMESTAMP_VALIDATION_ENABLED, Boolean.toString(false)); + props.put(QueryServices.PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED, Boolean.toString(false)); + props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, + Long.toString(Long.MAX_VALUE)); + props.put(DISABLE_VIEW_SUBTREE_VALIDATION, "true"); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @BeforeClass + public static synchronized void doSetup() throws Exception { + initCluster(); + } + + public static class TestMetaDataEndpointImpl extends MetaDataEndpointImpl { + + @Override + public void getTable(RpcController controller, MetaDataProtos.GetTableRequest request, + RpcCallback<MetaDataProtos.MetaDataResponse> done) { + LOGGER.error("Not expected to get getTable() for {}", + Bytes.toString(request.getTableName().toByteArray())); + ProtobufUtil.setControllerException(controller, + ClientUtil.createIOException( + SchemaUtil.getPhysicalTableName( + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, + false).toString(), + new DoNotRetryIOException("Not allowed"))); + + } + + } + + @Test + public void testUcfWithNoGetTableCalls() throws Throwable { + final String tableName = generateUniqueName(); + final String indexName = "IDX_" + tableName; + final String view01 = "v01_" + tableName; + final String view02 = "v02_" + tableName; + final String index_view01 = "idx_v01_" + tableName; + final String index_view02 = "idx_v02_" + tableName; + final String index_view03 = "idx_v03_" + tableName; + final String index_view04 = "idx_v04_" + tableName; + + try (Connection conn = DriverManager.getConnection(getUrl())) { + final Statement stmt = conn.createStatement(); + + stmt.execute("CREATE TABLE " + tableName + + " (COL1 CHAR(10) NOT NULL, COL2 CHAR(5) NOT NULL, COL3 VARCHAR," + + " COL4 VARCHAR CONSTRAINT pk PRIMARY KEY(COL1, COL2))" + + " UPDATE_CACHE_FREQUENCY=20000"); + stmt.execute("CREATE INDEX " + indexName + + " ON " + tableName + " (COL3) INCLUDE (COL4)"); + stmt.execute("CREATE VIEW " + view01 + + " (VCOL1 CHAR(8), COL5 VARCHAR) AS SELECT * FROM " + tableName + + " WHERE COL1 = 'col1'"); + stmt.execute("CREATE VIEW " + view02 + " (VCOL2 CHAR(10), COL6 VARCHAR)" + + " AS SELECT * FROM " + view01 + " WHERE VCOL1 = 'vcol1'"); + + stmt.execute("CREATE INDEX " + index_view01 + " ON " + view01 + " (COL5) INCLUDE " + + "(COL1, COL2, COL3)"); + stmt.execute("CREATE INDEX " + index_view02 + " ON " + view02 + " (COL6) INCLUDE " + + "(COL1, COL2, COL3)"); + stmt.execute("CREATE INDEX " + index_view03 + " ON " + view01 + " (COL5) INCLUDE " + + "(COL2, COL1)"); + stmt.execute("CREATE INDEX " + index_view04 + " ON " + view02 + " (COL6) INCLUDE " + + "(COL2, COL1)"); + + stmt.execute("UPSERT INTO " + view02 + + " (col2, vcol2, col5, col6) values ('0001', 'vcol2_01', 'col5_01', " + + "'col6_01')"); + conn.commit(); + + TestUtil.removeCoprocessor(conn, "SYSTEM.CATALOG", MetaDataEndpointImpl.class); + TestUtil.addCoprocessor(conn, "SYSTEM.CATALOG", TestMetaDataEndpointImpl.class); + + stmt.execute("UPSERT INTO " + view02 + + + " (col2, vcol2, col5, col6) values ('0002', 'vcol2_02', 'col5_02', 'col6_02')"); + stmt.execute("UPSERT INTO " + view02 + + + " (col2, vcol2, col5, col6) values ('0003', 'vcol2_03', 'col5_03', 'col6_03')"); + stmt.execute("UPSERT INTO " + view01 + " (col2, vcol1, col3, col4, col5) values " + + "('0004', 'vcol2', 'col3_04', 'col4_04', 'col5_04')"); + stmt.execute("UPSERT INTO " + view01 + " (col2, vcol1, col3, col4, col5) values " + + "('0005', 'vcol-2', 'col3_05', 'col4_05', 'col5_05')"); + stmt.execute("UPSERT INTO " + view01 + " (col2, vcol1, col3, col4, col5) values " + + "('0006', 'vcol-1', 'col3_06', 'col4_06', 'col5_06')"); + stmt.execute("UPSERT INTO " + view01 + " (col2, vcol1, col3, col4, col5) values " + + "('0007', 'vcol1', 'col3_07', 'col4_07', 'col5_07')"); + stmt.execute("UPSERT INTO " + view02 + + + " (col2, vcol2, col5, col6) values ('0008', 'vcol2_08', 'col5_08', 'col6_02')"); + conn.commit(); + + final Statement statement = conn.createStatement(); + ResultSet rs = + statement.executeQuery( + "SELECT COL2, VCOL1, VCOL2, COL5, COL6 FROM " + view02); + Assert.assertTrue(rs.next()); + Assert.assertTrue(rs.next()); + Assert.assertTrue(rs.next()); + Assert.assertTrue(rs.next()); + Assert.assertTrue(rs.next()); + Assert.assertFalse(rs.next()); + + String query = "SELECT COL1, COL2, COL3, COL4 FROM " + tableName + " WHERE COL3 = " + + "'col3_04'"; + rs = statement.executeQuery(query); + Assert.assertTrue(rs.next()); + Assert.assertFalse(rs.next()); + + ExplainPlan explainPlan = conn.prepareStatement(query) + .unwrap(PhoenixPreparedStatement.class).optimizeQuery() + .getExplainPlan(); + ExplainPlanAttributes explainPlanAttributes = + explainPlan.getPlanStepsAsAttributes(); + Assert.assertEquals(indexName, explainPlanAttributes.getTableName()); + + TestUtil.removeCoprocessor(conn, "SYSTEM.CATALOG", TestMetaDataEndpointImpl.class); + TestUtil.addCoprocessor(conn, "SYSTEM.CATALOG", MetaDataEndpointImpl.class); + } + } + + @Test + public void testUcfWithDisabledIndex() throws Throwable { + final String tableName = generateUniqueName(); + final String indexName = "IDX_" + tableName; + + Connection conn = DriverManager.getConnection(getUrl()); + try { + final Statement stmt = conn.createStatement(); + + stmt.execute("CREATE TABLE " + tableName + + " (COL1 CHAR(10) NOT NULL, COL2 CHAR(5) NOT NULL, COL3 VARCHAR," + + " COL4 VARCHAR CONSTRAINT pk PRIMARY KEY(COL1, COL2))" + + " UPDATE_CACHE_FREQUENCY=10000"); + stmt.execute("CREATE INDEX " + indexName + + " ON " + tableName + " (COL3) INCLUDE (COL4)"); + + stmt.execute("ALTER INDEX " + indexName + " ON " + tableName + " DISABLE"); + + Thread.sleep(11000); + stmt.execute("UPSERT INTO " + tableName + + " (col1, col2, col3, col4) values ('c001', " + + "'c002', 'c003', 'c004')"); + conn.commit(); + + TestUtil.removeCoprocessor(conn, "SYSTEM.CATALOG", MetaDataEndpointImpl.class); + TestUtil.addCoprocessor(conn, "SYSTEM.CATALOG", TestMetaDataEndpointImpl.class); + + stmt.execute("UPSERT INTO " + tableName + + " (col1, col2, col3, col4) values ('c011', " + + "'c012', 'c013', 'c014')"); + conn.commit(); + throw new RuntimeException("Should not reach here"); + } catch (PhoenixIOException e) { Review Comment: Except for Index jobs, do we any way to directly work on Index PTable object, bypassing data table PTable? If we rely on index jobs, it would be difficult to simulate when to switch over to new test coproc in the middle of job execution right? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
