ChinmaySKulkarni commented on a change in pull request #972: URL: https://github.com/apache/phoenix/pull/972#discussion_r530697105
########## File path: phoenix-core/src/main/java/org/apache/phoenix/coprocessor/SyscatRegionObserver.java ########## @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.coprocessor; + +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.phoenix.filter.SystemCatalogViewIndexIdFilter; +import org.apache.phoenix.util.ScanUtil; + +import java.io.IOException; + +import static org.apache.phoenix.util.ScanUtil.UNKNOWN_CLIENT_VERSION; + +/** + * Coprocessor that checks whether the VIEW_INDEX_ID needs to retrieve. + */ +public class SyscatRegionObserver extends BaseRegionObserver { + @Override public void start(CoprocessorEnvironment e) throws IOException { + super.start(e); + } + + @Override public void stop(CoprocessorEnvironment e) throws IOException { + super.stop(e); + } + + @Override + public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, + RegionScanner s) throws IOException { + int clientVersion = ScanUtil.getClientVersion(scan); + /* + ScanUtil.getClientVersion returns UNKNOWN_CLIENT_VERSION if the phoenix client version + didn't set. We only want to retrieve the data based on the client version, and we don't Review comment: nit: *isn't ########## File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIndexIdRetrieveIT.java ########## @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE; +import static org.apache.phoenix.query.QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB; +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.Types; +import java.util.Map; +import java.util.Properties; + +import com.google.common.collect.Maps; +import org.apache.hadoop.hbase.regionserver.ScanInfoUtil; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +/* + After 4.15 release, Phoenix introduced VIEW_INDEX_ID_COLUMN_TYPE and changed VIEW_INDEX_ID data + type from SMALLINT to BIGINT. However, SELECT from syscat doesn't have the right view index id + because the VIEW_INDEX_ID column always assume the data type is BIGINT. PHOENIX-5712 introduced + a coproc that checks the client request version and send it back to the client. + For more information, please see PHOENIX-3547, PHOENIX-5712 + */ +public class ViewIndexIdRetrieveIT extends BaseOwnClusterIT { Review comment: The comment in `BaseOwnClusterIT` says "Any new integration tests that need their own mini cluster should be extending {@link BaseUniqueNamesOwnClusterIT} class directly" so maybe we want to do that and then use uniqueNames for all table/view/index names etc. ########## File path: phoenix-core/src/main/java/org/apache/phoenix/filter/SystemCatalogViewIndexIdFilter.java ########## @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.filter; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.io.Writable; +import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.util.KeyValueUtil; +import org.apache.phoenix.util.ViewIndexIdRetrieveUtil; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.sql.Types; +import java.util.Collections; +import java.util.List; + + +import static org.apache.phoenix.coprocessor.MetaDataProtocol.MIN_SPLITTABLE_SYSTEM_CATALOG; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE_BYTES; +import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.NULL_DATA_TYPE_VALUE; +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN; +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_SMALLINT_TYPE_VALUE_LEN; + +public class SystemCatalogViewIndexIdFilter extends FilterBase implements Writable { + private int clientVersion; + + public SystemCatalogViewIndexIdFilter() { + } + + public SystemCatalogViewIndexIdFilter(int clientVersion) { + this.clientVersion = clientVersion; + } + + @Override + public ReturnCode filterKeyValue(Cell keyValue) { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + + @Override + public boolean hasFilterRow() { + return true; + } + + @Override + public void filterRowCells(List<Cell> kvs) throws IOException { + Cell viewIndexIdCell = KeyValueUtil.getColumnLatest( + GenericKeyValueBuilder.INSTANCE, kvs, + DEFAULT_COLUMN_FAMILY_BYTES, VIEW_INDEX_ID_BYTES); + + /* + We retrieve the VIEW_INDEX_ID cell from SMALLINT to BIGINT or BIGINT to SMALLINT if and + only if VIEW_INDEX_ID is included as part of the projected column. + + This is combination of diff client created view index looks like: + + client VIEW_INDEX_ID(Cell number of bytes) VIEW_INDEX_ID_DATA_TYPE + pre-4.14 2 bytes NULL + post-4.15[config smallint] 2 bytes 5(smallint) + post-4.15[config bigint] 8 bytes -5(bigint) + */ + if (viewIndexIdCell != null) { + int type = NULL_DATA_TYPE_VALUE; + Cell viewIndexIdDataTypeCell = KeyValueUtil.getColumnLatest( + GenericKeyValueBuilder.INSTANCE, kvs, + DEFAULT_COLUMN_FAMILY_BYTES, VIEW_INDEX_ID_DATA_TYPE_BYTES); + if (viewIndexIdDataTypeCell != null) { + type = (Integer) PInteger.INSTANCE.toObject( + viewIndexIdDataTypeCell.getValueArray(), + viewIndexIdDataTypeCell.getValueOffset(), + viewIndexIdDataTypeCell.getValueLength(), + PInteger.INSTANCE, + SortOrder.ASC); + } + if (this.clientVersion < MIN_SPLITTABLE_SYSTEM_CATALOG) { + /* + For pre-4.15 client select query cannot include VIEW_INDEX_ID_DATA_TYPE as part + of the projected columns; for this reason, the TYPE will always be NULL. Since + the pre-4.15 client always assume the VIEW_INDEX_ID column is type of SMALLINT, + we need to retrieve the BIGINT cell to SMALLINT cell. + + VIEW_INDEX_ID_DATA_TYPE, VIEW_INDEX_ID(Cell representation of the data) + NULL, SMALLINT -> DO NOT RETRIEVE Review comment: By DO NOT RETRIEVE, you mean DO NOT CONVERT right? Can you please clarify this in comments? ########## File path: phoenix-core/src/main/java/org/apache/phoenix/filter/SystemCatalogViewIndexIdFilter.java ########## @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.filter; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.filter.FilterBase; +import org.apache.hadoop.hbase.util.Writables; +import org.apache.hadoop.io.Writable; +import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.util.KeyValueUtil; +import org.apache.phoenix.util.ViewIndexIdRetrieveUtil; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.sql.Types; +import java.util.Collections; +import java.util.List; + + +import static org.apache.phoenix.coprocessor.MetaDataProtocol.MIN_SPLITTABLE_SYSTEM_CATALOG; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE_BYTES; +import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES; +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.NULL_DATA_TYPE_VALUE; +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN; +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_SMALLINT_TYPE_VALUE_LEN; + +public class SystemCatalogViewIndexIdFilter extends FilterBase implements Writable { + private int clientVersion; + + public SystemCatalogViewIndexIdFilter() { + } + + public SystemCatalogViewIndexIdFilter(int clientVersion) { + this.clientVersion = clientVersion; + } + + @Override + public ReturnCode filterKeyValue(Cell keyValue) { + return ReturnCode.INCLUDE_AND_NEXT_COL; + } + + @Override + public boolean hasFilterRow() { + return true; + } + + @Override + public void filterRowCells(List<Cell> kvs) throws IOException { + Cell viewIndexIdCell = KeyValueUtil.getColumnLatest( + GenericKeyValueBuilder.INSTANCE, kvs, + DEFAULT_COLUMN_FAMILY_BYTES, VIEW_INDEX_ID_BYTES); + + /* + We retrieve the VIEW_INDEX_ID cell from SMALLINT to BIGINT or BIGINT to SMALLINT if and + only if VIEW_INDEX_ID is included as part of the projected column. + + This is combination of diff client created view index looks like: + + client VIEW_INDEX_ID(Cell number of bytes) VIEW_INDEX_ID_DATA_TYPE + pre-4.14 2 bytes NULL Review comment: this should be pre-4.15 right? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
