This is an automated email from the ASF dual-hosted git repository.
yanxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new ad5e61c PHOENIX-5712 Got SYSCAT ILLEGAL_DATA exception after created
tenant index on view
ad5e61c is described below
commit ad5e61c3427c4c91ea91e0970361f844f293b27e
Author: Xinyi Yan <[email protected]>
AuthorDate: Wed Dec 2 22:19:23 2020 -0800
PHOENIX-5712 Got SYSCAT ILLEGAL_DATA exception after created tenant index
on view
Signed-off-by: Xinyi Yan <[email protected]>
---
.../phoenix/end2end/BackwardCompatibilityIT.java | 34 ++++
.../end2end/BackwardCompatibilityTestUtil.java | 3 +
.../phoenix/end2end/ViewIndexIdRetrieveIT.java | 175 +++++++++++++++++++++
.../resources/gold_files/gold_query_view_index.txt | 20 +++
.../src/it/resources/sql_files/add_view_index.sql | 21 +++
.../it/resources/sql_files/query_view_index.sql | 19 +++
.../coprocessor/SystemCatalogRegionObserver.java | 58 +++++++
.../filter/SystemCatalogViewIndexIdFilter.java | 156 ++++++++++++++++++
.../phoenix/query/ConnectionQueryServicesImpl.java | 22 +++
.../phoenix/util/ViewIndexIdRetrieveUtil.java | 66 ++++++++
10 files changed, 574 insertions(+)
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
index 238c574..5158a60 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.end2end;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.ADD_DATA;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.ADD_DELETE;
+import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.ADD_VIEW_INDEX;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_ADD;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_DIVERGED_VIEW;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.INDEX_REBUILD_ASYNC;
@@ -27,6 +28,7 @@ import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_ADD
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_CREATE_ADD;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_CREATE_DIVERGED_VIEW;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_INDEX_REBUILD_ASYNC;
+import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_VIEW_INDEX;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.assertExpectedOutput;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.checkForPreConditions;
import static
org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.computeClientVersions;
@@ -46,9 +48,11 @@ import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.phoenix.coprocessor.SystemCatalogRegionObserver;
import org.apache.phoenix.coprocessor.TaskMetaDataEndpoint;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixDriver;
@@ -377,4 +381,34 @@ public class BackwardCompatibilityIT {
}
}
+ @Test
+ public void testViewIndexIdCreatedWithOldClient() throws Exception {
+ executeQueryWithClientVersion(compatibleClientVersion, ADD_VIEW_INDEX,
zkQuorum);
+ try (org.apache.hadoop.hbase.client.Connection conn =
+ hbaseTestUtil.getConnection(); Admin admin =
conn.getAdmin()) {
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(
+
TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME));
+ assertFalse("Coprocessor " +
SystemCatalogRegionObserver.class.getName()
+ + " has been added with compatible client version: "
+ + compatibleClientVersion, tableDescriptor.hasCoprocessor(
+ SystemCatalogRegionObserver.class.getName()));
+
+ executeQueriesWithCurrentVersion(QUERY_VIEW_INDEX, url, NONE);
+ assertExpectedOutput(QUERY_VIEW_INDEX);
+
+ tableDescriptor = admin.getTableDescriptor(
+
TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME));
+ assertTrue("Coprocessor " +
SystemCatalogRegionObserver.class.getName()
+ + " has been added with compatible client version: "
+ + compatibleClientVersion, tableDescriptor.hasCoprocessor(
+ SystemCatalogRegionObserver.class.getName()));
+ }
+ }
+
+ @Test
+ public void testViewIndexIdCreatedWithNewClient() throws Exception {
+ executeQueriesWithCurrentVersion(ADD_VIEW_INDEX, url, NONE);
+ executeQueryWithClientVersion(compatibleClientVersion,
QUERY_VIEW_INDEX, zkQuorum);
+ assertExpectedOutput(QUERY_VIEW_INDEX);
+ }
}
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
index 300fbcc..049b6f0 100644
---
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
@@ -75,7 +75,9 @@ public final class BackwardCompatibilityTestUtil {
public static final String CREATE_DIVERGED_VIEW = "create_diverged_view";
public static final String ADD_DATA = "add_data";
public static final String ADD_DELETE = "add_delete";
+ public static final String ADD_VIEW_INDEX = "add_view_index";
public static final String DELETE = "delete";
+ public static final String VIEW_INDEX = "view_index";
public static final String SELECT_AND_DROP_TABLE = "select_and_drop_table";
public static final String QUERY_CREATE_ADD = QUERY_PREFIX + CREATE_ADD;
public static final String QUERY_ADD_DATA = QUERY_PREFIX + ADD_DATA;
@@ -83,6 +85,7 @@ public final class BackwardCompatibilityTestUtil {
public static final String QUERY_DELETE = QUERY_PREFIX + DELETE;
public static final String QUERY_SELECT_AND_DROP_TABLE = QUERY_PREFIX +
SELECT_AND_DROP_TABLE;
public static final String QUERY_CREATE_DIVERGED_VIEW = QUERY_PREFIX +
CREATE_DIVERGED_VIEW;
+ public static final String QUERY_VIEW_INDEX = QUERY_PREFIX + VIEW_INDEX;
public static final String INDEX_REBUILD_ASYNC = "index_rebuild_async";
public static final String QUERY_INDEX_REBUILD_ASYNC = QUERY_PREFIX
+ INDEX_REBUILD_ASYNC;
diff --git
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIndexIdRetrieveIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIndexIdRetrieveIT.java
new file mode 100644
index 0000000..486a535
--- /dev/null
+++
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIndexIdRetrieveIT.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE;
+import static
org.apache.phoenix.query.QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.sql.Types;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/*
+ After 4.15 release, Phoenix introduced VIEW_INDEX_ID_COLUMN_TYPE and
changed VIEW_INDEX_ID data
+ type from SMALLINT to BIGINT. However, SELECT from syscat doesn't have the
right view index id
+ because the VIEW_INDEX_ID column always assume the data type is BIGINT.
PHOENIX-5712 introduced
+ a coproc that checks the client request version and send it back to the
client.
+ For more information, please see PHOENIX-3547, PHOENIX-5712
+ */
+public class ViewIndexIdRetrieveIT extends BaseUniqueNamesOwnClusterIT {
+ private final String BASE_TABLE_DDL = "CREATE TABLE %s (TENANT_ID CHAR(15)
NOT NULL, " +
+ "ID CHAR(3) NOT NULL, NUM BIGINT CONSTRAINT PK PRIMARY KEY
(TENANT_ID, ID))" +
+ " MULTI_TENANT = true, COLUMN_ENCODED_BYTES=0 ";
+ private final String VIEW_DDL = "CREATE VIEW %s (A BIGINT PRIMARY KEY, B
BIGINT)" +
+ " AS SELECT * FROM %s WHERE ID='ABC'";
+ private final String VIEW_INDEX_DDL =
+ "CREATE INDEX %s ON %s (B DESC) INCLUDE (NUM)";
+ private final String SELECT_ALL = "SELECT * FROM SYSTEM.CATALOG";
+ private final String SELECT_ROW = "SELECT
VIEW_INDEX_ID,VIEW_INDEX_ID_DATA_TYPE" +
+ " FROM SYSTEM.CATALOG WHERE TABLE_NAME='%s' AND COLUMN_COUNT IS
NOT NULL";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Test
+ public void testSelectViewIndexIdAsLong() throws Exception {
+ testSelectViewIndexId(true);
+ }
+
+ @Test
+ public void testSelectViewIndexIdAsShort() throws Exception {
+ testSelectViewIndexId(false);
+ }
+
+ private void testSelectViewIndexId(boolean isTestingLongViewIndexId)
throws Exception {
+ String val = isTestingLongViewIndexId ? "true" : "false";
+ int expectedDataType = isTestingLongViewIndexId ? Types.BIGINT :
Types.SMALLINT;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(LONG_VIEW_INDEX_ENABLED_ATTRIB, val);
+ String schema = generateUniqueName();
+ String baseTable = generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(schema, baseTable);
+ String viewName = generateUniqueName();
+ String viewFullName = SchemaUtil.getTableName(schema, viewName);
+ String viewIndexName = generateUniqueName();
+ try (final Connection conn = DriverManager.getConnection(url,props);
+ final Statement stmt = conn.createStatement()) {
+ stmt.execute(String.format(BASE_TABLE_DDL, fullTableName));
+ stmt.execute(String.format(VIEW_DDL, viewFullName, fullTableName));
+ stmt.execute(String.format(VIEW_INDEX_DDL, viewIndexName,
viewFullName));
+
+ ResultSet rs = stmt.executeQuery(String.format(SELECT_ROW,
viewIndexName));
+ rs.next();
+ // even we enabled longViewIndex config, but the sequence always
starts at smallest short
+ assertEquals(Short.MIN_VALUE,rs.getLong(1));
+ assertEquals(expectedDataType, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testMixedCase() throws Exception {
+ // mixed case
+ Properties propsForLongType = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ propsForLongType.setProperty(LONG_VIEW_INDEX_ENABLED_ATTRIB, "true");
+ String schema = generateUniqueName();
+ String baseTable = generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(schema, baseTable);
+ String viewName = generateUniqueName();
+ String viewFullName = SchemaUtil.getTableName(schema, viewName);
+ String viewIndexName1 = generateUniqueName();
+
+ // view index id data type is long
+ try (final Connection conn =
DriverManager.getConnection(url,propsForLongType);
+ final Statement stmt = conn.createStatement()) {
+ stmt.execute(String.format(BASE_TABLE_DDL, fullTableName));
+ stmt.execute(String.format(VIEW_DDL, viewFullName, fullTableName));
+ stmt.execute(String.format(VIEW_INDEX_DDL, viewIndexName1,
viewFullName));
+
+ ResultSet rs = stmt.executeQuery(String.format(SELECT_ROW,
viewIndexName1));
+ rs.next();
+ assertEquals(Short.MIN_VALUE,rs.getLong(1));
+ assertEquals(Types.BIGINT, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+
+ // view index id data type is short
+ String viewIndexName2 = generateUniqueName();
+ Properties propsForShortType =
PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ propsForShortType.setProperty(LONG_VIEW_INDEX_ENABLED_ATTRIB, "false");
+ try (final Connection conn =
DriverManager.getConnection(url,propsForShortType);
+ final Statement stmt = conn.createStatement()) {
+ stmt.execute(String.format(VIEW_INDEX_DDL, viewIndexName2,
viewFullName));
+
+ ResultSet rs = stmt.executeQuery(String.format(SELECT_ROW,
viewIndexName2));
+ rs.next();
+ assertEquals(Short.MIN_VALUE + 1,rs.getLong(1));
+ assertEquals(Types.SMALLINT, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+
+ // check select * from syscat
+ try (final Connection conn = DriverManager.getConnection(url);
+ final Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery(String.format(SELECT_ALL));
+ boolean checkShort = false;
+ boolean checkLong = false;
+ while (rs.next()) {
+
+ String schemaName = rs.getString(TABLE_SCHEM);
+ long viewIndexId = rs.getLong(VIEW_INDEX_ID);
+ if (schemaName != null && schemaName.equals(schema) &&
viewIndexId != 0) {
+ int viewIndexIdDataType =
rs.getInt(VIEW_INDEX_ID_DATA_TYPE);
+ String tableName = rs.getString(TABLE_NAME);
+ if (tableName.equals(viewIndexName1)) {
+ assertEquals(Short.MIN_VALUE, viewIndexId);
+ assertEquals(Types.BIGINT, viewIndexIdDataType);
+ checkLong = true;
+ } else if (tableName.equals(viewIndexName2)) {
+ assertEquals(Short.MIN_VALUE + 1, viewIndexId);
+ assertEquals(Types.SMALLINT, viewIndexIdDataType);
+ checkShort = true;
+ }
+ }
+ }
+ assertTrue(checkLong);
+ assertTrue(checkShort);
+ }
+ }
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/resources/gold_files/gold_query_view_index.txt
b/phoenix-core/src/it/resources/gold_files/gold_query_view_index.txt
new file mode 100644
index 0000000..5ea0549
--- /dev/null
+++ b/phoenix-core/src/it/resources/gold_files/gold_query_view_index.txt
@@ -0,0 +1,20 @@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ 'VIEW_INDEX_ID'
+'-32768'
\ No newline at end of file
diff --git a/phoenix-core/src/it/resources/sql_files/add_view_index.sql
b/phoenix-core/src/it/resources/sql_files/add_view_index.sql
new file mode 100644
index 0000000..7cbfac2
--- /dev/null
+++ b/phoenix-core/src/it/resources/sql_files/add_view_index.sql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE MY_SCHEMA.VIEW_INDEX_BASE_TABLE (TENANT_ID CHAR(15) NOT NULL, ID
CHAR(3) NOT NULL, NUM BIGINT CONSTRAINT PK PRIMARY KEY (TENANT_ID, ID))
MULTI_TENANT = true;
+CREATE VIEW MY_SCHEMA.GLOBAL_VIEW (A BIGINT PRIMARY KEY, B BIGINT) AS SELECT *
FROM MY_SCHEMA.VIEW_INDEX_BASE_TABLE WHERE ID='ABC';
+CREATE INDEX MY_SCHEMA_VIEW_INDEX ON MY_SCHEMA.GLOBAL_VIEW (B DESC) INCLUDE
(NUM);
\ No newline at end of file
diff --git a/phoenix-core/src/it/resources/sql_files/query_view_index.sql
b/phoenix-core/src/it/resources/sql_files/query_view_index.sql
new file mode 100644
index 0000000..9f30f05
--- /dev/null
+++ b/phoenix-core/src/it/resources/sql_files/query_view_index.sql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+SELECT VIEW_INDEX_ID FROM SYSTEM.CATALOG WHERE
TABLE_NAME='MY_SCHEMA_VIEW_INDEX' AND TABLE_SCHEM='MY_SCHEMA' AND COLUMN_COUNT
IS NOT NULL;
\ No newline at end of file
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/SystemCatalogRegionObserver.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/SystemCatalogRegionObserver.java
new file mode 100644
index 0000000..19ebe4a
--- /dev/null
+++
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/SystemCatalogRegionObserver.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.coprocessor;
+
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.phoenix.filter.SystemCatalogViewIndexIdFilter;
+import org.apache.phoenix.util.ScanUtil;
+
+import java.io.IOException;
+
+import static org.apache.phoenix.util.ScanUtil.UNKNOWN_CLIENT_VERSION;
+
+/**
+ * Coprocessor that checks whether the VIEW_INDEX_ID needs to retrieve.
+ */
+public class SystemCatalogRegionObserver extends BaseRegionObserver {
+ @Override public void start(CoprocessorEnvironment e) throws IOException {
+ super.start(e);
+ }
+
+ @Override public void stop(CoprocessorEnvironment e) throws IOException {
+ super.stop(e);
+ }
+
+ @Override
+ public void preScannerOpen(ObserverContext<RegionCoprocessorEnvironment>
e, Scan scan)
+ throws IOException {
+ int clientVersion = ScanUtil.getClientVersion(scan);
+ /*
+ ScanUtil.getClientVersion returns UNKNOWN_CLIENT_VERSION if the
phoenix client version
+ isn't set. We only want to retrieve the data based on the client
version, and we don't
+ want to change the behavior other than Phoenix env.
+ */
+ if (clientVersion != UNKNOWN_CLIENT_VERSION) {
+ ScanUtil.andFilterAtBeginning(scan, new
SystemCatalogViewIndexIdFilter(clientVersion));
+ }
+ }
+}
\ No newline at end of file
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/filter/SystemCatalogViewIndexIdFilter.java
b/phoenix-core/src/main/java/org/apache/phoenix/filter/SystemCatalogViewIndexIdFilter.java
new file mode 100644
index 0000000..dbd230c
--- /dev/null
+++
b/phoenix-core/src/main/java/org/apache/phoenix/filter/SystemCatalogViewIndexIdFilter.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Writable;
+import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
+import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.PhoenixKeyValueUtil;
+import org.apache.phoenix.util.ViewIndexIdRetrieveUtil;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.sql.Types;
+import java.util.Collections;
+import java.util.List;
+
+
+import static
org.apache.phoenix.coprocessor.MetaDataProtocol.MIN_SPLITTABLE_SYSTEM_CATALOG;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE_BYTES;
+import static
org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
+import static
org.apache.phoenix.util.ViewIndexIdRetrieveUtil.NULL_DATA_TYPE_VALUE;
+import static
org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN;
+import static
org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_SMALLINT_TYPE_VALUE_LEN;
+
+public class SystemCatalogViewIndexIdFilter extends FilterBase implements
Writable {
+ private int clientVersion;
+
+ public SystemCatalogViewIndexIdFilter() {
+ }
+
+ public SystemCatalogViewIndexIdFilter(int clientVersion) {
+ this.clientVersion = clientVersion;
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(Cell keyValue) {
+ return ReturnCode.INCLUDE_AND_NEXT_COL;
+ }
+
+ @Override
+ public boolean hasFilterRow() {
+ return true;
+ }
+
+ @Override
+ public void filterRowCells(List<Cell> kvs) throws IOException {
+ Cell viewIndexIdCell = PhoenixKeyValueUtil.getColumnLatest(
+ GenericKeyValueBuilder.INSTANCE, kvs,
+ DEFAULT_COLUMN_FAMILY_BYTES, VIEW_INDEX_ID_BYTES);
+
+ /*
+ We retrieve the VIEW_INDEX_ID cell from SMALLINT to BIGINT or
BIGINT to SMALLINT if and
+ only if VIEW_INDEX_ID is included as part of the projected column.
+ This is combination of diff client created view index looks like:
+ client VIEW_INDEX_ID(Cell number of bytes)
VIEW_INDEX_ID_DATA_TYPE
+ pre-4.15 2 bytes
NULL
+ post-4.15[config smallint] 2 bytes
5(smallint)
+ post-4.15[config bigint] 8 bytes
-5(bigint)
+ */
+ if (viewIndexIdCell != null) {
+ int type = NULL_DATA_TYPE_VALUE;
+ Cell viewIndexIdDataTypeCell = PhoenixKeyValueUtil.getColumnLatest(
+ GenericKeyValueBuilder.INSTANCE, kvs,
+ DEFAULT_COLUMN_FAMILY_BYTES,
VIEW_INDEX_ID_DATA_TYPE_BYTES);
+ if (viewIndexIdDataTypeCell != null) {
+ type = (Integer) PInteger.INSTANCE.toObject(
+ viewIndexIdDataTypeCell.getValueArray(),
+ viewIndexIdDataTypeCell.getValueOffset(),
+ viewIndexIdDataTypeCell.getValueLength(),
+ PInteger.INSTANCE,
+ SortOrder.ASC);
+ }
+ if (this.clientVersion < MIN_SPLITTABLE_SYSTEM_CATALOG) {
+ /*
+ For pre-4.15 client select query cannot include
VIEW_INDEX_ID_DATA_TYPE as part
+ of the projected columns; for this reason, the TYPE will
always be NULL. Since
+ the pre-4.15 client always assume the VIEW_INDEX_ID column
is type of SMALLINT,
+ we need to retrieve the BIGINT cell to SMALLINT cell.
+ VIEW_INDEX_ID_DATA_TYPE, VIEW_INDEX_ID(Cell
representation of the data)
+ NULL, SMALLINT -> DO
NOT CONVERT
+ SMALLINT, SMALLINT -> DO
NOT CONVERT
+ BIGINT, BIGINT ->
RETRIEVE AND SEND SMALLINT BACK
+ */
+ if (type == NULL_DATA_TYPE_VALUE &&
viewIndexIdCell.getValueLength() >
+ VIEW_INDEX_ID_SMALLINT_TYPE_VALUE_LEN) {
+ Cell keyValue = ViewIndexIdRetrieveUtil.
+ getRetrievedViewIndexIdCell(viewIndexIdCell,
false);
+ Collections.replaceAll(kvs, viewIndexIdCell, keyValue);
+ }
+ } else {
+ /*
+ For post-4.15 client select query needs to include
VIEW_INDEX_ID_DATA_TYPE as
+ part of the projected columns, and VIEW_INDEX_ID depends
on it.
+ VIEW_INDEX_ID_DATA_TYPE, VIEW_INDEX_ID(Cell
representation of the data)
+ NULL, SMALLINT ->
RETRIEVE AND SEND BIGINT BACK
+ SMALLINT, SMALLINT ->
RETRIEVE AND SEND BIGINT BACK
+ BIGINT, BIGINT -> DO
NOT RETRIEVE
+ */
+ if (type != Types.BIGINT && viewIndexIdCell.getValueLength() <
+ VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN) {
+ Cell keyValue = ViewIndexIdRetrieveUtil.
+ getRetrievedViewIndexIdCell(viewIndexIdCell, true);
+ Collections.replaceAll(kvs, viewIndexIdCell, keyValue);
+ }
+ }
+ }
+ }
+
+ public static SystemCatalogViewIndexIdFilter parseFrom(final byte []
pbBytes) throws DeserializationException {
+ try {
+ SystemCatalogViewIndexIdFilter writable =
(SystemCatalogViewIndexIdFilter)
+ Writables.getWritable(pbBytes, new
SystemCatalogViewIndexIdFilter());
+ return writable;
+ } catch (IOException e) {
+ throw new DeserializationException(e);
+ }
+ }
+
+ @Override
+ public byte[] toByteArray() throws IOException {
+ return Writables.getBytes(this);
+ }
+
+ @Override
+ public void readFields(DataInput input) throws IOException {
+ this.clientVersion = input.readInt();
+ }
+
+ @Override
+ public void write(DataOutput output) throws IOException {
+ output.writeInt(this.clientVersion);
+ }
+
+}
\ No newline at end of file
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index a3d096e..037c8d8 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -122,6 +122,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
@@ -134,6 +135,7 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
@@ -171,6 +173,7 @@ import org.apache.phoenix.coprocessor.ScanRegionObserver;
import org.apache.phoenix.coprocessor.SequenceRegionObserver;
import org.apache.phoenix.coprocessor.ServerCachingEndpointImpl;
import org.apache.phoenix.coprocessor.PhoenixTTLRegionObserver;
+import org.apache.phoenix.coprocessor.SystemCatalogRegionObserver;
import org.apache.phoenix.coprocessor.TaskMetaDataEndpoint;
import org.apache.phoenix.coprocessor.TaskRegionObserver;
import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
@@ -1139,6 +1142,12 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
PhoenixTTLRegionObserver.class.getName(), null,
priority-2, null);
}
}
+ if (Arrays.equals(tableName, SYSTEM_CATALOG_NAME_BYTES)) {
+ if
(!newDesc.hasCoprocessor(SystemCatalogRegionObserver.class.getName())) {
+ builder.addCoprocessor(
+ SystemCatalogRegionObserver.class.getName(), null,
priority, null);
+ }
+ }
} catch (IOException e) {
throw ServerUtil.parseServerException(e);
@@ -3796,6 +3805,19 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
} else {
LOGGER.info("Updating VIEW_INDEX_ID data type is not needed.");
}
+ try (Admin admin = metaConnection.getQueryServices().getAdmin()) {
+ HTableDescriptor htd;
+ TableName syscatPhysicalTableName =
SchemaUtil.getPhysicalTableName(
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, props);
+ htd = admin.getTableDescriptor(syscatPhysicalTableName);
+ if
(!htd.hasCoprocessor(SystemCatalogRegionObserver.class.getName())) {
+ int priority =
props.getInt(QueryServices.COPROCESSOR_PRIORITY_ATTRIB,
+ QueryServicesOptions.DEFAULT_COPROCESSOR_PRIORITY);
+
htd.addCoprocessor(SystemCatalogRegionObserver.class.getName(), null, priority,
null);
+ admin.modifyTable(syscatPhysicalTableName, htd);
+ pollForUpdatedTableDescriptor(admin, htd,
syscatPhysicalTableName.getName());
+ }
+ }
}
return metaConnection;
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/util/ViewIndexIdRetrieveUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/util/ViewIndexIdRetrieveUtil.java
new file mode 100644
index 0000000..d5b0fea
--- /dev/null
+++
b/phoenix-core/src/main/java/org/apache/phoenix/util/ViewIndexIdRetrieveUtil.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PSmallint;
+
+public final class ViewIndexIdRetrieveUtil {
+ public static final int VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN = 9;
+ public static final int VIEW_INDEX_ID_SMALLINT_TYPE_VALUE_LEN = 3;
+ public static final int NULL_DATA_TYPE_VALUE = 0;
+
+ private ViewIndexIdRetrieveUtil() {
+
+ }
+
+ public static Cell buildNewCell(Cell viewIndexIdCell, byte[] newVal) {
+ KeyValue keyValue = new KeyValue(
+ viewIndexIdCell.getRowArray(), viewIndexIdCell.getRowOffset(),
+ viewIndexIdCell.getRowLength(),
+ viewIndexIdCell.getFamilyArray(),
viewIndexIdCell.getFamilyOffset(),
+ viewIndexIdCell.getFamilyLength(),
+ viewIndexIdCell.getQualifierArray(),
viewIndexIdCell.getQualifierOffset(),
+ viewIndexIdCell.getQualifierLength(),
+ viewIndexIdCell.getTimestamp(),KeyValue.Type.Put,
+ newVal, 0,newVal.length);
+ keyValue.setSequenceId(viewIndexIdCell.getSequenceId());
+ return keyValue;
+ }
+
+ public static Cell getRetrievedViewIndexIdCell(Cell viewIndexIdCell,
boolean isShortToLong) {
+
+ ImmutableBytesWritable columnValue =
+ new
ImmutableBytesWritable(CellUtil.cloneValue(viewIndexIdCell));
+ ImmutableBytesWritable newValue = new ImmutableBytesWritable();
+
+ byte[] newBytes;
+
+ if (isShortToLong) {
+ newBytes =
PLong.INSTANCE.toBytes(PSmallint.INSTANCE.toObject(columnValue.get()));
+ } else {
+ newBytes =
PSmallint.INSTANCE.toBytes(PLong.INSTANCE.toObject(columnValue.get()));
+ }
+ newValue.set(newBytes);
+ return buildNewCell(viewIndexIdCell, newValue.get());
+ }
+}
\ No newline at end of file
