This is an automated email from the ASF dual-hosted git repository.
yanxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new cf630b5 PHOENIX-6172 Updating VIEW_INDEX_ID column type and ts in
Syscat with a 4.16 upgrade script
cf630b5 is described below
commit cf630b51df6f80f0247ad196a7e3f90c34427464
Author: Xinyi Yan <x...@salesforce.com>
AuthorDate: Fri Oct 16 21:41:18 2020 -0700
PHOENIX-6172 Updating VIEW_INDEX_ID column type and ts in Syscat with a
4.16 upgrade script
Signed-off-by: Xinyi Yan <yanxi...@apache.org>
---
.../java/org/apache/phoenix/end2end/UpgradeIT.java | 75 ++++++++++++++++++++++
.../phoenix/query/ConnectionQueryServicesImpl.java | 24 +++++++
.../java/org/apache/phoenix/util/UpgradeUtil.java | 40 ++++++++++++
3 files changed, 139 insertions(+)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
index d2bd456..4ea90b0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpgradeIT.java
@@ -17,7 +17,11 @@
*/
package org.apache.phoenix.end2end;
+import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
+import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME;
import static
org.apache.phoenix.thirdparty.com.google.common.base.Preconditions.checkNotNull;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
@@ -31,6 +35,7 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Types;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -40,6 +45,15 @@ import java.util.concurrent.FutureTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.hbase.TableName;
@@ -672,4 +686,65 @@ public class UpgradeIT extends ParallelStatsDisabledIT {
return key;
}
+ @Test
+ public void testUpgradeViewIndexIdDataType() throws Exception {
+ byte[] rowKey = SchemaUtil.getColumnKey(null,
+ SYSTEM_SCHEMA_NAME, SYSTEM_CATALOG_TABLE, VIEW_INDEX_ID,
+ PhoenixDatabaseMetaData.TABLE_FAMILY);
+ byte[] syscatBytes =
PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME.getBytes();
+ byte[] viewIndexIdTypeCellValueIn414 =
PInteger.INSTANCE.toBytes(Types.SMALLINT);
+ byte[] viewIndexIdTypeCellValueIn416 =
PInteger.INSTANCE.toBytes(Types.BIGINT);
+
+ try (PhoenixConnection conn = getConnection(false, null).
+ unwrap(PhoenixConnection.class)) {
+ // update the VIEW_INDEX_ID 0:DATAT_TYPE cell value to SMALLINT
+ // (4.14 and prior version is a SMALLINT column)
+ updateViewIndexIdColumnValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn414);
+
assertTrue(UpgradeUtil.isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded(
+ conn, rowKey, syscatBytes));
+ verifyExpectedCellValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn414);
+ // calling UpgradeUtil to mock the upgrade VIEW_INDEX_ID data type
to BIGINT
+ UpgradeUtil.updateViewIndexIdColumnDataTypeFromShortToLong(conn,
rowKey, syscatBytes);
+ verifyExpectedCellValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn416);
+
assertFalse(UpgradeUtil.isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded(
+ conn, rowKey, syscatBytes));
+ } finally {
+ updateViewIndexIdColumnValue(rowKey, syscatBytes,
viewIndexIdTypeCellValueIn416);
+ }
+ }
+
+ private void updateViewIndexIdColumnValue(byte[] rowKey, byte[]
syscatBytes,
+ byte[] newColumnValue) throws
Exception {
+
+ try (PhoenixConnection conn =
+
DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class);
+ Table sysTable = conn.getQueryServices().getTable(syscatBytes)) {
+ KeyValue viewIndexIdKV = new KeyValue(rowKey,
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP,
+ newColumnValue);
+ Put viewIndexIdPut = new Put(rowKey);
+ viewIndexIdPut.add(viewIndexIdKV);
+ sysTable.put(viewIndexIdPut);
+ }
+ }
+
+ private void verifyExpectedCellValue(byte[] rowKey, byte[] syscatBytes,
+ byte[] expectedDateTypeBytes) throws
Exception {
+ try(PhoenixConnection conn = getConnection(false, null).
+ unwrap(PhoenixConnection.class);
+ Table sysTable = conn.getQueryServices().getTable(syscatBytes)) {
+ Scan s = new Scan();
+ s.setRowPrefixFilter(rowKey);
+ s.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ ResultScanner scanner = sysTable.getScanner(s);
+ Result result= scanner.next();
+ Cell cell = result.getColumnLatestCell(
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ assertArrayEquals(expectedDateTypeBytes,
CellUtil.cloneValue(cell));
+ }
+ }
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index ea46cd6..92794dc 100644
---
a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++
b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -61,6 +61,7 @@ import static
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_FOR_MUTEX;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
import static
org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_HCONNECTIONS_COUNTER;
import static
org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_PHOENIX_CONNECTIONS_THROTTLED_COUNTER;
import static
org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_SERVICES_COUNTER;
@@ -285,6 +286,7 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.ServerUtil;
+import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TimeKeeper;
import org.apache.phoenix.util.UpgradeUtil;
import org.slf4j.Logger;
@@ -3746,6 +3748,28 @@ public class ConnectionQueryServicesImpl extends
DelegateQueryServices implement
MIN_SYSTEM_TABLE_TIMESTAMP_4_16_0,
PhoenixDatabaseMetaData.PHOENIX_TTL_HWM + " "
+ PInteger.INSTANCE.getSqlTypeName());
+
+ boolean isNamespaceMapping =
+ SchemaUtil.isNamespaceMappingEnabled(null,
getConfiguration());
+ String tableName = PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
+ if (isNamespaceMapping) {
+ tableName = tableName.replace(
+ QueryConstants.NAME_SEPARATOR,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ }
+ byte[] tableBytes = StringUtil.toBytes(tableName);
+ byte[] rowKey = SchemaUtil.getColumnKey(null,
+ QueryConstants.SYSTEM_SCHEMA_NAME,
+ SYSTEM_CATALOG_TABLE, VIEW_INDEX_ID,
+ PhoenixDatabaseMetaData.TABLE_FAMILY);
+ if
(UpgradeUtil.isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded
+ (metaConnection, rowKey, tableBytes)) {
+ LOGGER.info("Updating VIEW_INDEX_ID data type to BIGINT.");
+ UpgradeUtil.updateViewIndexIdColumnDataTypeFromShortToLong(
+ metaConnection, rowKey, tableBytes);
+ } else {
+ LOGGER.info("Updating VIEW_INDEX_ID data type is not needed.");
+ }
}
return metaConnection;
}
diff --git
a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index 907bddd..9e0f5bf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -63,6 +63,7 @@ import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Types;
import java.text.Format;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -2451,4 +2452,43 @@ public class UpgradeUtil {
public static void doNotUpgradeOnFirstConnection(Properties props) {
props.setProperty(DO_NOT_UPGRADE, String.valueOf(true));
}
+
+ public static boolean
isUpdateViewIndexIdColumnDataTypeFromShortToLongNeeded(
+ PhoenixConnection metaConnection, byte[] rowKey, byte[]
syscatBytes) {
+ try (Table sysTable =
metaConnection.getQueryServices().getTable(syscatBytes)) {
+ Scan s = new Scan();
+ s.setRowPrefixFilter(rowKey);
+ s.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ ResultScanner scanner = sysTable.getScanner(s);
+ Result result = scanner.next();
+ Cell cell = result.getColumnLatestCell(
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES);
+ return Bytes.compareTo(CellUtil.cloneValue(cell),
+ PInteger.INSTANCE.toBytes(Types.SMALLINT)) == 0 ? true :
false;
+ } catch (Exception e) {
+ LOGGER.error(String.format(
+ "Checking VIEW_INDEX_ID data type for upgrade failed: %s.
", e.getMessage()));
+ }
+ return false;
+ }
+
+ public static void updateViewIndexIdColumnDataTypeFromShortToLong(
+ PhoenixConnection metaConnection, byte[] rowKey, byte[]
syscatBytes) {
+ try(Table sysTable =
metaConnection.getQueryServices().getTable(syscatBytes)) {
+ KeyValue viewIndexIdKV = new KeyValue(rowKey,
+ PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ PhoenixDatabaseMetaData.DATA_TYPE_BYTES,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP,
+ PInteger.INSTANCE.toBytes(Types.BIGINT));
+ Put viewIndexIdPut = new Put(rowKey);
+ viewIndexIdPut.add(viewIndexIdKV);
+ sysTable.put(viewIndexIdPut);
+ LOGGER.info("Updated VIEW_INDEX_ID data type from SMALLINT TO
BIGINT.");
+ } catch (Exception e) {
+ LOGGER.error(String.format(
+ "Upgrade/change VIEW_INDEX_ID data type failed: %s.
",e.getMessage()));
+ }
+ }
}
