Repository: phoenix Updated Branches: refs/heads/encodecolumns2 5469fd4cb -> 31a371312
PHOENIX-3447 Add upgrade code to add the required metadata columns for supporting column encoding Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/31a37131 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/31a37131 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/31a37131 Branch: refs/heads/encodecolumns2 Commit: 31a371312fc61e06cac61ad629ad28401a4168a1 Parents: 5469fd4 Author: Samarth <[email protected]> Authored: Fri Jan 13 14:01:45 2017 -0800 Committer: Samarth <[email protected]> Committed: Fri Jan 13 14:01:45 2017 -0800 ---------------------------------------------------------------------- .../phoenix/coprocessor/MetaDataProtocol.java | 4 +- .../expression/ArrayColumnExpression.java | 1 - .../query/ConnectionQueryServicesImpl.java | 133 ++++++++++++++++++- .../tuple/EncodedColumnQualiferCellsList.java | 7 +- 4 files changed, 138 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a37131/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java index 83290db..dd445ce 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java @@ -85,8 +85,9 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 = MIN_TABLE_TIMESTAMP + 18; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1 = MIN_TABLE_TIMESTAMP + 18; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0 = MIN_TABLE_TIMESTAMP + 20; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 = MIN_TABLE_TIMESTAMP + 25; // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants - public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0; // ALWAYS update this map whenever rolling out a new release (major, minor or patch release). // Key is the SYSTEM.CATALOG timestamp for the version and value is the version string. @@ -101,6 +102,7 @@ public abstract class MetaDataProtocol extends MetaDataService { TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, "4.7.x"); TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0, "4.8.x"); TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, "4.9.x"); + TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0, "4.10.x"); } public static final String CURRENT_CLIENT_VERSION = PHOENIX_MAJOR_VERSION + "." + PHOENIX_MINOR_VERSION + "." + PHOENIX_PATCH_NUMBER; http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a37131/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java index e52f667..330aee0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/ArrayColumnExpression.java @@ -36,7 +36,6 @@ import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PArrayDataType; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PVarbinary; -import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.SchemaUtil; import com.google.common.base.Preconditions; http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a37131/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 5a00372..076b788 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -24,9 +24,32 @@ import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MAJOR_VERS import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MINOR_VERSION; import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_PATCH_NUMBER; import static org.apache.phoenix.coprocessor.MetaDataProtocol.getVersion; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IS_VIEW_REFERENCED; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.KEY_SEQ; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_STATS_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; +import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_ENABLED; import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE; @@ -40,6 +63,7 @@ import java.lang.ref.WeakReference; import java.sql.PreparedStatement; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -94,7 +118,6 @@ import org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto; import org.apache.hadoop.hbase.regionserver.IndexHalfStoreFileReaderGenerator; -import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; @@ -160,11 +183,13 @@ import org.apache.phoenix.schema.ColumnAlreadyExistsException; import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.EmptySequenceCacheException; import org.apache.phoenix.schema.FunctionNotFoundException; +import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.MetaDataSplitPolicy; import org.apache.phoenix.schema.NewerSchemaAlreadyExistsException; import org.apache.phoenix.schema.NewerTableAlreadyExistsException; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnFamily; +import org.apache.phoenix.schema.PColumnImpl; import org.apache.phoenix.schema.PMetaData; import org.apache.phoenix.schema.PMetaDataImpl; import org.apache.phoenix.schema.PName; @@ -178,6 +203,7 @@ import org.apache.phoenix.schema.SaltingUtil; import org.apache.phoenix.schema.Sequence; import org.apache.phoenix.schema.SequenceAllocation; import org.apache.phoenix.schema.SequenceKey; +import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableAlreadyExistsException; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableProperty; @@ -187,7 +213,9 @@ import org.apache.phoenix.schema.types.PBoolean; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PTinyint; import org.apache.phoenix.schema.types.PUnsignedTinyint; +import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.Closeables; @@ -2688,6 +2716,31 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0); clearCache(); } + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) { + metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3); + metaConnection = addColumnsIfNotExists( + metaConnection, + PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2, + PhoenixDatabaseMetaData.STORAGE_SCHEME + " " + + PTinyint.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists( + metaConnection, + PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1, + PhoenixDatabaseMetaData.ENCODING_SCHEME + " " + + PTinyint.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists( + metaConnection, + PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0, + PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " " + + PInteger.INSTANCE.getSqlTypeName()); + ConnectionQueryServicesImpl.this.removeTable(null, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0); + clearCache(); + } } @@ -2825,6 +2878,84 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } } + + // Special method for adding the column qualifier column for 4.10. + private PhoenixConnection addColumnQualifierColumn(PhoenixConnection oldMetaConnection, Long timestamp) throws SQLException { + Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo()); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp)); + // Cannot go through DriverManager or you end up in an infinite loop because it'll call init again + PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props); + PTable sysCatalogPTable = metaConnection.getTable(new PTableKey(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)); + int numColumns = sysCatalogPTable.getColumns().size(); + try (PreparedStatement mutateTable = metaConnection.prepareStatement(MetaDataClient.MUTATE_TABLE)) { + mutateTable.setString(1, null); + mutateTable.setString(2, SYSTEM_CATALOG_SCHEMA); + mutateTable.setString(3, SYSTEM_CATALOG_TABLE); + mutateTable.setString(4, PTableType.SYSTEM.getSerializedValue()); + mutateTable.setLong(5, sysCatalogPTable.getSequenceNumber() + 1); + mutateTable.setInt(6, numColumns + 1); + mutateTable.execute(); + } + List<Mutation> tableMetadata = new ArrayList<>(); + tableMetadata.addAll(metaConnection.getMutationState().toMutations(metaConnection.getSCN()).next().getSecond()); + metaConnection.rollback(); + PColumn column = new PColumnImpl(PNameFactory.newName("COLUMN_QUALIFIER"), + PNameFactory.newName(DEFAULT_COLUMN_FAMILY_NAME), PVarbinary.INSTANCE, null, null, true, numColumns, + SortOrder.ASC, null, null, false, null, false, false, + Bytes.toBytes("COLUMN_QUALIFIER")); + String upsertColumnMetadata = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + + TENANT_ID + "," + + TABLE_SCHEM + "," + + TABLE_NAME + "," + + COLUMN_NAME + "," + + COLUMN_FAMILY + "," + + DATA_TYPE + "," + + NULLABLE + "," + + COLUMN_SIZE + "," + + DECIMAL_DIGITS + "," + + ORDINAL_POSITION + "," + + SORT_ORDER + "," + + DATA_TABLE_NAME + "," + + ARRAY_SIZE + "," + + VIEW_CONSTANT + "," + + IS_VIEW_REFERENCED + "," + + PK_NAME + "," + + KEY_SEQ + "," + + COLUMN_DEF + "," + + IS_ROW_TIMESTAMP + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + try (PreparedStatement colUpsert = metaConnection.prepareStatement(upsertColumnMetadata)) { + colUpsert.setString(1, null); + colUpsert.setString(2, SYSTEM_CATALOG_SCHEMA); + colUpsert.setString(3, SYSTEM_CATALOG_TABLE); + colUpsert.setString(4, "COLUMN_QUALIFIER"); + colUpsert.setString(5, DEFAULT_COLUMN_FAMILY); + colUpsert.setInt(6, column.getDataType().getSqlType()); + colUpsert.setInt(7, ResultSetMetaData.columnNullable); + colUpsert.setNull(8, Types.INTEGER); + colUpsert.setNull(9, Types.INTEGER); + colUpsert.setInt(10, sysCatalogPTable.getBucketNum() != null ? numColumns : (numColumns + 1)); + colUpsert.setInt(11, SortOrder.ASC.getSystemValue()); + colUpsert.setString(12, null); + colUpsert.setNull(13, Types.INTEGER); + colUpsert.setBytes(14, null); + colUpsert.setBoolean(15, false); + colUpsert.setString(16, sysCatalogPTable.getPKName() == null ? null : sysCatalogPTable.getPKName().getString()); + colUpsert.setNull(17, Types.SMALLINT); + colUpsert.setNull(18, Types.VARCHAR); + colUpsert.setBoolean(19, false); + colUpsert.execute(); + } + tableMetadata.addAll(metaConnection.getMutationState().toMutations(metaConnection.getSCN()).next().getSecond()); + metaConnection.rollback(); + metaConnection.getQueryServices().addColumn(tableMetadata, sysCatalogPTable, Collections.<String,List<Pair<String,Object>>>emptyMap(), Collections.<String>emptySet(), Lists.newArrayList(column)); + metaConnection.removeTable(null, SYSTEM_CATALOG_NAME, null, timestamp); + ConnectionQueryServicesImpl.this.removeTable(null, + SYSTEM_CATALOG_NAME, null, + timestamp); + clearCache(); + return metaConnection; + } private void createSnapshot(String snapshotName, String tableName) throws SQLException { http://git-wip-us.apache.org/repos/asf/phoenix/blob/31a37131/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java index 484aacb..d3065a7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java @@ -32,11 +32,10 @@ import javax.annotation.concurrent.NotThreadSafe; import org.apache.hadoop.hbase.Cell; import org.apache.phoenix.schema.PTable.QualifierEncodingScheme; -import org.apache.phoenix.schema.PTable.StorageScheme; /** * List implementation that provides indexed based look up when the cell column qualifiers are positive numbers. - * These qualifiers are generated by using one of the column qualifier encoding schemes specified in {@link StorageScheme}. + * These qualifiers are generated by using one of the column qualifier encoding schemes specified in {@link QualifierEncodingScheme}. * The api methods in this list assume that the caller wants to see * and add only non null elements in the list. * <p> @@ -49,7 +48,7 @@ import org.apache.phoenix.schema.PTable.StorageScheme; * is an O(n) operation. This makes iterating through the list using {@link #get(int)} an O(n^2) operation. * Instead, for iterating through the list, one should use the iterators created through {@link #iterator()} or * {@link #listIterator()}. Do note that getting an element using {@link #getCellForColumnQualifier(int)} is an O(1) operation - * and should generally be the way for getting elements out of the list. + * and should generally be the way for accessing elements in the list. * </p> */ @NotThreadSafe @@ -270,7 +269,7 @@ public class EncodedColumnQualiferCellsList implements List<Cell> { public Cell get(int index) { rangeCheck(index); int numNonNullElementsFound = 0; - for (int i = 0; i < array.length; i++) { + for (int i = firstNonNullElementIdx; i < array.length; i++) { if (array[i] != null) { numNonNullElementsFound++; if (numNonNullElementsFound == index + 1) {
