Repository: phoenix Updated Branches: refs/heads/4.x-HBase-0.98 366e7a779 -> 7e432be79
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index 523f684..dbc65d4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -41,7 +41,9 @@ import java.util.TreeSet; import javax.annotation.Nullable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.exception.SQLExceptionCode; @@ -52,14 +54,19 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.AmbiguousColumnException; import org.apache.phoenix.schema.ColumnFamilyNotFoundException; import org.apache.phoenix.schema.ColumnNotFoundException; +import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.PName; +import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.RowKeySchema.RowKeySchemaBuilder; import org.apache.phoenix.schema.SaltingUtil; @@ -115,6 +122,10 @@ public class SchemaUtil { }; public static final RowKeySchema VAR_BINARY_SCHEMA = new RowKeySchemaBuilder(1).addField(VAR_BINARY_DATUM, false, SortOrder.getDefault()).build(); + public static final String SCHEMA_FOR_DEFAULT_NAMESPACE = "DEFAULT"; + public static final String HBASE_NAMESPACE = "HBASE"; + public static final List<String> NOT_ALLOWED_SCHEMA_LIST = Arrays.asList(SCHEMA_FOR_DEFAULT_NAMESPACE, + HBASE_NAMESPACE); /** * May not be instantiated @@ -236,6 +247,12 @@ public class SchemaUtil { return ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY, functionName); } + public static byte[] getKeyForSchema(String tenantId, String schemaName) { + return ByteUtil.concat(tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(tenantId), + QueryConstants.SEPARATOR_BYTE_ARRAY, + schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName)); + } + public static byte[] getTableKey(String tenantId, String schemaName, String tableName) { return ByteUtil.concat(tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(tenantId), QueryConstants.SEPARATOR_BYTE_ARRAY, schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(tableName)); } @@ -253,6 +270,11 @@ public class SchemaUtil { return cf + QueryConstants.NAME_SEPARATOR + cq; } + private static String getName(String name, boolean caseSensitive) { + String cq = caseSensitive ? "\"" + name + "\"" : name; + return cq; + } + public static String getTableName(byte[] schemaName, byte[] tableName) { return getName(schemaName, tableName); } @@ -276,9 +298,13 @@ public class SchemaUtil { } return getName(familyName, columnName, false); } + if ((familyName == null || familyName.isEmpty()) && (columnName == null || columnName.isEmpty()) + && (tableName == null || tableName.equals(MetaDataClient.EMPTY_TABLE))) { return getName(schemaName, + false); } if ((familyName == null || familyName.isEmpty()) && (columnName == null || columnName.isEmpty())) { return getName(schemaName, tableName, false); } + return getName(getName(schemaName, tableName, false), getName(familyName, columnName, false), false); } @@ -414,19 +440,24 @@ public class SchemaUtil { } public static boolean isMetaTable(byte[] tableName) { - return Bytes.compareTo(tableName, SYSTEM_CATALOG_NAME_BYTES) == 0; + return Bytes.compareTo(tableName, SYSTEM_CATALOG_NAME_BYTES) == 0 || Bytes.compareTo(tableName, + SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true).getName()) == 0; } public static boolean isFunctionTable(byte[] tableName) { - return Bytes.compareTo(tableName, SYSTEM_FUNCTION_NAME_BYTES) == 0; + return Bytes.compareTo(tableName, SYSTEM_FUNCTION_NAME_BYTES) == 0 || Bytes.compareTo(tableName, + SchemaUtil.getPhysicalTableName(SYSTEM_FUNCTION_NAME_BYTES, true).getName()) == 0; } public static boolean isStatsTable(byte[] tableName) { - return Bytes.compareTo(tableName, SYSTEM_STATS_NAME_BYTES) == 0; + return Bytes.compareTo(tableName, SYSTEM_STATS_NAME_BYTES) == 0 || Bytes.compareTo(tableName, + SchemaUtil.getPhysicalTableName(SYSTEM_STATS_NAME_BYTES, true).getName()) == 0; } - + public static boolean isSequenceTable(byte[] tableName) { - return Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES) == 0; + return Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES) == 0 + || Bytes.compareTo(tableName, SchemaUtil + .getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES, true).getName()) == 0; } public static boolean isSequenceTable(PTable table) { @@ -564,9 +595,15 @@ public class SchemaUtil { return true; } } - + public static String getSchemaNameFromFullName(String tableName) { - int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR); + if (tableName + .contains(":")) { return getSchemaNameFromFullName(tableName, QueryConstants.NAMESPACE_SEPARATOR); } + return getSchemaNameFromFullName(tableName, QueryConstants.NAME_SEPARATOR); + } + + public static String getSchemaNameFromFullName(String tableName, String separator) { + int index = tableName.indexOf(separator); if (index < 0) { return StringUtil.EMPTY_STRING; } @@ -588,7 +625,8 @@ public class SchemaUtil { } int index = indexOf(tableName, QueryConstants.NAME_SEPARATOR_BYTE); if (index < 0) { - return StringUtil.EMPTY_STRING; + index = indexOf(tableName, QueryConstants.NAMESPACE_SEPARATOR_BYTE); + if (index < 0) { return StringUtil.EMPTY_STRING; } } return Bytes.toString(tableName, 0, index); } @@ -599,13 +637,19 @@ public class SchemaUtil { } int index = indexOf(tableName, QueryConstants.NAME_SEPARATOR_BYTE); if (index < 0) { - return Bytes.toString(tableName); + index = indexOf(tableName, QueryConstants.NAMESPACE_SEPARATOR_BYTE); + if (index < 0) { return Bytes.toString(tableName); } } return Bytes.toString(tableName, index+1, tableName.length - index - 1); } public static String getTableNameFromFullName(String tableName) { - int index = tableName.indexOf(QueryConstants.NAME_SEPARATOR); + if (tableName.contains(":")) { return getTableNameFromFullName(tableName, QueryConstants.NAMESPACE_SEPARATOR); } + return getTableNameFromFullName(tableName, QueryConstants.NAME_SEPARATOR); + } + + public static String getTableNameFromFullName(String tableName, String separator) { + int index = tableName.indexOf(separator); if (index < 0) { return tableName; } @@ -615,7 +659,8 @@ public class SchemaUtil { public static byte[] getTableKeyFromFullName(String fullTableName) { int index = fullTableName.indexOf(QueryConstants.NAME_SEPARATOR); if (index < 0) { - return getTableKey(null, null, fullTableName); + index = fullTableName.indexOf(QueryConstants.NAMESPACE_SEPARATOR); + if (index < 0) { return getTableKey(null, null, fullTableName); } } String schemaName = fullTableName.substring(0, index); String tableName = fullTableName.substring(index+1); @@ -897,4 +942,78 @@ public class SchemaUtil { PName schemaName = dataTable.getSchemaName(); return getTableKey(tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : tenantId.getBytes(), schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : schemaName.getBytes(), dataTable.getTableName().getBytes()); } + + public static byte[] getSchemaKey(String schemaName) { + return SchemaUtil.getTableKey(null, schemaName, MetaDataClient.EMPTY_TABLE); + } + + public static PName getPhysicalHBaseTableName(PName pName, boolean isNamespaceMapped, PTableType type) { + return getPhysicalHBaseTableName(pName.toString(), isNamespaceMapped, type); + } + + public static PName getPhysicalHBaseTableName(byte[] tableName, boolean isNamespaceMapped, PTableType type) { + return getPhysicalHBaseTableName(Bytes.toString(tableName), isNamespaceMapped, type); + } + + public static TableName getPhysicalTableName(String fullTableName, ReadOnlyProps readOnlyProps) { + return getPhysicalName(Bytes.toBytes(fullTableName), readOnlyProps); + } + + public static TableName getPhysicalTableName(byte[] fullTableName, Configuration conf) { + return getPhysicalTableName(fullTableName, isNamespaceMappingEnabled( + isSystemTable(fullTableName) ? PTableType.SYSTEM : null, new ReadOnlyProps(conf.iterator()))); + } + + public static TableName getPhysicalName(byte[] fullTableName, ReadOnlyProps readOnlyProps) { + return getPhysicalTableName(fullTableName, + isNamespaceMappingEnabled(isSystemTable(fullTableName) ? PTableType.SYSTEM : null, readOnlyProps)); + } + + public static TableName getPhysicalTableName(byte[] fullTableName, boolean isNamespaceMappingEnabled) { + if (indexOf(fullTableName, QueryConstants.NAMESPACE_SEPARATOR_BYTE) > 0 + || !isNamespaceMappingEnabled) { return TableName.valueOf(fullTableName); } + String tableName = getTableNameFromFullName(fullTableName); + String schemaName = getSchemaNameFromFullName(fullTableName); + return TableName.valueOf(schemaName, tableName); + } + + public static PName getPhysicalHBaseTableName(String tableName, boolean isNamespaceMapped, PTableType type) { + if (!isNamespaceMapped) { return PNameFactory.newName(tableName); } + return PNameFactory + .newName(tableName.replace(QueryConstants.NAME_SEPARATOR, QueryConstants.NAMESPACE_SEPARATOR)); + } + + public static boolean isSchemaCheckRequired(PTableType tableType, ReadOnlyProps props) { + return PTableType.TABLE.equals(tableType) && isNamespaceMappingEnabled(tableType, props); + } + + public static boolean isNamespaceMappingEnabled(PTableType type, ReadOnlyProps readOnlyProps) { + return readOnlyProps.getBoolean(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, + QueryServicesOptions.DEFAULT_IS_NAMESPACE_MAPPING_ENABLED) + && (type == null || !PTableType.SYSTEM.equals(type) + || readOnlyProps.getBoolean(QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE, + QueryServicesOptions.DEFAULT_IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE)); + } + + public static byte[] getParentTableNameFromIndexTable(byte[] physicalTableName, String indexPrefix) { + String tableName = Bytes.toString(physicalTableName); + return getParentTableNameFromIndexTable(tableName, indexPrefix).getBytes(); + } + + public static String getParentTableNameFromIndexTable(String physicalTableName, String indexPrefix) { + if (physicalTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) { + String schemaNameFromFullName = getSchemaNameFromFullName(physicalTableName, + QueryConstants.NAMESPACE_SEPARATOR); + String tableNameFromFullName = getTableNameFromFullName(physicalTableName, + QueryConstants.NAMESPACE_SEPARATOR); + return schemaNameFromFullName + QueryConstants.NAMESPACE_SEPARATOR + + getStrippedName(tableNameFromFullName, indexPrefix); + } + return getStrippedName(physicalTableName, indexPrefix); + } + + private static String getStrippedName(String physicalTableName, String indexPrefix) { + return physicalTableName.indexOf(indexPrefix) == 0 ? physicalTableName.substring(indexPrefix.length()) + : physicalTableName; + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index de1a2c7..b82e0c1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -28,8 +28,11 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT; import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT; @@ -67,12 +70,17 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataEndpointImpl; import org.apache.phoenix.coprocessor.MetaDataProtocol; +import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; +import org.apache.phoenix.coprocessor.MetaDataProtocol.MutationCode; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; @@ -81,6 +89,7 @@ import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.SaltingUtil; import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.types.PBinary; import org.apache.phoenix.schema.types.PBoolean; import org.apache.phoenix.schema.types.PChar; @@ -124,6 +133,22 @@ public class UpgradeUtil { + "TABLE_NAME = ? " ; + private static final String UPDATE_LINK = + "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + + TENANT_ID + "," + + TABLE_SCHEM + "," + + TABLE_NAME + "," + + COLUMN_FAMILY + "," + + LINK_TYPE + "," + + TABLE_SEQ_NUM + + ") SELECT " + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + ",'%s' AS " + + COLUMN_FAMILY + " ," + LINK_TYPE + "," + TABLE_SEQ_NUM + " FROM " + SYSTEM_CATALOG_SCHEMA + ".\"" + + SYSTEM_CATALOG_TABLE + "\" WHERE " + COLUMN_FAMILY + "=? AND " + LINK_TYPE + " = " + + LinkType.PHYSICAL_TABLE.getSerializedValue(); + + private static final String DELETE_LINK = "DELETE FROM " + SYSTEM_CATALOG_SCHEMA + "." + SYSTEM_CATALOG_TABLE + + " WHERE " + COLUMN_FAMILY + "=? AND " + LINK_TYPE + " = " + LinkType.PHYSICAL_TABLE.getSerializedValue(); + private UpgradeUtil() { } @@ -1279,4 +1304,156 @@ public class UpgradeUtil { } return false; } + + private static void mapTableToNamespace(HBaseAdmin admin, HTableInterface metatable, String srcTableName, + String destTableName, ReadOnlyProps props, Long ts, String phoenixTableName, PTableType pTableType) + throws SnapshotCreationException, IllegalArgumentException, IOException, InterruptedException, + SQLException { + srcTableName = SchemaUtil.normalizeIdentifier(srcTableName); + if (!SchemaUtil.isNamespaceMappingEnabled(pTableType, + props)) { throw new IllegalArgumentException(SchemaUtil.isSystemTable(srcTableName.getBytes()) + ? "For system table " + QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE + + " also needs to be enabled along with " + QueryServices.IS_NAMESPACE_MAPPING_ENABLED + : QueryServices.IS_NAMESPACE_MAPPING_ENABLED + " is not enabled"); } + // we need to move physical table in actual namespace for TABLE and Index + if (admin.tableExists(srcTableName) && (PTableType.TABLE.equals(pTableType) + || PTableType.INDEX.equals(pTableType) || PTableType.SYSTEM.equals(pTableType))) { + String snapshotName = QueryConstants.UPGRADE_TABLE_SNAPSHOT_PREFIX + srcTableName; + logger.info("Disabling table " + srcTableName + " .."); + admin.disableTable(srcTableName); + logger.info(String.format("Taking snapshot %s of table %s..", snapshotName, srcTableName)); + admin.snapshot(snapshotName, srcTableName); + logger.info(String.format("Restoring snapshot %s in destination table %s..", snapshotName, destTableName)); + admin.cloneSnapshot(Bytes.toBytes(snapshotName), Bytes.toBytes(destTableName)); + logger.info(String.format("deleting old table %s..", srcTableName)); + admin.deleteTable(srcTableName); + logger.info(String.format("deleting snapshot %s..", snapshotName)); + admin.deleteSnapshot(snapshotName); + } + // Update flag to represent table is mapped to namespace + logger.info(String.format("Updating meta information of phoenix table '%s' to map to namespace..", phoenixTableName)); + Put put = new Put(SchemaUtil.getTableKey(null, SchemaUtil.getSchemaNameFromFullName(phoenixTableName), + SchemaUtil.getTableNameFromFullName(phoenixTableName)), ts); + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED_BYTES, + PBoolean.INSTANCE.toBytes(Boolean.TRUE)); + metatable.put(put); + } + + /* + * Method to map existing phoenix table to a namespace. Should not be use if tables has views and indexes ,instead + * use map table utility in psql.py + */ + public static void mapTableToNamespace(HBaseAdmin admin, HTableInterface metatable, String tableName, + ReadOnlyProps props, Long ts, PTableType pTableType) throws SnapshotCreationException, + IllegalArgumentException, IOException, InterruptedException, SQLException { + String destTablename = SchemaUtil + .normalizeIdentifier(SchemaUtil.getPhysicalTableName(tableName, props).getNameAsString()); + mapTableToNamespace(admin, metatable, tableName, destTablename, props, ts, tableName, pTableType); + } + + public static void upgradeTable(PhoenixConnection conn, String srcTable) throws SQLException, + SnapshotCreationException, IllegalArgumentException, IOException, InterruptedException { + ReadOnlyProps readOnlyProps = conn.getQueryServices().getProps(); + if (conn.getClientInfo(PhoenixRuntime.TENANT_ID_ATTRIB) != null) { throw new SQLException( + "May not specify the TENANT_ID_ATTRIB property when upgrading"); } + if (conn.getSchema() != null) { throw new IllegalArgumentException( + "Schema should not be set for connection!!"); } + try (HBaseAdmin admin = conn.getQueryServices().getAdmin(); + HTableInterface metatable = conn.getQueryServices() + .getTable(SchemaUtil + .getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, readOnlyProps) + .getName());) { + String tableName = SchemaUtil.normalizeIdentifier(srcTable); + String schemaName = SchemaUtil.getSchemaNameFromFullName(tableName); + + // Upgrade is not required if schemaName is not present. + if (schemaName.equals("")) { throw new IllegalArgumentException("Table doesn't have schema name"); } + + // Confirm table is not already upgraded + PTable table = PhoenixRuntime.getTable(conn, tableName); + if (table.isNamespaceMapped()) { throw new IllegalArgumentException("Table is already upgraded"); } + logger.info(String.format("Creating schema %s..", schemaName)); + conn.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + schemaName); + String newPhysicalTablename = SchemaUtil.normalizeIdentifier(SchemaUtil + .getPhysicalTableName(table.getPhysicalName().getString(), readOnlyProps).getNameAsString()); + logger.info(String.format("Upgrading %s %s..", table.getType(), tableName)); + // Upgrade the data or main table + mapTableToNamespace(admin, metatable, tableName, newPhysicalTablename, readOnlyProps, + PhoenixRuntime.getCurrentScn(readOnlyProps), tableName, table.getType()); + + // clear the cache and get new table + conn.getQueryServices().clearTableFromCache(ByteUtil.EMPTY_BYTE_ARRAY, table.getSchemaName().getBytes(), + table.getTableName().getBytes(), PhoenixRuntime.getCurrentScn(readOnlyProps)); + MetaDataMutationResult result = new MetaDataClient(conn).updateCache(schemaName, + SchemaUtil.getTableNameFromFullName(tableName)); + if (result.getMutationCode() != MutationCode.TABLE_ALREADY_EXISTS) { throw new TableNotFoundException( + tableName); } + table = result.getTable(); + // check whether table is properly upgraded before upgrading indexes + if (table.isNamespaceMapped()) { + for (PTable index : table.getIndexes()) { + String srcTableName = index.getPhysicalName().getString(); + if (srcTableName.contains(QueryConstants.NAMESPACE_SEPARATOR) + || (!MetaDataUtil.isViewIndex(srcTableName) && PTableType.VIEW.equals(table.getType()))) { + // this condition occurs in case of multiple views on same table + // as all view indexes uses the same physical table, so if one view is already migrated then we + // can skip migrating the physical table again + logger.info(String.format("skipping as it seems index '%s' is already upgraded..", index.getName())); + continue; + } + String destTableName = null; + String phoenixTableName = index.getName().getString(); + boolean updateLink = false; + if (MetaDataUtil.isLocalIndex(srcTableName)) { + logger.info(String.format("local index '%s' found with physical hbase table name ''..", + index.getName(), srcTableName)); + destTableName = Bytes + .toString(MetaDataUtil.getLocalIndexPhysicalName(newPhysicalTablename.getBytes())); + // update parent_table property in local index table descriptor + conn.createStatement() + .execute(String.format("ALTER TABLE %s set " + MetaDataUtil.PARENT_TABLE_KEY + "='%s'", + phoenixTableName, table.getPhysicalName())); + updateLink = true; + } else if (MetaDataUtil.isViewIndex(srcTableName)) { + logger.info(String.format("View index '%s' found with physical hbase table name ''..", + index.getName(), srcTableName)); + destTableName = Bytes + .toString(MetaDataUtil.getViewIndexPhysicalName(newPhysicalTablename.getBytes())); + updateLink = true; + } else { + logger.info(String.format("Global index '%s' found with physical hbase table name ''..", + index.getName(), srcTableName)); + destTableName = SchemaUtil + .getPhysicalTableName(index.getPhysicalName().getString(), readOnlyProps) + .getNameAsString(); + } + logger.info(String.format("Upgrading index %s..", index.getName())); + mapTableToNamespace(admin, metatable, srcTableName, destTableName, readOnlyProps, + PhoenixRuntime.getCurrentScn(readOnlyProps), phoenixTableName, index.getType()); + if (updateLink) { + logger.info(String.format("Updating link information for index '%s' ..", index.getName())); + updateLink(conn, srcTableName, destTableName); + } + conn.getQueryServices().clearTableFromCache(ByteUtil.EMPTY_BYTE_ARRAY, + index.getSchemaName().getBytes(), index.getTableName().getBytes(), + PhoenixRuntime.getCurrentScn(readOnlyProps)); + } + + } else { + throw new RuntimeException("Error: problem occured during upgrade. Table is not upgraded successfully"); + } + } + } + + private static void updateLink(PhoenixConnection conn, String srcTableName, String destTableName) + throws SQLException { + PreparedStatement deleteLinkStatment = conn.prepareStatement(DELETE_LINK); + deleteLinkStatment.setString(1, srcTableName); + PreparedStatement updateLinkStatment = conn.prepareStatement(String.format(UPDATE_LINK, destTableName)); + updateLinkStatment.setString(1, srcTableName); + deleteLinkStatment.execute(); + updateLinkStatment.execute(); + conn.commit(); + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java index 6d0930d..4660e38 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java @@ -256,7 +256,7 @@ public class CorrelatePlanTest { PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.<PTable>emptyList(), false, Collections.<PName>emptyList(), null, null, false, false, false, null, - null, null, true, false, 0, 0L); + null, null, true, false, 0, 0L, Boolean.FALSE); TableRef sourceTable = new TableRef(pTable); List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java index 8ecf0ee..aba4be4 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java @@ -177,7 +177,7 @@ public class LiteralResultIteratorPlanTest { PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false, - false, null, null, null, true, false, 0, 0L); + false, null, null, null, true, false, 0, 0L, false); TableRef sourceTable = new TableRef(pTable); List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index e3bd4a0..d8651ae 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -118,9 +118,7 @@ import java.util.concurrent.TimeoutException; import javax.annotation.Nonnull; - import org.apache.commons.lang.RandomStringUtils; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -128,7 +126,6 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.IntegrationTestingUtility; -import org.apache.hadoop.hbase.RandomStringGeneratorImpl; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; @@ -147,6 +144,7 @@ import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer; import org.apache.phoenix.hbase.index.master.IndexMasterObserver; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixDriver; import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver; @@ -829,6 +827,7 @@ public abstract class BaseTest { protected static void ensureTableCreated(String url, String tableName, byte[][] splits, Long ts) throws SQLException { String ddl = tableDDLMap.get(tableName); + createSchema(url,tableName, ts); createTestTable(url, ddl, splits, ts); } @@ -843,7 +842,23 @@ public abstract class BaseTest { protected static void createTestTable(String url, String ddl, byte[][] splits, Long ts) throws SQLException { createTestTable(url, ddl, splits, ts, true); } - + + public static void createSchema(String url, String tableName, Long ts) throws SQLException { + String schema = SchemaUtil.getSchemaNameFromFullName(tableName); + if (!schema.equals("")) { + Properties props = new Properties(); + if (ts != null) { + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); + } + try (Connection conn = DriverManager.getConnection(url, props);) { + if (SchemaUtil.isNamespaceMappingEnabled(null, + conn.unwrap(PhoenixConnection.class).getQueryServices().getProps())) { + conn.createStatement().executeUpdate("CREATE SCHEMA IF NOT EXISTS " + schema); + } + } + } + } + protected static void createTestTable(String url, String ddl, byte[][] splits, Long ts, boolean swallowTableAlreadyExistsException) throws SQLException { assertNotNull(ddl); StringBuilder buf = new StringBuilder(ddl); http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java index ff06ff9..47101b2 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ParallelIteratorsSplitTest.java @@ -54,6 +54,7 @@ import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; import org.apache.phoenix.parse.FilterableStatement; import org.apache.phoenix.parse.PFunction; +import org.apache.phoenix.parse.PSchema; import org.apache.phoenix.parse.SelectStatement; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.PDatum; @@ -348,6 +349,16 @@ public class ParallelIteratorsSplitTest extends BaseConnectionlessQueryTest { public boolean hasUDFs() { return false; } + + @Override + public PSchema resolveSchema(String schemaName) throws SQLException { + return null; + } + + @Override + public List<PSchema> getSchemas() { + return null; + } }; PhoenixConnection connection = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class); final PhoenixStatement statement = new PhoenixStatement(connection); http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-core/src/test/java/org/apache/phoenix/util/JDBCUtilTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/JDBCUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/JDBCUtilTest.java index 74b397f..8c270c3 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/JDBCUtilTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/JDBCUtilTest.java @@ -21,6 +21,7 @@ import static org.apache.phoenix.util.PhoenixRuntime.ANNOTATION_ATTRIB_PREFIX; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.util.Map; @@ -89,6 +90,13 @@ public class JDBCUtilTest { } @Test + public void testSchema() { + assertTrue(JDBCUtil.getSchema("localhost;schema=TEST", new Properties(), null).equals("TEST")); + assertNull(JDBCUtil.getSchema("localhost;schema=", new Properties(), null)); + assertNull(JDBCUtil.getSchema("localhost;", new Properties(), null)); + } + + @Test public void testGetAutoCommit_TrueInProperties() { Properties props = new Properties(); props.setProperty("AutoCommit", "true"); http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-protocol/src/main/MetaDataService.proto ---------------------------------------------------------------------- diff --git a/phoenix-protocol/src/main/MetaDataService.proto b/phoenix-protocol/src/main/MetaDataService.proto index d1dcf9a..9d7b209 100644 --- a/phoenix-protocol/src/main/MetaDataService.proto +++ b/phoenix-protocol/src/main/MetaDataService.proto @@ -26,6 +26,7 @@ option optimize_for = SPEED; import "PTable.proto"; import "PFunction.proto"; +import "PSchema.proto"; enum MutationCode { TABLE_ALREADY_EXISTS = 0; @@ -42,6 +43,12 @@ enum MutationCode { FUNCTION_NOT_FOUND = 11; NEWER_FUNCTION_FOUND = 12; FUNCTION_NOT_IN_REGION = 13; + SCHEMA_ALREADY_EXISTS = 14; + NEWER_SCHEMA_FOUND = 15; + SCHEMA_NOT_FOUND = 16; + SCHEMA_NOT_IN_REGION = 17; + TABLES_EXIST_ON_SCHEMA = 18; + UNALLOWED_SCHEMA_MUTATION = 19; }; message SharedTableState { @@ -63,6 +70,7 @@ message MetaDataResponse { optional bytes functionName = 7; repeated PFunction function = 8; repeated SharedTableState sharedTablesToDelete = 9; + optional PSchema schema = 10; } message GetTableRequest { @@ -82,6 +90,12 @@ message GetFunctionsRequest { optional int32 clientVersion = 5; } +message GetSchemaRequest { + required string schemaName = 1; + required int64 clientTimestamp = 2; + required int32 clientVersion = 3; +} + // each byte array represents a MutationProto instance message CreateTableRequest { repeated bytes tableMetadataMutations = 1; @@ -96,6 +110,12 @@ message CreateFunctionRequest { optional int32 clientVersion = 4; } +message CreateSchemaRequest { + repeated bytes tableMetadataMutations = 1; + required string schemaName = 2; + required int32 clientVersion = 3; +} + message DropTableRequest { repeated bytes tableMetadataMutations = 1; required string tableType = 2; @@ -103,6 +123,12 @@ message DropTableRequest { optional int32 clientVersion = 4; } +message DropSchemaRequest { + repeated bytes schemaMetadataMutations = 1; + required string schemaName = 2; + required int32 clientVersion = 3; +} + message AddColumnRequest { repeated bytes tableMetadataMutations = 1; optional int32 clientVersion = 2; @@ -158,15 +184,24 @@ service MetaDataService { rpc getFunctions(GetFunctionsRequest) returns (MetaDataResponse); + rpc getSchema(GetSchemaRequest) + returns (MetaDataResponse); + rpc createTable(CreateTableRequest) returns (MetaDataResponse); rpc createFunction(CreateFunctionRequest) returns (MetaDataResponse); - + + rpc createSchema(CreateSchemaRequest) + returns (MetaDataResponse); + rpc dropTable(DropTableRequest) returns (MetaDataResponse); - + + rpc dropSchema(DropSchemaRequest) + returns (MetaDataResponse); + rpc dropFunction(DropFunctionRequest) returns (MetaDataResponse); http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-protocol/src/main/PSchema.proto ---------------------------------------------------------------------- diff --git a/phoenix-protocol/src/main/PSchema.proto b/phoenix-protocol/src/main/PSchema.proto new file mode 100644 index 0000000..91e12f59 --- /dev/null +++ b/phoenix-protocol/src/main/PSchema.proto @@ -0,0 +1,30 @@ +/* + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.phoenix.coprocessor.generated"; +option java_outer_classname = "PSchemaProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message PSchema { + required string schemaName = 1; + required int64 timeStamp = 2; +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/7e432be7/phoenix-protocol/src/main/PTable.proto ---------------------------------------------------------------------- diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 09bdeb6..c86f6b6 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -90,4 +90,5 @@ message PTable { optional bool transactional = 27; optional int64 updateCacheFrequency = 28; optional int64 indexDisableTimestamp = 29; + optional bool isNamespaceMapped = 30; }
