Repository: phoenix Updated Branches: refs/heads/master 9a818dd43 -> 86fe2fc5d
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86fe2fc5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 08f7310..87925db 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -19,6 +19,8 @@ package org.apache.phoenix.query; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.MAX_VERSIONS; import static org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.TTL; +import static org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.REPLICATION_SCOPE; +import static org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS; import static org.apache.phoenix.coprocessor.MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP; import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MAJOR_VERSION; import static org.apache.phoenix.coprocessor.MetaDataProtocol.PHOENIX_MINOR_VERSION; @@ -63,6 +65,7 @@ import static org.apache.phoenix.util.UpgradeUtil.addViewIndexToParentLinks; import static org.apache.phoenix.util.UpgradeUtil.getSysCatalogSnapshotName; import static org.apache.phoenix.util.UpgradeUtil.moveChildLinks; import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0; +import static org.apache.phoenix.util.UpgradeUtil.syncTableAndIndexProperties; import java.io.IOException; import java.lang.management.ManagementFactory; @@ -103,9 +106,11 @@ import java.util.regex.Pattern; import javax.annotation.concurrent.GuardedBy; +import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.KeepDeletedCells; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.TableExistsException; @@ -782,7 +787,36 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement String defaultFamilyName = (String)tableProps.remove(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME); TableDescriptorBuilder tableDescriptorBuilder = (existingDesc != null) ?TableDescriptorBuilder.newBuilder(existingDesc) : TableDescriptorBuilder.newBuilder(TableName.valueOf(physicalTableName)); + + ColumnFamilyDescriptor dataTableColDescForIndexTablePropSyncing = null; + if (tableType == PTableType.INDEX || MetaDataUtil.isViewIndex(Bytes.toString(physicalTableName))) { + byte[] defaultFamilyBytes = + defaultFamilyName == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : Bytes.toBytes(defaultFamilyName); + + final TableDescriptor baseTableDesc; + if (MetaDataUtil.isViewIndex(Bytes.toString(physicalTableName))) { + // Handles indexes created on views for single-tenant tables and + // global indexes created on views of multi-tenant tables + baseTableDesc = this.getTableDescriptor(Bytes.toBytes(MetaDataUtil.getViewIndexUserTableName(Bytes.toString(physicalTableName)))); + } else if (existingDesc == null) { + // Global/local index creation on top of a physical base table + baseTableDesc = this.getTableDescriptor(SchemaUtil.getPhysicalTableName( + Bytes.toBytes((String) tableProps.get(PhoenixDatabaseMetaData.DATA_TABLE_NAME)), isNamespaceMapped) + .getName()); + } else { + // In case this a local index created on a view of a multi-tenant table, the + // DATA_TABLE_NAME points to the name of the view instead of the physical base table + baseTableDesc = existingDesc; + } + dataTableColDescForIndexTablePropSyncing = baseTableDesc.getColumnFamily(defaultFamilyBytes); + // It's possible that the table has specific column families and none of them are declared + // to be the DEFAULT_COLUMN_FAMILY, so we choose the first column family for syncing properties + if (dataTableColDescForIndexTablePropSyncing == null) { + dataTableColDescForIndexTablePropSyncing = baseTableDesc.getColumnFamilies()[0]; + } + } // By default, do not automatically rebuild/catch up an index on a write failure + // Add table-specific properties to the table descriptor for (Entry<String,Object> entry : tableProps.entrySet()) { String key = entry.getKey(); if (!TableProperty.isPhoenixTableProperty(key)) { @@ -790,38 +824,40 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement tableDescriptorBuilder.setValue(key, value == null ? null : value.toString()); } } - if (families.isEmpty()) { - if (tableType != PTableType.VIEW) { - byte[] defaultFamilyByes = defaultFamilyName == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : Bytes.toBytes(defaultFamilyName); - // Add dummy column family so we have key values for tables that - ColumnFamilyDescriptor columnDescriptor = generateColumnFamilyDescriptor(new Pair<byte[],Map<String,Object>>(defaultFamilyByes,Collections.<String,Object>emptyMap()), tableType); - tableDescriptorBuilder.addColumnFamily(columnDescriptor); - } - } else { - for (Pair<byte[],Map<String,Object>> family : families) { - // If family is only in phoenix description, add it. otherwise, modify its property accordingly. - byte[] familyByte = family.getFirst(); - if (tableDescriptorBuilder.build().getColumnFamily(familyByte) == null) { - if (tableType == PTableType.VIEW) { - String fullTableName = Bytes.toString(physicalTableName); - throw new ReadOnlyTableException( - "The HBase column families for a read-only table must already exist", - SchemaUtil.getSchemaNameFromFullName(fullTableName), - SchemaUtil.getTableNameFromFullName(fullTableName), - Bytes.toString(familyByte)); - } - ColumnFamilyDescriptor columnDescriptor = generateColumnFamilyDescriptor(family, tableType); - tableDescriptorBuilder.addColumnFamily(columnDescriptor); - } else { - if (tableType != PTableType.VIEW) { - ColumnFamilyDescriptor columnDescriptor = tableDescriptorBuilder.build().getColumnFamily(familyByte); - if (columnDescriptor == null) { - throw new IllegalArgumentException("Unable to find column descriptor with family name " + Bytes.toString(family.getFirst())); - } - ColumnFamilyDescriptorBuilder columnDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(columnDescriptor); - modifyColumnFamilyDescriptor(columnDescriptorBuilder, family.getSecond()); - tableDescriptorBuilder.modifyColumnFamily(columnDescriptorBuilder.build()); + + Map<String, Object> syncedProps = MetaDataUtil.getSyncedProps(dataTableColDescForIndexTablePropSyncing); + // Add column family-specific properties to the table descriptor + for (Pair<byte[],Map<String,Object>> family : families) { + // If family is only in phoenix description, add it. otherwise, modify its property accordingly. + byte[] familyByte = family.getFirst(); + if (tableDescriptorBuilder.build().getColumnFamily(familyByte) == null) { + if (tableType == PTableType.VIEW) { + String fullTableName = Bytes.toString(physicalTableName); + throw new ReadOnlyTableException( + "The HBase column families for a read-only table must already exist", + SchemaUtil.getSchemaNameFromFullName(fullTableName), + SchemaUtil.getTableNameFromFullName(fullTableName), + Bytes.toString(familyByte)); + } + + ColumnFamilyDescriptor columnDescriptor = generateColumnFamilyDescriptor(family, tableType); + // Keep certain index column family properties in sync with the base table + if ((tableType == PTableType.INDEX || MetaDataUtil.isViewIndex(Bytes.toString(physicalTableName))) && + (syncedProps != null && !syncedProps.isEmpty())) { + ColumnFamilyDescriptorBuilder colFamDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(columnDescriptor); + modifyColumnFamilyDescriptor(colFamDescBuilder, syncedProps); + columnDescriptor = colFamDescBuilder.build(); + } + tableDescriptorBuilder.setColumnFamily(columnDescriptor); + } else { + if (tableType != PTableType.VIEW) { + ColumnFamilyDescriptor columnDescriptor = tableDescriptorBuilder.build().getColumnFamily(familyByte); + if (columnDescriptor == null) { + throw new IllegalArgumentException("Unable to find column descriptor with family name " + Bytes.toString(family.getFirst())); } + ColumnFamilyDescriptorBuilder columnDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(columnDescriptor); + modifyColumnFamilyDescriptor(columnDescriptorBuilder, family.getSecond()); + tableDescriptorBuilder.modifyColumnFamily(columnDescriptorBuilder.build()); } } } @@ -1863,16 +1899,19 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement @Override public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, PTable table, Map<String, List<Pair<String,Object>>> stmtProperties, Set<String> colFamiliesForPColumnsToBeAdded, List<PColumn> columns) throws SQLException { List<Pair<byte[], Map<String, Object>>> families = new ArrayList<>(stmtProperties.size()); - Map<String, Object> tableProps = new HashMap<String, Object>(); + Map<String, Object> tableProps = new HashMap<>(); Set<TableDescriptor> tableDescriptors = Collections.emptySet(); - Set<TableDescriptor> origTableDescriptors = Collections.emptySet(); boolean nonTxToTx = false; - Pair<TableDescriptor,TableDescriptor> tableDescriptorPair = separateAndValidateProperties(table, stmtProperties, colFamiliesForPColumnsToBeAdded, tableProps); - TableDescriptor tableDescriptor = tableDescriptorPair.getSecond(); - TableDescriptor origTableDescriptor = tableDescriptorPair.getFirst(); + + Map<TableDescriptor, TableDescriptor> oldToNewTableDescriptors = + separateAndValidateProperties(table, stmtProperties, colFamiliesForPColumnsToBeAdded, tableProps); + Set<TableDescriptor> origTableDescriptors = new HashSet<>(oldToNewTableDescriptors.keySet()); + + TableDescriptor baseTableOrigDesc = this.getTableDescriptor(table.getPhysicalName().getBytes()); + TableDescriptor tableDescriptor = oldToNewTableDescriptors.get(baseTableOrigDesc); + if (tableDescriptor != null) { tableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size()); - origTableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size()); nonTxToTx = Boolean.TRUE.equals(tableProps.get(PhoenixTransactionContext.READ_NON_TX_DATA)); /* * If the table was transitioned from non transactional to transactional, we need @@ -1881,11 +1920,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement TableDescriptorBuilder tableDescriptorBuilder = TableDescriptorBuilder.newBuilder(tableDescriptor); if (nonTxToTx) { - updateDescriptorForTx(table, tableProps, tableDescriptorBuilder, Boolean.TRUE.toString(), tableDescriptors, origTableDescriptors); + updateDescriptorForTx(table, tableProps, tableDescriptorBuilder, Boolean.TRUE.toString(), + tableDescriptors, origTableDescriptors, oldToNewTableDescriptors); + tableDescriptor = tableDescriptorBuilder.build(); + tableDescriptors.add(tableDescriptor); + } else { + tableDescriptors = new HashSet<>(oldToNewTableDescriptors.values()); } - tableDescriptor=tableDescriptorBuilder.build(); - tableDescriptors.add(tableDescriptor); - origTableDescriptors.add(origTableDescriptor); } boolean success = false; @@ -1978,23 +2019,36 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } return result; } + private void updateDescriptorForTx(PTable table, Map<String, Object> tableProps, TableDescriptorBuilder tableDescriptorBuilder, - String txValue, Set<TableDescriptor> descriptorsToUpdate, Set<TableDescriptor> origDescriptors) throws SQLException { + String txValue, Set<TableDescriptor> descriptorsToUpdate, Set<TableDescriptor> origDescriptors, + Map<TableDescriptor, TableDescriptor> oldToNewTableDescriptors) throws SQLException { byte[] physicalTableName = table.getPhysicalName().getBytes(); try (Admin admin = getAdmin()) { setTransactional(physicalTableName, tableDescriptorBuilder, table.getType(), txValue, tableProps); Map<String, Object> indexTableProps; if (txValue == null) { - indexTableProps = Collections.<String,Object>emptyMap(); + indexTableProps = Collections.emptyMap(); } else { indexTableProps = Maps.newHashMapWithExpectedSize(1); indexTableProps.put(PhoenixTransactionContext.READ_NON_TX_DATA, Boolean.valueOf(txValue)); indexTableProps.put(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER, tableProps.get(PhoenixDatabaseMetaData.TRANSACTION_PROVIDER)); } for (PTable index : table.getIndexes()) { - TableDescriptor indexDesc = admin.getDescriptor(TableName.valueOf(index.getPhysicalName().getBytes())); - origDescriptors.add(indexDesc); - TableDescriptorBuilder indexDescriptorBuilder = TableDescriptorBuilder.newBuilder(indexDesc); + TableDescriptor origIndexDesc = admin.getDescriptor(TableName.valueOf(index.getPhysicalName().getBytes())); + TableDescriptor intermedIndexDesc = origIndexDesc; + // If we already wished to make modifications to this index table descriptor previously, we use the updated + // table descriptor to carry out further modifications + // See {@link ConnectionQueryServicesImpl#separateAndValidateProperties(PTable, Map, Set, Map)} + if (origDescriptors.contains(origIndexDesc)) { + intermedIndexDesc = oldToNewTableDescriptors.get(origIndexDesc); + // Remove any previous modification for this table descriptor because we will add + // the combined modification done in this method as well + descriptorsToUpdate.remove(intermedIndexDesc); + } else { + origDescriptors.add(origIndexDesc); + } + TableDescriptorBuilder indexDescriptorBuilder = TableDescriptorBuilder.newBuilder(intermedIndexDesc); if (index.getColumnFamilies().isEmpty()) { byte[] dataFamilyName = SchemaUtil.getEmptyColumnFamily(table); byte[] indexFamilyName = SchemaUtil.getEmptyColumnFamily(index); @@ -2004,7 +2058,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement indexColDescriptor.setValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL), tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(indexFamilyName); - indexDescriptorBuilder.addColumnFamily(indexColDescriptor.build()); + indexDescriptorBuilder.setColumnFamily(indexColDescriptor.build()); } else { for (PColumnFamily family : index.getColumnFamilies()) { byte[] familyName = family.getName().getBytes(); @@ -2014,16 +2068,22 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement indexColDescriptor.setValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL), tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName); - indexDescriptorBuilder.addColumnFamily(indexColDescriptor.build()); + indexDescriptorBuilder.setColumnFamily(indexColDescriptor.build()); } } setTransactional(index.getPhysicalName().getBytes(), indexDescriptorBuilder, index.getType(), txValue, indexTableProps); descriptorsToUpdate.add(indexDescriptorBuilder.build()); } try { - TableDescriptor indexDescriptor = admin.getDescriptor(TableName.valueOf(MetaDataUtil.getViewIndexPhysicalName(physicalTableName))); - origDescriptors.add(indexDescriptor); - TableDescriptorBuilder indexDescriptorBuilder = TableDescriptorBuilder.newBuilder(indexDescriptor); + TableDescriptor origIndexDesc = admin.getDescriptor(TableName.valueOf(MetaDataUtil.getViewIndexPhysicalName(physicalTableName))); + TableDescriptor intermedIndexDesc = origIndexDesc; + if (origDescriptors.contains(origIndexDesc)) { + intermedIndexDesc = oldToNewTableDescriptors.get(origIndexDesc); + descriptorsToUpdate.remove(intermedIndexDesc); + } else { + origDescriptors.add(origIndexDesc); + } + TableDescriptorBuilder indexDescriptorBuilder = TableDescriptorBuilder.newBuilder(intermedIndexDesc); setSharedIndexMaxVersion(table, tableDescriptorBuilder.build(), indexDescriptorBuilder); setTransactional(MetaDataUtil.getViewIndexPhysicalName(physicalTableName), indexDescriptorBuilder, PTableType.INDEX, txValue, indexTableProps); descriptorsToUpdate.add(indexDescriptorBuilder.build()); @@ -2031,20 +2091,26 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement // Ignore, as we may never have created a view index table } try { - TableDescriptor indexDescriptor = admin.getDescriptor(TableName.valueOf(MetaDataUtil.getLocalIndexPhysicalName(physicalTableName))); - origDescriptors.add(indexDescriptor); - TableDescriptorBuilder indexDescriptorBuilder = TableDescriptorBuilder.newBuilder(indexDescriptor); - + TableDescriptor origIndexDesc = admin.getDescriptor(TableName.valueOf(MetaDataUtil.getLocalIndexPhysicalName(physicalTableName))); + TableDescriptor intermedIndexDesc = origIndexDesc; + if (origDescriptors.contains(origIndexDesc)) { + intermedIndexDesc = oldToNewTableDescriptors.get(origIndexDesc); + descriptorsToUpdate.remove(intermedIndexDesc); + } else { + origDescriptors.add(origIndexDesc); + } + TableDescriptorBuilder indexDescriptorBuilder = TableDescriptorBuilder.newBuilder(intermedIndexDesc); setSharedIndexMaxVersion(table, tableDescriptorBuilder.build(), indexDescriptorBuilder); setTransactional(MetaDataUtil.getViewIndexPhysicalName(physicalTableName), indexDescriptorBuilder, PTableType.INDEX, txValue, indexTableProps); descriptorsToUpdate.add(indexDescriptorBuilder.build()); } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) { - // Ignore, as we may never have created a view index table + // Ignore, as we may never have created a local index } } catch (IOException e) { throw ServerUtil.parseServerException(e); } } + private void setSharedIndexMaxVersion(PTable table, TableDescriptor tableDescriptor, TableDescriptorBuilder indexDescriptorBuilder) { if (table.getColumnFamilies().isEmpty()) { @@ -2100,7 +2166,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement this.addCoprocessors(physicalTableName, tableDescriptorBuilder, tableType, tableProps); } - private Pair<TableDescriptor, TableDescriptor> separateAndValidateProperties(PTable table, + private Map<TableDescriptor, TableDescriptor> separateAndValidateProperties(PTable table, Map<String, List<Pair<String, Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, Map<String, Object> tableProps) throws SQLException { Map<String, Map<String, Object>> stmtFamiliesPropsMap = new HashMap<>(properties.size()); @@ -2112,11 +2178,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement boolean willBeTransactional = false; boolean isOrWillBeTransactional = isTransactional; Integer newTTL = null; + Integer newReplicationScope = null; + KeepDeletedCells newKeepDeletedCells = null; TransactionFactory.Provider txProvider = null; for (String family : properties.keySet()) { List<Pair<String, Object>> propsList = properties.get(family); if (propsList != null && propsList.size() > 0) { - Map<String, Object> colFamilyPropsMap = new HashMap<String, Object>(propsList.size()); + Map<String, Object> colFamilyPropsMap = new HashMap<>(propsList.size()); for (Pair<String, Object> prop : propsList) { String propName = prop.getFirst(); Object propValue = prop.getSecond(); @@ -2145,10 +2213,15 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement TableProperty tableProp = TableProperty.valueOf(propName); tableProp.validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType()); if (propName.equals(TTL)) { - newTTL = ((Number)prop.getSecond()).intValue(); + if (table.getType() == PTableType.INDEX) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX) + .setMessage("Property: " + propName).build() + .buildException(); + } + newTTL = ((Number)propValue).intValue(); // Even though TTL is really a HColumnProperty we treat it specially. // We enforce that all column families have the same TTL. - commonFamilyProps.put(propName, prop.getSecond()); + commonFamilyProps.put(propName, propValue); } else if (propName.equals(PhoenixDatabaseMetaData.TRANSACTIONAL) && Boolean.TRUE.equals(propValue)) { willBeTransactional = isOrWillBeTransactional = true; tableProps.put(PhoenixTransactionContext.READ_NON_TX_DATA, propValue); @@ -2160,8 +2233,28 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } } else { if (MetaDataUtil.isHColumnProperty(propName)) { + if (table.getType() == PTableType.INDEX && MetaDataUtil.propertyNotAllowedToBeOutOfSync(propName)) { + // We disallow index tables from overriding TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE, + // in order to avoid situations where indexes are not in sync with their data table + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX) + .setMessage("Property: " + propName).build() + .buildException(); + } if (family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY)) { + if (propName.equals(KEEP_DELETED_CELLS)) { + newKeepDeletedCells = + Boolean.valueOf(propValue.toString()) ? KeepDeletedCells.TRUE : KeepDeletedCells.FALSE; + } + if (propName.equals(REPLICATION_SCOPE)) { + newReplicationScope = ((Number)propValue).intValue(); + } commonFamilyProps.put(propName, propValue); + } else if (MetaDataUtil.propertyNotAllowedToBeOutOfSync(propName)) { + // Don't allow specifying column families for TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE. + // These properties can only be applied for all column families of a table and can't be column family specific. + throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY) + .setMessage("Property: " + propName).build() + .buildException(); } else { colFamilyPropsMap.put(propName, propValue); } @@ -2202,7 +2295,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement if (!addingColumns) { // Add the common family props to all existing column families for (String existingColFamily : existingColumnFamilies) { - Map<String, Object> m = new HashMap<String, Object>(commonFamilyProps.size()); + Map<String, Object> m = new HashMap<>(commonFamilyProps.size()); m.putAll(commonFamilyProps); allFamiliesProps.put(existingColFamily, m); } @@ -2211,7 +2304,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement for (String colFamily : colFamiliesForPColumnsToBeAdded) { if (colFamily != null) { // only set properties for key value columns - Map<String, Object> m = new HashMap<String, Object>(commonFamilyProps.size()); + Map<String, Object> m = new HashMap<>(commonFamilyProps.size()); m.putAll(commonFamilyProps); allFamiliesProps.put(colFamily, m); } else if (isAddingPkColOnly) { @@ -2276,30 +2369,30 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement TableDescriptorBuilder newTableDescriptorBuilder = null; TableDescriptor origTableDescriptor = null; + // Store all old to new table descriptor mappings for the table as well as its global indexes + Map<TableDescriptor, TableDescriptor> tableAndIndexDescriptorMappings = Collections.emptyMap(); if (!allFamiliesProps.isEmpty() || !tableProps.isEmpty()) { - byte[] tableNameBytes = Bytes.toBytes(table.getPhysicalName().getString()); - TableDescriptor existingTableDescriptor = origTableDescriptor = this.getTableDescriptor(tableNameBytes); + tableAndIndexDescriptorMappings = Maps.newHashMapWithExpectedSize(3 + table.getIndexes().size()); + TableDescriptor existingTableDescriptor = origTableDescriptor = this.getTableDescriptor(table.getPhysicalName().getBytes()); newTableDescriptorBuilder = TableDescriptorBuilder.newBuilder(existingTableDescriptor); if (!tableProps.isEmpty()) { - // add all the table properties to the existing table descriptor + // add all the table properties to the new table descriptor for (Entry<String, Object> entry : tableProps.entrySet()) { newTableDescriptorBuilder.setValue(entry.getKey(), entry.getValue() != null ? entry.getValue().toString() : null); } } if (addingColumns) { - // Make sure that all the CFs of the table have the same TTL as the empty CF. - setTTLForNewCFs(allFamiliesProps, table, newTableDescriptorBuilder, newTTL); - } - // Set TTL on all table column families, even if they're not referenced here - if (newTTL != null) { - for (PColumnFamily family : table.getColumnFamilies()) { - if (!allFamiliesProps.containsKey(family.getName().getString())) { - Map<String,Object> familyProps = Maps.newHashMapWithExpectedSize(1); - familyProps.put(TTL, newTTL); - allFamiliesProps.put(family.getName().getString(), familyProps); - } - } + // Make sure that TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE for the new column family to be added stays in sync + // with the table's existing column families. Note that we use the new values for these properties in case we are + // altering their values. We also propagate these altered values to existing column families and indexes on the table below + setSyncedPropsForNewColumnFamilies(allFamiliesProps, table, newTableDescriptorBuilder, newTTL, newKeepDeletedCells, newReplicationScope); + } + if (newTTL != null || newKeepDeletedCells != null || newReplicationScope != null) { + // Set properties to be kept in sync on all table column families of this table, even if they are not referenced here + setSyncedPropsForUnreferencedColumnFamilies(this.getTableDescriptor(table.getPhysicalName().getBytes()), + allFamiliesProps, newTTL, newKeepDeletedCells, newReplicationScope); } + Integer defaultTxMaxVersions = null; if (isOrWillBeTransactional) { // Calculate default for max versions @@ -2328,21 +2421,20 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } } } - } - // Set Tephra's TTL property based on HBase property if we're - // transitioning to become transactional or setting TTL on - // an already transactional table. - if (isOrWillBeTransactional) { + // Set Tephra's TTL property based on HBase property if we're + // transitioning to become transactional or setting TTL on + // an already transactional table. int ttl = getTTL(table, newTableDescriptorBuilder.build(), newTTL); if (ttl != ColumnFamilyDescriptorBuilder.DEFAULT_TTL) { for (Map.Entry<String, Map<String, Object>> entry : allFamiliesProps.entrySet()) { Map<String, Object> props = entry.getValue(); if (props == null) { - props = new HashMap<String, Object>(); + allFamiliesProps.put(entry.getKey(), new HashMap<>()); + props = allFamiliesProps.get(entry.getKey()); } else { - props = new HashMap<String, Object>(props); + props = new HashMap<>(props); } - props.put(PhoenixTransactionContext.PROPERTY_TTL, new Integer(ttl)); + props.put(PhoenixTransactionContext.PROPERTY_TTL, ttl); // Remove HBase TTL if we're not transitioning an existing table to become transactional // or if the existing transactional table wasn't originally non transactional. if (!willBeTransactional && !Boolean.valueOf(newTableDescriptorBuilder.build().getValue(PhoenixTransactionContext.READ_NON_TX_DATA))) { @@ -2364,21 +2456,30 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement if (colDescriptor == null) { // new column family colDescriptor = generateColumnFamilyDescriptor(new Pair<>(cf, familyProps), table.getType()); - newTableDescriptorBuilder.addColumnFamily(colDescriptor); + newTableDescriptorBuilder.setColumnFamily(colDescriptor); } else { ColumnFamilyDescriptorBuilder colDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(colDescriptor); modifyColumnFamilyDescriptor(colDescriptorBuilder, familyProps); colDescriptor = colDescriptorBuilder.build(); newTableDescriptorBuilder.removeColumnFamily(cf); - newTableDescriptorBuilder.addColumnFamily(colDescriptor); + newTableDescriptorBuilder.setColumnFamily(colDescriptor); } if (isOrWillBeTransactional) { checkTransactionalVersionsValue(colDescriptor); } } } - return new Pair<>(origTableDescriptor, newTableDescriptorBuilder == null ? null - : newTableDescriptorBuilder.build()); + if (origTableDescriptor != null && newTableDescriptorBuilder != null) { + // Add the table descriptor mapping for the base table + tableAndIndexDescriptorMappings.put(origTableDescriptor, newTableDescriptorBuilder.build()); + } + + Map<String, Object> applyPropsToAllIndexColFams = getNewSyncedPropsMap(newTTL, newKeepDeletedCells, newReplicationScope); + // Copy properties that need to be synced from the default column family of the base table to + // the column families of each of its indexes (including indexes on this base table's views) + // and store those table descriptor mappings as well + setSyncedPropertiesForTableIndexes(table, tableAndIndexDescriptorMappings, applyPropsToAllIndexColFams); + return tableAndIndexDescriptorMappings; } private void checkTransactionalVersionsValue(ColumnFamilyDescriptor colDescriptor) throws SQLException { @@ -2405,23 +2506,146 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement return cfNames; } + private static KeepDeletedCells getKeepDeletedCells(PTable table, TableDescriptor tableDesc, + KeepDeletedCells newKeepDeletedCells) throws SQLException { + // If we're setting KEEP_DELETED_CELLS now, then use that value. Otherwise, use the empty column family value + return (newKeepDeletedCells != null) ? + newKeepDeletedCells : + tableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(table)).getKeepDeletedCells(); + } + + private static int getReplicationScope(PTable table, TableDescriptor tableDesc, + Integer newReplicationScope) throws SQLException { + // If we're setting replication scope now, then use that value. Otherwise, use the empty column family value + return (newReplicationScope != null) ? + newReplicationScope : + tableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(table)).getScope(); + } + private static int getTTL(PTable table, TableDescriptor tableDesc, Integer newTTL) throws SQLException { // If we're setting TTL now, then use that value. Otherwise, use empty column family value - int ttl = newTTL != null ? newTTL - : tableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(table)).getTimeToLive(); - return ttl; + return (newTTL != null) ? + newTTL : + tableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(table)).getTimeToLive(); } - private static void setTTLForNewCFs(Map<String, Map<String, Object>> familyProps, PTable table, - TableDescriptorBuilder tableDescBuilder, Integer newTTL) throws SQLException { - if (!familyProps.isEmpty()) { + /** + * Keep the TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE properties of new column families + * in sync with the existing column families. Note that we use the new values for these properties in case they + * are passed from our alter table command, if not, we use the default column family's value for each property + * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} + * @param allFamiliesProps Map of all column family properties + * @param table original table + * @param tableDescBuilder new table descriptor builder + * @param newTTL new value of TTL + * @param newKeepDeletedCells new value of KEEP_DELETED_CELLS + * @param newReplicationScope new value of REPLICATION_SCOPE + * @throws SQLException + */ + private void setSyncedPropsForNewColumnFamilies(Map<String, Map<String, Object>> allFamiliesProps, PTable table, + TableDescriptorBuilder tableDescBuilder, Integer newTTL, KeepDeletedCells newKeepDeletedCells, + Integer newReplicationScope) throws SQLException { + if (!allFamiliesProps.isEmpty()) { int ttl = getTTL(table, tableDescBuilder.build(), newTTL); - for (Map.Entry<String, Map<String, Object>> entry : familyProps.entrySet()) { + int replicationScope = getReplicationScope(table, tableDescBuilder.build(), newReplicationScope); + KeepDeletedCells keepDeletedCells = getKeepDeletedCells(table, tableDescBuilder.build(), newKeepDeletedCells); + for (Map.Entry<String, Map<String, Object>> entry : allFamiliesProps.entrySet()) { Map<String, Object> props = entry.getValue(); if (props == null) { - props = new HashMap<String, Object>(); + allFamiliesProps.put(entry.getKey(), new HashMap<>()); + props = allFamiliesProps.get(entry.getKey()); } props.put(TTL, ttl); + props.put(KEEP_DELETED_CELLS, keepDeletedCells); + props.put(REPLICATION_SCOPE, replicationScope); + } + } + } + + private void setPropIfNotNull(Map<String, Object> propMap, String propName, Object propVal) { + if (propName!= null && propVal != null) { + propMap.put(propName, propVal); + } + } + + private Map<String, Object> getNewSyncedPropsMap(Integer newTTL, KeepDeletedCells newKeepDeletedCells, Integer newReplicationScope) { + Map<String,Object> newSyncedProps = Maps.newHashMapWithExpectedSize(3); + setPropIfNotNull(newSyncedProps, TTL, newTTL); + setPropIfNotNull(newSyncedProps,KEEP_DELETED_CELLS, newKeepDeletedCells); + setPropIfNotNull(newSyncedProps, REPLICATION_SCOPE, newReplicationScope); + return newSyncedProps; + } + + /** + * Set the new values for properties that are to be kept in sync amongst those column families of the table which are + * not referenced in the context of our alter table command, including the local index column family if it exists + * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} + * @param tableDesc original table descriptor + * @param allFamiliesProps Map of all column family properties + * @param newTTL new value of TTL + * @param newKeepDeletedCells new value of KEEP_DELETED_CELLS + * @param newReplicationScope new value of REPLICATION_SCOPE + * @return + */ + private void setSyncedPropsForUnreferencedColumnFamilies(TableDescriptor tableDesc, Map<String, Map<String, Object>> allFamiliesProps, + Integer newTTL, KeepDeletedCells newKeepDeletedCells, Integer newReplicationScope) { + for (ColumnFamilyDescriptor family: tableDesc.getColumnFamilies()) { + if (!allFamiliesProps.containsKey(family.getNameAsString())) { + allFamiliesProps.put(family.getNameAsString(), + getNewSyncedPropsMap(newTTL, newKeepDeletedCells, newReplicationScope)); + } + } + } + + /** + * Set properties to be kept in sync for global indexes of a table, as well as + * the physical table corresponding to indexes created on views of a table + * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} and + * @param table base table + * @param tableAndIndexDescriptorMappings old to new table descriptor mappings + * @param applyPropsToAllIndexesDefaultCF new properties to apply to all index column families + * @throws SQLException + */ + private void setSyncedPropertiesForTableIndexes(PTable table, + Map<TableDescriptor, TableDescriptor> tableAndIndexDescriptorMappings, + Map<String, Object> applyPropsToAllIndexesDefaultCF) throws SQLException { + if (applyPropsToAllIndexesDefaultCF == null || applyPropsToAllIndexesDefaultCF.isEmpty()) { + return; + } + + for (PTable indexTable: table.getIndexes()) { + if (indexTable.getIndexType() == PTable.IndexType.LOCAL) { + // local index tables are already handled when we sync all column families of a base table + continue; + } + TableDescriptor origIndexDescriptor = this.getTableDescriptor(indexTable.getPhysicalName().getBytes()); + TableDescriptorBuilder newIndexDescriptorBuilder = TableDescriptorBuilder.newBuilder(origIndexDescriptor); + + byte[] defaultIndexColFam = SchemaUtil.getEmptyColumnFamily(indexTable); + ColumnFamilyDescriptorBuilder indexDefaultColDescriptorBuilder = + ColumnFamilyDescriptorBuilder.newBuilder(origIndexDescriptor.getColumnFamily(defaultIndexColFam)); + modifyColumnFamilyDescriptor(indexDefaultColDescriptorBuilder, applyPropsToAllIndexesDefaultCF); + newIndexDescriptorBuilder.removeColumnFamily(defaultIndexColFam); + newIndexDescriptorBuilder.setColumnFamily(indexDefaultColDescriptorBuilder.build()); + tableAndIndexDescriptorMappings.put(origIndexDescriptor, newIndexDescriptorBuilder.build()); + } + // Also keep properties for the physical view index table in sync + String viewIndexName = MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getString()); + if (!Strings.isNullOrEmpty(viewIndexName)) { + try { + TableDescriptor origViewIndexTableDescriptor = this.getTableDescriptor(Bytes.toBytes(viewIndexName)); + TableDescriptorBuilder newViewIndexDescriptorBuilder = + TableDescriptorBuilder.newBuilder(origViewIndexTableDescriptor); + for (ColumnFamilyDescriptor cfd: origViewIndexTableDescriptor.getColumnFamilies()) { + ColumnFamilyDescriptorBuilder newCfd = + ColumnFamilyDescriptorBuilder.newBuilder(cfd); + modifyColumnFamilyDescriptor(newCfd, applyPropsToAllIndexesDefaultCF); + newViewIndexDescriptorBuilder.removeColumnFamily(cfd.getName()); + newViewIndexDescriptorBuilder.setColumnFamily(newCfd.build()); + } + tableAndIndexDescriptorMappings.put(origViewIndexTableDescriptor, newViewIndexDescriptorBuilder.build()); + } catch (TableNotFoundException ignore) { + // Ignore since this means that a view index table does not exist for this table } } } @@ -3197,6 +3421,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement // We will not reach here if we fail to acquire the lock, since it throws UpgradeInProgressException } metaConnection = upgradeSystemCatalogIfRequired(metaConnection, currentServerSideTableTimeStamp); + // Synchronize necessary properties amongst all column families of a base table and its indexes + // See PHOENIX-3955 + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_15_0) { + syncTableAndIndexProperties(metaConnection, getAdmin()); + } } int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt( @@ -3960,22 +4189,21 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement @Override public MetaDataMutationResult updateIndexState(final List<Mutation> tableMetaData, String parentTableName, Map<String, List<Pair<String,Object>>> stmtProperties, PTable table) throws SQLException { - if(stmtProperties==null) return updateIndexState(tableMetaData,parentTableName); + if(stmtProperties == null) { + return updateIndexState(tableMetaData,parentTableName); + } - Map<String, Object> tableProps = new HashMap<String, Object>(); - Pair<TableDescriptor,TableDescriptor> tableDescriptorPair = separateAndValidateProperties(table, stmtProperties, new HashSet<String>(), tableProps); - TableDescriptor tableDescriptor = tableDescriptorPair.getSecond(); - TableDescriptor origTableDescriptor = tableDescriptorPair.getFirst(); - Set<TableDescriptor> tableDescriptors = Collections.emptySet(); - Set<TableDescriptor> origTableDescriptors = Collections.emptySet(); - if (tableDescriptor != null) { - tableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size()); - origTableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size()); - tableDescriptors.add(tableDescriptor); - origTableDescriptors.add(origTableDescriptor); + Map<TableDescriptor, TableDescriptor> oldToNewTableDescriptors = + separateAndValidateProperties(table, stmtProperties, new HashSet<>(), new HashMap<>()); + TableDescriptor origTableDescriptor = this.getTableDescriptor(table.getPhysicalName().getBytes()); + TableDescriptor newTableDescriptor = oldToNewTableDescriptors.remove(origTableDescriptor); + Set<TableDescriptor> modifiedTableDescriptors = Collections.emptySet(); + if (newTableDescriptor != null) { + modifiedTableDescriptors = Sets.newHashSetWithExpectedSize(3 + table.getIndexes().size()); + modifiedTableDescriptors.add(newTableDescriptor); } - sendHBaseMetaData(tableDescriptors, true); - return updateIndexState(tableMetaData,parentTableName); + sendHBaseMetaData(modifiedTableDescriptors, true); + return updateIndexState(tableMetaData, parentTableName); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/86fe2fc5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index ef1f34a..144064c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -1065,7 +1065,7 @@ public class MetaDataClient { TableName tableName = statement.getTableName(); Map<String,Object> tableProps = Maps.newHashMapWithExpectedSize(statement.getProps().size()); Map<String,Object> commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size() + 1); - populatePropertyMaps(statement.getProps(), tableProps, commonFamilyProps); + populatePropertyMaps(statement.getProps(), tableProps, commonFamilyProps, statement.getTableType()); boolean isAppendOnlySchema = false; long updateCacheFrequency = connection.getQueryServices().getProps().getLong( @@ -1143,13 +1143,26 @@ public class MetaDataClient { return connection.getQueryServices().updateData(plan); } - private void populatePropertyMaps(ListMultimap<String,Pair<String,Object>> props, Map<String, Object> tableProps, - Map<String, Object> commonFamilyProps) { + /** + * Populate properties for the table and common properties for all column families of the table + * @param statementProps Properties specified in SQL statement + * @param tableProps Properties for an HTableDescriptor + * @param commonFamilyProps Properties common to all column families + * @param tableType Used to distinguish between index creation vs. base table creation paths + * @throws SQLException + */ + private void populatePropertyMaps(ListMultimap<String,Pair<String,Object>> statementProps, Map<String, Object> tableProps, + Map<String, Object> commonFamilyProps, PTableType tableType) throws SQLException { // Somewhat hacky way of determining if property is for HColumnDescriptor or HTableDescriptor ColumnFamilyDescriptor defaultDescriptor = ColumnFamilyDescriptorBuilder.of(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES); - if (!props.isEmpty()) { - Collection<Pair<String,Object>> propsList = props.get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY); + if (!statementProps.isEmpty()) { + Collection<Pair<String,Object>> propsList = statementProps.get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY); for (Pair<String,Object> prop : propsList) { + if (tableType == PTableType.INDEX && MetaDataUtil.propertyNotAllowedToBeOutOfSync(prop.getFirst())) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX) + .setMessage("Property: " + prop.getFirst()).build() + .buildException(); + } if (defaultDescriptor.getValue(Bytes.toBytes(prop.getFirst())) == null) { tableProps.put(prop.getFirst(), prop.getSecond()); } else { @@ -1502,7 +1515,7 @@ public class MetaDataClient { Map<String,Object> tableProps = Maps.newHashMapWithExpectedSize(statement.getProps().size()); Map<String,Object> commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size() + 1); - populatePropertyMaps(statement.getProps(), tableProps, commonFamilyProps); + populatePropertyMaps(statement.getProps(), tableProps, commonFamilyProps, PTableType.INDEX); List<Pair<ParseNode, SortOrder>> indexParseNodeAndSortOrderList = ik.getParseNodeAndSortOrderList(); List<ColumnName> includedColumns = statement.getIncludeColumns(); TableRef tableRef = null; @@ -1897,6 +1910,57 @@ public class MetaDataClient { connection.getQueryServices().deleteMutexCell(tenantId, schemaName, tableName, columnName, null); } + /** + * + * Populate the properties for each column family referenced in the create table statement + * @param familyNames column families referenced in the create table statement + * @param commonFamilyProps properties common to all column families + * @param statement create table statement + * @param defaultFamilyName the default column family name + * @param isLocalIndex true if in the create local index path + * @param familyPropList list containing pairs of column families and their corresponding properties + * @throws SQLException + */ + private void populateFamilyPropsList(Map<String, PName> familyNames, Map<String,Object> commonFamilyProps, + CreateTableStatement statement, String defaultFamilyName, boolean isLocalIndex, + final List<Pair<byte[],Map<String,Object>>> familyPropList) throws SQLException { + for (PName familyName : familyNames.values()) { + String fam = familyName.getString(); + Collection<Pair<String, Object>> propsForCF = + statement.getProps().get(IndexUtil.getActualColumnFamilyName(fam)); + // No specific properties for this column family, so add the common family properties + if (propsForCF.isEmpty()) { + familyPropList.add(new Pair<>(familyName.getBytes(),commonFamilyProps)); + } else { + Map<String,Object> combinedFamilyProps = Maps.newHashMapWithExpectedSize(propsForCF.size() + commonFamilyProps.size()); + combinedFamilyProps.putAll(commonFamilyProps); + for (Pair<String,Object> prop : propsForCF) { + // Don't allow specifying column families for TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE. + // These properties can only be applied for all column families of a table and can't be column family specific. + // See PHOENIX-3955 + if (!fam.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY) && MetaDataUtil.propertyNotAllowedToBeOutOfSync(prop.getFirst())) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY) + .setMessage("Property: " + prop.getFirst()) + .build() + .buildException(); + } + combinedFamilyProps.put(prop.getFirst(), prop.getSecond()); + } + familyPropList.add(new Pair<>(familyName.getBytes(),combinedFamilyProps)); + } + } + + if (familyNames.isEmpty()) { + // If there are no family names, use the default column family name. This also takes care of the case when + // the table ddl has only PK cols present (which means familyNames is empty). + byte[] cf = + defaultFamilyName == null ? (!isLocalIndex? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES + : QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES) + : Bytes.toBytes(defaultFamilyName); + familyPropList.add(new Pair<>(cf, commonFamilyProps)); + } + } + private PTable createTableInternal(CreateTableStatement statement, byte[][] splits, final PTable parent, String viewStatement, ViewType viewType, PDataType viewIndexType, final byte[][] viewColumnConstants, final BitSet isViewColumnReferenced, boolean allocateIndexId, @@ -1937,7 +2001,6 @@ public class MetaDataClient { ImmutableStorageScheme immutableStorageScheme = ONE_CELL_PER_COLUMN; if (parent != null && tableType == PTableType.INDEX) { timestamp = TransactionUtil.getTableTimestamp(connection, transactionProvider != null, transactionProvider); - storeNulls = parent.getStoreNulls(); isImmutableRows = parent.isImmutableRows(); isAppendOnlySchema = parent.isAppendOnlySchema(); @@ -2552,7 +2615,6 @@ public class MetaDataClient { .build().buildException(); } - List<Pair<byte[],Map<String,Object>>> familyPropList = Lists.newArrayListWithExpectedSize(familyNames.size()); if (!statement.getProps().isEmpty()) { for (String familyName : statement.getProps().keySet()) { if (!familyName.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY)) { @@ -2567,36 +2629,8 @@ public class MetaDataClient { } throwIfInsufficientColumns(schemaName, tableName, pkColumns, saltBucketNum!=null, multiTenant); - for (PName familyName : familyNames.values()) { - String fam = familyName.getString(); - Collection<Pair<String, Object>> props = - statement.getProps().get(IndexUtil.getActualColumnFamilyName(fam)); - if (props.isEmpty()) { - familyPropList.add(new Pair<byte[],Map<String,Object>>(familyName.getBytes(),commonFamilyProps)); - } else { - Map<String,Object> combinedFamilyProps = Maps.newHashMapWithExpectedSize(props.size() + commonFamilyProps.size()); - combinedFamilyProps.putAll(commonFamilyProps); - for (Pair<String,Object> prop : props) { - // Don't allow specifying column families for TTL. TTL can only apply for the all the column families of the table - // i.e. it can't be column family specific. - if (!familyName.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY) && prop.getFirst().equals(ColumnFamilyDescriptorBuilder.TTL)) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL).build().buildException(); - } - combinedFamilyProps.put(prop.getFirst(), prop.getSecond()); - } - familyPropList.add(new Pair<byte[],Map<String,Object>>(familyName.getBytes(),combinedFamilyProps)); - } - } - - if (familyNames.isEmpty()) { - //if there are no family names, use the default column family name. This also takes care of the case when - //the table ddl has only PK cols present (which means familyNames is empty). - byte[] cf = - defaultFamilyName == null ? (!isLocalIndex? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES - : QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES) - : Bytes.toBytes(defaultFamilyName); - familyPropList.add(new Pair<byte[],Map<String,Object>>(cf, commonFamilyProps)); - } + List<Pair<byte[],Map<String,Object>>> familyPropList = Lists.newArrayListWithExpectedSize(familyNames.size()); + populateFamilyPropsList(familyNames, commonFamilyProps, statement, defaultFamilyName, isLocalIndex, familyPropList); // Bootstrapping for our SYSTEM.TABLE that creates itself before it exists if (SchemaUtil.isMetaTable(schemaName,tableName)) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/86fe2fc5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java index 3d2e84e..ab667c2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java @@ -18,7 +18,7 @@ package org.apache.phoenix.schema; import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_ALTER_PROPERTY; -import static org.apache.phoenix.exception.SQLExceptionCode.COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL; +import static org.apache.phoenix.exception.SQLExceptionCode.COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY; import static org.apache.phoenix.exception.SQLExceptionCode.COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY; import static org.apache.phoenix.exception.SQLExceptionCode.DEFAULT_COLUMN_FAMILY_ONLY_ON_CREATE_TABLE; import static org.apache.phoenix.exception.SQLExceptionCode.SALT_ONLY_ON_CREATE_TABLE; @@ -74,7 +74,7 @@ public enum TableProperty { } }, - TTL(HColumnDescriptor.TTL, COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL, true, CANNOT_ALTER_PROPERTY, false, false) { + TTL(HColumnDescriptor.TTL, COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY, true, CANNOT_ALTER_PROPERTY, false, false) { @Override public Object getPTableValue(PTable table) { return null; http://git-wip-us.apache.org/repos/asf/phoenix/blob/86fe2fc5/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index 39c3976..c2c285b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -25,8 +25,10 @@ import java.sql.Types; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.NavigableMap; import org.apache.hadoop.conf.Configuration; @@ -80,6 +82,7 @@ import org.apache.phoenix.schema.types.PUnsignedTinyint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -93,8 +96,22 @@ public class MetaDataUtil { public static final byte[] VIEW_INDEX_SEQUENCE_PREFIX_BYTES = Bytes.toBytes(VIEW_INDEX_SEQUENCE_PREFIX); private static final String VIEW_INDEX_ID_COLUMN_NAME = "_INDEX_ID"; public static final String PARENT_TABLE_KEY = "PARENT_TABLE"; - public static final byte[] PARENT_TABLE_KEY_BYTES = Bytes.toBytes("PARENT_TABLE"); - + public static final String IS_VIEW_INDEX_TABLE_PROP_NAME = "IS_VIEW_INDEX_TABLE"; + public static final byte[] IS_VIEW_INDEX_TABLE_PROP_BYTES = Bytes.toBytes(IS_VIEW_INDEX_TABLE_PROP_NAME); + + public static final String IS_LOCAL_INDEX_TABLE_PROP_NAME = "IS_LOCAL_INDEX_TABLE"; + public static final byte[] IS_LOCAL_INDEX_TABLE_PROP_BYTES = Bytes.toBytes(IS_LOCAL_INDEX_TABLE_PROP_NAME); + + public static final String DATA_TABLE_NAME_PROP_NAME = "DATA_TABLE_NAME"; + + public static final byte[] DATA_TABLE_NAME_PROP_BYTES = Bytes.toBytes(DATA_TABLE_NAME_PROP_NAME); + + // See PHOENIX-3955 + public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES = ImmutableList.of( + ColumnFamilyDescriptorBuilder.TTL, + ColumnFamilyDescriptorBuilder.KEEP_DELETED_CELLS, + ColumnFamilyDescriptorBuilder.REPLICATION_SCOPE); + public static boolean areClientAndServerCompatible(long serverHBaseAndPhoenixVersion) { // As of 3.0, we allow a client and server to differ for the minor version. // Care has to be taken to upgrade the server before the client, as otherwise @@ -700,17 +717,20 @@ public class MetaDataUtil { return true; } - public static final String IS_VIEW_INDEX_TABLE_PROP_NAME = "IS_VIEW_INDEX_TABLE"; - public static final byte[] IS_VIEW_INDEX_TABLE_PROP_BYTES = Bytes.toBytes(IS_VIEW_INDEX_TABLE_PROP_NAME); - - public static final String IS_LOCAL_INDEX_TABLE_PROP_NAME = "IS_LOCAL_INDEX_TABLE"; - public static final byte[] IS_LOCAL_INDEX_TABLE_PROP_BYTES = Bytes.toBytes(IS_LOCAL_INDEX_TABLE_PROP_NAME); - - public static final String DATA_TABLE_NAME_PROP_NAME = "DATA_TABLE_NAME"; - - public static final byte[] DATA_TABLE_NAME_PROP_BYTES = Bytes.toBytes(DATA_TABLE_NAME_PROP_NAME); - + public static boolean propertyNotAllowedToBeOutOfSync(String colFamProp) { + return SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.contains(colFamProp); + } + public static Map<String, Object> getSyncedProps(ColumnFamilyDescriptor defaultCFDesc) { + Map<String, Object> syncedProps = new HashMap<>(); + if (defaultCFDesc != null) { + for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + syncedProps.put(propToKeepInSync, Bytes.toString( + defaultCFDesc.getValue(Bytes.toBytes(propToKeepInSync)))); + } + } + return syncedProps; + } public static Scan newTableRowsScan(byte[] key, long startTimeStamp, long stopTimeStamp){ return newTableRowsScan(key, null, startTimeStamp, stopTimeStamp); http://git-wip-us.apache.org/repos/asf/phoenix/blob/86fe2fc5/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index aa3e1a6..f86331c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -74,6 +74,7 @@ import java.util.concurrent.TimeoutException; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import com.google.common.base.Strings; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; @@ -104,6 +105,7 @@ import org.apache.phoenix.coprocessor.TableViewFinderResult; import org.apache.phoenix.coprocessor.ViewFinder; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.MetaDataClient; @@ -1222,7 +1224,135 @@ public class UpgradeUtil { queryConn.getQueryServices().clearCache(); } } - + + /** + * Synchronize column family properties using the default cf properties for a given table + * @param tableDesc table descriptor of table to modify + * @param defaultColFam default column family used as the baseline for property synchronization + * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor + * @return modified table descriptor builder + */ + private static TableDescriptorBuilder syncColFamProperties(TableDescriptor tableDesc, ColumnFamilyDescriptor defaultColFam, + Map<String, Object> syncedProps) { + TableDescriptorBuilder tableDescBuilder = TableDescriptorBuilder.newBuilder(tableDesc); + // Ensure that all column families have necessary properties in sync (including local index cf if present) + for (ColumnFamilyDescriptor currentColFam: tableDesc.getColumnFamilies()) { + if (!currentColFam.equals(defaultColFam)) { + ColumnFamilyDescriptorBuilder colFamDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(currentColFam); + for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) { + String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop))); + String expectedPropVal = syncedProps.get(prop).toString(); + if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) { + // Need to synchronize this property for the current column family descriptor + colFamDescBuilder.setValue(prop, expectedPropVal); + } + } + if (!colFamDescBuilder.equals(ColumnFamilyDescriptorBuilder.newBuilder(currentColFam))) { + tableDescBuilder.modifyColumnFamily(colFamDescBuilder.build()); + } + } + } + return tableDescBuilder; + } + + /** + * Add the table descriptor to the set of table descriptors to keep in sync, if it has been changed + * @param origTableDesc original table descriptor of the table in question + * @param defaultColFam column family to be used for synchronizing properties + * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor + * @param tableDescsToSync set of modified table descriptors + * @throws SQLException + */ + private static void addTableDescIfPropsChanged(TableDescriptor origTableDesc, ColumnFamilyDescriptor defaultColFam, + Map<String, Object> syncedProps, Set<TableDescriptor> tableDescsToSync) throws SQLException { + TableDescriptorBuilder tableDescBuilder = syncColFamProperties(origTableDesc, defaultColFam, syncedProps); + if (!origTableDesc.equals(tableDescBuilder.build())) { + tableDescsToSync.add(tableDescBuilder.build()); + } + } + + /** + * Synchronize certain properties across column families of global index tables for a given base table + * @param cqs CQS object to get table descriptor from PTable + * @param baseTable base table + * @param defaultColFam column family to be used for synchronizing properties + * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor + * @param tableDescsToSync set of modified table descriptors + */ + private static void syncGlobalIndexesForTable(ConnectionQueryServices cqs, PTable baseTable, ColumnFamilyDescriptor defaultColFam, + Map<String, Object> syncedProps, Set<TableDescriptor> tableDescsToSync) throws SQLException { + for (PTable indexTable: baseTable.getIndexes()) { + // We already handle local index property synchronization when considering all column families of the base table + if (indexTable.getIndexType() == IndexType.GLOBAL) { + addTableDescIfPropsChanged(cqs.getTableDescriptor(indexTable.getPhysicalName().getBytes()), + defaultColFam, syncedProps, tableDescsToSync); + } + } + } + + /** + * Synchronize certain properties across column families of view index tables for a given base table + * @param cqs CQS object to get table descriptor from PTable + * @param baseTable base table + * @param defaultColFam column family to be used for synchronizing properties + * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor + * @param tableDescsToSync set of modified table descriptors + */ + private static void syncViewIndexTable(ConnectionQueryServices cqs, PTable baseTable, ColumnFamilyDescriptor defaultColFam, + Map<String, Object> syncedProps, Set<TableDescriptor> tableDescsToSync) throws SQLException { + String viewIndexName = MetaDataUtil.getViewIndexPhysicalName(baseTable.getPhysicalName().getString()); + if (!Strings.isNullOrEmpty(viewIndexName)) { + try { + addTableDescIfPropsChanged(cqs.getTableDescriptor(Bytes.toBytes(viewIndexName)), + defaultColFam, syncedProps, tableDescsToSync); + } catch (TableNotFoundException ignore) { + // Ignore since this means that a view index table does not exist for this table + } + } + } + + /** + * Make sure that all tables have necessary column family properties in sync + * with each other and also in sync with all the table's indexes + * See PHOENIX-3955 + * @param conn Phoenix connection + * @param admin HBase admin used for getting existing tables and their descriptors + * @throws SQLException + * @throws IOException + */ + public static void syncTableAndIndexProperties(PhoenixConnection conn, Admin admin) + throws SQLException, IOException { + Set<TableDescriptor> tableDescriptorsToSynchronize = new HashSet<>(); + for (TableDescriptor origTableDesc : admin.listTableDescriptors()) { + if (MetaDataUtil.isViewIndex(origTableDesc.getTableName().getNameWithNamespaceInclAsString())) { + // Ignore physical view index tables since we handle them for each base table already + continue; + } + PTable table = null; + String tableName = origTableDesc.getTableName().getNameAsString(); + try { + table = PhoenixRuntime.getTable(conn, tableName); + } catch (TableNotFoundException e) { + // Ignore tables not mapped to a Phoenix Table + logger.warn("Error getting PTable for HBase table: " + tableName); + continue; + } + if (table.getType() == PTableType.INDEX) { + // Ignore global index tables since we handle them for each base table already + continue; + } + ColumnFamilyDescriptor defaultColFam = origTableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(table)); + Map<String, Object> syncedProps = MetaDataUtil.getSyncedProps(defaultColFam); + + addTableDescIfPropsChanged(origTableDesc, defaultColFam, syncedProps, tableDescriptorsToSynchronize); + syncGlobalIndexesForTable(conn.getQueryServices(), table, defaultColFam, syncedProps, tableDescriptorsToSynchronize); + syncViewIndexTable(conn.getQueryServices(), table, defaultColFam, syncedProps, tableDescriptorsToSynchronize); + } + for (TableDescriptor t: tableDescriptorsToSynchronize) { + admin.modifyTable(t); + } + } + private static void upsertBaseColumnCountInHeaderRow(PhoenixConnection metaConnection, String tenantId, String schemaName, String viewOrTableName, int baseColumnCount) throws SQLException {
