Repository: phoenix Updated Branches: refs/heads/encodecolumns 450dbc59a -> 2433c8d21
http://git-wip-us.apache.org/repos/asf/phoenix/blob/2433c8d2/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java index bfbfc54..43f5801 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java @@ -19,6 +19,7 @@ package org.apache.phoenix.util; import static com.google.common.base.Preconditions.checkArgument; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.PColumn; @@ -29,7 +30,11 @@ import org.apache.phoenix.schema.types.PInteger; public class EncodedColumnsUtil { public static boolean usesEncodedColumnNames(PTable table) { - return table.getStorageScheme() != null && table.getStorageScheme() == StorageScheme.ENCODED_COLUMN_NAMES; + return usesEncodedColumnNames(table.getStorageScheme()); + } + + public static boolean usesEncodedColumnNames(StorageScheme storageSchema) { + return storageSchema != null && storageSchema != StorageScheme.NON_ENCODED_COLUMN_NAMES; } public static byte[] getEncodedColumnQualifier(PColumn column) { @@ -41,6 +46,18 @@ public class EncodedColumnsUtil { public static byte[] getColumnQualifier(PColumn column, PTable table) { return EncodedColumnsUtil.getColumnQualifier(column, usesEncodedColumnNames(table)); } + + public static void setColumns(PColumn column, PTable table, Scan scan) { + if (table.getStorageScheme() == StorageScheme.COLUMNS_STORED_IN_SINGLE_CELL) { + // if a table storage scheme is COLUMNS_STORED_IN_SINGLE_CELL set then all columns of a column family are stored in a single cell + // (with the qualifier name being same as the family name), just project the column family here + // so that we can calculate estimatedByteSize correctly in ProjectionCompiler + scan.addFamily(column.getFamilyName().getBytes()); + } + else { + scan.addColumn(column.getFamilyName().getBytes(), EncodedColumnsUtil.getColumnQualifier(column, table)); + } + } public static byte[] getColumnQualifier(PColumn column, boolean encodedColumnName) { checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); http://git-wip-us.apache.org/repos/asf/phoenix/blob/2433c8d2/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index 76ec772..c6524f7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; @@ -57,7 +58,9 @@ import org.apache.phoenix.compile.WhereCompiler; import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; +import org.apache.phoenix.execute.MutationState.RowMutationState; import org.apache.phoenix.execute.TupleProjector; +import org.apache.phoenix.expression.ArrayColumnExpression; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.expression.KeyValueColumnExpression; import org.apache.phoenix.expression.RowKeyColumnExpression; @@ -81,13 +84,16 @@ import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnFamily; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableNotFoundException; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.ValueSchema.Field; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PBinary; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PDecimal; +import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PVarbinary; import org.apache.phoenix.schema.types.PVarchar; @@ -258,10 +264,10 @@ public class IndexUtil { } public static List<Mutation> generateIndexData(final PTable table, PTable index, - List<Mutation> dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection) + final Map<ImmutableBytesPtr, RowMutationState> valuesMap, List<Mutation> dataMutations, final KeyValueBuilder kvBuilder, PhoenixConnection connection) throws SQLException { try { - final ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + final ImmutableBytesPtr ptr = new ImmutableBytesPtr(); IndexMaintainer maintainer = index.getIndexMaintainer(table, connection); List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size()); for (final Mutation dataMutation : dataMutations) { @@ -290,20 +296,21 @@ public class IndexUtil { if (isEmptyKeyValue(table, ref)) { return null; } - Map<byte [], List<Cell>> familyMap = dataMutation.getFamilyCellMap(); byte[] family = ref.getFamily(); - List<Cell> kvs = familyMap.get(family); - if (kvs == null) { - return null; - } byte[] qualifier = ref.getQualifier(); - for (Cell kv : kvs) { - if (Bytes.compareTo(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), family, 0, family.length) == 0 && - Bytes.compareTo(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(), qualifier, 0, qualifier.length) == 0) { - ImmutableBytesPtr ptr = new ImmutableBytesPtr(); - kvBuilder.getValueAsPtr(kv, ptr); - return ptr; - } + RowMutationState rowMutationState = valuesMap.get(ptr); + PColumn column = null; + try { + column = table.getColumnFamily(family).getPColumnForColumnQualifier(qualifier); + } catch (ColumnNotFoundException e) { + } catch (ColumnFamilyNotFoundException e) { + } + if (rowMutationState!=null && column!=null) { + byte[] value = rowMutationState.getColumnValues().get(column); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(); + ptr.set(value==null ? ByteUtil.EMPTY_BYTE_ARRAY : value); + SchemaUtil.padData(table.getName().getString(), column, ptr); + return ptr; } return null; } @@ -316,7 +323,7 @@ public class IndexUtil { regionStartKey = tableRegionLocation.getRegionInfo().getStartKey(); regionEndkey = tableRegionLocation.getRegionInfo().getEndKey(); } - indexMutations.add(maintainer.buildUpdateMutation(kvBuilder, valueGetter, ptr, ts, regionStartKey, regionEndkey)); + indexMutations.add(maintainer.buildUpdateMutation(kvBuilder, valueGetter, ptr, ts, regionStartKey, regionEndkey, true)); } } return indexMutations; @@ -430,13 +437,18 @@ public class IndexUtil { public static TupleProjector getTupleProjector(Scan scan, ColumnReference[] dataColumns) { if (dataColumns != null && dataColumns.length != 0) { KeyValueSchema keyValueSchema = deserializeLocalIndexJoinSchemaFromScan(scan); - KeyValueColumnExpression[] keyValueColumns = new KeyValueColumnExpression[dataColumns.length]; + boolean storeColsInSingleCell = scan.getAttribute(BaseScannerRegionObserver.COLUMNS_STORED_IN_SINGLE_CELL)!=null; + Expression[] colExpressions = storeColsInSingleCell ? new ArrayColumnExpression[dataColumns.length] : new KeyValueColumnExpression[dataColumns.length]; for (int i = 0; i < dataColumns.length; i++) { - ColumnReference dataColumn = dataColumns[i]; - KeyValueColumnExpression dataColumnExpr = new KeyValueColumnExpression(keyValueSchema.getField(i), dataColumn.getFamily(), dataColumn.getQualifier()); - keyValueColumns[i] = dataColumnExpr; + byte[] family = dataColumns[i].getFamily(); + byte[] qualifier = dataColumns[i].getQualifier(); + Field field = keyValueSchema.getField(i); + Expression dataColumnExpr = + storeColsInSingleCell ? new ArrayColumnExpression(field, family, PInteger.INSTANCE.getCodec().decodeInt(qualifier, 0, SortOrder.getDefault())) + : new KeyValueColumnExpression(field, family, qualifier); + colExpressions[i] = dataColumnExpr; } - return new TupleProjector(keyValueSchema, keyValueColumns); + return new TupleProjector(keyValueSchema, colExpressions); } return null; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/2433c8d2/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index b13f4e3..00ad1ae 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -67,6 +67,7 @@ import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; +import org.apache.phoenix.schema.PTable.StorageScheme; import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.ValueSchema.Field; @@ -918,7 +919,8 @@ public class ScanUtil { } public static boolean setMinMaxQualifiersOnScan(PTable table) { - return EncodedColumnsUtil.usesEncodedColumnNames(table) && !table.isTransactional() && !hasDynamicColumns(table); + return table.getStorageScheme() != null && table.getStorageScheme() == StorageScheme.ENCODED_COLUMN_NAMES + && !table.isTransactional() && !hasDynamicColumns(table); } public static boolean hasDynamicColumns(PTable table) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/2433c8d2/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index 3071dc0..eb55f38 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -44,8 +44,10 @@ import javax.annotation.Nullable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.exception.DataExceedsCapacityException; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; import org.apache.phoenix.expression.Expression; @@ -1043,4 +1045,20 @@ public class SchemaUtil { } } + /** + * Pads the data in ptr by the required amount for fixed width data types + */ + public static void padData(String tableName, PColumn column, ImmutableBytesWritable ptr) { + PDataType type = column.getDataType(); + byte[] byteValue = ptr.get(); + boolean isNull = type.isNull(byteValue); + Integer maxLength = column.getMaxLength(); + if (!isNull && type.isFixedWidth() && maxLength != null) { + if (ptr.getLength() < maxLength) { + type.pad(ptr, maxLength, column.getSortOrder()); + } else if (ptr.getLength() > maxLength) { + throw new DataExceedsCapacityException(tableName + "." + column.getName().getString() + " may not exceed " + maxLength + " bytes (" + type.toObject(byteValue) + ")"); + } + } + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/2433c8d2/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java index 5887e5b..abcf0c8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java @@ -141,7 +141,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest { Mutation indexMutation = indexMutations.get(0); ImmutableBytesWritable indexKeyPtr = new ImmutableBytesWritable(indexMutation.getRow()); ptr.set(rowKeyPtr.get(), rowKeyPtr.getOffset(), rowKeyPtr.getLength()); - byte[] mutablelndexRowKey = im1.buildRowKey(valueGetter, ptr, null, null); + byte[] mutablelndexRowKey = im1.buildRowKey(valueGetter, ptr, null, null, false); byte[] immutableIndexRowKey = indexKeyPtr.copyBytes(); assertArrayEquals(immutableIndexRowKey, mutablelndexRowKey); for (ColumnReference ref : im1.getCoveredColumns()) {
