Repository: phoenix Updated Branches: refs/heads/encodecolumns 2433c8d21 -> 920be8fec (forced update)
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/UnboundedSkipNullCellsList.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/UnboundedSkipNullCellsList.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/UnboundedSkipNullCellsList.java new file mode 100644 index 0000000..02a85a5 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/UnboundedSkipNullCellsList.java @@ -0,0 +1,488 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema.tuple; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.NoSuchElementException; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.types.PInteger; + +import com.google.common.base.Preconditions; + +public class UnboundedSkipNullCellsList implements List<Cell> { + private int minQualifier; + private int maxQualifier; + private Cell[] array; + private int numNonNullElements; + private int firstNonNullElementIdx = -1; + private int leftBoundary; + private int rightBoundary; + + // extra capacity we have either at the start or at the end or at at both extremes + // to accommodate column qualifiers outside of the range (minQualifier, maxQualifier) + private static final int INIITAL_EXTRA_BUFFER = 10; + + public UnboundedSkipNullCellsList(int minQualifier, int maxQualifier) { + checkArgument(maxQualifier - minQualifier > 0, "Illegal arguments. MinQualifier: " + minQualifier + ". MaxQualifier: " + maxQualifier); + this.minQualifier = minQualifier; + this.maxQualifier = maxQualifier; + int minIndex = Math.max(0, minQualifier - INIITAL_EXTRA_BUFFER); + int maxIndex = maxQualifier + INIITAL_EXTRA_BUFFER; + int size = maxIndex - minIndex + 1; + this.array = new Cell[size]; + } + + + @Override + public int size() { + return numNonNullElements; + } + + @Override + public boolean isEmpty() { + return numNonNullElements == 0; + } + + @Override + public boolean contains(Object o) { + return indexOf(o) >= 0; + } + + + /** + * This implementation only returns an array of non-null elements in the list. + */ + @Override + public Object[] toArray() { + Object[] toReturn = new Object[numNonNullElements]; + int counter = 0; + for (int i = 0; i < array.length; i++) { + if (array[i] != null) { + toReturn[counter++] = array[i]; + } + } + return toReturn; + } + + + /** + * This implementation only returns an array of non-null elements in the list. + * This is not the most efficient way of copying elemts into an array + */ + @Override + @SuppressWarnings("unchecked") + public <T> T[] toArray(T[] a) { + T[] toReturn = (T[])java.lang.reflect.Array.newInstance(a.getClass().getComponentType(), numNonNullElements); + int counter = 0; + for (int i = 0; i < array.length; i++) { + if (array[i] != null) { + toReturn[counter++] = (T)array[i]; + } + } + return toReturn; + } + + @Override + public boolean add(Cell e) { + if (e == null) { + throw new NullPointerException(); + } + int columnQualifier = PInteger.INSTANCE.getCodec().decodeInt(e.getQualifierArray(), e.getQualifierOffset(), SortOrder.ASC); + if (columnQualifier < 0) { + throw new IllegalArgumentException("Invalid column qualifier " + columnQualifier + " for cell " + e); + } + ensureCapacity(columnQualifier); + int idx = getArrayIndex(columnQualifier); + array[idx] = e; + numNonNullElements++; + if (firstNonNullElementIdx == -1) { + firstNonNullElementIdx = idx; + } + return true; + } + + @Override + public boolean remove(Object o) { + if (o == null) { + return false; + } + Cell e = (Cell)o; + int i = 0; + while (i < array.length) { + if (array[i] != null && array[i].equals(e)) { + array[i] = null; + numNonNullElements--; + if (numNonNullElements == 0) { + firstNonNullElementIdx = -1; + } else if (firstNonNullElementIdx == i) { + // the element being removed was the first non-null element we knew + while (i < array.length && (array[i]) == null) { + i++; + } + if (i < array.length) { + firstNonNullElementIdx = i; + } else { + firstNonNullElementIdx = -1; + } + } + return true; + } + i++; + } + return false; + } + + @Override + public boolean containsAll(Collection<?> c) { + boolean containsAll = true; + Iterator<?> itr = c.iterator(); + while (itr.hasNext()) { + containsAll &= (indexOf(itr.next()) >= 0); + } + return containsAll; + } + + @Override + public boolean addAll(Collection<? extends Cell> c) { + boolean changed = false; + for (Cell cell : c) { + if (c == null) { + throw new NullPointerException(); + } + changed |= add(cell); + } + return changed; + } + + @Override + public boolean addAll(int index, Collection<? extends Cell> c) { + throwUnsupportedOperationException(); + return false; + } + + @Override + public boolean removeAll(Collection<?> c) { + Iterator<?> itr = c.iterator(); + boolean changed = false; + while (itr.hasNext()) { + changed |= remove(itr.next()); + } + return changed; + } + + @Override + public boolean retainAll(Collection<?> c) { + throwUnsupportedOperationException(); + return false; + } + + @Override + public void clear() { + Arrays.fill(array, null); + } + + @Override + public Cell get(int index) { + rangeCheck(index); + int counter = 0; + for (; counter < array.length; counter++) { + if (array[counter] != null && counter == index) { + break; + } + } + return array[counter]; + } + + @Override + public Cell set(int index, Cell element) { + throwUnsupportedOperationException(); + return null; + } + + @Override + public void add(int index, Cell element) { + throwUnsupportedOperationException(); + } + + @Override + public Cell remove(int index) { + throwUnsupportedOperationException(); + return null; + } + + @Override + public int indexOf(Object o) { + if (o == null) { + return -1; + } else { + for (int i = 0; i < array.length; i++) + if (o.equals(array[i])) { + return i; + } + } + return -1; + } + + @Override + public int lastIndexOf(Object o) { + if (o == null) { + return -1; + } + for (int i = array.length - 1; i >=0 ; i--) { + if (o.equals(array[i])) { + return i; + } + } + return -1; + } + + @Override + public ListIterator<Cell> listIterator() { + throwUnsupportedOperationException(); + return null; + } + + @Override + public ListIterator<Cell> listIterator(int index) { + throwUnsupportedOperationException(); + return null; + } + + @Override + public List<Cell> subList(int fromIndex, int toIndex) { + throwUnsupportedOperationException(); + return null; + } + + @Override + public Iterator<Cell> iterator() { + return new Itr(); + } + + public Cell getCellForColumnQualifier(int columnQualifier) { + int idx = getArrayIndex(columnQualifier); + return array[idx]; + } + + //TODO: samarth need to handle overflow conditions and integer growing beyond sizeofint. + private void ensureCapacity(int qualifier) { + if (qualifier >= 0 && qualifier < leftBoundary) { + // This should happen very rarely. + //TODO: samarth implement this case. + } else if (qualifier >= 0 && qualifier > rightBoundary) { + // TODO: samarth implement this case. + } + } + + private void rangeCheck(int index) { + if (index < 0 || index > size() - 1) { + throw new IndexOutOfBoundsException(); + } + } + + private void throwUnsupportedOperationException() { + throw new UnsupportedOperationException("Operation cannot be supported because it violates invariance"); + } + + private class Itr implements Iterator<Cell> { + private Cell current; + private int currentIdx = 0; + private boolean exhausted = false; + private Itr() { + moveToNextNonNullCell(true); + } + + @Override + public boolean hasNext() { + return !exhausted; + } + + @Override + public Cell next() { + if (exhausted) { + return null; + } + Cell next = current; + moveToNextNonNullCell(false); + return next; + } + + @Override + public void remove() { + throwUnsupportedOperationException(); + } + + private void moveToNextNonNullCell(boolean init) { + int i = init ? minQualifier : currentIdx + 1; + while (i < array.length && (current = array[i]) == null) { + i++; + } + if (i < array.length) { + currentIdx = i; + } else { + currentIdx = -1; + exhausted = true; + } + } + + } + + public Cell getFirstCell() { + if (firstNonNullElementIdx == -1) { + throw new IllegalStateException("List doesn't have any non-null cell present"); + } + return array[firstNonNullElementIdx]; + } + + private int getArrayIndex(int columnQualifier) { + return columnQualifier - minQualifier; + } + + private class ListItr implements ListIterator<Cell> { + private int previousIndex; + private int nextIndex; + private Cell previous; + private Cell next; + + private ListItr() { + movePointersForward(true); + previous = null; + if (nextIndex != -1) { + next = array[nextIndex]; + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Cell next() { + Cell toReturn = next; + if (toReturn == null) { + throw new NoSuchElementException(); + } + movePointersForward(false); + return toReturn; + } + + @Override + public boolean hasPrevious() { + return previous != null; + } + + @Override + public Cell previous() { + Cell toReturn = previous; + if (toReturn == null) { + throw new NoSuchElementException(); + } + movePointersBackward(false); + return toReturn; + } + + @Override + public int nextIndex() { + return nextIndex; + } + + @Override + public int previousIndex() { + return previousIndex; + } + + @Override + public void remove() { + // TODO Auto-generated method stub + + } + + // TODO: samarth this is one of these ouch methods that can make our implementation frgaile. + // It is a non-optional method and can't really be supported + @Override + public void set(Cell e) { + // TODO Auto-generated method stub + + } + + @Override + public void add(Cell e) { + // TODO Auto-generated method stub + + } + + private void movePointersForward(boolean init) { + int i = init ? 0 : nextIndex; + if (!init) { + previousIndex = nextIndex; + previous = next; + } else { + previousIndex = -1; + previous = null; + } + while (i < array.length && (array[i]) == null) { + i++; + } + if (i < array.length) { + nextIndex = i; + next = array[i]; + } else { + nextIndex = -1; + next = null; + } + } + + private void movePointersBackward(boolean init) { + int i = init ? 0 : previousIndex; + } + + } + + public static void main (String args[]) throws Exception { + UnboundedSkipNullCellsList list = new UnboundedSkipNullCellsList(0, 3); // list of eleven elements + System.out.println(list.size()); + byte[] row = Bytes.toBytes("row"); + byte[] cf = Bytes.toBytes("cf"); + list.add(KeyValue.createFirstOnRow(row, cf, PInteger.INSTANCE.toBytes(0))); + list.add(KeyValue.createFirstOnRow(row, cf, PInteger.INSTANCE.toBytes(5))); + list.add(KeyValue.createFirstOnRow(row, cf, PInteger.INSTANCE.toBytes(10))); + + for (Cell c : list) { + System.out.println(c); + } + System.out.println(list.size()); + System.out.println(list.get(0)); + System.out.println(list.get(5)); + System.out.println(list.get(10)); + System.out.println(list.get(1)); + System.out.println(list.remove(KeyValue.createFirstOnRow(row, cf, PInteger.INSTANCE.toBytes(5)))); + System.out.println(list.get(5)); + System.out.println(list.size()); + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java new file mode 100644 index 0000000..bfbfc54 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/EncodedColumnsUtil.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.StorageScheme; +import org.apache.phoenix.schema.types.PInteger; + +public class EncodedColumnsUtil { + + public static boolean usesEncodedColumnNames(PTable table) { + return table.getStorageScheme() != null && table.getStorageScheme() == StorageScheme.ENCODED_COLUMN_NAMES; + } + + public static byte[] getEncodedColumnQualifier(PColumn column) { + checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); + checkArgument(!column.isDynamic(), "No encoded column qualifiers for dynamic columns"); + return PInteger.INSTANCE.toBytes(column.getEncodedColumnQualifier()); + } + + public static byte[] getColumnQualifier(PColumn column, PTable table) { + return EncodedColumnsUtil.getColumnQualifier(column, usesEncodedColumnNames(table)); + } + + public static byte[] getColumnQualifier(PColumn column, boolean encodedColumnName) { + checkArgument(!SchemaUtil.isPKColumn(column), "No column qualifiers for PK columns"); + if (column.isDynamic()) { // Dynamic column names don't have encoded column names + return column.getName().getBytes(); + } + return encodedColumnName ? PInteger.INSTANCE.toBytes(column.getEncodedColumnQualifier()) : column.getName().getBytes(); + } + + /** + * @return pair of byte arrays. The first part of the pair is the empty key value's column qualifier, and the second + * part is the value to use for it. + */ + public static Pair<byte[], byte[]> getEmptyKeyValueInfo(PTable table) { + return usesEncodedColumnNames(table) ? new Pair<>(QueryConstants.ENCODED_EMPTY_COLUMN_BYTES, + QueryConstants.ENCODED_EMPTY_COLUMN_VALUE_BYTES) : new Pair<>(QueryConstants.EMPTY_COLUMN_BYTES, + QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + } + + /** + * @return pair of byte arrays. The first part of the pair is the empty key value's column qualifier, and the second + * part is the value to use for it. + */ + public static Pair<byte[], byte[]> getEmptyKeyValueInfo(boolean usesEncodedColumnNames) { + return usesEncodedColumnNames ? new Pair<>(QueryConstants.ENCODED_EMPTY_COLUMN_BYTES, + QueryConstants.ENCODED_EMPTY_COLUMN_VALUE_BYTES) : new Pair<>(QueryConstants.EMPTY_COLUMN_BYTES, + QueryConstants.EMPTY_COLUMN_VALUE_BYTES); + } + + public static boolean hasEncodedColumnName(PColumn column){ + return !SchemaUtil.isPKColumn(column) && !column.isDynamic() && column.getEncodedColumnQualifier() != null; + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index 7cd7eb3..76ec772 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -18,6 +18,9 @@ package org.apache.phoenix.util; import static org.apache.phoenix.query.QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_QUALIFIER; + import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.IOException; @@ -192,7 +195,7 @@ public class IndexUtil { throw new IllegalArgumentException("Could not find column family \"" + indexColumnName.substring(0, pos) + "\" in index column name of \"" + indexColumnName + "\"", e); } try { - return family.getColumn(indexColumnName.substring(pos+1)); + return family.getPColumnForColumnName(indexColumnName.substring(pos+1)); } catch (ColumnNotFoundException e) { throw new IllegalArgumentException("Could not find column \"" + indexColumnName.substring(pos+1) + "\" in index column name of \"" + indexColumnName + "\"", e); } @@ -219,10 +222,11 @@ public class IndexUtil { private static boolean isEmptyKeyValue(PTable table, ColumnReference ref) { byte[] emptyKeyValueCF = SchemaUtil.getEmptyColumnFamily(table); + byte[] emptyKeyValueQualifier = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst(); return (Bytes.compareTo(emptyKeyValueCF, 0, emptyKeyValueCF.length, ref.getFamilyWritable() .get(), ref.getFamilyWritable().getOffset(), ref.getFamilyWritable().getLength()) == 0 && Bytes - .compareTo(QueryConstants.EMPTY_COLUMN_BYTES, 0, - QueryConstants.EMPTY_COLUMN_BYTES.length, ref.getQualifierWritable().get(), ref + .compareTo(emptyKeyValueQualifier, 0, + emptyKeyValueQualifier.length, ref.getQualifierWritable().get(), ref .getQualifierWritable().getOffset(), ref.getQualifierWritable() .getLength()) == 0); } @@ -501,139 +505,142 @@ public class IndexUtil { } // TODO: handle null case (but shouldn't happen) + //TODO: samarth confirm if this is the right thing to do here i.e. pass false for look up. Tuple joinTuple = new ResultTuple(joinResult); // This will create a byte[] that captures all of the values from the data table byte[] value = tupleProjector.getSchema().toBytes(joinTuple, tupleProjector.getExpressions(), tupleProjector.getValueBitSet(), ptr); KeyValue keyValue = - KeyValueUtil.newKeyValue(firstCell.getRowArray(),firstCell.getRowOffset(),firstCell.getRowLength(), TupleProjector.VALUE_COLUMN_FAMILY, - TupleProjector.VALUE_COLUMN_QUALIFIER, firstCell.getTimestamp(), value, 0, value.length); + KeyValueUtil.newKeyValue(firstCell.getRowArray(),firstCell.getRowOffset(),firstCell.getRowLength(), VALUE_COLUMN_FAMILY, + VALUE_COLUMN_QUALIFIER, firstCell.getTimestamp(), value, 0, value.length); result.add(keyValue); } for (int i = 0; i < result.size(); i++) { final Cell cell = result.get(i); - // TODO: Create DelegateCell class instead - Cell newCell = new Cell() { + if (cell != null) { + // TODO: Create DelegateCell class instead + Cell newCell = new Cell() { - @Override - public byte[] getRowArray() { - return cell.getRowArray(); - } + @Override + public byte[] getRowArray() { + return cell.getRowArray(); + } - @Override - public int getRowOffset() { - return cell.getRowOffset() + offset; - } + @Override + public int getRowOffset() { + return cell.getRowOffset() + offset; + } - @Override - public short getRowLength() { - return (short)(cell.getRowLength() - offset); - } + @Override + public short getRowLength() { + return (short)(cell.getRowLength() - offset); + } - @Override - public byte[] getFamilyArray() { - return cell.getFamilyArray(); - } + @Override + public byte[] getFamilyArray() { + return cell.getFamilyArray(); + } - @Override - public int getFamilyOffset() { - return cell.getFamilyOffset(); - } + @Override + public int getFamilyOffset() { + return cell.getFamilyOffset(); + } - @Override - public byte getFamilyLength() { - return cell.getFamilyLength(); - } + @Override + public byte getFamilyLength() { + return cell.getFamilyLength(); + } - @Override - public byte[] getQualifierArray() { - return cell.getQualifierArray(); - } + @Override + public byte[] getQualifierArray() { + return cell.getQualifierArray(); + } - @Override - public int getQualifierOffset() { - return cell.getQualifierOffset(); - } + @Override + public int getQualifierOffset() { + return cell.getQualifierOffset(); + } - @Override - public int getQualifierLength() { - return cell.getQualifierLength(); - } + @Override + public int getQualifierLength() { + return cell.getQualifierLength(); + } - @Override - public long getTimestamp() { - return cell.getTimestamp(); - } + @Override + public long getTimestamp() { + return cell.getTimestamp(); + } - @Override - public byte getTypeByte() { - return cell.getTypeByte(); - } + @Override + public byte getTypeByte() { + return cell.getTypeByte(); + } - @Override - public long getMvccVersion() { - return cell.getMvccVersion(); - } + @Override + public long getMvccVersion() { + return cell.getMvccVersion(); + } - @Override - public byte[] getValueArray() { - return cell.getValueArray(); - } + @Override + public byte[] getValueArray() { + return cell.getValueArray(); + } - @Override - public int getValueOffset() { - return cell.getValueOffset(); - } + @Override + public int getValueOffset() { + return cell.getValueOffset(); + } - @Override - public int getValueLength() { - return cell.getValueLength(); - } + @Override + public int getValueLength() { + return cell.getValueLength(); + } - @Override - public byte[] getTagsArray() { - return cell.getTagsArray(); - } + @Override + public byte[] getTagsArray() { + return cell.getTagsArray(); + } - @Override - public int getTagsOffset() { - return cell.getTagsOffset(); - } + @Override + public int getTagsOffset() { + return cell.getTagsOffset(); + } - @Override - public short getTagsLength() { - return cell.getTagsLength(); - } + @Override + public short getTagsLength() { + return cell.getTagsLength(); + } - @Override - public byte[] getValue() { - return cell.getValue(); - } + @Override + public byte[] getValue() { + return cell.getValue(); + } - @Override - public byte[] getFamily() { - return cell.getFamily(); - } + @Override + public byte[] getFamily() { + return cell.getFamily(); + } - @Override - public byte[] getQualifier() { - return cell.getQualifier(); - } + @Override + public byte[] getQualifier() { + return cell.getQualifier(); + } - @Override - public byte[] getRow() { - return cell.getRow(); - } + @Override + public byte[] getRow() { + return cell.getRow(); + } - @Override - @Deprecated - public int getTagsLengthUnsigned() { - return cell.getTagsLengthUnsigned(); - } - }; - // Wrap cell in cell that offsets row key - result.set(i, newCell); + @Override + @Deprecated + public int getTagsLengthUnsigned() { + return cell.getTagsLengthUnsigned(); + } + }; + // Wrap cell in cell that offsets row key + result.set(i, newCell); + } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java index 685879d..8a0ad84 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixRuntime.java @@ -495,9 +495,9 @@ public class PhoenixRuntime { String familyName = tokens[0]; String familyColumn = tokens[1]; PColumnFamily family = table.getColumnFamily(familyName); - pColumn = family.getColumn(familyColumn); + pColumn = family.getPColumnForColumnName(familyColumn); } else { - pColumn = table.getColumn(columnName); + pColumn = table.getPColumnForColumnName(columnName); } return getColumnInfo(pColumn); } @@ -1028,9 +1028,9 @@ public class PhoenixRuntime { PColumn pColumn = null; if (familyName != null) { PColumnFamily family = table.getColumnFamily(familyName); - pColumn = family.getColumn(columnName); + pColumn = family.getPColumnForColumnName(columnName); } else { - pColumn = table.getColumn(columnName); + pColumn = table.getPColumnForColumnName(columnName); } return pColumn; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java index dba6550..f97230b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ResultUtil.java @@ -18,7 +18,6 @@ package org.apache.phoenix.util; import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -129,63 +128,4 @@ public class ResultUtil { return Bytes.compareTo(getRawBytes(r1), getKeyOffset(r1), getKeyLength(r1), getRawBytes(r2), getKeyOffset(r2), getKeyLength(r2)); } - /** - * Binary search for latest column value without allocating memory in the process - */ - public static KeyValue getColumnLatest(Result r, byte[] family, byte[] qualifier) { - byte[] rbytes = getRawBytes(r); - int roffset = getKeyOffset(r); - int rlength = getKeyLength(r); - return getColumnLatest(r, rbytes, roffset, rlength, family, 0, family.length, qualifier, 0, qualifier.length); - } - - public static KeyValue getSearchTerm(Result r, byte[] family, byte[] qualifier) { - byte[] rbytes = getRawBytes(r); - int roffset = getKeyOffset(r); - int rlength = getKeyLength(r); - return KeyValue.createFirstOnRow(rbytes, roffset, rlength, family, 0, family.length, qualifier, 0, qualifier.length); - } - /** - * Binary search for latest column value without allocating memory in the process - */ - public static KeyValue getColumnLatest(Result r, byte[] row, int roffset, int rlength, byte[] family, int foffset, int flength, byte[] qualifier, int qoffset, int qlength) { - KeyValue searchTerm = KeyValue.createFirstOnRow(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, qlength); - return getColumnLatest(r,searchTerm); - - } - - /** - * Binary search for latest column value without allocating memory in the process - * @param r - * @param searchTerm - */ - @SuppressWarnings("deprecation") - public static KeyValue getColumnLatest(Result r, KeyValue searchTerm) { - KeyValue [] kvs = r.raw(); // side effect possibly. - if (kvs == null || kvs.length == 0) { - return null; - } - - // pos === ( -(insertion point) - 1) - int pos = Arrays.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR); - // never will exact match - if (pos < 0) { - pos = (pos+1) * -1; - // pos is now insertion point - } - if (pos == kvs.length) { - return null; // doesn't exist - } - - KeyValue kv = kvs[pos]; - if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), - searchTerm.getBuffer(), searchTerm.getFamilyOffset(), searchTerm.getFamilyLength()) != 0) { - return null; - } - if (Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), - searchTerm.getBuffer(), searchTerm.getQualifierOffset(), searchTerm.getQualifierLength()) != 0) { - return null; - } - return kv; - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java index d7f6f2f..b13f4e3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ScanUtil.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.io.WritableComparator; import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.ScanRanges; @@ -62,6 +63,7 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.IllegalDataException; +import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTable.IndexType; @@ -69,6 +71,7 @@ import org.apache.phoenix.schema.RowKeySchema; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.ValueSchema.Field; import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PVarbinary; import com.google.common.collect.Iterators; @@ -893,5 +896,38 @@ public class ScanUtil { } return true; } + + public static Pair<Integer, Integer> getMinMaxQualifiersFromScan(Scan scan) { + Integer minQ = null, maxQ = null; + byte[] minQualifier = scan.getAttribute(BaseScannerRegionObserver.MIN_QUALIFIER); + if (minQualifier != null) { + minQ = (Integer)PInteger.INSTANCE.toObject(minQualifier); + } + byte[] maxQualifier = scan.getAttribute(BaseScannerRegionObserver.MAX_QUALIFIER); + if (maxQualifier != null) { + maxQ = (Integer)PInteger.INSTANCE.toObject(maxQualifier); + } + if (minQualifier == null) { + return null; + } + return new Pair<>(minQ, maxQ); + } + + public static boolean useQualifierAsIndex(Pair<Integer, Integer> minMaxQualifiers, boolean isJoin) { + return minMaxQualifiers != null && !isJoin; + } + + public static boolean setMinMaxQualifiersOnScan(PTable table) { + return EncodedColumnsUtil.usesEncodedColumnNames(table) && !table.isTransactional() && !hasDynamicColumns(table); + } + + public static boolean hasDynamicColumns(PTable table) { + for (PColumn col : table.getColumns()) { + if (col.isDynamic()) { + return true; + } + } + return false; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index b53daea..3071dc0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -154,8 +154,9 @@ public class SchemaUtil { rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, column.getFamilyName().getBytes().length, column.getName().getBytes().length, valueLength); } } + byte[] emptyKeyValueKV = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst(); // Empty key value - rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, getEmptyColumnFamily(table).length, QueryConstants.EMPTY_COLUMN_BYTES.length, 0); + rowSize += KeyValue.getKeyValueDataStructureSize(keyLength, getEmptyColumnFamily(table).length, emptyKeyValueKV.length, 0); return rowSize; } @@ -394,7 +395,7 @@ public class SchemaUtil { } } else { try { - return table.getColumnFamily(familyName.getString()).getColumn(column.getName().getString()).getName().getString(); + return table.getColumnFamily(familyName.getString()).getPColumnForColumnName(column.getName().getString()).getName().getString(); } catch (ColumnFamilyNotFoundException e) { continue; // Shouldn't happen } catch (ColumnNotFoundException e) { @@ -591,7 +592,7 @@ public class SchemaUtil { public static boolean columnExists(PTable table, String columnName) { try { - table.getColumn(columnName); + table.getPColumnForColumnName(columnName); return true; } catch (ColumnNotFoundException e) { return false; @@ -1020,7 +1021,7 @@ public class SchemaUtil { } return getStrippedName(physicalTableName, indexPrefix); } - + private static String getStrippedName(String physicalTableName, String indexPrefix) { return physicalTableName.indexOf(indexPrefix) == 0 ? physicalTableName.substring(indexPrefix.length()) : physicalTableName; @@ -1041,4 +1042,5 @@ public class SchemaUtil { return normalizeIdentifier(tableName); } } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java index 1c7477d..ae2bd14 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/HavingCompilerTest.java @@ -156,7 +156,7 @@ public class HavingCompilerTest extends BaseConnectionlessQueryTest { String query = "select count(1) from atable group by a_string having count(1) >= 1 or a_string = 'foo'"; List<Object> binds = Collections.emptyList(); Expressions expressions = compileStatement(query,binds); - PColumn aCol = ATABLE.getColumn("A_STRING"); + PColumn aCol = ATABLE.getPColumnForColumnName("A_STRING"); Expression h = or( constantComparison(CompareOp.GREATER_OR_EQUAL, new CountAggregateFunction(),1L), constantComparison(CompareOp.EQUAL, http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index c91b855..ec36ed2 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -169,7 +169,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { String query = "CREATE TABLE t1 (k integer not null primary key, a.k decimal, b.k decimal)"; conn.createStatement().execute(query); PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - PColumn c = pconn.getTable(new PTableKey(pconn.getTenantId(), "T1")).getColumn("K"); + PColumn c = pconn.getTable(new PTableKey(pconn.getTenantId(), "T1")).getPColumnForColumnName("K"); assertTrue(SchemaUtil.isPKColumn(c)); } finally { conn.close(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java index 2f1a369..7b0a905 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/WhereCompilerTest.java @@ -67,8 +67,10 @@ import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SaltingUtil; +import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.types.PChar; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.types.PVarchar; @@ -118,9 +120,9 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - ColumnExpression idExpression = new ColumnRef(plan.getTableRef(), plan.getTableRef().getTable().getColumn("ID").getPosition()).newColumnExpression(); + ColumnExpression idExpression = new ColumnRef(plan.getTableRef(), plan.getTableRef().getTable().getPColumnForColumnName("ID").getPosition()).newColumnExpression(); Expression id = new RowKeyColumnExpression(idExpression,new RowKeyValueAccessor(plan.getTableRef().getTable().getPKColumns(),0)); - Expression company = new KeyValueColumnExpression(plan.getTableRef().getTable().getColumn("COMPANY")); + Expression company = new KeyValueColumnExpression(plan.getTableRef().getTable().getPColumnForColumnName("COMPANY"), false); // FilterList has no equals implementation assertTrue(filter instanceof FilterList); FilterList filterList = (FilterList)filter; @@ -148,11 +150,11 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - PColumn column = plan.getTableRef().getTable().getColumn("COMPANY"); + PColumn column = plan.getTableRef().getTable().getPColumnForColumnName("COMPANY"); assertEquals( singleKVFilter(constantComparison( CompareOp.EQUAL, - new KeyValueColumnExpression(column), + new KeyValueColumnExpression(column, false), "c3")), filter); } @@ -939,16 +941,18 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - + PTable table = plan.getTableRef().getTable(); + ColumnExpression aInteger = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_INTEGER").getPosition()).newColumnExpression(); + ColumnExpression aString = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_STRING").getPosition()).newColumnExpression(); assertEquals( multiKVFilter(and( constantComparison( CompareOp.EQUAL, - A_INTEGER, + aInteger, 0), constantComparison( CompareOp.EQUAL, - A_STRING, + aString, "foo"))), filter); @@ -971,16 +975,18 @@ public class WhereCompilerTest extends BaseConnectionlessQueryTest { QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Filter filter = scan.getFilter(); - + PTable table = plan.getTableRef().getTable(); + ColumnExpression aInteger = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_INTEGER").getPosition()).newColumnExpression(); + ColumnExpression aString = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_STRING").getPosition()).newColumnExpression(); assertEquals( multiKVFilter(and( constantComparison( CompareOp.EQUAL, - A_INTEGER, + aInteger, 0), constantComparison( CompareOp.EQUAL, - A_STRING, + aString, "foo"))), filter); http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java index 62aafa5..680dc57 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; @@ -55,6 +56,7 @@ import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.JoinTableNode.JoinType; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.parse.SelectStatement; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; @@ -247,16 +249,16 @@ public class CorrelatePlanTest { for (int i = 0; i < row.length; i++) { String name = ParseNodeFactory.createTempAlias(); Expression expr = LiteralExpression.newConstant(row[i]); - columns.add(new PColumnImpl(PNameFactory.newName(name), PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), + columns.add(new PColumnImpl(PNameFactory.newName(name), PNameFactory.newName(VALUE_COLUMN_FAMILY), expr.getDataType(), expr.getMaxLength(), expr.getScale(), expr.isNullable(), - i, expr.getSortOrder(), null, null, false, name, false, false)); + i, expr.getSortOrder(), null, null, false, name, false, false, null)); } try { PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.<PTable>emptyList(), false, Collections.<PName>emptyList(), null, null, false, false, false, null, - null, null, true, false, 0, 0L, Boolean.FALSE, null, false); + null, null, true, false, 0, 0L, Boolean.FALSE, null, false, null, null); TableRef sourceTable = new TableRef(pTable); List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java index 1b16d40..bae4dd5 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; @@ -49,6 +50,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.parse.ParseNodeFactory; import org.apache.phoenix.parse.SelectStatement; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.ColumnRef; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PColumnImpl; @@ -170,14 +172,14 @@ public class LiteralResultIteratorPlanTest { String name = ParseNodeFactory.createTempAlias(); Expression expr = LiteralExpression.newConstant(row[i]); columns.add(new PColumnImpl(PNameFactory.newName(name), - PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), expr.getDataType(), expr.getMaxLength(), - expr.getScale(), expr.isNullable(), i, expr.getSortOrder(), null, null, false, name, false, false)); + PNameFactory.newName(VALUE_COLUMN_FAMILY), expr.getDataType(), expr.getMaxLength(), + expr.getScale(), expr.isNullable(), i, expr.getSortOrder(), null, null, false, name, false, false, null)); } try { PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false, - false, null, null, null, true, false, 0, 0L, false, null, false); + false, null, null, null, true, false, 0, 0L, false, null, false, null, null); TableRef sourceTable = new TableRef(pTable); List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java index 50e2721..ff62f63 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/UnnestArrayPlanTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.execute; +import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY; import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR; @@ -117,8 +118,8 @@ public class UnnestArrayPlanTest { LiteralExpression dummy = LiteralExpression.newConstant(null, arrayType); RowKeyValueAccessor accessor = new RowKeyValueAccessor(Arrays.asList(dummy), 0); UnnestArrayPlan plan = new UnnestArrayPlan(subPlan, new RowKeyColumnExpression(dummy, accessor), withOrdinality); - PColumn elemColumn = new PColumnImpl(PNameFactory.newName("ELEM"), PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), baseType, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false); - PColumn indexColumn = withOrdinality ? new PColumnImpl(PNameFactory.newName("IDX"), PNameFactory.newName(TupleProjector.VALUE_COLUMN_FAMILY), PInteger.INSTANCE, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false) : null; + PColumn elemColumn = new PColumnImpl(PNameFactory.newName("ELEM"), PNameFactory.newName(VALUE_COLUMN_FAMILY), baseType, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false, null); + PColumn indexColumn = withOrdinality ? new PColumnImpl(PNameFactory.newName("IDX"), PNameFactory.newName(VALUE_COLUMN_FAMILY), PInteger.INSTANCE, null, null, true, 0, SortOrder.getDefault(), null, null, false, "", false, false, null) : null; List<PColumn> columns = withOrdinality ? Arrays.asList(elemColumn, indexColumn) : Arrays.asList(elemColumn); ProjectedColumnExpression elemExpr = new ProjectedColumnExpression(elemColumn, columns, 0, elemColumn.getName().getString()); ProjectedColumnExpression indexExpr = withOrdinality ? new ProjectedColumnExpression(indexColumn, columns, 1, indexColumn.getName().getString()) : null; http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java index 7ee579c..98c2495 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/ColumnExpressionTest.java @@ -41,8 +41,8 @@ public class ColumnExpressionTest { int maxLen = 30; int scale = 5; PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PDecimal.INSTANCE, maxLen, scale, - true, 20, SortOrder.getDefault(), 0, null, false, null, false, false); - ColumnExpression colExp = new KeyValueColumnExpression(column); + true, 20, SortOrder.getDefault(), 0, null, false, null, false, false, 0); + ColumnExpression colExp = new KeyValueColumnExpression(column, true); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); colExp.write(dOut); @@ -61,8 +61,8 @@ public class ColumnExpressionTest { public void testSerializationWithNullScale() throws Exception { int maxLen = 30; PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PBinary.INSTANCE, maxLen, null, - true, 20, SortOrder.getDefault(), 0, null, false, null, false, false); - ColumnExpression colExp = new KeyValueColumnExpression(column); + true, 20, SortOrder.getDefault(), 0, null, false, null, false, false, 0); + ColumnExpression colExp = new KeyValueColumnExpression(column, true); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); colExp.write(dOut); @@ -81,8 +81,8 @@ public class ColumnExpressionTest { public void testSerializationWithNullMaxLength() throws Exception { int scale = 5; PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PVarchar.INSTANCE, null, scale, - true, 20, SortOrder.getDefault(), 0, null, false, null, false, false); - ColumnExpression colExp = new KeyValueColumnExpression(column); + true, 20, SortOrder.getDefault(), 0, null, false, null, false, false, 0); + ColumnExpression colExp = new KeyValueColumnExpression(column, true); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); colExp.write(dOut); @@ -100,8 +100,8 @@ public class ColumnExpressionTest { @Test public void testSerializationWithNullScaleAndMaxLength() throws Exception { PColumn column = new PColumnImpl(PNameFactory.newName("c1"), PNameFactory.newName("f1"), PDecimal.INSTANCE, null, null, true, - 20, SortOrder.getDefault(), 0, null, false, null, false, false); - ColumnExpression colExp = new KeyValueColumnExpression(column); + 20, SortOrder.getDefault(), 0, null, false, null, false, false, 0); + ColumnExpression colExp = new KeyValueColumnExpression(column, true); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dOut = new DataOutputStream(baos); colExp.write(dOut); http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java index 112109e..5887e5b 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/index/IndexMaintainerTest.java @@ -135,8 +135,7 @@ public class IndexMaintainerTest extends BaseConnectionlessQueryTest { } ValueGetter valueGetter = newValueGetter(row, valueMap); - List<Mutation> indexMutations = - IndexTestUtil.generateIndexData(index, table, dataMutation, ptr, builder); + List<Mutation> indexMutations = IndexTestUtil.generateIndexData(index, table, dataMutation, ptr, builder); assertEquals(1,indexMutations.size()); assertTrue(indexMutations.get(0) instanceof Put); Mutation indexMutation = indexMutations.get(0); http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java index 791eb23..ddd8241 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/iterate/AggregateResultScannerTest.java @@ -74,4 +74,4 @@ public class AggregateResultScannerTest extends BaseConnectionlessQueryTest { ResultIterator scanner = new GroupedAggregatingResultIterator(new MergeSortRowKeyResultIterator(iterators), aggregators); AssertResults.assertResults(scanner, expectedResults); } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java index 452ea4d..0844d9b 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseConnectionlessQueryTest.java @@ -125,15 +125,15 @@ public class BaseConnectionlessQueryTest extends BaseTest { try { PTable table = conn.getTable(new PTableKey(null, ATABLE_NAME)); ATABLE = table; - ORGANIZATION_ID = new ColumnRef(new TableRef(table), table.getColumn("ORGANIZATION_ID").getPosition()).newColumnExpression(); - ENTITY_ID = new ColumnRef(new TableRef(table), table.getColumn("ENTITY_ID").getPosition()).newColumnExpression(); - A_INTEGER = new ColumnRef(new TableRef(table), table.getColumn("A_INTEGER").getPosition()).newColumnExpression(); - A_STRING = new ColumnRef(new TableRef(table), table.getColumn("A_STRING").getPosition()).newColumnExpression(); - B_STRING = new ColumnRef(new TableRef(table), table.getColumn("B_STRING").getPosition()).newColumnExpression(); - A_DATE = new ColumnRef(new TableRef(table), table.getColumn("A_DATE").getPosition()).newColumnExpression(); - A_TIME = new ColumnRef(new TableRef(table), table.getColumn("A_TIME").getPosition()).newColumnExpression(); - A_TIMESTAMP = new ColumnRef(new TableRef(table), table.getColumn("A_TIMESTAMP").getPosition()).newColumnExpression(); - X_DECIMAL = new ColumnRef(new TableRef(table), table.getColumn("X_DECIMAL").getPosition()).newColumnExpression(); + ORGANIZATION_ID = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("ORGANIZATION_ID").getPosition()).newColumnExpression(); + ENTITY_ID = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("ENTITY_ID").getPosition()).newColumnExpression(); + A_INTEGER = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_INTEGER").getPosition()).newColumnExpression(); + A_STRING = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_STRING").getPosition()).newColumnExpression(); + B_STRING = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("B_STRING").getPosition()).newColumnExpression(); + A_DATE = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_DATE").getPosition()).newColumnExpression(); + A_TIME = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_TIME").getPosition()).newColumnExpression(); + A_TIMESTAMP = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("A_TIMESTAMP").getPosition()).newColumnExpression(); + X_DECIMAL = new ColumnRef(new TableRef(table), table.getPColumnForColumnName("X_DECIMAL").getPosition()).newColumnExpression(); } finally { conn.close(); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java index 3396cf8..089c5f1 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java @@ -136,7 +136,6 @@ public class ConnectionlessTest { conn.rollback(); // to clear the list of mutations for the next } - @SuppressWarnings("deprecation") private static void assertRow1(Iterator<KeyValue> iterator, byte[] expectedRowKey1) { KeyValue kv; assertTrue(iterator.hasNext()); @@ -153,7 +152,6 @@ public class ConnectionlessTest { assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, PVarchar.INSTANCE.toObject(kv.getValue())); } - @SuppressWarnings("deprecation") private static void assertRow2(Iterator<KeyValue> iterator, byte[] expectedRowKey2) { KeyValue kv; assertTrue(iterator.hasNext()); http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index 0a11977..fb17ae6 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -269,11 +269,11 @@ public class TestUtil { } public static Expression constantComparison(CompareOp op, PColumn c, Object o) { - return new ComparisonExpression(Arrays.<Expression>asList(new KeyValueColumnExpression(c), LiteralExpression.newConstant(o)), op); + return new ComparisonExpression(Arrays.<Expression>asList(new KeyValueColumnExpression(c, true), LiteralExpression.newConstant(o)), op); } public static Expression kvColumn(PColumn c) { - return new KeyValueColumnExpression(c); + return new KeyValueColumnExpression(c, true); } public static Expression pkColumn(PColumn c, List<PColumn> columns) { @@ -569,7 +569,7 @@ public class TestUtil { } public static void analyzeTable(Connection conn, String tableName) throws IOException, SQLException { - analyzeTable(conn, tableName, false); + analyzeTable(conn, tableName, false); } public static void analyzeTable(Connection conn, String tableName, boolean transactional) throws IOException, SQLException { @@ -616,17 +616,17 @@ public class TestUtil { Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY); stmt.setDate(6, date); } - + public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException { - assertTrue(rs.next()); - assertEquals(rs.getString(1), "varchar" + String.valueOf(i)); - assertEquals(rs.getString(2), "char" + String.valueOf(i)); - assertEquals(rs.getInt(3), i); - assertEquals(rs.getInt(4), i); - assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d)); - Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY); - assertEquals(rs.getDate(6), date); - } + assertTrue(rs.next()); + assertEquals(rs.getString(1), "varchar" + String.valueOf(i)); + assertEquals(rs.getString(2), "char" + String.valueOf(i)); + assertEquals(rs.getInt(3), i); + assertEquals(rs.getInt(4), i); + assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d)); + Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY); + assertEquals(rs.getDate(6), date); + } public static String getTableName(Boolean mutable, Boolean transactional) { StringBuilder tableNameBuilder = new StringBuilder(DEFAULT_DATA_TABLE_NAME); @@ -658,7 +658,7 @@ public class TestUtil { @Override public SortOrder getSortOrder() { - return SortOrder.getDefault(); + return SortOrder.getDefault(); } @Override @@ -684,15 +684,18 @@ public class TestUtil { public boolean isRowTimestamp() { return false; } - @Override - public boolean isDynamic() { - return false; - } - })), null); + @Override + public boolean isDynamic() { + return false; + } + @Override + public Integer getEncodedColumnQualifier() { + return null; + } + }, false)), null); aggregationManager.setAggregators(new ClientAggregators(Collections.<SingleAggregateFunction>singletonList(func), 1)); ClientAggregators aggregators = aggregationManager.getAggregators(); return aggregators; } } } - http://git-wip-us.apache.org/repos/asf/phoenix/blob/cc472a8e/phoenix-protocol/src/main/PTable.proto ---------------------------------------------------------------------- diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 1eec5d9..cb9cb83 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -47,6 +47,7 @@ message PColumn { optional string expression = 12; optional bool isRowTimestamp = 13; optional bool isDynamic = 14; + optional int32 columnQualifier = 15; } message PTableStats { @@ -91,6 +92,13 @@ message PTable { optional int64 updateCacheFrequency = 28; optional int64 indexDisableTimestamp = 29; optional bool isNamespaceMapped = 30; - optional string autoParititonSeqName = 31; - optional bool isAppendOnlySchema = 32; + optional string autoParititonSeqName = 32; + optional bool isAppendOnlySchema = 33; + optional bytes storageScheme = 34; + repeated EncodedCQCounter encodedCQCounters = 35; } + +message EncodedCQCounter { + required string colFamily = 1; + required int32 counter = 2; +} \ No newline at end of file
