Repository: drill Updated Branches: refs/heads/master 6446e56f2 -> 63e243378
http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java index 170c606..0f8d90c 100644 --- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java @@ -251,8 +251,8 @@ public final class ${className} extends BaseDataValueVector implements <#if type bits.zeroVector(); values.zeroVector(); } - </#if> + </#if> @Override public void load(SerializedField metadata, DrillBuf buffer) { clear(); @@ -374,6 +374,27 @@ public final class ${className} extends BaseDataValueVector implements <#if type values.copyFromSafe(fromIndex, thisIndex, from.values); } + @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + Nullable${minor.class}Vector fromVector = (Nullable${minor.class}Vector) from; + <#if type.major == "VarLen"> + + // This method is to be called only for loading the vector + // sequentially, so there should be no empties to fill. + + </#if> + bits.copyFromSafe(fromIndex, toIndex, fromVector.bits); + values.copyFromSafe(fromIndex, toIndex, fromVector.values); + } + + @Override + public void exchange(ValueVector other) { + ${className} target = (${className}) other; + bits.exchange(target.bits); + values.exchange(target.values); + mutator.exchange(other.getMutator()); + } + public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> { final UInt1Vector.Accessor bAccessor = bits.getAccessor(); final ${valuesName}.Accessor vAccessor = values.getAccessor(); @@ -383,7 +404,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type * * @param index position of the value * @return value of the element, if not null - * @throws NullValueException if the value is null + * @throws IllegalStateException if the value is null */ public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) { if (isNull(index)) { @@ -410,8 +431,8 @@ public final class ${className} extends BaseDataValueVector implements <#if type public int getValueLength(int index) { return values.getAccessor().getValueLength(index); } - </#if> + </#if> public void get(int index, Nullable${minor.class}Holder holder){ vAccessor.get(index, holder); holder.isSet = bAccessor.get(index); @@ -439,22 +460,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type return vAccessor.getAsStringBuilder(index); } } - </#if> + </#if> @Override - public int getValueCount(){ + public int getValueCount() { return bits.getAccessor().getValueCount(); } - public void reset(){} + public void reset() {} } public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> { private int setCount; - <#if type.major = "VarLen"> private int lastSet = -1;</#if> + <#if type.major = "VarLen">private int lastSet = -1;</#if> - private Mutator(){ - } + private Mutator() { } public ${valuesName} getVectorWithValues(){ return values; @@ -466,11 +486,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type } /** - * Set the variable length element at the specified index to the supplied byte array. + * Set the variable length element at the specified index to the supplied value. * * @param index position of the bit to set - * @param bytes array of bytes to write + * @param value value to write */ + public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) { setCount++; final ${valuesName}.Mutator valuesMutator = values.getMutator(); @@ -486,7 +507,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type } <#if type.major == "VarLen"> - private void fillEmpties(int index){ + private void fillEmpties(int index) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); for (int i = lastSet; i < index; i++) { valuesMutator.setSafe(i + 1, emptyByteArray); @@ -502,27 +523,29 @@ public final class ${className} extends BaseDataValueVector implements <#if type values.getMutator().setValueLengthSafe(index, length); lastSet = index; } - </#if> public void setSafe(int index, byte[] value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> - if (index > lastSet + 1) { + if (index > lastSet + 1) { fillEmpties(index); } bits.getMutator().setSafe(index, 1); values.getMutator().setSafe(index, value, start, length); setCount++; - <#if type.major == "VarLen">lastSet = index;</#if> - </#if> + lastSet = index; + } + + public void setScalar(int index, byte[] value, int start, int length) throws VectorOverflowException { + if (index > lastSet + 1) { + fillEmpties(index); // Filling empties cannot overflow the vector + } + values.getMutator().setScalar(index, value, start, length); + bits.getMutator().setSafe(index, 1); + setCount++; + lastSet = index; } public void setSafe(int index, ByteBuffer value, int start, int length) { - <#if type.major != "VarLen"> - throw new UnsupportedOperationException(); - <#else> if (index > lastSet + 1) { fillEmpties(index); } @@ -530,23 +553,38 @@ public final class ${className} extends BaseDataValueVector implements <#if type bits.getMutator().setSafe(index, 1); values.getMutator().setSafe(index, value, start, length); setCount++; - <#if type.major == "VarLen">lastSet = index;</#if> - </#if> + lastSet = index; } - public void setNull(int index){ + public void setScalar(int index, DrillBuf value, int start, int length) throws VectorOverflowException { + if (index > lastSet + 1) { + fillEmpties(index); // Filling empties cannot overflow the vector + } + + values.getMutator().setScalar(index, value, start, length); + bits.getMutator().setSafe(index, 1); + setCount++; + lastSet = index; + } + + </#if> + public void setNull(int index) { bits.getMutator().setSafe(index, 0); } - public void setSkipNull(int index, ${minor.class}Holder holder){ + public void setSkipNull(int index, ${minor.class}Holder holder) { values.getMutator().set(index, holder); } - public void setSkipNull(int index, Nullable${minor.class}Holder holder){ + public void setSkipNull(int index, Nullable${minor.class}Holder holder) { values.getMutator().set(index, holder); } - public void set(int index, Nullable${minor.class}Holder holder){ + public void setNullBounded(int index) throws VectorOverflowException { + bits.getMutator().setScalar(index, 0); + } + + public void set(int index, Nullable${minor.class}Holder holder) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -558,7 +596,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major == "VarLen">lastSet = index;</#if> } - public void set(int index, ${minor.class}Holder holder){ + public void set(int index, ${minor.class}Holder holder) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -575,7 +613,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type } <#assign fields = minor.fields!type.fields /> - public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){ + public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) { final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { @@ -593,11 +631,22 @@ public final class ${className} extends BaseDataValueVector implements <#if type fillEmpties(index); } </#if> - bits.getMutator().setSafe(index, isSet); values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>); setCount++; <#if type.major == "VarLen">lastSet = index;</#if> + } + + public void setScalar(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) throws VectorOverflowException { + <#if type.major == "VarLen"> + if (index > lastSet + 1) { + fillEmpties(index); + } + </#if> + values.getMutator().setScalar(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>); + bits.getMutator().setSafe(index, isSet); + setCount++; + <#if type.major == "VarLen">lastSet = index;</#if> } public void setSafe(int index, Nullable${minor.class}Holder value) { @@ -612,6 +661,18 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major == "VarLen">lastSet = index;</#if> } + public void setScalar(int index, Nullable${minor.class}Holder value) throws VectorOverflowException { + <#if type.major == "VarLen"> + if (index > lastSet + 1) { + fillEmpties(index); + } + </#if> + values.getMutator().setScalar(index, value); + bits.getMutator().setSafe(index, value.isSet); + setCount++; + <#if type.major == "VarLen">lastSet = index;</#if> + } + public void setSafe(int index, ${minor.class}Holder value) { <#if type.major == "VarLen"> if (index > lastSet + 1) { @@ -624,6 +685,18 @@ public final class ${className} extends BaseDataValueVector implements <#if type <#if type.major == "VarLen">lastSet = index;</#if> } + public void setScalar(int index, ${minor.class}Holder value) throws VectorOverflowException { + <#if type.major == "VarLen"> + if (index > lastSet + 1) { + fillEmpties(index); + } + </#if> + values.getMutator().setScalar(index, value); + bits.getMutator().setSafe(index, 1); + setCount++; + <#if type.major == "VarLen">lastSet = index;</#if> + } + <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")> public void setSafe(int index, ${minor.javaType!type.javaType} value) { <#if type.major == "VarLen"> @@ -636,6 +709,17 @@ public final class ${className} extends BaseDataValueVector implements <#if type setCount++; } + public void setScalar(int index, ${minor.javaType!type.javaType} value) throws VectorOverflowException { + <#if type.major == "VarLen"> + if (index > lastSet + 1) { + fillEmpties(index); + } + </#if> + values.getMutator().setScalar(index, value); + bits.getMutator().setSafe(index, 1); + setCount++; + } + </#if> <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse"> public void set(int index, BigDecimal value) { @@ -650,6 +734,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type setCount++; } + public void setScalar(int index, BigDecimal value) throws VectorOverflowException { + values.getMutator().setScalar(index, value); + bits.getMutator().setSafe(index, 1); + setCount++; + } + </#if> @Override public void setValueCount(int valueCount) { @@ -674,6 +764,17 @@ public final class ${className} extends BaseDataValueVector implements <#if type setCount = 0; <#if type.major = "VarLen">lastSet = -1;</#if> } + + // For nullable vectors, exchanging buffers (done elsewhere) + // requires also exchanging mutator state (done here.) + + @Override + public void exchange(ValueVector.Mutator other) { + final Mutator target = (Mutator) other; + int temp = setCount; + setCount = target.setCount; + target.setCount = temp; + } } } </#list> http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java index 159a8e7..9780b7d 100644 --- a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java +++ b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -36,10 +36,10 @@ package org.apache.drill.exec.vector; <#include "/@includes/vv_imports.ftl" /> /** - * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or - * repeated protobuf field). The implementation uses two additional value vectors; one to convert - * the index offset to the underlying element offset, and another to store the number of values - * in the vector. + * Repeated${minor.class} implements a vector with multiple values per row (e.g. JSON array or + * repeated protobuf field). The implementation uses an additional value vectors to convert + * the index offset to the underlying element offset. The count of values comes from subtracting + * two successive offsets. * * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @@ -178,11 +178,16 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector } } + @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (Repeated${minor.class}Vector) from); + } + public boolean allocateNewSafe() { - /* boolean to keep track if all the memory allocation were successful + /* boolean to keep track if all the memory allocations were successful. * Used in the case of composite vectors when we need to allocate multiple * buffers for multiple vectors. If one of the allocations failed we need to - * clear all the memory that we allocated + * clear all the memory that we allocated. */ boolean success = false; try { @@ -239,12 +244,6 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector @Override public void allocateNew(int valueCount, int innerValueCount) { clear(); - /* boolean to keep track if all the memory allocation were successful - * Used in the case of composite vectors when we need to allocate multiple - * buffers for multiple vectors. If one of the allocations failed we need to// - * clear all the memory that we allocated - */ - boolean success = false; try { offsets.allocateNew(valueCount + 1); values.allocateNew(innerValueCount); @@ -258,9 +257,9 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector </#if> // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for - // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge + // variable length vectors, as they should have a consistent interface as much as possible, if they need to diverge // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable - // and we should refer to each in the generation template + // and we should refer to each in the generation template. public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor { @Override public List<${friendlyType}> getObject(int index) { @@ -281,7 +280,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector } /** - * Get a value for the given record. Each element in the repeated field is accessed by + * Get a value for the given record. Each element in the repeated field is accessed by * the positionIndex param. * * @param index record containing the repeated field @@ -329,7 +328,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector * @param value value to add to the given row */ public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) { - int nextOffset = offsets.getAccessor().get(index+1); + final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().set(nextOffset, value); offsets.getMutator().set(index+1, nextOffset+1); } @@ -339,12 +338,25 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector addSafe(index, bytes, 0, bytes.length); } + public void addEntry(int index, byte[] bytes) throws VectorOverflowException { + addEntry(index, bytes, 0, bytes.length); + } + public void addSafe(int index, byte[] bytes, int start, int length) { final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, bytes, start, length); offsets.getMutator().setSafe(index+1, nextOffset+1); } + public void addEntry(int index, byte[] bytes, int start, int length) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + final int nextOffset = offsets.getAccessor().get(index+1); + values.getMutator().setArrayItem(nextOffset, bytes, start, length); + offsets.getMutator().setSafe(index+1, nextOffset+1); + } + <#else> public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) { final int nextOffset = offsets.getAccessor().get(index+1); @@ -352,6 +364,15 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector offsets.getMutator().setSafe(index+1, nextOffset+1); } + public void addEntry(int index, ${minor.javaType!type.javaType} srcValue) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + final int nextOffset = offsets.getAccessor().get(index+1); + values.getMutator().setArrayItem(nextOffset, srcValue); + offsets.getMutator().setSafe(index+1, nextOffset+1); + } + </#if> public void setSafe(int index, Repeated${minor.class}Holder h) { final ${minor.class}Holder ih = new ${minor.class}Holder(); @@ -364,43 +385,104 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector } public void addSafe(int index, ${minor.class}Holder holder) { - int nextOffset = offsets.getAccessor().get(index+1); + final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, holder); offsets.getMutator().setSafe(index+1, nextOffset+1); } + public void addEntry(int index, ${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + final int nextOffset = offsets.getAccessor().get(index+1); + values.getMutator().setArrayItem(nextOffset, holder); + offsets.getMutator().setSafe(index+1, nextOffset+1); + } + public void addSafe(int index, Nullable${minor.class}Holder holder) { final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, holder); offsets.getMutator().setSafe(index+1, nextOffset+1); } + public void addEntry(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + final int nextOffset = offsets.getAccessor().get(index+1); + values.getMutator().setArrayItem(nextOffset, holder); + offsets.getMutator().setSafe(index+1, nextOffset+1); + } + + /** + * Backfill missing offsets from the given last written position to the + * given current write position. Used by the "new" size-safe column + * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt> + * <b>do not</b> fill empties. See DRILL-5529. + * @param lastWrite the position of the last valid write: the offset + * to be copied forward + * @param index the current write position to be initialized + */ + + public void fillEmptiesBounded(int lastWrite, int index) + throws VectorOverflowException { + if (index >= UInt4Vector.MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + // If last write was 2, offsets are [0, 3, 6] + // If next write is 4, offsets must be: [0, 3, 6, 6, 6] + // Remember the offsets are one more than row count. + final int fillOffset = offsets.getAccessor().get(lastWrite+1); + final UInt4Vector.Mutator offsetMutator = offsets.getMutator(); + for (int i = lastWrite; i < index; i++) { + offsetMutator.setSafe(i + 1, fillOffset); + } + } + <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")> - public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) { - int nextOffset = offsets.getAccessor().get(arrayIndex+1); + public void addSafe(int rowIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) { + final int nextOffset = offsets.getAccessor().get(rowIndex+1); values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); - offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1); + offsets.getMutator().setSafe(rowIndex+1, nextOffset+1); + } + + public void addEntry(int rowIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) throws VectorOverflowException { + if (rowIndex >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + final int nextOffset = offsets.getAccessor().get(rowIndex+1); + values.getMutator().setArrayItem(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); + offsets.getMutator().setSafe(rowIndex+1, nextOffset+1); } </#if> <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse"> public void addSafe(int index, BigDecimal value) { - int nextOffset = offsets.getAccessor().get(index+1); + final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().setSafe(nextOffset, value); offsets.getMutator().setSafe(index+1, nextOffset+1); } + public void addEntry(int index, BigDecimal value) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + final int nextOffset = offsets.getAccessor().get(index+1); + values.getMutator().setArrayItem(nextOffset, value); + offsets.getMutator().setSafe(index+1, nextOffset+1); + } + </#if> protected void add(int index, ${minor.class}Holder holder) { - int nextOffset = offsets.getAccessor().get(index+1); + final int nextOffset = offsets.getAccessor().get(index+1); values.getMutator().set(nextOffset, holder); offsets.getMutator().set(index+1, nextOffset+1); } public void add(int index, Repeated${minor.class}Holder holder) { - ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor(); - ${minor.class}Holder innerHolder = new ${minor.class}Holder(); + final ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor(); + final ${minor.class}Holder innerHolder = new ${minor.class}Holder(); for(int i = holder.start; i < holder.end; i++) { accessor.get(i, innerHolder); http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/UnionVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/UnionVector.java b/exec/vector/src/main/codegen/templates/UnionVector.java index 93854e7..207e55a 100644 --- a/exec/vector/src/main/codegen/templates/UnionVector.java +++ b/exec/vector/src/main/codegen/templates/UnionVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -80,6 +80,7 @@ public class UnionVector implements ValueVector { this.callBack = callBack; } + @Override public BufferAllocator getAllocator() { return allocator; } @@ -248,6 +249,11 @@ public class UnionVector implements ValueVector { copyFrom(inIndex, outIndex, from); } + @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (UnionVector) from); + } + public ValueVector addVector(ValueVector v) { String name = v.getField().getType().getMinorType().name().toLowerCase(); MajorType type = v.getField().getType(); @@ -485,4 +491,9 @@ public class UnionVector implements ValueVector { @Override public void generateTestData(int values) { } } + + @Override + public void exchange(ValueVector other) { + throw new UnsupportedOperationException("Union vector does not yet support exchange()"); + } } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/VariableLengthVectors.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java index 581a9f8..9a9e178 100644 --- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java +++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java @@ -42,20 +42,23 @@ package org.apache.drill.exec.vector; * ${minor.class}Vector implements a vector of variable width values. Elements in the vector * are accessed by position from the logical start of the vector. A fixed width offsetVector * is used to convert an element's position to it's offset from the start of the (0-based) - * DrillBuf. Size is inferred by adjacent elements. - * The width of each element is ${type.width} byte(s) - * The equivalent Java primitive is '${minor.javaType!type.javaType}' - * + * DrillBuf. Size is inferred from adjacent elements. + * <ul> + * <li>The width of each element is ${type.width} byte(s). Note that the actual width is + * variable, this width is used as a guess for certain calculations.</li> + * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'<li> + * </ul> * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{ + +public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); private static final int DEFAULT_RECORD_BYTE_COUNT = 8; - private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT; + private static final int INITIAL_BYTE_COUNT = Math.min(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT, MAX_BUFFER_SIZE); private static final int MIN_BYTE_COUNT = 4096; - public final static String OFFSETS_VECTOR_NAME = "$offsets$"; + private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4)); private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator); private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this); @@ -239,6 +242,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (${minor.class}Vector) from); + } + + @Override public int getAllocatedByteCount() { return offsetVector.getAllocatedByteCount() + super.getAllocatedByteCount(); } @@ -406,6 +414,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V return mutator; } + @Override + public void exchange(ValueVector other) { + super.exchange(other); + ${minor.class}Vector target = (${minor.class}Vector) other; + offsetVector.exchange(target.offsetVector); + } + public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor { final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor(); public long getStartEnd(int index){ @@ -519,6 +534,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } } + public void setScalar(int index, byte[] bytes) throws VectorOverflowException { + setScalar(index, bytes, 0, bytes.length); + } + /** * Set the variable length element at the specified index to the supplied byte array. * @@ -537,7 +556,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V public void setSafe(int index, ByteBuffer bytes, int start, int length) { assert index >= 0; - int currentOffset = offsetVector.getAccessor().get(index); + final int currentOffset = offsetVector.getAccessor().get(index); offsetVector.getMutator().setSafe(index + 1, currentOffset + length); try { data.setBytes(currentOffset, bytes, start, length); @@ -549,6 +568,23 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } } + public void setScalar(int index, DrillBuf bytes, int start, int length) throws VectorOverflowException { + assert index >= 0; + + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + int currentOffset = offsetVector.getAccessor().get(index); + final int newSize = currentOffset + length; + if (newSize > MAX_BUFFER_SIZE) { + throw new VectorOverflowException(); + } + while (! data.setBytesBounded(currentOffset, bytes, start, length)) { + reAlloc(); + } + offsetVector.getMutator().setSafe(index + 1, newSize); + } + public void setSafe(int index, byte[] bytes, int start, int length) { assert index >= 0; @@ -565,6 +601,28 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } } + public void setScalar(int index, byte[] bytes, int start, int length) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + setArrayItem(index, bytes, start, length); + } + + public void setArrayItem(int index, byte[] bytes, int start, int length) throws VectorOverflowException { + assert index >= 0; + + final int currentOffset = offsetVector.getAccessor().get(index); + final int newSize = currentOffset + length; + if (newSize > MAX_BUFFER_SIZE) { + throw new VectorOverflowException(); + } + + while (! data.setBytesBounded(currentOffset, bytes, start, length)) { + reAlloc(); + } + offsetVector.getMutator().setSafe(index + 1, newSize); + } + @Override public void setValueLengthSafe(int index, int length) { final int offset = offsetVector.getAccessor().get(index); @@ -574,12 +632,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length); } - - public void setSafe(int index, int start, int end, DrillBuf buffer){ + public void setSafe(int index, int start, int end, DrillBuf buffer) { final int len = end - start; final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); - offsetVector.getMutator().setSafe( index+1, outputStart + len); + offsetVector.getMutator().setSafe(index+1, outputStart + len); try{ buffer.getBytes(start, data, outputStart, len); } catch (IndexOutOfBoundsException e) { @@ -588,17 +645,42 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } buffer.getBytes(start, data, outputStart, len); } + } + + public void setScalar(int index, int start, int end, DrillBuf buffer) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + setArrayItem(index, start, end, buffer); + } + public void setArrayItem(int index, int start, int end, DrillBuf buffer) throws VectorOverflowException { + final int len = end - start; + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + final int newSize = outputStart + len; + if (newSize > MAX_BUFFER_SIZE) { + throw new VectorOverflowException(); + } + + offsetVector.getMutator().setSafe(index+1, newSize); + try{ + buffer.getBytes(start, data, outputStart, len); + } catch (IndexOutOfBoundsException e) { + while (data.capacity() < newSize) { + reAlloc(); + } + buffer.getBytes(start, data, outputStart, len); + } } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { assert holder.isSet == 1; final int start = holder.start; final int end = holder.end; final int len = end - start; - int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); try { holder.buffer.getBytes(start, data, outputStart, len); @@ -608,15 +690,45 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } holder.buffer.getBytes(start, data, outputStart, len); } - offsetVector.getMutator().setSafe( index+1, outputStart + len); + offsetVector.getMutator().setSafe(index+1, outputStart + len); + } + + public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + setArrayItem(index, holder); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException { + assert holder.isSet == 1; + final int start = holder.start; final int end = holder.end; final int len = end - start; + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + final int newSize = outputStart + len; + if (newSize > MAX_BUFFER_SIZE) { + throw new VectorOverflowException(); + } + + try { + holder.buffer.getBytes(start, data, outputStart, len); + } catch (IndexOutOfBoundsException e) { + while (data.capacity() < newSize) { + reAlloc(); + } + holder.buffer.getBytes(start, data, outputStart, len); + } + offsetVector.getMutator().setSafe(index+1, newSize); + } + public void setSafe(int index, ${minor.class}Holder holder) { + final int start = holder.start; + final int end = holder.end; + final int len = end - start; + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); try { holder.buffer.getBytes(start, data, outputStart, len); @@ -629,6 +741,68 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V offsetVector.getMutator().setSafe( index+1, outputStart + len); } + public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + setArrayItem(index, holder); + } + + public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException { + final int start = holder.start; + final int end = holder.end; + final int len = end - start; + final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); + final int newSize = outputStart + len; + if (newSize > MAX_BUFFER_SIZE) { + throw new VectorOverflowException(); + } + + try { + holder.buffer.getBytes(start, data, outputStart, len); + } catch (IndexOutOfBoundsException e) { + while(data.capacity() < newSize) { + reAlloc(); + } + holder.buffer.getBytes(start, data, outputStart, len); + } + offsetVector.getMutator().setSafe( index+1, newSize); + } + + /** + * Backfill missing offsets from the given last written position to the + * given current write position. Used by the "new" size-safe column + * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt> + * <b>do not</b> fill empties. See DRILL-5529. + * @param lastWrite the position of the last valid write: the offset + * to be copied forward + * @param index the current write position filling occurs up to, + * but not including, this position + * @throws VectorOverflowException if the item was written, false if the index would + * overfill the vector + */ + + public void fillEmptiesBounded(int lastWrite, int index) + throws VectorOverflowException { + + // Index is the next write index, which might be "virtual", + // that is, past the last row at EOF. This check only protects + // the actual data written here, which is up to index-1. + + if (index > UInt4Vector.MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + // If last write was 2, offsets are [0, 3, 6] + // If next write is 4, offsets must be: [0, 3, 6, 6, 6] + // Remember the offsets are one more than row count. + + final int fillOffset = offsetVector.getAccessor().get(lastWrite+1); + final UInt4Vector.Mutator offsetMutator = offsetVector.getMutator(); + for (int i = lastWrite; i < index; i++) { + offsetMutator.setSafe(i + 1, fillOffset); + } + } + protected void set(int index, int start, int length, DrillBuf buffer){ assert index >= 0; final int currentOffset = offsetVector.getAccessor().get(index); @@ -651,6 +825,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V data.setBytes(currentOffset, holder.buffer, holder.start, length); } + <#if (minor.class == "VarChar")> + public void setScalar(int index, String value) throws VectorOverflowException { + if (index >= MAX_ROW_COUNT) { + throw new VectorOverflowException(); + } + // Treat a null string as an empty string. + if (value != null) { + byte encoded[] = value.getBytes(Charsets.UTF_8); + setScalar(index, encoded, 0, encoded.length); + } + } + + </#if> @Override public void setValueCount(int valueCount) { final int currentByteCapacity = getByteCapacity(); @@ -685,7 +872,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V } } } - </#if> <#-- type.major --> </#list> </#list> http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java index 4def5b8..5ce58ed 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,7 +24,7 @@ import org.apache.drill.exec.record.MaterializedField; public abstract class BaseDataValueVector extends BaseValueVector { - protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this + protected final static byte[] emptyByteArray = new byte[0]; // Nullable vectors use this protected DrillBuf data; @@ -78,9 +78,7 @@ public abstract class BaseDataValueVector extends BaseValueVector { return data.writerIndex(); } - public DrillBuf getBuffer() { - return data; - } + public DrillBuf getBuffer() { return data; } /** * This method has a similar effect of allocateNew() without actually clearing and reallocating @@ -89,7 +87,16 @@ public abstract class BaseDataValueVector extends BaseValueVector { public void reset() {} @Override - public int getAllocatedByteCount() { - return data.capacity(); + public int getAllocatedByteCount() { return data.capacity(); } + + @Override + public void exchange(ValueVector other) { + BaseDataValueVector target = (BaseDataValueVector) other; + DrillBuf temp = data; + data = target.data; + target.data = temp; + getReader().reset(); + getMutator().exchange(target.getMutator()); + // No state in an Accessor to reset } } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java index a0d5f65..f4a5847 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,6 +32,13 @@ import org.apache.drill.exec.record.TransferPair; public abstract class BaseValueVector implements ValueVector { // private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class); + /** + * Physical maximum allocation. This is the value prior to Drill 1.11. + * This size causes memory fragmentation. Please use + * {@link ValueVector#MAX_BUFFER_SIZE} in new code. + */ + + @Deprecated public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE; public static final int INITIAL_VALUE_ALLOCATION = 4096; @@ -101,6 +108,10 @@ public abstract class BaseValueVector implements ValueVector { //TODO: consider making mutator stateless(if possible) on another issue. @Override public void reset() {} + + // TODO: If mutator becomes stateless, remove this method. + @Override + public void exchange(ValueVector.Mutator other) { } } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java index a6c0cea..0062e77 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -39,6 +39,38 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader; public final class BitVector extends BaseDataValueVector implements FixedWidthVector { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class); + /** + * Width of each fixed-width value. + */ + + public static final int VALUE_WIDTH = 1; + + /** + * Maximum number of values that this fixed-width vector can hold + * and stay below the maximum vector size limit. This is the limit + * enforced when the vector is used to hold values in a repeated + * vector. + */ + + public static final int MAX_CAPACITY = MAX_BUFFER_SIZE / VALUE_WIDTH; + + /** + * Maximum number of values that this fixed-width vector can hold + * and stay below the maximum vector size limit and/or stay below + * the maximum item count. This lis the limit enforced when the + * vector is used to hold required or nullable values. + */ + + public static final int MAX_COUNT = Math.min(MAX_ROW_COUNT, MAX_CAPACITY); + + /** + * Actual maximum vector size, in bytes, given the number of fixed-width + * values that either fit in the maximum overall vector size, or that + * is no larger than the maximum vector item count. + */ + + public static final int NET_MAX_SIZE = VALUE_WIDTH * MAX_COUNT; + private final FieldReader reader = new BitReaderImpl(BitVector.this); private final Accessor accessor = new Accessor(); private final Mutator mutator = new Mutator(); @@ -72,7 +104,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe @Override public int getValueCapacity() { - return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L); + return (int) Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L); } private int getByteIndex(int index) { @@ -183,6 +215,11 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe } @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFrom(fromIndex, toIndex, (BitVector) from); + } + + @Override public void load(SerializedField metadata, DrillBuf buffer) { Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata); final int valueCount = metadata.getValueCount(); @@ -221,7 +258,6 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe return new TransferImpl((BitVector) to); } - public void transferTo(BitVector target) { target.clear(); if (target.data != null) { @@ -401,6 +437,20 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe set(index, value); } + public void setScalar(int index, int value) throws VectorOverflowException { + if (index >= MAX_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, value); + } + + public void setArrayItem(int index, int value) throws VectorOverflowException { + if (index >= MAX_CAPACITY) { + throw new VectorOverflowException(); + } + setSafe(index, value); + } + public void setSafe(int index, BitHolder holder) { while(index >= getValueCapacity()) { reAlloc(); @@ -408,6 +458,20 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe set(index, holder.value); } + public void setScalar(int index, BitHolder holder) throws VectorOverflowException { + if (index >= MAX_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, holder); + } + + public void setArrayItem(int index, BitHolder holder) throws VectorOverflowException { + if (index >= MAX_CAPACITY) { + throw new VectorOverflowException(); + } + setSafe(index, holder); + } + public void setSafe(int index, NullableBitHolder holder) { while(index >= getValueCapacity()) { reAlloc(); @@ -415,6 +479,20 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe set(index, holder.value); } + public void setScalar(int index, NullableBitHolder holder) throws VectorOverflowException { + if (index >= MAX_COUNT) { + throw new VectorOverflowException(); + } + setSafe(index, holder); + } + + public void setArrayItem(int index, NullableBitHolder holder) throws VectorOverflowException { + if (index >= MAX_CAPACITY) { + throw new VectorOverflowException(); + } + setSafe(index, holder); + } + @Override public final void setValueCount(int valueCount) { int currentValueCapacity = getValueCapacity(); @@ -441,7 +519,6 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe } setValueCount(values); } - } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java index c2781eb..a9a1631 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,7 +18,7 @@ package org.apache.drill.exec.vector; -public interface FixedWidthVector extends ValueVector{ +public interface FixedWidthVector extends ValueVector { /** * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. @@ -31,5 +31,4 @@ public interface FixedWidthVector extends ValueVector{ * Zero out the underlying buffer backing this vector. */ void zeroVector(); - } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java index f69dc98..bd8566d 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,11 +32,12 @@ import org.apache.drill.exec.record.TransferPair; import org.apache.drill.exec.vector.complex.reader.FieldReader; public class ObjectVector extends BaseValueVector { + private final int ALLOCATION_SIZE = 4096; + private final Accessor accessor = new Accessor(); private final Mutator mutator = new Mutator(); private int maxCount = 0; private int count = 0; - private int allocationSize = 4096; private List<Object[]> objectArrayList = new ArrayList<>(); @@ -45,8 +46,8 @@ public class ObjectVector extends BaseValueVector { } public void addNewArray() { - objectArrayList.add(new Object[allocationSize]); - maxCount += allocationSize; + objectArrayList.add(new Object[ALLOCATION_SIZE]); + maxCount += ALLOCATION_SIZE; } @Override @@ -57,11 +58,11 @@ public class ObjectVector extends BaseValueVector { public final class Mutator implements ValueVector.Mutator { public void set(int index, Object obj) { - int listOffset = index / allocationSize; + int listOffset = index / ALLOCATION_SIZE; if (listOffset >= objectArrayList.size()) { addNewArray(); } - objectArrayList.get(listOffset)[index % allocationSize] = obj; + objectArrayList.get(listOffset)[index % ALLOCATION_SIZE] = obj; } public boolean setSafe(int index, long value) { @@ -94,6 +95,9 @@ public class ObjectVector extends BaseValueVector { @Override public void generateTestData(int values) { } + + @Override + public void exchange(ValueVector.Mutator other) { } } @Override @@ -161,6 +165,11 @@ public class ObjectVector extends BaseValueVector { } @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + throw new UnsupportedOperationException("ObjectVector does not support this"); + } + + @Override public int getValueCapacity() { return maxCount; } @@ -198,11 +207,11 @@ public class ObjectVector extends BaseValueVector { public final class Accessor extends BaseAccessor { @Override public Object getObject(int index) { - int listOffset = index / allocationSize; + int listOffset = index / ALLOCATION_SIZE; if (listOffset >= objectArrayList.size()) { addNewArray(); } - return objectArrayList.get(listOffset)[index % allocationSize]; + return objectArrayList.get(listOffset)[index % ALLOCATION_SIZE]; } @Override @@ -230,4 +239,18 @@ public class ObjectVector extends BaseValueVector { // Values not stored in direct memory? return 0; } + + @Override + public void exchange(ValueVector other) { + ObjectVector target = (ObjectVector) other; + List<Object[]> tempList = objectArrayList; + objectArrayList = target.objectArrayList; + target.objectArrayList = tempList; + int tempCount = count; + count = target.count; + target.count = tempCount; + tempCount = maxCount; + maxCount = target.maxCount; + target.maxCount = tempCount; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java index f4c7935..3bc43fa 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -40,14 +40,17 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader; * There are a few "rules" around vectors: * * <ul> - * <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li> - * <li>null vectors start with all values as null before writing anything</li> - * <li>for variable width types, the offset vector should be all zeros before writing</li> - * <li>you must call setValueCount before a vector can be read</li> - * <li>you should never write to a vector once it has been read.</li> + * <li>Values need to be written in order (e.g. index 0, 1, 2, 5).</li> + * <li>Null vectors start with all values as null before writing anything.</li> + * <li>For variable width types, the offset vector should be all zeros before writing.</li> + * <li>You must call setValueCount before a vector can be read.</li> + * <li>You should never write to a vector once it has been read.</li> + * <li>Vectors may not grow larger than the number of bytes specified + * in {@link #MAX_BUFFER_SIZE} to prevent memory fragmentation. Use the + * <tt>setBounded()</tt> methods in the mutator to enforce this rule.</li> * </ul> * - * Please note that the current implementation doesn't enfore those rules, hence we may find few places that + * Please note that the current implementation doesn't enforce those rules, hence we may find few places that * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector) * * This interface "should" strive to guarantee this order of operation: @@ -56,6 +59,29 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader; * </blockquote> */ public interface ValueVector extends Closeable, Iterable<ValueVector> { + + /** + * Maximum allowed size of the buffer backing a value vector. + */ + + int MAX_BUFFER_SIZE = VectorUtils.maxSize(); + + /** + * Debug-time system option that artificially limits vector lengths + * for testing. Must be set prior to the first reference to this + * class. (Made deliberately difficult to prevent misuse...) + */ + + String MAX_BUFFER_SIZE_KEY = "drill.max_vector"; + + /** + * Maximum allowed row count in a vector. Repeated vectors + * may have more items, but can have no more than this number + * or arrays. Limited by 2-byte length in SV2: 65536 = 2<sup>16</sup>. + */ + + int MAX_ROW_COUNT = Character.MAX_VALUE + 1; + /** * Allocate new buffers. ValueVector implements logic to determine how much to allocate. * @throws OutOfMemoryException Thrown if no memory can be allocated. @@ -64,7 +90,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { /** * Allocates new buffers. ValueVector implements logic to determine how much to allocate. - * @return Returns true if allocation was succesful. + * @return Returns true if allocation was successful. */ boolean allocateNewSafe(); @@ -175,6 +201,8 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { */ void load(SerializedField metadata, DrillBuf buffer); + void copyEntry(int toIndex, ValueVector from, int fromIndex); + /** * Return the total memory consumed by all buffers within this vector. */ @@ -188,6 +216,13 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { int getPayloadByteCount(); /** + * Exchange state with another value vector of the same type. + * Used to implement look-ahead writers. + */ + + void exchange(ValueVector other); + + /** * An abstraction that is used to read from this vector instance. */ interface Accessor { @@ -211,7 +246,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { } /** - * An abstractiong that is used to write into this vector instance. + * An abstraction that is used to write into this vector instance. */ interface Mutator { /** @@ -231,5 +266,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { */ @Deprecated void generateTestData(int values); + + /** + * Exchanges state with the mutator of another mutator. Used when exchanging + * state with another vector. + */ + + void exchange(Mutator other); } } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java new file mode 100644 index 0000000..8b54966 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector; + +/** + * Indicates that an attempt to write to a vector overflowed the vector + * bounds: either the limit on values or the size of the buffer backing + * the vector. This is an expected exception: code must catch it and + * properly handle the partially-written, incomplete last row. + */ + +@SuppressWarnings("serial") +public class VectorOverflowException extends Exception { + +} http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java new file mode 100644 index 0000000..6b29eb2 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector; + +public class VectorUtils { + + /** + * Vectors cannot be any larger than the Netty memory allocation + * block size. + */ + + private static final int ABSOLUTE_MAX_SIZE = 16 * 1024 * 1024; + + /** + * Minimum size selected to prevent pathological performance if vectors + * are limited to an unusably small size. This limit is a judgment call, + * not based on any known limits. + */ + + private static final int ABSOLUTE_MIN_SIZE = 16 * 1024; + + private VectorUtils() { } + + /** + * Static function called once per run to compute the maximum + * vector size, in bytes. Normally uses the hard-coded limit, + * but allows setting a system property to override the limit + * for testing. The configured value must be within reasonable + * bounds. + * @return the maximum vector size, in bytes + */ + + static int maxSize() { + String prop = System.getProperty( ValueVector.MAX_BUFFER_SIZE_KEY ); + int value = ABSOLUTE_MAX_SIZE; + if (prop != null) { + try { + value = Integer.parseInt(prop); + value = Math.max(value, ABSOLUTE_MIN_SIZE); + value = Math.min(value, ABSOLUTE_MAX_SIZE); + } catch (NumberFormatException e) { + // Ignore + } + } + return value; + } + +} http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java index 9181f20..9a0b6be 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -53,19 +53,13 @@ public class ZeroVector implements ValueVector { private final Accessor defaultAccessor = new Accessor() { @Override - public Object getObject(int index) { - return null; - } + public Object getObject(int index) { return null; } @Override - public int getValueCount() { - return 0; - } + public int getValueCount() { return 0; } @Override - public boolean isNull(int index) { - return true; - } + public boolean isNull(int index) { return true; } }; private final Mutator defaultMutator = new Mutator() { @@ -77,6 +71,9 @@ public class ZeroVector implements ValueVector { @Override public void generateTestData(int values) { } + + @Override + public void exchange(Mutator other) { } }; public ZeroVector() { } @@ -88,9 +85,7 @@ public class ZeroVector implements ValueVector { public void clear() { } @Override - public MaterializedField getField() { - return field; - } + public MaterializedField getField() { return field; } @Override public TransferPair getTransferPair(BufferAllocator allocator) { @@ -112,14 +107,10 @@ public class ZeroVector implements ValueVector { } @Override - public int getBufferSize() { - return 0; - } + public int getBufferSize() { return 0; } @Override - public int getBufferSizeFor(final int valueCount) { - return 0; - } + public int getBufferSizeFor(final int valueCount) { return 0; } @Override public DrillBuf[] getBuffers(boolean clear) { @@ -132,9 +123,7 @@ public class ZeroVector implements ValueVector { } @Override - public boolean allocateNewSafe() { - return true; - } + public boolean allocateNewSafe() { return true; } @Override public BufferAllocator getAllocator() { @@ -145,45 +134,35 @@ public class ZeroVector implements ValueVector { public void setInitialCapacity(int numRecords) { } @Override - public int getValueCapacity() { - return 0; - } + public int getValueCapacity() { return 0; } @Override - public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - return defaultPair; - } + public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return defaultPair; } @Override - public TransferPair makeTransferPair(ValueVector target) { - return defaultPair; - } + public TransferPair makeTransferPair(ValueVector target) { return defaultPair; } @Override - public Accessor getAccessor() { - return defaultAccessor; - } + public Accessor getAccessor() { return defaultAccessor; } @Override - public Mutator getMutator() { - return defaultMutator; - } + public Mutator getMutator() { return defaultMutator; } @Override - public FieldReader getReader() { - return NullReader.INSTANCE; - } + public FieldReader getReader() { return NullReader.INSTANCE; } @Override public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { } @Override - public int getAllocatedByteCount() { - return 0; - } + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { } @Override - public int getPayloadByteCount() { - return 0; - } + public int getAllocatedByteCount() { return 0; } + + @Override + public int getPayloadByteCount() { return 0; } + + @Override + public void exchange(ValueVector other) { } } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java index 8a54535..5b8f44d 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -209,7 +209,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements vector = v; } - @Override public int getAllocatedByteCount() { return offsets.getAllocatedByteCount() + vector.getAllocatedByteCount(); @@ -220,6 +219,13 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements return offsets.getPayloadByteCount() + vector.getPayloadByteCount(); } + @Override + public void exchange(ValueVector other) { + BaseRepeatedValueVector target = (BaseRepeatedValueVector) other; + vector.exchange(target.vector); + offsets.exchange(target.offsets); + } + public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor { @Override @@ -259,6 +265,14 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements setValueCount(index+1); } + public boolean startNewValueBounded(int index) { + if (index >= MAX_ROW_COUNT) { + return false; + } + startNewValue(index); + return true; + } + @Override public void setValueCount(int valueCount) { // TODO: populate offset end points http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java index f71baa7..7f0e939 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java @@ -1,5 +1,4 @@ -/******************************************************************************* - +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -95,6 +94,11 @@ public class ListVector extends BaseRepeatedValueVector { } @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (ListVector) from); + } + + @Override public ValueVector getDataVector() { return vector; } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index af1ec8e..f755081 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -90,6 +90,11 @@ public class MapVector extends AbstractMapVector { } @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (MapVector) from); + } + + @Override protected boolean supportsDirectRead() { return true; } @@ -302,6 +307,13 @@ public class MapVector extends AbstractMapVector { return mutator; } + @Override + public void exchange(ValueVector other) { + // Exchange is used for look-ahead writers, but writers manage + // map member vectors directly. + throw new UnsupportedOperationException("Exchange() not supported for maps"); + } + public class Accessor extends BaseValueVector.BaseAccessor { @Override http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java index b5c97bf..969c141 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -118,7 +118,6 @@ public class RepeatedListVector extends AbstractContainerVector } } - public class DelegateTransferPair implements TransferPair { private final DelegateRepeatedVector target; private final TransferPair[] children; @@ -218,6 +217,10 @@ public class RepeatedListVector extends AbstractContainerVector ephPair.copyValueSafe(fromIndex, thisIndex); } + @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (DelegateRepeatedVector) from); + } } protected class RepeatedListTransferPair implements TransferPair { @@ -428,6 +431,11 @@ public class RepeatedListVector extends AbstractContainerVector } @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (RepeatedListVector) from); + } + + @Override public int getAllocatedByteCount() { return delegate.getAllocatedByteCount(); } @@ -436,4 +444,11 @@ public class RepeatedListVector extends AbstractContainerVector public int getPayloadByteCount() { return delegate.getPayloadByteCount(); } + + @Override + public void exchange(ValueVector other) { + // TODO: Figure out how to test this scenario, then what to do... + throw new UnsupportedOperationException("Exchange() not yet supported for repeated lists"); + } + } http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index d930728..7ff36a7 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -390,7 +390,6 @@ public class RepeatedMapVector extends AbstractMapVector } } - transient private RepeatedMapTransferPair ephPair; public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) { @@ -401,6 +400,11 @@ public class RepeatedMapVector extends AbstractMapVector } @Override + public void copyEntry(int toIndex, ValueVector from, int fromIndex) { + copyFromSafe(fromIndex, toIndex, (RepeatedMapVector) from); + } + + @Override public int getValueCapacity() { return Math.max(offsets.getValueCapacity() - 1, 0); } @@ -411,6 +415,13 @@ public class RepeatedMapVector extends AbstractMapVector } @Override + public void exchange(ValueVector other) { + // Exchange is used for look-ahead writers, but writers manage + // map member vectors directly. + throw new UnsupportedOperationException("Exchange() not supported for maps"); + } + + @Override public DrillBuf[] getBuffers(boolean clear) { //final int expectedBufferSize = getBufferSize(); //final int actualBufferSize = super.getBufferSize(); @@ -418,7 +429,6 @@ public class RepeatedMapVector extends AbstractMapVector return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear)); } - @Override public void load(SerializedField metadata, DrillBuf buffer) { final List<SerializedField> children = metadata.getChildList(); @@ -572,6 +582,9 @@ public class RepeatedMapVector extends AbstractMapVector offsets.getMutator().setSafe(index + 1, prevEnd + 1); return prevEnd; } + + @Override + public void exchange(ValueVector.Mutator other) { } } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java ---------------------------------------------------------------------- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java index 05fe63f..fdcfcd6 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java @@ -38,18 +38,12 @@ abstract class AbstractBaseReader implements FieldReader{ super(); } - public void setPosition(int index){ - this.index = index; - } + public void setPosition(int index) { this.index = index; } - int idx(){ - return index; - } + int idx() { return index; } @Override - public void reset() { - index = 0; - } + public void reset() { index = 0; } @Override public Iterator<String> iterator() {
