DRILL-1942-templates: template changes with a few related dependencies. This closes #108
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/deee06e3 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/deee06e3 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/deee06e3 Branch: refs/heads/master Commit: deee06e307931da79029bf64d2498509b68b46df Parents: b55e232 Author: Chris Westin <cwes...@yahoo.com> Authored: Tue Aug 11 15:18:36 2015 -0700 Committer: Parth Chandra <par...@apache.org> Committed: Wed Aug 19 21:58:32 2015 -0700 ---------------------------------------------------------------------- .../codegen/templates/AbstractFieldWriter.java | 80 +++--- .../src/main/codegen/templates/BaseWriter.java | 33 +-- .../main/codegen/templates/ComplexWriters.java | 73 ++--- .../codegen/templates/FixedValueVectors.java | 287 ++++++++++--------- .../templates/JsonOutputRecordWriter.java | 3 +- .../src/main/codegen/templates/ListWriters.java | 136 +++++---- .../src/main/codegen/templates/MapWriters.java | 114 ++++---- .../codegen/templates/NullableValueVectors.java | 184 +++++++----- .../templates/ParquetOutputRecordWriter.java | 3 +- .../codegen/templates/RepeatedValueVectors.java | 175 +++++------ .../templates/StringOutputRecordWriter.java | 3 +- .../templates/VariableLengthVectors.java | 258 +++++++++-------- .../drill/exec/vector/BaseDataValueVector.java | 21 +- .../vector/complex/impl/ComplexWriterImpl.java | 29 +- .../complex/impl/VectorContainerWriter.java | 27 +- 15 files changed, 788 insertions(+), 638 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java b/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java index 1b5dad1..9b67304 100644 --- a/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java +++ b/exec/java-exec/src/main/codegen/templates/AbstractFieldWriter.java @@ -26,58 +26,66 @@ package org.apache.drill.exec.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> -/* This class is generated using freemarker and the AbstractFieldWriter.java template */ +/* + * This class is generated using freemarker and the ${.template_name} template. + */ @SuppressWarnings("unused") -abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter{ - - AbstractFieldWriter(FieldWriter parent){ +abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter { + AbstractFieldWriter(FieldWriter parent) { super(parent); } - - public void start(){ + + @Override + public void start() { throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); } - - public void end(){ + + @Override + public void end() { throw new IllegalStateException(String.format("You tried to end when you are using a ValueWriter of type %s.", this.getClass().getSimpleName())); } - + <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> - public void write(${name}Holder holder){ + @Override + public void write(${name}Holder holder) { fail("${name}"); } - - public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>){ + + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) { fail("${name}"); } - + </#list></#list> - public void writeNull(){ + public void writeNull() { fail("${name}"); } - - public MapWriter map(){ + + @Override + public MapWriter map() { fail("Map"); return null; } - - public ListWriter list(){ + + @Override + public ListWriter list() { fail("List"); return null; } - - public MapWriter map(String name){ + + @Override + public MapWriter map(String name) { fail("Map"); return null; } - - public ListWriter list(String name){ + + @Override + public ListWriter list(String name) { fail("List"); return null; } - + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> @@ -89,32 +97,30 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr return null; } </#if> + + @Override public ${capName}Writer ${lowerName}(String name) { fail("${capName}"); return null; } - public ${capName}Writer ${lowerName}(){ + + @Override + public ${capName}Writer ${lowerName}() { fail("${capName}"); return null; } - - + </#list></#list> - - - public void copyReader(FieldReader reader){ + + public void copyReader(FieldReader reader) { fail("Copy FieldReader"); } - public void copyReaderToField(String name, FieldReader reader){ + + public void copyReaderToField(String name, FieldReader reader) { fail("Copy FieldReader to STring"); } - - private void fail(String name){ + + private void fail(String name) { throw new IllegalArgumentException(String.format("You tried to write a %s type when you are using a ValueWriter of type %s.", name, this.getClass().getSimpleName())); } - - } - - - http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/BaseWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/BaseWriter.java b/exec/java-exec/src/main/codegen/templates/BaseWriter.java index ada410d..7697880 100644 --- a/exec/java-exec/src/main/codegen/templates/BaseWriter.java +++ b/exec/java-exec/src/main/codegen/templates/BaseWriter.java @@ -26,13 +26,15 @@ package org.apache.drill.exec.vector.complex.writer; <#include "/@includes/vv_imports.ftl" /> -/* This class is generated using freemarker and the BaseWriter.java template */ +/* + * File generated from ${.template_name} using FreeMarker. + */ @SuppressWarnings("unused") -public interface BaseWriter extends Positionable{ + public interface BaseWriter extends AutoCloseable, Positionable { FieldWriter getParent(); int getValueCapacity(); - public interface MapWriter extends BaseWriter{ + public interface MapWriter extends BaseWriter { MaterializedField getField(); @@ -46,21 +48,21 @@ public interface BaseWriter extends Positionable{ </#if> ${capName}Writer ${lowerName}(String name); </#list></#list> - + void copyReaderToField(String name, FieldReader reader); MapWriter map(String name); ListWriter list(String name); void start(); void end(); } - - public interface ListWriter extends BaseWriter{ + + public interface ListWriter extends BaseWriter { void start(); void end(); MapWriter map(); ListWriter list(); void copyReader(FieldReader reader); - + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> @@ -69,20 +71,19 @@ public interface BaseWriter extends Positionable{ ${capName}Writer ${lowerName}(); </#list></#list> } - - public interface ScalarWriter extends + + public interface ScalarWriter extends <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Writer, </#list></#list> BaseWriter {} - - public interface ComplexWriter{ + + public interface ComplexWriter { void allocate(); void clear(); void copyReader(FieldReader reader); MapWriter rootAsMap(); ListWriter rootAsList(); - - public void setPosition(int index); - public void setValueCount(int count); - public void reset(); + + void setPosition(int index); + void setValueCount(int count); + void reset(); } } - http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/ComplexWriters.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ComplexWriters.java b/exec/java-exec/src/main/codegen/templates/ComplexWriters.java index 980f9ac..71a7f9a 100644 --- a/exec/java-exec/src/main/codegen/templates/ComplexWriters.java +++ b/exec/java-exec/src/main/codegen/templates/ComplexWriters.java @@ -40,92 +40,103 @@ package org.apache.drill.exec.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> -/* This class is generated using freemarker and the ComplexWriters.java template */ +/* + * This class is generated using FreeMarker on the ${.template_name} template. + */ @SuppressWarnings("unused") public class ${eName}WriterImpl extends AbstractFieldWriter { - + private final ${name}Vector.Mutator mutator; final ${name}Vector vector; - - public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent){ + + public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent) { super(parent); this.mutator = vector.getMutator(); this.vector = vector; } - public MaterializedField getField(){ + @Override + public MaterializedField getField() { return vector.getField(); } + @Override public int getValueCapacity() { return vector.getValueCapacity(); } - public void allocate(){ + @Override + public void allocate() { vector.allocateNew(); } - - public void clear(){ + + @Override + public void close() { + vector.close(); + } + + @Override + public void clear() { vector.clear(); } - - protected int idx(){ + + @Override + protected int idx() { return super.idx(); } - + <#if mode == "Repeated"> - public void write(${minor.class?cap_first}Holder h){ + public void write(${minor.class?cap_first}Holder h) { mutator.addSafe(idx(), h); vector.getMutator().setValueCount(idx()+1); } - - public void write(Nullable${minor.class?cap_first}Holder h){ + + public void write(Nullable${minor.class?cap_first}Holder h) { mutator.addSafe(idx(), h); vector.getMutator().setValueCount(idx()+1); } <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")> - public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>){ + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) { mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); vector.getMutator().setValueCount(idx()+1); } </#if> - - public void setPosition(int idx){ + + public void setPosition(int idx) { super.setPosition(idx); mutator.startNewValue(idx); } - - + + <#else> - - public void write(${minor.class}Holder h){ + + public void write(${minor.class}Holder h) { mutator.setSafe(idx(), h); vector.getMutator().setValueCount(idx()+1); } - - public void write(Nullable${minor.class}Holder h){ + + public void write(Nullable${minor.class}Holder h) { mutator.setSafe(idx(), h); vector.getMutator().setValueCount(idx()+1); } <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")> - public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>){ + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) { mutator.setSafe(idx(), <#if mode == "Nullable">1, </#if><#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); vector.getMutator().setValueCount(idx()+1); } <#if mode == "Nullable"> - public void writeNull(){ + + public void writeNull() { mutator.setNull(idx()); vector.getMutator().setValueCount(idx()+1); } </#if> </#if> - </#if> - } <@pp.changeOutputFile name="/org/apache/drill/exec/vector/complex/writer/${eName}Writer.java" /> @@ -135,18 +146,14 @@ package org.apache.drill.exec.vector.complex.writer; <#include "/@includes/vv_imports.ftl" /> @SuppressWarnings("unused") -public interface ${eName}Writer extends BaseWriter{ +public interface ${eName}Writer extends BaseWriter { public void write(${minor.class}Holder h); - + <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")> public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>); </#if> } - - </#list> </#list> </#list> - - http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java index cadcfd9..7bb76fa 100644 --- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java @@ -38,10 +38,9 @@ package org.apache.drill.exec.vector; * The width of each element is ${type.width} byte(s) * The equivalent Java primitive is '${minor.javaType!type.javaType}' * - * Source code generated using FreeMarker template ${.template_name} + * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ -@SuppressWarnings("unused") -public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{ +public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class); private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this); @@ -56,19 +55,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } @Override - public FieldReader getReader(){ + public FieldReader getReader() { return reader; } - public int getValueCapacity(){ - return (int) (data.capacity() *1.0 / ${type.width}); + @Override + public int getValueCapacity() { + return (int) (data.capacity() * 1.0 / ${type.width}); } - public Accessor getAccessor(){ + @Override + public Accessor getAccessor() { return accessor; } - public Mutator getMutator(){ + @Override + public Mutator getMutator() { return mutator; } @@ -81,12 +83,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F allocationSizeInBytes = (int)size; } + @Override public void allocateNew() { - if(!allocateNewSafe()){ + if(!allocateNewSafe()) { throw new OutOfMemoryRuntimeException("Failure while allocating buffer."); } } + @Override public boolean allocateNewSafe() { long curAllocationSize = allocationSizeInBytes; if (allocationMonitor > 10) { @@ -97,7 +101,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F allocationMonitor = 0; } - try{ + try { allocateBytes(curAllocationSize); } catch (DrillRuntimeException ex) { return false; @@ -114,6 +118,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F * @param valueCount * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the new buffer */ + @Override public void allocateNew(final int valueCount) { allocateBytes(valueCount * ${type.width}); } @@ -142,25 +147,33 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize); - final DrillBuf newBuf = allocator.buffer((int)newAllocationSize); + final DrillBuf newBuf = allocator.buffer((int) newAllocationSize); newBuf.setBytes(0, data, 0, data.capacity()); - newBuf.setZero(newBuf.capacity() / 2, newBuf.capacity() / 2); + final int halfNewCapacity = newBuf.capacity() / 2; + newBuf.setZero(halfNewCapacity, halfNewCapacity); newBuf.writerIndex(data.writerIndex()); - data.release(); + data.release(1); data = newBuf; - allocationSizeInBytes = (int)newAllocationSize; + allocationSizeInBytes = (int) newAllocationSize; } + /** + * {@inheritDoc} + */ + @Override public void zeroVector() { data.setZero(0, data.capacity()); } @Override - public int load(int valueCount, DrillBuf buf){ + public int load(int valueCount, DrillBuf buf) { clear(); - int len = valueCount * ${type.width}; + final int len = valueCount * ${type.width}; + if (data != null) { + data.release(1); + } data = buf.slice(0, len); - data.retain(); + data.retain(1); data.writerIndex(len); return len; } @@ -168,58 +181,64 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public void load(SerializedField metadata, DrillBuf buffer) { assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata); - int loaded = load(metadata.getValueCount(), buffer); + final int loaded = load(metadata.getValueCount(), buffer); assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded); } - public TransferPair getTransferPair(){ + @Override + public TransferPair getTransferPair() { return new TransferImpl(getField()); } + + @Override public TransferPair getTransferPair(FieldReference ref){ return new TransferImpl(getField().withPath(ref)); } + @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((${minor.class}Vector) to); } - public void transferTo(${minor.class}Vector target){ + public void transferTo(${minor.class}Vector target) { target.clear(); target.data = data; - target.data.retain(); + target.data.retain(1); target.data.writerIndex(data.writerIndex()); clear(); } public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) { - int currentWriterIndex = data.writerIndex(); - int startPoint = startIndex * ${type.width}; - int sliceLength = length * ${type.width}; + final int startPoint = startIndex * ${type.width}; + final int sliceLength = length * ${type.width}; target.clear(); - target.data = this.data.slice(startPoint, sliceLength); + target.data = data.slice(startPoint, sliceLength); + target.data.retain(1); target.data.writerIndex(sliceLength); - target.data.retain(); } - private class TransferImpl implements TransferPair{ - ${minor.class}Vector to; + private class TransferImpl implements TransferPair { + private ${minor.class}Vector to; - public TransferImpl(MaterializedField field){ - this.to = new ${minor.class}Vector(field, allocator); + public TransferImpl(MaterializedField field) { + to = new ${minor.class}Vector(field, allocator); } public TransferImpl(${minor.class}Vector to) { this.to = to; } - public ${minor.class}Vector getTo(){ + @Override + public ${minor.class}Vector getTo() { return to; } - public void transfer(){ + @Override + public void transfer() { transferTo(to); } + @Override public void splitAndTransfer(int startIndex, int length) { splitAndTransferTo(startIndex, length, to); } @@ -230,7 +249,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } } - public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) { <#if (type.width > 8)> from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width}); <#else> <#-- type.width <= 8 --> @@ -240,7 +259,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F </#if> <#-- type.width --> } - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { while(thisIndex >= getValueCapacity()) { reAlloc(); } @@ -259,12 +278,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } public final class Accessor extends BaseDataValueVector.BaseAccessor { - + @Override public int getValueCount() { return data.writerIndex() / ${type.width}; } - public boolean isNull(int index){ + @Override + public boolean isNull(int index) { return false; } @@ -275,16 +295,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } <#if (minor.class == "Interval")> - public void get(int index, ${minor.class}Holder holder){ + public void get(int index, ${minor.class}Holder holder) { - int offsetIndex = index * ${type.width}; + final int offsetIndex = index * ${type.width}; holder.months = data.getInt(offsetIndex); holder.days = data.getInt(offsetIndex + ${minor.daysOffset}); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } - public void get(int index, Nullable${minor.class}Holder holder){ - int offsetIndex = index * ${type.width}; + public void get(int index, Nullable${minor.class}Holder holder) { + final int offsetIndex = index * ${type.width}; holder.isSet = 1; holder.months = data.getInt(offsetIndex); holder.days = data.getInt(offsetIndex + ${minor.daysOffset}); @@ -293,37 +313,37 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public ${friendlyType} getObject(int index) { - int offsetIndex = index * ${type.width}; - int months = data.getInt(offsetIndex); - int days = data.getInt(offsetIndex + ${minor.daysOffset}); - int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - Period p = new Period(); + final int offsetIndex = index * ${type.width}; + final int months = data.getInt(offsetIndex); + final int days = data.getInt(offsetIndex + ${minor.daysOffset}); + final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); + final Period p = new Period(); return p.plusMonths(months).plusDays(days).plusMillis(millis); } public StringBuilder getAsStringBuilder(int index) { - int offsetIndex = index * ${type.width}; + final int offsetIndex = index * ${type.width}; int months = data.getInt(offsetIndex); - int days = data.getInt(offsetIndex + ${minor.daysOffset}); + final int days = data.getInt(offsetIndex + ${minor.daysOffset}); int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); + final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); + final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); - int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); + final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); - long seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); + final long seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); - String yearString = (Math.abs(years) == 1) ? " year " : " years "; - String monthString = (Math.abs(months) == 1) ? " month " : " months "; - String dayString = (Math.abs(days) == 1) ? " day " : " days "; + final String yearString = (Math.abs(years) == 1) ? " year " : " years "; + final String monthString = (Math.abs(months) == 1) ? " month " : " months "; + final String dayString = (Math.abs(days) == 1) ? " day " : " days "; return(new StringBuilder(). @@ -337,15 +357,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } <#elseif (minor.class == "IntervalDay")> - public void get(int index, ${minor.class}Holder holder){ + public void get(int index, ${minor.class}Holder holder) { - int offsetIndex = index * ${type.width}; + final int offsetIndex = index * ${type.width}; holder.days = data.getInt(offsetIndex); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); } - public void get(int index, Nullable${minor.class}Holder holder){ - int offsetIndex = index * ${type.width}; + public void get(int index, Nullable${minor.class}Holder holder) { + final int offsetIndex = index * ${type.width}; holder.isSet = 1; holder.days = data.getInt(offsetIndex); holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset}); @@ -353,30 +373,30 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F @Override public ${friendlyType} getObject(int index) { - int offsetIndex = index * ${type.width}; - int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - int days = data.getInt(offsetIndex); - Period p = new Period(); + final int offsetIndex = index * ${type.width}; + final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); + final int days = data.getInt(offsetIndex); + final Period p = new Period(); return p.plusDays(days).plusMillis(millis); } public StringBuilder getAsStringBuilder(int index) { - int offsetIndex = index * ${type.width}; + final int offsetIndex = index * ${type.width}; int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset}); - int days = data.getInt(offsetIndex); + final int days = data.getInt(offsetIndex); - int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); + final int hours = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.hoursToMillis); - int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); + final int minutes = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.minutesToMillis); - int seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); + final int seconds = millis / (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); millis = millis % (org.apache.drill.exec.expr.fn.impl.DateUtility.secondsToMillis); - String dayString = (Math.abs(days) == 1) ? " day " : " days "; + final String dayString = (Math.abs(days) == 1) ? " day " : " days "; return(new StringBuilder(). append(days).append(dayString). @@ -414,12 +434,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } <#else> - public void get(int index, ${minor.class}Holder holder){ + public void get(int index, ${minor.class}Holder holder) { holder.buffer = data; holder.start = index * ${type.width}; } - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { holder.isSet = 1; holder.buffer = data; holder.start = index * ${type.width}; @@ -445,6 +465,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F </#if> <#if minor.class == "Date"> + @Override public ${friendlyType} getObject(int index) { org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); @@ -452,6 +473,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } <#elseif minor.class == "TimeStamp"> + @Override public ${friendlyType} getObject(int index) { org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC); date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault()); @@ -459,13 +481,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } <#elseif minor.class == "IntervalYear"> + @Override public ${friendlyType} getObject(int index) { - int value = get(index); + final int value = get(index); - int years = (value / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - int months = (value % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - Period p = new Period(); + final int years = (value / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); + final int months = (value % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); + final Period p = new Period(); return p.plusYears(years).plusMonths(months); } @@ -473,11 +496,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F int months = data.getInt(index); - int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); + final int years = (months / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); months = (months % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths); - String yearString = (Math.abs(years) == 1) ? " year " : " years "; - String monthString = (Math.abs(months) == 1) ? " month " : " months "; + final String yearString = (Math.abs(years) == 1) ? " year " : " years "; + final String monthString = (Math.abs(months) == 1) ? " month " : " months "; return(new StringBuilder(). append(years).append(yearString). @@ -493,17 +516,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F return time; } - - <#elseif minor.class == "Decimal9" || minor.class == "Decimal18"> @Override public ${friendlyType} getObject(int index) { - BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value()); + final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value()); return new BigDecimal(value, getField().getScale()); } <#else> + @Override public ${friendlyType} getObject(int index) { return get(index); } @@ -512,7 +534,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } </#if> - public void get(int index, ${minor.class}Holder holder){ + public void get(int index, ${minor.class}Holder holder) { <#if minor.class.startsWith("Decimal")> holder.scale = getField().getScale(); holder.precision = getField().getPrecision(); @@ -521,7 +543,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); } - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { holder.isSet = 1; holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}); } @@ -541,7 +563,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F */ public final class Mutator extends BaseDataValueVector.BaseMutator { - private Mutator(){}; + private Mutator() {}; /** * Set the element at the given index to the given value. Note that widths smaller than * 32 bits are handled by the DrillBuf interface. @@ -562,122 +584,122 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F } <#if (minor.class == "Interval")> - public void set(int index, int months, int days, int milliseconds){ - int offsetIndex = index * ${type.width}; + public void set(int index, int months, int days, int milliseconds) { + final int offsetIndex = index * ${type.width}; data.setInt(offsetIndex, months); data.setInt((offsetIndex + ${minor.daysOffset}), days); data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); } - protected void set(int index, ${minor.class}Holder holder){ + protected void set(int index, ${minor.class}Holder holder) { set(index, holder.months, holder.days, holder.milliseconds); } - protected void set(int index, Nullable${minor.class}Holder holder){ + protected void set(int index, Nullable${minor.class}Holder holder) { set(index, holder.months, holder.days, holder.milliseconds); } - public void setSafe(int index, int months, int days, int milliseconds){ + public void setSafe(int index, int months, int days, int milliseconds) { while(index >= getValueCapacity()) { reAlloc(); } set(index, months, days, milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.months, holder.days, holder.milliseconds); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setSafe(int index, ${minor.class}Holder holder) { setSafe(index, holder.months, holder.days, holder.milliseconds); } <#elseif (minor.class == "IntervalDay")> - public void set(int index, int days, int milliseconds){ - int offsetIndex = index * ${type.width}; + public void set(int index, int days, int milliseconds) { + final int offsetIndex = index * ${type.width}; data.setInt(offsetIndex, days); data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds); } - protected void set(int index, ${minor.class}Holder holder){ + protected void set(int index, ${minor.class}Holder holder) { set(index, holder.days, holder.milliseconds); } - protected void set(int index, Nullable${minor.class}Holder holder){ + protected void set(int index, Nullable${minor.class}Holder holder) { set(index, holder.days, holder.milliseconds); } - public void setSafe(int index, int days, int milliseconds){ + public void setSafe(int index, int days, int milliseconds) { while(index >= getValueCapacity()) { reAlloc(); } set(index, days, milliseconds); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setSafe(int index, ${minor.class}Holder holder) { setSafe(index, holder.days, holder.milliseconds); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.days, holder.milliseconds); } <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")> - public void set(int index, ${minor.class}Holder holder){ + public void set(int index, ${minor.class}Holder holder) { set(index, holder.start, holder.buffer); } - void set(int index, Nullable${minor.class}Holder holder){ + void set(int index, Nullable${minor.class}Holder holder) { set(index, holder.start, holder.buffer); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setSafe(int index, ${minor.class}Holder holder) { setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, int start, DrillBuf buffer){ + public void setSafe(int index, int start, DrillBuf buffer) { while(index >= getValueCapacity()) { reAlloc(); } set(index, start, buffer); } - public void set(int index, int start, DrillBuf buffer){ + public void set(int index, int start, DrillBuf buffer) { data.setBytes(index * ${type.width}, buffer, start, ${type.width}); } <#else> - protected void set(int index, ${minor.class}Holder holder){ + protected void set(int index, ${minor.class}Holder holder) { set(index, holder.start, holder.buffer); } - public void set(int index, Nullable${minor.class}Holder holder){ + public void set(int index, Nullable${minor.class}Holder holder) { set(index, holder.start, holder.buffer); } - public void set(int index, int start, DrillBuf buffer){ + public void set(int index, int start, DrillBuf buffer) { data.setBytes(index * ${type.width}, buffer, start, ${type.width}); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setSafe(int index, ${minor.class}Holder holder) { setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { setSafe(index, holder.start, holder.buffer); } - public void setSafe(int index, int start, DrillBuf buffer){ + public void setSafe(int index, int start, DrillBuf buffer) { while(index >= getValueCapacity()) { reAlloc(); } set(index, holder); } - public void set(int index, Nullable${minor.class}Holder holder){ + public void set(int index, Nullable${minor.class}Holder holder) { data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width}); } </#if> @@ -686,17 +708,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void generateTestData(int count) { setValueCount(count); boolean even = true; - for(int i =0; i < getAccessor().getValueCount(); i++, even = !even){ - byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE; - for(int w = 0; w < ${type.width}; w++){ + final int valueCount = getAccessor().getValueCount(); + for(int i = 0; i < valueCount; i++, even = !even) { + final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE; + for(int w = 0; w < ${type.width}; w++) { data.setByte(i + w, b); } } } - - - <#else> <#-- type.width <= 8 --> public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) { data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value); @@ -709,22 +729,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F set(index, value); } - protected void set(int index, ${minor.class}Holder holder){ + protected void set(int index, ${minor.class}Holder holder) { data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); } - public void setSafe(int index, ${minor.class}Holder holder){ + public void setSafe(int index, ${minor.class}Holder holder) { while(index >= getValueCapacity()) { reAlloc(); } set(index, holder); } - protected void set(int index, Nullable${minor.class}Holder holder){ + protected void set(int index, Nullable${minor.class}Holder holder) { data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value); } - public void setSafe(int index, Nullable${minor.class}Holder holder){ + public void setSafe(int index, Nullable${minor.class}Holder holder) { while(index >= getValueCapacity()) { reAlloc(); } @@ -735,23 +755,24 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F public void generateTestData(int size) { setValueCount(size); boolean even = true; - for(int i =0; i < getAccessor().getValueCount(); i++, even = !even){ - if(even){ + final int valueCount = getAccessor().getValueCount(); + for(int i = 0; i < valueCount; i++, even = !even) { + if (even) { set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE); - }else{ + } else { set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE); } } } - public void generateTestDataAlt(int size) { setValueCount(size); boolean even = true; - for(int i =0; i < getAccessor().getValueCount(); i++, even = !even){ - if(even){ + final int valueCount = getAccessor().getValueCount(); + for(int i = 0; i < valueCount; i++, even = !even) { + if (even) { set(i, (${(minor.javaType!type.javaType)}) 1); - }else{ + } else { set(i, (${(minor.javaType!type.javaType)}) 0); } } @@ -759,11 +780,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F </#if> <#-- type.width --> - - + @Override public void setValueCount(int valueCount) { - int currentValueCapacity = getValueCapacity(); - int idx = (${type.width} * valueCount); + final int currentValueCapacity = getValueCapacity(); + final int idx = (${type.width} * valueCount); while(valueCount > getValueCapacity()) { reAlloc(); } @@ -775,11 +795,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F VectorTrimmer.trim(data, idx); data.writerIndex(valueCount * ${type.width}); } - - - - - } } http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java index ea643f0..6a713a3 100644 --- a/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/JsonOutputRecordWriter.java @@ -37,7 +37,6 @@ import parquet.io.api.RecordConsumer; import parquet.schema.MessageType; import parquet.io.api.Binary; import io.netty.buffer.ByteBuf; -import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.vector.complex.fn.JsonOutput; @@ -61,6 +60,8 @@ import java.util.Map; * to output the data in string format instead of implementing addField for each type holder. * * This is useful for text format writers such as CSV, TSV etc. + * + * NB: Source code generated using FreeMarker template ${.template_name} */ public abstract class JSONOutputRecordWriter extends AbstractRecordWriter implements RecordWriter { http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/ListWriters.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ListWriters.java b/exec/java-exec/src/main/codegen/templates/ListWriters.java index 6d26204..f2683a3 100644 --- a/exec/java-exec/src/main/codegen/templates/ListWriters.java +++ b/exec/java-exec/src/main/codegen/templates/ListWriters.java @@ -36,9 +36,11 @@ package org.apache.drill.exec.vector.complex.impl; <#include "/@includes/vv_imports.ftl" /> -/* This class is generated using freemarker and the ListWriters.java template */ +/* + * This class is generated using FreeMarker and the ${.template_name} template. + */ @SuppressWarnings("unused") -public class ${mode}ListWriter extends AbstractFieldWriter{ +public class ${mode}ListWriter extends AbstractFieldWriter { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class); static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> } @@ -48,7 +50,7 @@ public class ${mode}ListWriter extends AbstractFieldWriter{ private Mode mode = Mode.INIT; private FieldWriter writer; protected RepeatedValueVector innerVector; - + <#if mode == "Repeated">private int currentChildIndex = 0;</#if> public ${mode}ListWriter(String name, ${containerClass} container, FieldWriter parent){ super(parent); @@ -62,36 +64,53 @@ public class ${mode}ListWriter extends AbstractFieldWriter{ this.container = container; } - public void allocate(){ - if(writer != null){ + @Override + public void allocate() { + if(writer != null) { writer.allocate(); } - + <#if mode == "Repeated"> container.allocateNew(); </#if> } - - public void clear(){ - writer.clear(); + + @Override + public void clear() { + if (writer != null) { + writer.clear(); + } + } + + @Override + public void close() { + clear(); + container.close(); + if (innerVector != null) { + innerVector.close(); + } } + @Override public int getValueCapacity() { - return innerVector==null ? 0:innerVector.getValueCapacity(); + return innerVector == null ? 0 : innerVector.getValueCapacity(); } public void setValueCount(int count){ if(innerVector != null) innerVector.getMutator().setValueCount(count); } - - public MapWriter map(){ - switch(mode){ + + @Override + public MapWriter map() { + switch(mode) { case INIT: int vectorCount = container.size(); - RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class); + final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class); innerVector = vector; writer = new RepeatedMapWriter(vector, this); - if(vectorCount != container.size()) writer.allocate(); + if(vectorCount != container.size()) { + writer.allocate(); + } writer.setPosition(${index}); mode = Mode.IN_MAP; return writer; @@ -102,15 +121,18 @@ public class ${mode}ListWriter extends AbstractFieldWriter{ throw UserException.unsupportedError().message(getUnsupportedErrorMsg("MAP", mode.name())).build(logger); } - - public ListWriter list(){ - switch(mode){ + + @Override + public ListWriter list() { + switch(mode) { case INIT: - int vectorCount = container.size(); - RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class); + final int vectorCount = container.size(); + final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class); innerVector = vector; writer = new RepeatedListWriter(null, vector, this); - if(vectorCount != container.size()) writer.allocate(); + if(vectorCount != container.size()) { + writer.allocate(); + } writer.setPosition(${index}); mode = Mode.IN_LIST; return writer; @@ -121,23 +143,26 @@ public class ${mode}ListWriter extends AbstractFieldWriter{ throw UserException.unsupportedError().message(getUnsupportedErrorMsg("LIST", mode.name())).build(logger); } - + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> - + private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName}); - - public ${capName}Writer ${lowerName}(){ - switch(mode){ + + @Override + public ${capName}Writer ${lowerName}() { + switch(mode) { case INIT: - int vectorCount = container.size(); - Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class); + final int vectorCount = container.size(); + final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class); innerVector = vector; writer = new Repeated${capName}WriterImpl(vector, this); - if(vectorCount != container.size()) writer.allocate(); + if(vectorCount != container.size()) { + writer.allocate(); + } writer.setPosition(${index}); mode = Mode.IN_${upperName}; return writer; @@ -155,60 +180,55 @@ public class ${mode}ListWriter extends AbstractFieldWriter{ } <#if mode == "Repeated"> - - public void start(){ - + + public void start() { final RepeatedListVector list = (RepeatedListVector) container; final RepeatedListVector.RepeatedMutator mutator = list.getMutator(); - + // make sure that the current vector can support the end position of this list. - if(container.getValueCapacity() <= idx()){ + if(container.getValueCapacity() <= idx()) { mutator.setValueCount(idx()+1); } - + // update the repeated vector to state that there is current+1 objects. - RepeatedListHolder h = new RepeatedListHolder(); + final RepeatedListHolder h = new RepeatedListHolder(); list.getAccessor().get(idx(), h); - if(h.start >= h.end){ - mutator.startNewValue(idx()); + if (h.start >= h.end) { + mutator.startNewValue(idx()); } currentChildIndex = container.getMutator().add(idx()); - if(writer != null){ - writer.setPosition(currentChildIndex); + if(writer != null) { + writer.setPosition(currentChildIndex); } } - - - - public void end(){ + + public void end() { // noop, we initialize state at start rather than end. } <#else> - - - public void setPosition(int index){ + + public void setPosition(int index) { super.setPosition(index); - if(writer != null) writer.setPosition(index); + if(writer != null) { + writer.setPosition(index); + } } - - public void start(){ + + public void start() { // noop } - - public void end(){ + + public void end() { // noop } </#if> - private String getUnsupportedErrorMsg(String expected, String found ){ - String f = found.substring(3); + private String getUnsupportedErrorMsg(String expected, String found) { + final String f = found.substring(3); return String.format("In a list of type %s, encountered a value of type %s. "+ "Drill does not support lists of different types.", f, expected ); } - - } +} </#list> - - http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/MapWriters.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/MapWriters.java b/exec/java-exec/src/main/codegen/templates/MapWriters.java index 06a6813..9683338 100644 --- a/exec/java-exec/src/main/codegen/templates/MapWriters.java +++ b/exec/java-exec/src/main/codegen/templates/MapWriters.java @@ -42,112 +42,130 @@ import org.apache.drill.exec.vector.complex.writer.FieldWriter; import com.google.common.collect.Maps; -/* This class is generated using freemarker and the MapWriters.java template */ +/* + * This class is generated using FreeMarker and the ${.template_name} template. + */ @SuppressWarnings("unused") -public class ${mode}MapWriter extends AbstractFieldWriter{ - +public class ${mode}MapWriter extends AbstractFieldWriter { + protected final ${containerClass} container; private final Map<String, FieldWriter> fields = Maps.newHashMap(); <#if mode == "Repeated">private int currentChildIndex = 0;</#if> - + public ${mode}MapWriter(${containerClass} container, FieldWriter parent) { super(parent); this.container = container; } + @Override public int getValueCapacity() { return container.getValueCapacity(); } + @Override public MaterializedField getField() { return container.getField(); } - public MapWriter map(String name){ - FieldWriter writer = fields.get(name.toLowerCase()); - if(writer == null){ + @Override + public MapWriter map(String name) { + FieldWriter writer = fields.get(name.toLowerCase()); + if(writer == null) { int vectorCount = container.size(); MapVector vector = container.addOrGet(name, MapVector.TYPE, MapVector.class); writer = new SingleMapWriter(vector, this); - if(vectorCount != container.size()) writer.allocate(); + if(vectorCount != container.size()) { + writer.allocate(); + } writer.setPosition(${index}); fields.put(name.toLowerCase(), writer); } return writer; } - - public void allocate(){ + + @Override + public void close() throws Exception { + clear(); + container.close(); + } + + @Override + public void allocate() { container.allocateNew(); - for(FieldWriter w : fields.values()){ + for(final FieldWriter w : fields.values()) { w.allocate(); } } - - public void clear(){ + + @Override + public void clear() { container.clear(); - for(FieldWriter w : fields.values()){ + for(final FieldWriter w : fields.values()) { w.clear(); } } - - public ListWriter list(String name){ + + @Override + public ListWriter list(String name) { FieldWriter writer = fields.get(name.toLowerCase()); - if(writer == null){ + if(writer == null) { writer = new SingleListWriter(name, container, this); writer.setPosition(${index}); fields.put(name.toLowerCase(), writer); } return writer; } - <#if mode == "Repeated"> - public void start(){ + public void start() { // update the repeated vector to state that there is current+1 objects. - final RepeatedMapHolder h = new RepeatedMapHolder(); final RepeatedMapVector map = (RepeatedMapVector) container; final RepeatedMapVector.Mutator mutator = map.getMutator(); - - // make sure that the current vector can support the end position of this list. - if(container.getValueCapacity() <= idx()){ + + // Make sure that the current vector can support the end position of this list. + if(container.getValueCapacity() <= idx()) { mutator.setValueCount(idx()+1); } map.getAccessor().get(idx(), h); - if(h.start >= h.end){ - container.getMutator().startNewValue(idx()); + if (h.start >= h.end) { + container.getMutator().startNewValue(idx()); } currentChildIndex = container.getMutator().add(idx()); - for(FieldWriter w: fields.values()){ - w.setPosition(currentChildIndex); + for(final FieldWriter w : fields.values()) { + w.setPosition(currentChildIndex); } } - - public void end(){ + + public void end() { // noop } <#else> - public void setValueCount(int count){ + public void setValueCount(int count) { container.getMutator().setValueCount(count); } - public void setPosition(int index){ + @Override + public void setPosition(int index) { super.setPosition(index); - for(FieldWriter w: fields.values()){ - w.setPosition(index); + for(final FieldWriter w: fields.values()) { + w.setPosition(index); } } - public void start(){ + + @Override + public void start() { } - - public void end(){ - // noop + + @Override + public void end() { } + </#if> - + <#list vv.types as type><#list type.minor as minor> <#assign lowerName = minor.class?uncap_first /> <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> @@ -157,22 +175,23 @@ public class ${mode}MapWriter extends AbstractFieldWriter{ <#assign vectName = "Nullable${capName}" /> <#if minor.class?starts_with("Decimal") > - public ${minor.class}Writer ${lowerName}(String name){ + public ${minor.class}Writer ${lowerName}(String name) { // returns existing writer - FieldWriter writer = fields.get(name.toLowerCase()); + final FieldWriter writer = fields.get(name.toLowerCase()); assert writer != null; return writer; } - public ${minor.class}Writer ${lowerName}(String name, int scale, int precision){ + public ${minor.class}Writer ${lowerName}(String name, int scale, int precision) { final MajorType ${upperName}_TYPE = Types.withScaleAndPrecision(MinorType.${upperName}, DataMode.OPTIONAL, scale, precision); <#else> private static final MajorType ${upperName}_TYPE = Types.optional(MinorType.${upperName}); - public ${minor.class}Writer ${lowerName}(String name){ + @Override + public ${minor.class}Writer ${lowerName}(String name) { </#if> FieldWriter writer = fields.get(name.toLowerCase()); - if(writer == null){ - ${vectName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class); + if(writer == null) { + final ${vectName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, ${vectName}Vector.class); vector.allocateNewSafe(); writer = new ${vectName}WriterImpl(vector, this); writer.setPosition(${index}); @@ -180,13 +199,8 @@ public class ${mode}MapWriter extends AbstractFieldWriter{ } return writer; } - - - </#list></#list> - + </#list></#list> } </#list> - - http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java index 7fa0d55..4bdde16 100644 --- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java +++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java @@ -42,10 +42,10 @@ package org.apache.drill.exec.vector; * are first checked against a fixed length vector of boolean values. Then the element is retrieved * from the base class (if not null). * - * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker. + * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker. */ @SuppressWarnings("unused") -public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{ +public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class); private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this); @@ -60,20 +60,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type } @Override - public FieldReader getReader(){ + public FieldReader getReader() { return reader; } - public int getValueCapacity(){ + @Override + public int getValueCapacity() { return Math.min(bits.getValueCapacity(), values.getValueCapacity()); } @Override public DrillBuf[] getBuffers(boolean clear) { - DrillBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), DrillBuf.class); + final DrillBuf[] buffers = ObjectArrays.concat(bits.getBuffers(false), values.getBuffers(false), DrillBuf.class); if (clear) { - for (DrillBuf buffer:buffers) { - buffer.retain(); + for (final DrillBuf buffer:buffers) { + buffer.retain(1); } clear(); } @@ -81,13 +82,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type } @Override + public void close() { + bits.close(); + values.close(); + super.close(); + } + + @Override public void clear() { bits.clear(); values.clear(); super.clear(); } - public int getBufferSize(){ + @Override + public int getBufferSize() { return values.getBufferSize() + bits.getBufferSize(); } @@ -96,6 +105,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type return values.getBuffer(); } + @Override public ${valuesName} getValuesVector() { return values; } @@ -116,8 +126,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type .build(); } + @Override public void allocateNew() { - if(!allocateNewSafe()){ + if(!allocateNewSafe()) { throw new OutOfMemoryRuntimeException("Failure while allocating buffer."); } } @@ -158,7 +169,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type } @Override - public int load(int dataBytes, int valueCount, DrillBuf buf){ + public int load(int dataBytes, int valueCount, DrillBuf buf) { clear(); int loaded = bits.load(valueCount, buf); @@ -166,24 +177,24 @@ public final class ${className} extends BaseDataValueVector implements <#if type buf = buf.slice(loaded, buf.capacity() - loaded); dataBytes -= loaded; loaded += values.load(dataBytes, valueCount, buf); - this.mutator.lastSet = valueCount; + mutator.lastSet = valueCount; return loaded; } @Override public void load(SerializedField metadata, DrillBuf buffer) { - assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata); - int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer); + assert field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", field, metadata); + final int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer); assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded); } @Override - public int getByteCapacity(){ + public int getByteCapacity() { return values.getByteCapacity(); } @Override - public int getCurrentSizeInBytes(){ + public int getCurrentSizeInBytes() { return values.getCurrentSizeInBytes(); } @@ -242,13 +253,14 @@ public final class ${className} extends BaseDataValueVector implements <#if type /** * {@inheritDoc} */ + @Override public void zeroVector() { - this.values.zeroVector(); - this.bits.zeroVector(); + values.zeroVector(); + bits.zeroVector(); } @Override - public int load(int valueCount, DrillBuf buf){ + public int load(int valueCount, DrillBuf buf) { clear(); int loaded = bits.load(valueCount, buf); @@ -260,26 +272,29 @@ public final class ${className} extends BaseDataValueVector implements <#if type @Override public void load(SerializedField metadata, DrillBuf buffer) { - assert this.field.matches(metadata); - int loaded = load(metadata.getValueCount(), buffer); + assert field.matches(metadata); + final int loaded = load(metadata.getValueCount(), buffer); assert metadata.getBufferLength() == loaded; } </#if> - public TransferPair getTransferPair(){ + @Override + public TransferPair getTransferPair() { return new TransferImpl(getField()); } + + @Override public TransferPair getTransferPair(FieldReference ref){ return new TransferImpl(getField().withPath(ref)); } + @Override public TransferPair makeTransferPair(ValueVector to) { return new TransferImpl((Nullable${minor.class}Vector) to); } - - public void transferTo(Nullable${minor.class}Vector target){ + public void transferTo(Nullable${minor.class}Vector target) { bits.transferTo(target.bits); values.transferTo(target.values); <#if type.major == "VarLen"> @@ -296,25 +311,28 @@ public final class ${className} extends BaseDataValueVector implements <#if type </#if> } - private class TransferImpl implements TransferPair{ + private class TransferImpl implements TransferPair { Nullable${minor.class}Vector to; - public TransferImpl(MaterializedField field){ - this.to = new Nullable${minor.class}Vector(field, allocator); + public TransferImpl(MaterializedField field) { + to = new Nullable${minor.class}Vector(field, allocator); } - public TransferImpl(Nullable${minor.class}Vector to){ + public TransferImpl(Nullable${minor.class}Vector to) { this.to = to; } - public Nullable${minor.class}Vector getTo(){ + @Override + public Nullable${minor.class}Vector getTo() { return to; } - public void transfer(){ + @Override + public void transfer() { transferTo(to); } + @Override public void splitAndTransfer(int startIndex, int length) { splitAndTransferTo(startIndex, length, to); } @@ -325,31 +343,35 @@ public final class ${className} extends BaseDataValueVector implements <#if type } } - public Accessor getAccessor(){ + @Override + public Accessor getAccessor() { return accessor; } - public Mutator getMutator(){ + @Override + public Mutator getMutator() { return mutator; } - public ${minor.class}Vector convertToRequiredVector(){ + public ${minor.class}Vector convertToRequiredVector() { ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator); + if (v.data != null) { + v.data.release(1); + } v.data = values.data; - v.data.retain(); + v.data.retain(1); clear(); return v; } - - public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){ - if (!from.getAccessor().isNull(fromIndex)) { - mutator.set(thisIndex, from.getAccessor().get(fromIndex)); + public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from) { + final Accessor fromAccessor = from.getAccessor(); + if (!fromAccessor.isNull(fromIndex)) { + mutator.set(thisIndex, fromAccessor.get(fromIndex)); } } - - public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){ + public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) { <#if type.major == "VarLen"> mutator.fillEmpties(thisIndex); </#if> @@ -357,7 +379,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type bits.getMutator().setSafe(thisIndex, 1); } - public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){ + public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from) { <#if type.major == "VarLen"> mutator.fillEmpties(thisIndex); </#if> @@ -366,7 +388,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type } public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> { - final UInt1Vector.Accessor bAccessor = bits.getAccessor(); final ${valuesName}.Accessor vAccessor = values.getAccessor(); @@ -379,30 +400,32 @@ public final class ${className} extends BaseDataValueVector implements <#if type */ public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) { if (isNull(index)) { - throw new IllegalStateException("Can't get a null value"); + throw new IllegalStateException("Can't get a null value"); } return vAccessor.get(index); } + @Override public boolean isNull(int index) { return isSet(index) == 0; } - public int isSet(int index){ + public int isSet(int index) { return bAccessor.get(index); } <#if type.major == "VarLen"> - public long getStartEnd(int index){ + public long getStartEnd(int index) { return vAccessor.getStartEnd(index); } + @Override public int getValueLength(int index) { return values.getAccessor().getValueLength(index); } </#if> - public void get(int index, Nullable${minor.class}Holder holder){ + public void get(int index, Nullable${minor.class}Holder holder) { vAccessor.get(index, holder); holder.isSet = bAccessor.get(index); @@ -416,7 +439,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type public ${friendlyType} getObject(int index) { if (isNull(index)) { return null; - }else{ + } else { return vAccessor.getObject(index); } } @@ -425,32 +448,33 @@ public final class ${className} extends BaseDataValueVector implements <#if type public StringBuilder getAsStringBuilder(int index) { if (isNull(index)) { return null; - }else{ + } else { return vAccessor.getAsStringBuilder(index); } } </#if> - public int getValueCount(){ + @Override + public int getValueCount() { return bits.getAccessor().getValueCount(); } - public void reset(){} + public void reset() {} } public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> { - private int setCount; <#if type.major = "VarLen"> private int lastSet = -1;</#if> - private Mutator(){ + private Mutator() { } - public ${valuesName} getVectorWithValues(){ + public ${valuesName} getVectorWithValues() { return values; } - public void setIndexDefined(int index){ + @Override + public void setIndexDefined(int index) { bits.getMutator().set(index, 1); } @@ -462,27 +486,32 @@ public final class ${className} extends BaseDataValueVector implements <#if type */ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) { setCount++; + final ${valuesName}.Mutator valuesMutator = values.getMutator(); + final UInt1Vector.Mutator bitsMutator = bits.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { - values.getMutator().set(i, new byte[]{}); + valuesMutator.set(i, emptyByteArray); } </#if> - bits.getMutator().set(index, 1); - values.getMutator().set(index, value); + bitsMutator.set(index, 1); + valuesMutator.set(index, value); <#if type.major == "VarLen">lastSet = index;</#if> } <#if type.major == "VarLen"> - private void fillEmpties(int index){ - for (int i = lastSet; i < index; i++) { - values.getMutator().setSafe(i+1, new byte[]{}); + + private void fillEmpties(int index) { + final ${valuesName}.Mutator valuesMutator = values.getMutator(); + for(int i = lastSet; i < index; i++) { + valuesMutator.setSafe(i + 1, emptyByteArray); } - if (index > bits.getValueCapacity()) { + while(index > bits.getValueCapacity()) { bits.reAlloc(); } lastSet = index; } + @Override public void setValueLengthSafe(int index, int length) { values.getMutator().setValueLengthSafe(index, length); } @@ -514,38 +543,40 @@ public final class ${className} extends BaseDataValueVector implements <#if type </#if> } - public void setNull(int index){ + public void setNull(int index) { bits.getMutator().setSafe(index, 0); } - public void setSkipNull(int index, ${minor.class}Holder holder){ + public void setSkipNull(int index, ${minor.class}Holder holder) { values.getMutator().set(index, holder); } - public void setSkipNull(int index, Nullable${minor.class}Holder holder){ + public void setSkipNull(int index, Nullable${minor.class}Holder holder) { values.getMutator().set(index, holder); } - public void set(int index, Nullable${minor.class}Holder holder){ + public void set(int index, Nullable${minor.class}Holder holder) { + final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { - values.getMutator().set(i, new byte[]{}); + valuesMutator.set(i, emptyByteArray); } </#if> bits.getMutator().set(index, holder.isSet); - values.getMutator().set(index, holder); + valuesMutator.set(index, holder); <#if type.major == "VarLen">lastSet = index;</#if> } - public void set(int index, ${minor.class}Holder holder){ + public void set(int index, ${minor.class}Holder holder) { + final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { - values.getMutator().set(i, new byte[]{}); + valuesMutator.set(i, emptyByteArray); } </#if> bits.getMutator().set(index, 1); - values.getMutator().set(index, holder); + valuesMutator.set(index, holder); <#if type.major == "VarLen">lastSet = index;</#if> } @@ -554,14 +585,15 @@ public final class ${className} extends BaseDataValueVector implements <#if type } <#assign fields = minor.fields!type.fields /> - public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){ + public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) { + final ${valuesName}.Mutator valuesMutator = values.getMutator(); <#if type.major == "VarLen"> for (int i = lastSet + 1; i < index; i++) { - values.getMutator().set(i, new byte[]{}); + valuesMutator.set(i, emptyByteArray); } </#if> bits.getMutator().set(index, isSet); - values.getMutator().set(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>); + valuesMutator.set(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>); <#if type.major == "VarLen">lastSet = index;</#if> } @@ -611,6 +643,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type </#if> + @Override public void setValueCount(int valueCount) { assert valueCount >= 0; <#if type.major == "VarLen"> @@ -620,18 +653,19 @@ public final class ${className} extends BaseDataValueVector implements <#if type bits.getMutator().setValueCount(valueCount); } - public void generateTestData(int valueCount){ + @Override + public void generateTestData(int valueCount) { bits.getMutator().generateTestDataAlt(valueCount); values.getMutator().generateTestData(valueCount); <#if type.major = "VarLen">lastSet = valueCount;</#if> setValueCount(valueCount); } - public void reset(){ + @Override + public void reset() { setCount = 0; <#if type.major = "VarLen">lastSet = -1;</#if> } - } } </#list> http://git-wip-us.apache.org/repos/asf/drill/blob/deee06e3/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java index 35777b0..787edc2 100644 --- a/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java +++ b/exec/java-exec/src/main/codegen/templates/ParquetOutputRecordWriter.java @@ -43,7 +43,6 @@ import parquet.io.api.RecordConsumer; import parquet.schema.MessageType; import parquet.io.api.Binary; import io.netty.buffer.DrillBuf; -import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.MaterializedField; @@ -65,6 +64,8 @@ import java.util.Map; * to output the data in string format instead of implementing addField for each type holder. * * This is useful for text format writers such as CSV, TSV etc. + * + * NB: Source code generated using FreeMarker template ${.template_name} */ public abstract class ParquetOutputRecordWriter extends AbstractRecordWriter implements RecordWriter {
