http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java index a2a191d..f9a3c9f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java @@ -19,7 +19,6 @@ package org.apache.drill.exec.store.parquet.columnreaders; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.exception.SchemaChangeException; - import org.apache.drill.exec.vector.BigIntVector; import org.apache.drill.exec.vector.Decimal28SparseVector; import org.apache.drill.exec.vector.Decimal38SparseVector; @@ -38,6 +37,7 @@ import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VarBinaryVector; import org.apache.drill.exec.vector.VarCharVector; import org.apache.drill.exec.vector.VariableWidthVector; + import parquet.column.ColumnDescriptor; import parquet.column.Encoding; import parquet.format.ConvertedType;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java index bfbefdb..f467f8c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedByteAlignedReader.java @@ -20,6 +20,8 @@ package org.apache.drill.exec.store.parquet.columnreaders; import io.netty.buffer.ByteBuf; import io.netty.buffer.DrillBuf; +import java.math.BigDecimal; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.expr.holders.Decimal28SparseHolder; import org.apache.drill.exec.expr.holders.Decimal38SparseHolder; @@ -36,8 +38,6 @@ import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; -import java.math.BigDecimal; - class FixedByteAlignedReader extends ColumnReader { protected DrillBuf bytebuf; @@ -79,6 +79,7 @@ class FixedByteAlignedReader extends ColumnReader { castedVector = v; } + @Override protected void readField(long recordsToReadInThisPass) { // we can use the standard read method to transfer the data super.readField(recordsToReadInThisPass); @@ -101,6 +102,7 @@ class FixedByteAlignedReader extends ColumnReader { super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement); } + @Override public void writeData() { dataTypeLengthInBytes = (int) Math.ceil(dataTypeLengthInBits / 8.0); for (int i = 0; i < recordsReadInThisIteration; i++) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java index 18618e6..2fc3d6e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/FixedWidthRepeatedReader.java @@ -17,15 +17,16 @@ ******************************************************************************/ package org.apache.drill.exec.store.parquet.columnreaders; -import org.apache.drill.exec.vector.RepeatedFixedWidthVector; +import java.io.IOException; + import org.apache.drill.common.exceptions.ExecutionSetupException; +import org.apache.drill.exec.vector.RepeatedFixedWidthVector; import org.apache.drill.exec.vector.ValueVector; + import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; -import java.io.IOException; - public class FixedWidthRepeatedReader extends VarLengthColumn { RepeatedFixedWidthVector castedRepeatedVector; @@ -59,6 +60,7 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { notFishedReadingList = false; } + @Override public void reset() { bytesReadInCurrentPass = 0; valuesReadInCurrentPass = 0; @@ -68,6 +70,7 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { repeatedGroupsReadInCurrentPass = 0; } + @Override public int getRecordsReadInCurrentPass() { return repeatedGroupsReadInCurrentPass; } @@ -77,10 +80,12 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { //To change body of implemented methods use File | Settings | File Templates. } + @Override public boolean skipReadyToReadPositionUpdate() { return false; } + @Override public void updateReadyToReadPosition() { valuesToRead += repeatedValuesInCurrentList; pageReader.valuesReadyToRead += repeatedValuesInCurrentList; @@ -90,17 +95,20 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { repeatedValuesInCurrentList = -1; } + @Override public void updatePosition() { pageReader.readPosInBytes += dataTypeLengthInBits; bytesReadInCurrentPass += dataTypeLengthInBits; valuesReadInCurrentPass++; } + @Override public void hitRowGroupEnd() { pageReader.valuesReadyToRead = 0; definitionLevelsRead = 0; } + @Override public void postPageRead() { super.postPageRead(); // this is no longer correct as we figured out that lists can reach across pages @@ -109,6 +117,7 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { definitionLevelsRead = 0; } + @Override protected int totalValuesReadAndReadyToReadInPage() { // we need to prevent the page reader from getting rid of the current page in the case where we have a repeated // value split across a page boundary @@ -118,6 +127,7 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { return definitionLevelsRead; } + @Override protected boolean checkVectorCapacityReached() { boolean doneReading = super.checkVectorCapacityReached(); if (doneReading) @@ -128,6 +138,7 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { return false; } + @Override protected boolean readAndStoreValueSizeInformation() { boolean readingValsAcrossPageBoundary = false; int numLeftoverVals = 0; @@ -196,6 +207,7 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { return false; } + @Override protected void readRecords(int valuesToRead) { if (valuesToRead == 0) return; // TODO - validate that this works in all cases, it fixes a bug when reading from multiple pages into @@ -211,6 +223,7 @@ public class FixedWidthRepeatedReader extends VarLengthColumn { return castedRepeatedVector.getMutator().getDataVector().getData().capacity(); } + @Override public void clear() { super.clear(); dataReader.clear(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableBitReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableBitReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableBitReader.java index fbf1dee..b9b808b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableBitReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableBitReader.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.store.parquet.columnreaders; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.NullableBitVector; import org.apache.drill.exec.vector.ValueVector; + import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableColumnReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableColumnReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableColumnReader.java index 8551ee0..c7ada6b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableColumnReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableColumnReader.java @@ -17,16 +17,17 @@ */ package org.apache.drill.exec.store.parquet.columnreaders; +import java.io.IOException; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.BaseValueVector; import org.apache.drill.exec.vector.NullableVectorDefinitionSetter; import org.apache.drill.exec.vector.ValueVector; + import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; -import java.io.IOException; - abstract class NullableColumnReader<V extends ValueVector> extends ColumnReader<V>{ int nullsFound; @@ -48,6 +49,7 @@ abstract class NullableColumnReader<V extends ValueVector> extends ColumnReader< } + @Override public void processPages(long recordsToReadInThisPass) throws IOException { int indexInOutputVector = 0; readStartInBytes = 0; @@ -138,5 +140,6 @@ abstract class NullableColumnReader<V extends ValueVector> extends ColumnReader< valuesReadInCurrentPass); } + @Override protected abstract void readField(long recordsToRead); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java index fc1df79..63ac0ee 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java @@ -20,27 +20,27 @@ package org.apache.drill.exec.store.parquet.columnreaders; import io.netty.buffer.ByteBuf; import io.netty.buffer.DrillBuf; +import java.math.BigDecimal; + import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.exec.util.DecimalUtility; -import org.apache.drill.exec.vector.NullableBigIntVector; -import org.apache.drill.exec.vector.NullableFloat4Vector; -import org.apache.drill.exec.vector.NullableFloat8Vector; -import org.apache.drill.exec.vector.NullableIntVector; -import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.expr.holders.NullableDecimal28SparseHolder; import org.apache.drill.exec.expr.holders.NullableDecimal38SparseHolder; import org.apache.drill.exec.store.ParquetOutputRecordWriter; +import org.apache.drill.exec.util.DecimalUtility; +import org.apache.drill.exec.vector.NullableBigIntVector; import org.apache.drill.exec.vector.NullableDateVector; import org.apache.drill.exec.vector.NullableDecimal28SparseVector; import org.apache.drill.exec.vector.NullableDecimal38SparseVector; +import org.apache.drill.exec.vector.NullableFloat4Vector; +import org.apache.drill.exec.vector.NullableFloat8Vector; +import org.apache.drill.exec.vector.NullableIntVector; +import org.apache.drill.exec.vector.ValueVector; import org.joda.time.DateTimeUtils; import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; -import java.math.BigDecimal; - public class NullableFixedByteAlignedReaders { static class NullableFixedByteAlignedReader extends NullableColumnReader { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java index 8bff0b0..47d64bc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableVarLengthValuesColumn.java @@ -17,19 +17,17 @@ ******************************************************************************/ package org.apache.drill.exec.store.parquet.columnreaders; -import io.netty.buffer.ByteBuf; import io.netty.buffer.DrillBuf; +import java.io.IOException; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.ValueVector; -import parquet.bytes.BytesUtils; import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; -import java.io.IOException; - public abstract class NullableVarLengthValuesColumn<V extends ValueVector> extends VarLengthValuesColumn<V> { int nullsRead; @@ -41,10 +39,13 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement); } + @Override public abstract boolean setSafe(int index, DrillBuf value, int start, int length); + @Override public abstract int capacity(); + @Override public void reset() { bytesReadInCurrentPass = 0; valuesReadInCurrentPass = 0; @@ -52,11 +53,13 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten pageReader.valuesReadyToRead = 0; } + @Override protected void postPageRead() { currLengthDeterminingDictVal = null; pageReader.valuesReadyToRead = 0; } + @Override protected boolean readAndStoreValueSizeInformation() throws IOException { // we need to read all of the lengths to determine if this value will fit in the current vector, // as we can only read each definition level once, we have to store the last one as we will need it @@ -95,6 +98,7 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten return false; } + @Override public void updateReadyToReadPosition() { if (! currentValNull){ pageReader.readyToReadPosInBytes += dataTypeLengthInBits + 4; @@ -103,6 +107,7 @@ public abstract class NullableVarLengthValuesColumn<V extends ValueVector> exten currLengthDeterminingDictVal = null; } + @Override public void updatePosition() { if (! currentValNull){ pageReader.readPosInBytes += dataTypeLengthInBits + 4; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java index 7b77e0c..ef30c19 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/PageReader.java @@ -17,14 +17,13 @@ */ package org.apache.drill.exec.store.parquet.columnreaders; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; + import java.io.IOException; import java.util.ArrayList; import java.util.List; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.DrillBuf; -import io.netty.buffer.Unpooled; - import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.store.parquet.ColumnDataReader; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java index bad6e6a..02b583a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java @@ -22,12 +22,10 @@ import org.apache.drill.exec.vector.BigIntVector; import org.apache.drill.exec.vector.Float4Vector; import org.apache.drill.exec.vector.Float8Vector; import org.apache.drill.exec.vector.IntVector; -import org.apache.drill.exec.vector.NullableIntVector; -import org.apache.drill.exec.vector.ValueVector; + import parquet.column.ColumnDescriptor; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; -import parquet.schema.PrimitiveType; public class ParquetFixedWidthDictionaryReaders { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java index 9f2012a..bdfc860 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetRecordReader.java @@ -20,14 +20,9 @@ package org.apache.drill.exec.store.parquet.columnreaders; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; @@ -40,31 +35,29 @@ import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.memory.OutOfMemoryException; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; -import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.MaterializedField.Key; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.store.RecordReader; -import org.apache.drill.exec.vector.NullableBitVector; import org.apache.drill.exec.vector.AllocationHelper; -import org.apache.drill.exec.vector.ValueVector; +import org.apache.drill.exec.vector.NullableBitVector; import org.apache.drill.exec.vector.RepeatedFixedWidthVector; +import org.apache.drill.exec.vector.ValueVector; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import parquet.column.ColumnDescriptor; -import parquet.format.ConvertedType; import parquet.format.FileMetaData; import parquet.format.SchemaElement; import parquet.format.converter.ParquetMetadataConverter; import parquet.hadoop.CodecFactoryExposer; import parquet.hadoop.ParquetFileWriter; -import parquet.hadoop.metadata.BlockMetaData; import parquet.hadoop.metadata.ColumnChunkMetaData; import parquet.hadoop.metadata.ParquetMetadata; import parquet.schema.PrimitiveType; +import com.google.common.collect.Lists; + public class ParquetRecordReader extends AbstractRecordReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReader.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java index 481b289..15f648c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java @@ -17,12 +17,13 @@ ******************************************************************************/ package org.apache.drill.exec.store.parquet.columnreaders; -import org.apache.drill.common.types.TypeProtos; -import static org.apache.drill.common.types.TypeProtos.MinorType; -import static org.apache.drill.common.types.TypeProtos.DataMode; import static parquet.Preconditions.checkArgument; +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.types.TypeProtos.DataMode; +import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; + import parquet.format.ConvertedType; import parquet.format.SchemaElement; import parquet.schema.PrimitiveType; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumn.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumn.java index 14ee631..8e72bff 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumn.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumn.java @@ -17,16 +17,17 @@ ******************************************************************************/ package org.apache.drill.exec.store.parquet.columnreaders; +import java.io.IOException; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.ValueVector; + import parquet.column.ColumnDescriptor; import parquet.format.Encoding; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; import parquet.io.api.Binary; -import java.io.IOException; - public abstract class VarLengthColumn<V extends ValueVector> extends ColumnReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarLengthColumn.class); @@ -44,10 +45,12 @@ public abstract class VarLengthColumn<V extends ValueVector> extends ColumnReade } } + @Override protected boolean processPageData(int recordsToReadInThisPass) throws IOException { return readAndStoreValueSizeInformation(); } + @Override public void reset() { super.reset(); pageReader.valuesReadyToRead = 0; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java index 2f3711d..99e6f53 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthColumnReaders.java @@ -17,9 +17,7 @@ ******************************************************************************/ package org.apache.drill.exec.store.parquet.columnreaders; -import io.netty.buffer.ByteBuf; import io.netty.buffer.DrillBuf; -import io.netty.buffer.Unpooled; import java.math.BigDecimal; @@ -27,7 +25,6 @@ import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.expr.holders.Decimal28SparseHolder; import org.apache.drill.exec.expr.holders.Decimal38SparseHolder; import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder; -import org.apache.drill.exec.expr.holders.NullableVarCharHolder; import org.apache.drill.exec.expr.holders.VarBinaryHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; import org.apache.drill.exec.util.DecimalUtility; @@ -225,6 +222,7 @@ public class VarLengthColumnReaders { this.mutator = vector.getMutator(); } + @Override public boolean setSafe(int index, DrillBuf value, int start, int length) { boolean success; if(index >= vector.getValueCapacity()) return false; @@ -302,6 +300,7 @@ public class VarLengthColumnReaders { nullableVarBinaryVector = v; } + @Override public boolean setSafe(int index, DrillBuf value, int start, int length) { boolean success; if(index >= nullableVarBinaryVector.getValueCapacity()) return false; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java index 4f02c70..7c4b33b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/VarLengthValuesColumn.java @@ -17,22 +17,20 @@ ******************************************************************************/ package org.apache.drill.exec.store.parquet.columnreaders; -import io.netty.buffer.ByteBuf; import io.netty.buffer.DrillBuf; +import java.io.IOException; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VariableWidthVector; -import parquet.bytes.BytesUtils; import parquet.column.ColumnDescriptor; import parquet.format.Encoding; import parquet.format.SchemaElement; import parquet.hadoop.metadata.ColumnChunkMetaData; import parquet.io.api.Binary; -import java.io.IOException; - public abstract class VarLengthValuesColumn<V extends ValueVector> extends VarLengthColumn { Binary currLengthDeterminingDictVal; @@ -64,22 +62,26 @@ public abstract class VarLengthValuesColumn<V extends ValueVector> extends VarLe updatePosition(); } + @Override public void updateReadyToReadPosition() { pageReader.readyToReadPosInBytes += dataTypeLengthInBits + 4; pageReader.valuesReadyToRead++; currLengthDeterminingDictVal = null; } + @Override public void updatePosition() { pageReader.readPosInBytes += dataTypeLengthInBits + 4; bytesReadInCurrentPass += dataTypeLengthInBits; valuesReadInCurrentPass++; } + @Override public boolean skipReadyToReadPositionUpdate() { return false; } + @Override protected boolean readAndStoreValueSizeInformation() throws IOException { // re-purposing this field here for length in BYTES to prevent repetitive multiplication/division if (usingDictionary) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java index 14075f3..6ce4e20 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java @@ -17,10 +17,13 @@ */ package org.apache.drill.exec.store.parquet2; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.expression.SchemaPath; @@ -29,17 +32,16 @@ import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField.Key; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.parquet.RowGroupReadEntry; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.BaseValueVector; -import org.apache.drill.exec.vector.RepeatedFixedWidthVector; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.VariableWidthVector; import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; + import parquet.hadoop.CodecFactoryExposer; import parquet.hadoop.ColumnChunkIncReadStore; import parquet.hadoop.metadata.BlockMetaData; @@ -53,12 +55,8 @@ import parquet.schema.GroupType; import parquet.schema.MessageType; import parquet.schema.Type; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; public class DrillParquetReader extends AbstractRecordReader { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/AbstractWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/AbstractWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/AbstractWriter.java index 0c9a1cd..c41a07a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/AbstractWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/AbstractWriter.java @@ -24,7 +24,6 @@ import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField; -import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.ValueVector; abstract class AbstractWriter<V extends ValueVector> implements PojoWriter{ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java index 8227695..b982c9e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/Writers.java @@ -117,6 +117,7 @@ public class Writers { data = data.reallocIfNeeded(len); } + @Override public void cleanup(){ } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java index 56d0485..fea0875 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AffinityCreator.java @@ -23,23 +23,23 @@ import java.util.concurrent.TimeUnit; import org.apache.drill.exec.physical.EndpointAffinity; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; -import com.google.common.collect.Lists; import com.carrotsearch.hppc.ObjectFloatOpenHashMap; import com.carrotsearch.hppc.cursors.ObjectFloatCursor; import com.carrotsearch.hppc.cursors.ObjectLongCursor; import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; public class AffinityCreator { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCreator.class); - + public static <T extends CompleteWork> List<EndpointAffinity> getAffinityMap(List<T> work){ Stopwatch watch = new Stopwatch(); - + long totalBytes = 0; for (CompleteWork entry : work) { totalBytes += entry.getTotalBytes(); } - + ObjectFloatOpenHashMap<DrillbitEndpoint> affinities = new ObjectFloatOpenHashMap<DrillbitEndpoint>(); for (CompleteWork entry : work) { for (ObjectLongCursor<DrillbitEndpoint> cursor : entry.getByteMap()) { @@ -49,7 +49,7 @@ public class AffinityCreator { affinities.putOrAdd(cursor.key, affinity, affinity); } } - + List<EndpointAffinity> affinityList = Lists.newLinkedList(); for (ObjectFloatCursor<DrillbitEndpoint> d : affinities) { logger.debug("Endpoint {} has affinity {}", d.key.getAddress(), d.value); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java index 4610d88..705688e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java @@ -41,18 +41,18 @@ public class AssignmentCreator<T extends CompleteWork> { private final ArrayListMultimap<Integer, T> mappings; private final List<DrillbitEndpoint> endpoints; - + /** * Given a set of endpoints to assign work to, attempt to evenly assign work based on affinity of work units to * Drillbits. - * + * * @param incomingEndpoints * The set of nodes to assign work to. Note that nodes can be listed multiple times if we want to have * multiple slices on a node working on the task simultaneously. * @param units * The work units to assign. - * @return ListMultimap of Integer > List<CompleteWork> (based on their incoming order) to with + * @return ListMultimap of Integer > List<CompleteWork> (based on their incoming order) to with */ public static <T extends CompleteWork> ListMultimap<Integer, T> getMappings(List<DrillbitEndpoint> incomingEndpoints, List<T> units) { @@ -63,7 +63,7 @@ public class AssignmentCreator<T extends CompleteWork> { private AssignmentCreator(List<DrillbitEndpoint> incomingEndpoints, List<T> units) { logger.debug("Assigning {} units to {} endpoints", units.size(), incomingEndpoints.size()); Stopwatch watch = new Stopwatch(); - + Preconditions.checkArgument(incomingEndpoints.size() <= units.size(), String.format("Incoming endpoints %d " + "is greater than number of row groups %d", incomingEndpoints.size(), units.size())); this.mappings = ArrayListMultimap.create(); @@ -83,7 +83,7 @@ public class AssignmentCreator<T extends CompleteWork> { } /** - * + * * @param mappings * the mapping between fragment/endpoint and rowGroup * @param endpoints http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java index 3c5d9a4..149f9e1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java @@ -33,14 +33,14 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.compress.CompressionCodecFactory; -import com.google.common.collect.Lists; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableRangeMap; +import com.google.common.collect.Lists; import com.google.common.collect.Range; -import org.apache.hadoop.io.compress.CompressionCodecFactory; public class BlockMapBuilder { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BlockMapBuilder.class); @@ -87,7 +87,7 @@ public class BlockMapBuilder { } return work; } - + private class FileStatusWork implements FileWork{ private FileStatus status; @@ -110,16 +110,16 @@ public class BlockMapBuilder { public long getLength() { return status.getLen(); } - - - + + + } - + private ImmutableRangeMap<Long,BlockLocation> buildBlockMap(Path path) throws IOException { FileStatus status = fs.getFileStatus(path); return buildBlockMap(status); } - + /** * Builds a mapping of block locations to file byte range */ @@ -140,7 +140,7 @@ public class BlockMapBuilder { context.stop(); return blockMap; } - + private ImmutableRangeMap<Long,BlockLocation> getBlockMap(Path path) throws IOException{ ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(path); if(blockMap == null) { @@ -148,7 +148,7 @@ public class BlockMapBuilder { } return blockMap; } - + private ImmutableRangeMap<Long,BlockLocation> getBlockMap(FileStatus status) throws IOException{ ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(status.getPath()); if(blockMap == null){ @@ -157,19 +157,19 @@ public class BlockMapBuilder { return blockMap; } - + /** * For a given FileWork, calculate how many bytes are available on each on drillbit endpoint * * @param work the FileWork to calculate endpoint bytes for - * @throws IOException + * @throws IOException */ public EndpointByteMap getEndpointByteMap(FileWork work) throws IOException { Stopwatch watch = new Stopwatch(); watch.start(); Path fileName = new Path(work.getPath()); - - + + ImmutableRangeMap<Long,BlockLocation> blockMap = getBlockMap(fileName); EndpointByteMapImpl endpointByteMap = new EndpointByteMapImpl(); long start = work.getStart(); @@ -203,7 +203,7 @@ public class BlockMapBuilder { } logger.debug("FileWork group ({},{}) max bytes {}", work.getPath(), work.getStart(), endpointByteMap.getMaxBytes()); - + logger.debug("Took {} ms to set endpoint bytes", watch.stop().elapsed(TimeUnit.MILLISECONDS)); return endpointByteMap; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java index cfa4fcb..04838bd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteFileWork.java @@ -29,7 +29,7 @@ public class CompleteFileWork implements FileWork, CompleteWork{ private long length; private String path; private EndpointByteMap byteMap; - + public CompleteFileWork(EndpointByteMap byteMap, long start, long length, String path) { super(); this.start = start; @@ -67,11 +67,11 @@ public class CompleteFileWork implements FileWork, CompleteWork{ public long getLength() { return length; } - + public FileWorkImpl getAsFileWork(){ return new FileWorkImpl(start, length, path); } - + public static class FileWorkImpl implements FileWork{ @JsonCreator @@ -85,7 +85,7 @@ public class CompleteFileWork implements FileWork, CompleteWork{ public long start; public long length; public String path; - + @Override public String getPath() { return path; @@ -100,7 +100,7 @@ public class CompleteFileWork implements FileWork, CompleteWork{ public long getLength() { return length; } - + } @Override http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java index 44e27d4..52cf0f7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/CompleteWork.java @@ -23,7 +23,7 @@ package org.apache.drill.exec.store.schedule; */ public interface CompleteWork extends Comparable<CompleteWork>{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CompleteWork.class); - + public long getTotalBytes(); public EndpointByteMap getByteMap(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java index 64e52eb..567175d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/EndpointByteMapImpl.java @@ -26,23 +26,23 @@ import com.carrotsearch.hppc.cursors.ObjectLongCursor; public class EndpointByteMapImpl implements EndpointByteMap{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EndpointByteMapImpl.class); - + private final ObjectLongOpenHashMap<DrillbitEndpoint> map = new ObjectLongOpenHashMap<>(); - + private long maxBytes; - + public boolean isSet(DrillbitEndpoint endpoint){ return map.containsKey(endpoint); } - + public long get(DrillbitEndpoint endpoint){ return map.get(endpoint); } - + public boolean isEmpty(){ return map.isEmpty(); } - + public void add(DrillbitEndpoint endpoint, long bytes){ assert endpoint != null; maxBytes = Math.max(maxBytes, map.putOrAdd(endpoint, bytes, bytes)+1); @@ -56,6 +56,6 @@ public class EndpointByteMapImpl implements EndpointByteMap{ public Iterator<ObjectLongCursor<DrillbitEndpoint>> iterator() { return map.iterator(); } - - + + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java index bb8d950..8080747 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/PartialWork.java @@ -21,23 +21,23 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; public class PartialWork { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PartialWork.class); - + private final long length; private final DrillbitEndpoint[] locations; - + public PartialWork(long length, DrillbitEndpoint[] locations) { super(); this.length = length; this.locations = locations; } - + public long getLength() { return length; } public DrillbitEndpoint[] getLocations() { return locations; } - - - + + + } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java index 4fe23aa..580d20a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/PStoreRegistry.java @@ -54,7 +54,7 @@ public class PStoreRegistry { Class<? extends PStoreProvider> storeProviderClass = (Class<? extends PStoreProvider>) Class.forName(storeProviderClassName); Constructor<? extends PStoreProvider> c = storeProviderClass.getConstructor(PStoreRegistry.class); return c.newInstance(this); - } catch (ConfigException.Missing | ClassNotFoundException | NoSuchMethodException | SecurityException + } catch (ConfigException.Missing | ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { logger.error(e.getMessage(), e); throw new ExecutionSetupException("A System Table provider was either not specified or could not be found or instantiated", e); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java index 4abd253..743ab53 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTablePlugin.java @@ -22,7 +22,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import com.google.common.collect.ImmutableList; import net.hydromatic.optiq.SchemaPlus; import org.apache.drill.common.JSONOptions; @@ -34,13 +33,13 @@ import org.apache.drill.exec.planner.logical.DrillTable; import org.apache.drill.exec.rpc.user.UserSession; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.DrillConfigIterator; -import org.apache.drill.exec.server.options.OptionValue; import org.apache.drill.exec.store.AbstractSchema; import org.apache.drill.exec.store.AbstractStoragePlugin; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; public class SystemTablePlugin extends AbstractStoragePlugin{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemTablePlugin.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java index d260927..76da554 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTableScan.java @@ -29,7 +29,6 @@ import org.apache.drill.exec.physical.base.AbstractGroupScan; import org.apache.drill.exec.physical.base.GroupScan; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.ScanStats; -import org.apache.drill.exec.physical.base.ScanStats.GroupScanProperty; import org.apache.drill.exec.physical.base.SubScan; import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint; import org.apache.drill.exec.proto.UserBitShared.CoreOperatorType; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java index ef65f2a..7b8761c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordReader.java @@ -21,8 +21,6 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterables; import org.apache.drill.common.exceptions.DrillRuntimeException; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.FieldReference; @@ -31,14 +29,11 @@ import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.SchemaChangeException; -import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.store.RecordReader; -import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.RepeatedVarCharVector; import org.apache.drill.exec.vector.ValueVector; import org.apache.hadoop.io.LongWritable; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordWriter.java index 23d95b8..b713a0c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/text/DrillTextRecordWriter.java @@ -23,7 +23,6 @@ import java.io.PrintStream; import java.util.List; import java.util.Map; -import org.apache.drill.exec.expr.holders.ComplexHolder; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.store.EventBasedRecordWriter.FieldConverter; import org.apache.drill.exec.store.StringOutputRecordWriter; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java index a10038a..e035e6b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/AssertionUtil.java @@ -19,16 +19,16 @@ package org.apache.drill.exec.util; public class AssertionUtil { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssertionUtil.class); - - + + private static final boolean ASSERT_ENABLED; - + static{ boolean isAssertEnabled = false; assert isAssertEnabled = true; ASSERT_ENABLED = isAssertEnabled; } - + public static boolean isAssertionsEnabled(){ return ASSERT_ENABLED; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/AtomicState.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AtomicState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/AtomicState.java index 2df199d..ec498d4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/AtomicState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/AtomicState.java @@ -27,9 +27,9 @@ import com.google.protobuf.Internal.EnumLite; */ public abstract class AtomicState<T extends EnumLite> { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AtomicState.class); - + private final AtomicInteger state = new AtomicInteger(); - + /** * Constructor that defines initial T state. * @param initial @@ -37,11 +37,11 @@ public abstract class AtomicState<T extends EnumLite> { public AtomicState(T initial){ state.set(initial.getNumber()); } - + protected abstract T getStateFromNumber(int i); - + /** - * Does an atomic conditional update from one state to another. + * Does an atomic conditional update from one state to another. * @param oldState The expected current state. * @param newState The desired new state. * @return Whether or not the update was successful. @@ -49,7 +49,7 @@ public abstract class AtomicState<T extends EnumLite> { public boolean updateState(T oldState, T newState){ return state.compareAndSet(oldState.getNumber(), newState.getNumber()); } - + public T getState(){ return getStateFromNumber(state.get()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java index 6222514..1029e61 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java @@ -17,18 +17,15 @@ */ package org.apache.drill.exec.util; -import com.google.common.collect.Lists; +import java.util.List; import org.apache.commons.lang.StringUtils; -import org.apache.drill.common.expression.ExpressionStringBuilder; -import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorWrapper; import org.apache.drill.exec.record.selection.SelectionVector4; -import org.apache.drill.exec.rpc.RpcException; import org.apache.drill.exec.vector.ValueVector; -import java.util.List; +import com.google.common.collect.Lists; /** * This is a tool for printing the content of record batches to screen. Used for debugging. http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java index 3d0a9de..a697d03 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java @@ -23,7 +23,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; -import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.util.CoreDecimalUtility; import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers; import org.apache.drill.exec.expr.holders.Decimal38SparseHolder; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java index 88478f2..d1f4c08 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java @@ -17,12 +17,13 @@ */ package org.apache.drill.exec.util; +import java.util.ArrayList; + +import org.apache.hadoop.io.Text; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import org.apache.hadoop.io.Text; - -import java.util.ArrayList; public class JsonStringArrayList<E> extends ArrayList<E> { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java index 507cb28..7ec870f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java @@ -17,14 +17,13 @@ */ package org.apache.drill.exec.util; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; +import java.util.LinkedHashMap; import org.apache.hadoop.io.Text; -import java.util.HashMap; -import java.util.LinkedHashMap; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; /* * Simple class that extends the regular java.util.HashMap but overrides the http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java index 3070592..87df102 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java @@ -17,13 +17,14 @@ */ package org.apache.drill.exec.util; +import java.io.IOException; + +import org.apache.hadoop.io.Text; + import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import org.apache.hadoop.io.Text; - -import java.io.IOException; public class TextSerializer extends StdSerializer<Text> { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java index 81fce5d..c0fb572 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java @@ -19,13 +19,13 @@ package org.apache.drill.exec.util; import java.util.List; -import com.google.common.base.Joiner; import org.apache.commons.lang.StringUtils; import org.apache.drill.common.util.DrillStringUtils; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorWrapper; +import com.google.common.base.Joiner; import com.google.common.collect.Lists; public class VectorUtil { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java index 5db0299..94e8300 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.vector; import io.netty.buffer.DrillBuf; -import io.netty.buffer.DrillBuf; import java.util.Iterator; @@ -55,10 +54,12 @@ public abstract class BaseDataValueVector extends BaseValueVector{ } } + @Override public void setCurrentValueCount(int count) { currentValueCount = count; } + @Override public int getCurrentValueCount() { return currentValueCount; } @@ -90,6 +91,7 @@ public abstract class BaseDataValueVector extends BaseValueVector{ @Override public abstract SerializedField getMetadata(); + @Override public DrillBuf getData(){ return data; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java index 82cc4c9..a28f919 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java @@ -269,6 +269,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe return new Boolean(get(index) != 0); } + @Override public final int getValueCount() { return valueCount; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/CopyUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/CopyUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/CopyUtil.java index c68f62f..1ed7f37 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/CopyUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/CopyUtil.java @@ -17,14 +17,15 @@ */ package org.apache.drill.exec.vector; -import com.sun.codemodel.JExpr; -import com.sun.codemodel.JExpression; -import com.sun.codemodel.JVar; import org.apache.drill.exec.expr.ClassGenerator; import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.record.VectorWrapper; +import com.sun.codemodel.JExpr; +import com.sun.codemodel.JExpression; +import com.sun.codemodel.JVar; + public class CopyUtil { public static void generateCopies(ClassGenerator g, VectorAccessible batch, boolean hyper){ // we have parallel ids for each value vector so we don't actually have to deal with managing the ids at all. http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java index 032ccc2..b68f089 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java @@ -17,9 +17,12 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.ByteBuf; import io.netty.buffer.DrillBuf; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + import org.apache.drill.common.expression.FieldReference; import org.apache.drill.exec.expr.holders.ObjectHolder; import org.apache.drill.exec.memory.BufferAllocator; @@ -29,10 +32,6 @@ import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.TransferPair; import org.apache.drill.exec.vector.complex.reader.FieldReader; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - public class ObjectVector extends BaseValueVector{ private Accessor accessor = new Accessor(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java index 3433537..df6a486 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java @@ -22,7 +22,6 @@ import io.netty.buffer.DrillBuf; import java.io.Closeable; import org.apache.drill.common.expression.FieldReference; -import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.memory.OutOfMemoryRuntimeException; import org.apache.drill.exec.proto.UserBitShared.SerializedField; import org.apache.drill.exec.record.MaterializedField; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java index 619dd95..7f977f2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java @@ -17,8 +17,8 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.DrillBuf; import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; public class VectorTrimmer { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorTrimmer.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java index 9884612..1605c7d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/AbstractSqlAccessor.java @@ -28,60 +28,60 @@ import java.sql.Timestamp; import org.apache.drill.common.types.TypeProtos.MajorType; abstract class AbstractSqlAccessor implements SqlAccessor { - + @Override public abstract boolean isNull(int index); - + @Override public BigDecimal getBigDecimal(int index) throws InvalidAccessException{ throw new InvalidAccessException("BigDecimal"); } - + @Override public boolean getBoolean(int index) throws InvalidAccessException{ throw new InvalidAccessException("boolean"); } - + @Override public byte getByte(int index) throws InvalidAccessException{ throw new InvalidAccessException("byte"); } - + @Override public byte[] getBytes(int index) throws InvalidAccessException{ throw new InvalidAccessException("byte[]"); } - + @Override public Date getDate(int index) throws InvalidAccessException{ throw new InvalidAccessException("Date"); } - + @Override public double getDouble(int index) throws InvalidAccessException{ throw new InvalidAccessException("double"); } - + @Override public float getFloat(int index) throws InvalidAccessException{ throw new InvalidAccessException("float"); } - + @Override public int getInt(int index) throws InvalidAccessException{ throw new InvalidAccessException("int"); } - + @Override public long getLong(int index) throws InvalidAccessException{ throw new InvalidAccessException("long"); } - + @Override public short getShort(int index) throws InvalidAccessException{ throw new InvalidAccessException("short"); } - + @Override public InputStream getStream(int index) throws InvalidAccessException{ throw new InvalidAccessException("InputStream"); @@ -91,8 +91,8 @@ abstract class AbstractSqlAccessor implements SqlAccessor { public char getChar(int index) throws InvalidAccessException{ throw new InvalidAccessException("Char"); } - - @Override + + @Override public Reader getReader(int index) throws InvalidAccessException{ throw new InvalidAccessException("Reader"); } @@ -113,8 +113,8 @@ abstract class AbstractSqlAccessor implements SqlAccessor { } abstract MajorType getType(); - - + + public class InvalidAccessException extends SQLException{ public InvalidAccessException(String name){ super(String.format("Requesting class of type %s for an object of type %s:%s is not allowed.", name, getType().getMinorType().name(), getType().getMode().name())); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java index 5611354..b8480b4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/SqlAccessor.java @@ -45,7 +45,7 @@ public interface SqlAccessor { public abstract float getFloat(int index) throws InvalidAccessException; public abstract char getChar(int index) throws InvalidAccessException; - + public abstract int getInt(int index) throws InvalidAccessException; public abstract long getLong(int index) throws InvalidAccessException; @@ -55,13 +55,13 @@ public interface SqlAccessor { public abstract InputStream getStream(int index) throws InvalidAccessException; public abstract Reader getReader(int index) throws InvalidAccessException; - + public abstract String getString(int index) throws InvalidAccessException; public abstract Time getTime(int index) throws InvalidAccessException; public abstract Timestamp getTimestamp(int index) throws InvalidAccessException; - + public abstract Object getObject(int index) throws InvalidAccessException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/sql/TimePrintMillis.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/sql/TimePrintMillis.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/sql/TimePrintMillis.java index 372bd86..349abd5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/sql/TimePrintMillis.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/accessor/sql/TimePrintMillis.java @@ -17,10 +17,10 @@ */ package org.apache.drill.exec.vector.accessor.sql; -import org.apache.drill.exec.expr.fn.impl.DateUtility; - import java.sql.Time; +import org.apache.drill.exec.expr.fn.impl.DateUtility; + public class TimePrintMillis extends Time { public TimePrintMillis(long time) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/FixedVectorAllocator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/FixedVectorAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/FixedVectorAllocator.java index 7b1351d..e783d02 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/FixedVectorAllocator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/FixedVectorAllocator.java @@ -27,6 +27,7 @@ public class FixedVectorAllocator extends VectorAllocator{ this.out = out; } + @Override public void alloc(int recordCount){ out.allocateNew(recordCount); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVariableEstimatedAllocator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVariableEstimatedAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVariableEstimatedAllocator.java index 2a0ca65..c234687 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVariableEstimatedAllocator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVariableEstimatedAllocator.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.vector.allocator; import org.apache.drill.exec.vector.RepeatedVariableWidthVector; -import org.apache.drill.exec.vector.VariableWidthVector; public class RepeatedVariableEstimatedAllocator extends VectorAllocator{ RepeatedVariableWidthVector out; @@ -29,7 +28,8 @@ public class RepeatedVariableEstimatedAllocator extends VectorAllocator{ this.out = out; this.avgWidth = avgWidth; } - + + @Override public void alloc(int recordCount){ out.allocateNew(avgWidth * recordCount, recordCount, recordCount); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVectorAllocator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVectorAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVectorAllocator.java index 95c1d4c..b4428df 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVectorAllocator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/RepeatedVectorAllocator.java @@ -18,7 +18,6 @@ package org.apache.drill.exec.vector.allocator; import org.apache.drill.exec.vector.RepeatedVariableWidthVector; -import org.apache.drill.exec.vector.VariableWidthVector; class RepeatedVectorAllocator extends VectorAllocator{ RepeatedVariableWidthVector in; @@ -30,6 +29,7 @@ class RepeatedVectorAllocator extends VectorAllocator{ this.out = out; } + @Override public void alloc(int recordCount){ out.allocateNew(in.getByteCapacity(), in.getAccessor().getValueCount(), in.getAccessor().getValueCount()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableEstimatedVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableEstimatedVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableEstimatedVector.java index b861582..815a531 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableEstimatedVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableEstimatedVector.java @@ -29,6 +29,7 @@ public class VariableEstimatedVector extends VectorAllocator{ this.avgWidth = avgWidth; } + @Override public void alloc(int recordCount){ out.allocateNew(avgWidth * recordCount, recordCount); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java index 47286ae..7934ecb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/allocator/VariableVectorAllocator.java @@ -29,6 +29,7 @@ class VariableVectorAllocator extends VectorAllocator{ this.out = out; } + @Override public void alloc(int recordCount){ out.allocateNew(in.getByteCapacity(), recordCount); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java index 5eb1358..a6df3a3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java @@ -17,6 +17,8 @@ */ package org.apache.drill.exec.vector.complex; +import java.util.List; + import org.apache.drill.common.expression.PathSegment; import org.apache.drill.common.types.TypeProtos.DataMode; import org.apache.drill.common.types.TypeProtos.MajorType; @@ -24,8 +26,6 @@ import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.TypedFieldId; import org.apache.drill.exec.vector.ValueVector; -import java.util.List; - public abstract class AbstractContainerVector implements ValueVector{ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index 834719c..85067a1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -41,12 +41,11 @@ import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier; import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl; import org.apache.drill.exec.vector.complex.reader.FieldReader; -import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; import com.carrotsearch.hppc.IntObjectOpenHashMap; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; public class MapVector extends AbstractContainerVector { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class); @@ -72,6 +71,7 @@ public class MapVector extends AbstractContainerVector { this.allocator = allocator; } + @Override public int size(){ return vectors.size(); } @@ -405,6 +405,7 @@ public class MapVector extends AbstractContainerVector { } } + @Override public VectorWithOrdinal getVectorWithOrdinal(String name){ return vectorIds.get(name); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java index fef416f..dd31316 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java @@ -46,8 +46,8 @@ import org.apache.drill.exec.vector.complex.impl.NullReader; import org.apache.drill.exec.vector.complex.impl.RepeatedListReaderImpl; import org.apache.drill.exec.vector.complex.reader.FieldReader; -import com.google.common.collect.Lists; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; public class RepeatedListVector extends AbstractContainerVector implements RepeatedFixedWidthVector{ @@ -74,6 +74,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea this.field = field; } + @Override public int size(){ return vector != null ? 1 : 0; } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index 678439b..df5fdaa 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -91,6 +91,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat return vectors.keySet().iterator(); } + @Override public int size(){ return vectors.size(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java index fa26b54..84c4902 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java @@ -18,10 +18,6 @@ package org.apache.drill.exec.vector.complex.fn; import io.netty.buffer.DrillBuf; -import com.google.common.base.Preconditions; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufInputStream; -import io.netty.buffer.UnpooledByteBufAllocator; import java.io.IOException; import java.io.Reader; @@ -36,9 +32,7 @@ import org.apache.drill.exec.expr.holders.BitHolder; import org.apache.drill.exec.expr.holders.Float8Holder; import org.apache.drill.exec.expr.holders.VarCharHolder; import org.apache.drill.exec.physical.base.GroupScan; -import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.store.AbstractRecordReader; -import org.apache.drill.exec.vector.complex.writer.BaseWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; @@ -49,6 +43,7 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonParser.Feature; import com.fasterxml.jackson.core.JsonToken; import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; public class JsonReader { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonReader.class); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/39990292/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/UTF8JsonRecordSplitter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/UTF8JsonRecordSplitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/UTF8JsonRecordSplitter.java index ee94015..3dca86e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/UTF8JsonRecordSplitter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/UTF8JsonRecordSplitter.java @@ -21,7 +21,6 @@ import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader;
