[01/11] drill git commit: DRILL-6279: Indicate operators that spilled in-memory data to disk on Web UI
Repository: drill Updated Branches: refs/heads/master 9a6cb59b9 -> da241134f DRILL-6279: Indicate operators that spilled in-memory data to disk on Web UI As part of this coomit added Bootstrap's Glyphicons. closes #1197 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cc440af0 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cc440af0 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cc440af0 Branch: refs/heads/master Commit: cc440af0397d0ec8b52227db9b7e09513b5ff157 Parents: 9a6cb59 Author: Kunal Khatua Authored: Sun Mar 25 22:05:06 2018 -0700 Committer: Arina Ielchiieva Committed: Fri Apr 6 11:52:22 2018 +0300 -- .../server/rest/profile/OperatorWrapper.java| 89 ++- .../exec/server/rest/profile/TableBuilder.java | 14 ++- .../src/main/resources/rest/profile/profile.ftl | 12 ++- .../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20127 bytes .../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 45404 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23424 bytes .../fonts/glyphicons-halflings-regular.woff2| Bin 0 -> 18028 bytes 7 files changed, 109 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/cc440af0/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java index afccbb7..3a60c71 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/profile/OperatorWrapper.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.rest.profile; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -38,7 +39,17 @@ import com.google.common.base.Preconditions; * Wrapper class for profiles of ALL operator instances of the same operator type within a major fragment. */ public class OperatorWrapper { + @SuppressWarnings("unused") + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OperatorWrapper.class); + + private static final String HTML_ATTRIB_SPILLS = "spills"; + private static final String HTML_ATTRIB_CLASS = "class"; + private static final String HTML_ATTRIB_STYLE = "style"; + private static final String HTML_ATTRIB_TITLE = "title"; + private static final DecimalFormat DECIMAL_FORMATTER = new DecimalFormat("#.##"); private static final String UNKNOWN_OPERATOR = "UNKNOWN_OPERATOR"; + //Negative valued constant used for denoting invalid index to indicate absence of metric + private static final int NO_SPILL_METRIC_INDEX = Integer.MIN_VALUE; private final int major; private final List, String>> opsAndHosts; // [(operatorProfile --> minorFragment number,host), ...] private final OperatorProfile firstProfile; @@ -146,6 +157,11 @@ public class OperatorWrapper { tb.appendCell(path, null, null, opTblBgColor); tb.appendCell(operatorName); +//Check if spill information is available +int spillCycleMetricIndex = getSpillCycleMetricIndex(operatorType); +boolean isSpillableOp = (spillCycleMetricIndex != NO_SPILL_METRIC_INDEX); +boolean hasSpilledToDisk = false; + //Get MajorFragment Busy+Wait Time Tally long majorBusyNanos = majorFragmentBusyTally.get(new OperatorPathBuilder().setMajor(major).build()); @@ -153,6 +169,8 @@ public class OperatorWrapper { double processSum = 0.0; double waitSum = 0.0; double memSum = 0.0; +double spillCycleSum = 0.0; +long spillCycleMax = 0L; long recordSum = 0L; //Construct list for sorting purposes (using legacy Comparators) @@ -168,6 +186,22 @@ public class OperatorWrapper { recordSum += sp.getRecords(); } opList.add(ip.getLeft()); + + //Capture Spill Info + //Check to ensure index < #metrics (old profiles have less metrics); else reset isSpillableOp + if (isSpillableOp) { +//NOTE: We get non-zero value for non-existent metrics, so we can't use getMetric(index) +//profile.getMetric(spillCycleMetricIndex).getLongValue(); +//Forced to iterate list +for (MetricValue metricVal : profile.getMetricList()) { + if (metricVal.getMetricId() == spillCycleMetricIndex) { +long spillCycles = metricVal.getLongValue(); +spillCycleMax = Math.max(spillCycles, spillCycleMax); +spillCycleSum += spillCycles; +hasSpilledToDisk = (spillCycle
[06/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java -- diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java deleted file mode 100644 index f32c101..000 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector.accessor.reader; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.List; - -import org.apache.drill.common.types.TypeProtos.MajorType; -import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.exec.vector.accessor.ColumnReaderIndex; -import org.apache.drill.exec.vector.accessor.ObjectType; -import org.apache.drill.exec.vector.accessor.ScalarElementReader; -import org.apache.drill.exec.vector.accessor.impl.AccessorUtilities; -import org.joda.time.Period; - -public abstract class BaseElementReader implements ScalarElementReader { - - public static class ScalarElementObjectReader extends AbstractObjectReader { - -private BaseElementReader elementReader; - -public ScalarElementObjectReader(BaseElementReader elementReader) { - this.elementReader = elementReader; -} - -@Override -public void bindIndex(ColumnReaderIndex index) { - elementReader.bindIndex((ElementReaderIndex) index); -} - -@Override -public ObjectType type() { - return ObjectType.SCALAR; -} - -@Override -public ScalarElementReader elements() { - return elementReader; -} - -@Override -public Object getObject() { - // Simple: return elements as an object list. - // If really needed, could return as a typed array, but that - // is a bit of a hassle. - - List elements = new ArrayList<>(); - for (int i = 0; i < elementReader.size(); i++) { -elements.add(elementReader.getObject(i)); - } - return elements; -} - -@Override -public String getAsString() { - StringBuilder buf = new StringBuilder(); - buf.append("["); - for (int i = 0; i < elementReader.size(); i++) { -if (i > 0) { - buf.append( ", " ); -} -buf.append(elementReader.getAsString(i)); - } - buf.append("]"); - return buf.toString(); -} - } - - protected ElementReaderIndex vectorIndex; - protected VectorAccessor vectorAccessor; - - public abstract void bindVector(ValueVector vector); - - public void bindVector(MajorType majorType, VectorAccessor va) { -vectorAccessor = va; - } - - protected void bindIndex(ElementReaderIndex rowIndex) { -this.vectorIndex = rowIndex; - } - - @Override - public int size() { return vectorIndex.size(); } - - @Override - public Object getObject(int index) { -if (isNull(index)) { - return "null"; -} -switch (valueType()) { -case BYTES: - return getBytes(index); -case DECIMAL: - return getDecimal(index); -case DOUBLE: - return getDouble(index); -case INTEGER: - return getInt(index); -case LONG: - return getLong(index); -case PERIOD: - return getPeriod(index); -case STRING: - return getString(index); -default: - throw new IllegalStateException("Unexpected type: " + valueType()); -} - } - - @Override - public String getAsString(int index) { -switch (valueType()) { -case BYTES: - return AccessorUtilities.bytesToString(getBytes(index)); -case DOUBLE: - return Double.toString(getDouble(index)); -case INTEGER: - return Integer.toString(getInt(index)); -case LONG: - return Long.toString(getLong(index)); -case STRING: - return "\"" + getString(index) + "\""; -case DECIMAL: - return getDecimal(index).toPlainString(); -case PERIOD: - return getPeriod(index).normalizedStandard().toString(); -default: - throw new Il
[09/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors
DRILL-6230: Extend row set readers to handle hyper vectors closes #1161 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4f2182e4 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4f2182e4 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4f2182e4 Branch: refs/heads/master Commit: 4f2182e41f4474ca42ae6d572a9c5d5ff274d984 Parents: 127e415 Author: Paul Rogers Authored: Sat Mar 10 23:43:36 2018 -0800 Committer: Arina Ielchiieva Committed: Fri Apr 6 12:05:14 2018 +0300 -- .../rowSet/model/AbstractReaderBuilder.java | 45 ++ .../exec/physical/rowSet/model/ReaderIndex.java | 28 +- .../physical/rowSet/model/SchemaInference.java | 62 --- .../rowSet/model/hyper/BaseReaderBuilder.java | 142 -- .../model/hyper/HyperSchemaInference.java | 72 +++ .../rowSet/model/single/BaseReaderBuilder.java | 81 ++-- .../rowSet/model/single/DirectRowIndex.java | 40 ++ .../model/single/SingleSchemaInference.java | 65 +++ .../physical/rowSet/impl/RowSetTestUtils.java | 59 +++ .../impl/TestResultSetLoaderMapArray.java | 135 +++--- .../rowSet/impl/TestResultSetLoaderMaps.java| 193 +--- .../impl/TestResultSetLoaderOmittedValues.java | 3 +- .../impl/TestResultSetLoaderOverflow.java | 173 --- .../impl/TestResultSetLoaderProjection.java | 139 +- .../impl/TestResultSetLoaderProtocol.java | 2 +- .../rowSet/impl/TestResultSetLoaderTorture.java | 16 +- .../drill/exec/record/TestVectorContainer.java | 3 +- .../apache/drill/test/rowSet/DirectRowSet.java | 31 +- .../drill/test/rowSet/HyperRowSetImpl.java | 102 +++- .../drill/test/rowSet/IndirectRowSet.java | 14 +- .../org/apache/drill/test/rowSet/RowSet.java| 7 + .../drill/test/rowSet/RowSetComparison.java | 73 +-- .../apache/drill/test/rowSet/RowSetPrinter.java | 18 +- .../apache/drill/test/rowSet/RowSetReader.java | 11 +- .../drill/test/rowSet/RowSetReaderImpl.java | 24 +- .../drill/test/rowSet/schema/UnionBuilder.java | 4 +- .../drill/test/rowSet/test/RowSetTest.java | 268 +-- .../drill/test/rowSet/test/TestFillEmpties.java | 12 +- .../rowSet/test/TestHyperVectorReaders.java | 365 ++ .../test/rowSet/test/TestScalarAccessors.java | 474 +++ .../drill/test/rowSet/test/VectorPrinter.java | 72 --- .../src/main/java/io/netty/buffer/DrillBuf.java | 15 + .../main/codegen/templates/ColumnAccessors.java | 251 +- .../drill/exec/vector/accessor/ArrayReader.java | 40 +- .../exec/vector/accessor/ColumnReader.java | 83 .../exec/vector/accessor/ColumnReaderIndex.java | 102 +++- .../exec/vector/accessor/ObjectReader.java | 26 +- .../vector/accessor/ScalarElementReader.java| 65 --- .../exec/vector/accessor/ScalarReader.java | 13 +- .../drill/exec/vector/accessor/TupleReader.java | 31 +- .../accessor/UnsupportedConversionError.java| 52 ++ .../drill/exec/vector/accessor/ValueType.java | 57 ++- .../vector/accessor/impl/VectorPrinter.java | 72 +++ .../exec/vector/accessor/package-info.java | 184 +++ .../accessor/reader/AbstractArrayReader.java| 188 .../accessor/reader/AbstractObjectReader.java | 21 +- .../accessor/reader/AbstractScalarReader.java | 205 .../accessor/reader/AbstractTupleReader.java| 85 ++-- .../vector/accessor/reader/ArrayReaderImpl.java | 357 ++ .../accessor/reader/BaseElementReader.java | 187 .../accessor/reader/BaseScalarReader.java | 198 .../accessor/reader/ColumnReaderFactory.java| 52 +- .../reader/FixedWidthElementReaderIndex.java| 38 -- .../exec/vector/accessor/reader/MapReader.java | 49 +- .../vector/accessor/reader/NullStateReader.java | 52 ++ .../accessor/reader/NullStateReaders.java | 193 .../accessor/reader/ObjectArrayReader.java | 159 --- .../accessor/reader/OffsetVectorReader.java | 70 +++ .../vector/accessor/reader/ReaderEvents.java| 31 ++ .../accessor/reader/ScalarArrayReader.java | 102 .../vector/accessor/reader/VectorAccessor.java | 5 +- .../vector/accessor/reader/VectorAccessors.java | 344 ++ .../vector/accessor/reader/package-info.java| 65 ++- .../accessor/writer/ColumnWriterFactory.java| 4 +- .../drill/exec/vector/complex/ListVector.java | 2 +- 65 files changed, 4122 insertions(+), 2009 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/model/AbstractReaderBuilder.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/rowSet/mode
[10/11] drill git commit: DRILL-6284: Add operator metrics for batch sizing for flatten
DRILL-6284: Add operator metrics for batch sizing for flatten Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/77f5e901 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/77f5e901 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/77f5e901 Branch: refs/heads/master Commit: 77f5e901370c4c23195ac226d140e8f3eca5f71d Parents: 4f2182e Author: Padma Penumarthy Authored: Tue Mar 20 13:44:50 2018 -0700 Committer: Arina Ielchiieva Committed: Fri Apr 6 12:06:49 2018 +0300 -- .../drill/exec/ops/OperatorMetricRegistry.java | 2 + .../drill/exec/physical/config/FlattenPOP.java | 4 +- .../impl/flatten/FlattenRecordBatch.java| 42 .../AbstractRecordBatchMemoryManager.java | 53 .../apache/drill/exec/proto/UserBitShared.java | 20 ++-- .../exec/proto/beans/CoreOperatorType.java | 4 +- protocol/src/main/protobuf/UserBitShared.proto | 1 + 7 files changed, 118 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/77f5e901/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java index 0424332..b029154 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java @@ -21,6 +21,7 @@ import org.apache.drill.exec.physical.impl.ScreenCreator; import org.apache.drill.exec.physical.impl.SingleSenderCreator; import org.apache.drill.exec.physical.impl.aggregate.HashAggTemplate; import org.apache.drill.exec.physical.impl.broadcastsender.BroadcastSenderRootExec; +import org.apache.drill.exec.physical.impl.flatten.FlattenRecordBatch; import org.apache.drill.exec.physical.impl.join.HashJoinBatch; import org.apache.drill.exec.physical.impl.mergereceiver.MergingRecordBatch; import org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec; @@ -49,6 +50,7 @@ public class OperatorMetricRegistry { register(CoreOperatorType.HASH_JOIN_VALUE, HashJoinBatch.Metric.class); register(CoreOperatorType.EXTERNAL_SORT_VALUE, ExternalSortBatch.Metric.class); register(CoreOperatorType.PARQUET_ROW_GROUP_SCAN_VALUE, ParquetRecordReader.Metric.class); +register(CoreOperatorType.FLATTEN_VALUE, FlattenRecordBatch.Metric.class); } private static void register(final int operatorType, final Class metricDef) { http://git-wip-us.apache.org/repos/asf/drill/blob/77f5e901/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java index 42e6870..f3499bf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/FlattenPOP.java @@ -27,6 +27,7 @@ import org.apache.drill.exec.physical.base.AbstractBase; import org.apache.drill.exec.physical.base.AbstractSingle; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.base.PhysicalVisitor; +import org.apache.drill.exec.proto.UserBitShared; import java.util.Iterator; import java.util.List; @@ -67,7 +68,6 @@ public class FlattenPOP extends AbstractSingle { @Override public int getOperatorType() { -// TODO - add this operator to the protobuf definition -return 0; +return UserBitShared.CoreOperatorType.FLATTEN_VALUE; } } http://git-wip-us.apache.org/repos/asf/drill/blob/77f5e901/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index 7509809..a1f783f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -40,6 +40,7 @@ import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.expr.ValueVectorReadExpression; import org.apache.drill.exec.expr.ValueVectorWriteExpression; import org.apache.drill.exec.o
[05/11] drill git commit: DRILL-6016: Fix for Error reading INT96 created by Apache Spark
DRILL-6016: Fix for Error reading INT96 created by Apache Spark closes #1166 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/127e4150 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/127e4150 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/127e4150 Branch: refs/heads/master Commit: 127e4150b9495c465f8c37a534dfd50512013765 Parents: 67669a0 Author: Rahul Raj Authored: Wed Mar 14 12:05:45 2018 +0530 Committer: Arina Ielchiieva Committed: Fri Apr 6 12:01:04 2018 +0300 -- .../columnreaders/ColumnReaderFactory.java | 7 +++- .../ParquetFixedWidthDictionaryReaders.java | 27 + .../physical/impl/writer/TestParquetWriter.java | 40 --- ...ark-generated-int96-timestamp.snappy.parquet | Bin 0 -> 2896 bytes .../testInt96DictChange/q1.tsv | 12 -- 5 files changed, 59 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/127e4150/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java index 09cdc5d..ba5f1de 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ColumnReaderFactory.java @@ -156,8 +156,13 @@ public class ColumnReaderFactory { case DOUBLE: return new ParquetFixedWidthDictionaryReaders.DictionaryFloat8Reader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (Float8Vector) v, schemaElement); case FIXED_LEN_BYTE_ARRAY: -case INT96: return new ParquetFixedWidthDictionaryReaders.DictionaryFixedBinaryReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarBinaryVector) v, schemaElement); +case INT96: + if (recordReader.getFragmentContext().getOptions().getOption(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP).bool_val) { +return new ParquetFixedWidthDictionaryReaders.DictionaryBinaryAsTimeStampReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (TimeStampVector) v, schemaElement); + } else { +return new ParquetFixedWidthDictionaryReaders.DictionaryFixedBinaryReader(recordReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, (VarBinaryVector) v, schemaElement); + } default: throw new ExecutionSetupException("Unsupported dictionary column type " + descriptor.getType().name() ); } http://git-wip-us.apache.org/repos/asf/drill/blob/127e4150/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java index 5fbac20..5033046 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetFixedWidthDictionaryReaders.java @@ -34,6 +34,8 @@ import org.apache.parquet.format.SchemaElement; import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; import org.apache.parquet.io.api.Binary; +import static org.apache.drill.exec.store.parquet.ParquetReaderUtility.NanoTimeUtils.getDateTimeValueFromBinary; + public class ParquetFixedWidthDictionaryReaders { static class DictionaryIntReader extends FixedByteAlignedReader { @@ -294,6 +296,31 @@ public class ParquetFixedWidthDictionaryReaders { } } + static class DictionaryBinaryAsTimeStampReader extends FixedByteAlignedReader { +DictionaryBinaryAsTimeStampReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, + ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, TimeStampVector v, + SchemaElement schemaElement) throws ExecutionSetupException { + super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement); +} + +// this method is called by its superclass during a
[04/11] drill git commit: DRILL-6271: Updated copyright range in NOTICE
DRILL-6271: Updated copyright range in NOTICE closes #1188 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/67669a00 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/67669a00 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/67669a00 Branch: refs/heads/master Commit: 67669a000c19a39510118f20366f3a189809200a Parents: a054945 Author: dvjyothsna Authored: Fri Mar 23 17:13:46 2018 -0700 Committer: Arina Ielchiieva Committed: Fri Apr 6 11:58:42 2018 +0300 -- NOTICE| 2 +- distribution/src/resources/NOTICE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/67669a00/NOTICE -- diff --git a/NOTICE b/NOTICE index abdcb91..21b3e34 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Drill -Copyright 2013-2014 The Apache Software Foundation +Copyright 2013-2018 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). http://git-wip-us.apache.org/repos/asf/drill/blob/67669a00/distribution/src/resources/NOTICE -- diff --git a/distribution/src/resources/NOTICE b/distribution/src/resources/NOTICE index bd17b61..8a85d07 100644 --- a/distribution/src/resources/NOTICE +++ b/distribution/src/resources/NOTICE @@ -1,5 +1,5 @@ Apache Drill -Copyright 2013-2014 The Apache Software Foundation +Copyright 2013-2018 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
[02/11] drill git commit: DRILL-6303: Provide a button to copy the Drillbit's JStack shown in /threads
DRILL-6303: Provide a button to copy the Drillbit's JStack shown in /threads Provides a button in the web UI to copy the thread stack to the user's clipboard. closes #1199 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2e1568ce Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2e1568ce Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2e1568ce Branch: refs/heads/master Commit: 2e1568cecd4aafb87c092c6b403a60366cc8f107 Parents: cc440af Author: Kunal Khatua Authored: Sun Mar 25 23:09:43 2018 -0700 Committer: Arina Ielchiieva Committed: Fri Apr 6 11:54:46 2018 +0300 -- .../src/main/resources/rest/threads/threads.ftl | 27 ++-- 1 file changed, 25 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/2e1568ce/exec/java-exec/src/main/resources/rest/threads/threads.ftl -- diff --git a/exec/java-exec/src/main/resources/rest/threads/threads.ftl b/exec/java-exec/src/main/resources/rest/threads/threads.ftl index e393164..14d22f2 100644 --- a/exec/java-exec/src/main/resources/rest/threads/threads.ftl +++ b/exec/java-exec/src/main/resources/rest/threads/threads.ftl @@ -17,18 +17,41 @@ back + Auto Refresh + + + + + +var statusUrl = location.protocol + "//" + location.host + "/status/threads"; +var refreshInterval = 3; +document.getElementById('refreshTimeLabel').innerHTML = refreshInterval+" sec"; var update = function() { - $.get("/status/threads", function(data) { + $.get(statusUrl, function(data) { $("#mainDiv").html("" + data + ""); }); }; +//Ref: https://stackoverflow.com/a/36640126/8323038 +function copyThreads() { + if (document.selection) { +var range = document.body.createTextRange(); +range.moveToElementText(document.getElementById("mainDiv")); +range.select().createTextRange(); +document.execCommand("copy"); + } else if (window.getSelection) { +var range = document.createRange(); +range.selectNode(document.getElementById("mainDiv")); +window.getSelection().addRange(range); +document.execCommand("copy"); + } +} update(); -setInterval(update, 2000); +setInterval(update, refreshInterval*1000);
[07/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java deleted file mode 100644 index 2056220..000 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/VectorPrinter.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.test.rowSet.test; - -import org.apache.drill.exec.vector.UInt4Vector; -import org.apache.drill.exec.vector.ValueVector; -import org.apache.drill.exec.vector.VarCharVector; - -import com.google.common.base.Charsets; - -/** - * Handy tool to visualize string and offset vectors for - * debugging. - */ - -public class VectorPrinter { - - public static void printOffsets(UInt4Vector vector, int start, int length) { -header(vector, start, length); -for (int i = start, j = 0; j < length; i++, j++) { - if (j > 0) { -System.out.print(" "); - } - System.out.print(vector.getAccessor().get(i)); -} -System.out.print("], addr = "); -System.out.println(vector.getBuffer().addr()); - } - - public static void printStrings(VarCharVector vector, int start, int length) { -printOffsets(vector.getOffsetVector(), start, length + 1); -header(vector, start, length); -System.out.println(); -for (int i = start, j = 0; j < length; i++, j++) { - System.out.print(" "); - System.out.print(i); - System.out.print(": \""); - System.out.print(stringAt(vector, i)); - System.out.println("\""); -} -System.out.println("]"); - } - - public static void header(ValueVector vector, int start, int length) { -System.out.print(vector.getClass()); -System.out.print(": ("); -System.out.print(start); -System.out.print(" - "); -System.out.print(start + length - 1); -System.out.print("): ["); - } - - public static String stringAt(VarCharVector vector, int i) { -return new String(vector.getAccessor().get(i), Charsets.UTF_8); - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java -- diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java index 109500a..513f50b 100644 --- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java +++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java @@ -823,4 +823,19 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable { historicalLog.buildHistory(sb, indent + 1, verbosity.includeStackTraces); } } + + /** + * Convenience method to read buffer bytes into a newly allocated byte + * array. + * + * @param srcOffset the offset into this buffer of the data to read + * @param length number of bytes to read + * @return byte array with the requested bytes + */ + + public byte[] unsafeGetMemory(int srcOffset, int length) { +byte buf[] = new byte[length]; +PlatformDependent.copyMemory(addr + srcOffset, buf, 0, length); +return buf; + } } http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/codegen/templates/ColumnAccessors.java -- diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java index 14ec1e8..4836099 100644 --- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java +++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java @@ -1,3 +1,4 @@ +<#macro copyright> /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -16,9 +17,10 @@ * limitations under the License. */ +// This class is generated using Freemarker and the ${.template_name} template. +
[03/11] drill git commit: DRILL-6287: apache-release profile should be disabled by default
DRILL-6287: apache-release profile should be disabled by default closes #1182 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a0549453 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a0549453 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a0549453 Branch: refs/heads/master Commit: a05494537494efe7d9489e1468245e1ca7b16708 Parents: 2e1568c Author: Vlad Rozov Authored: Thu Mar 22 06:54:56 2018 -0700 Committer: Arina Ielchiieva Committed: Fri Apr 6 11:56:55 2018 +0300 -- pom.xml | 11 --- 1 file changed, 4 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/a0549453/pom.xml -- diff --git a/pom.xml b/pom.xml index f07beaf..b9d395d 100644 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,7 @@ 4096 4096 +-Xdoclint:none @@ -1234,12 +1235,6 @@ apache-release - -[1.8,) - - --Xdoclint:none - @@ -1274,7 +1269,9 @@ true false - src/main/resources/assemblies/source-assembly.xml + + src/main/resources/assemblies/source-assembly.xml + apache-drill-${project.version}-src gnu
[11/11] drill git commit: DRILL-6296: Add operator metrics for batch sizing for merge join
DRILL-6296: Add operator metrics for batch sizing for merge join closes #1181 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/da241134 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/da241134 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/da241134 Branch: refs/heads/master Commit: da241134fb88464139437b05b1feaafbb3014bb0 Parents: 77f5e90 Author: Padma Penumarthy Authored: Thu Mar 29 19:05:07 2018 -0700 Committer: Arina Ielchiieva Committed: Fri Apr 6 12:07:30 2018 +0300 -- .../drill/exec/ops/OperatorMetricRegistry.java | 2 + .../impl/flatten/FlattenRecordBatch.java| 28 ++- .../exec/physical/impl/join/MergeJoinBatch.java | 90 +++- .../AbstractRecordBatchMemoryManager.java | 134 --- .../exec/record/RecordBatchMemoryManager.java | 228 +++ .../drill/exec/record/RecordIterator.java | 6 +- 6 files changed, 330 insertions(+), 158 deletions(-) -- http://git-wip-us.apache.org/repos/asf/drill/blob/da241134/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java index b029154..0b9aeb6 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/OperatorMetricRegistry.java @@ -23,6 +23,7 @@ import org.apache.drill.exec.physical.impl.aggregate.HashAggTemplate; import org.apache.drill.exec.physical.impl.broadcastsender.BroadcastSenderRootExec; import org.apache.drill.exec.physical.impl.flatten.FlattenRecordBatch; import org.apache.drill.exec.physical.impl.join.HashJoinBatch; +import org.apache.drill.exec.physical.impl.join.MergeJoinBatch; import org.apache.drill.exec.physical.impl.mergereceiver.MergingRecordBatch; import org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec; import org.apache.drill.exec.physical.impl.unorderedreceiver.UnorderedReceiverBatch; @@ -51,6 +52,7 @@ public class OperatorMetricRegistry { register(CoreOperatorType.EXTERNAL_SORT_VALUE, ExternalSortBatch.Metric.class); register(CoreOperatorType.PARQUET_ROW_GROUP_SCAN_VALUE, ParquetRecordReader.Metric.class); register(CoreOperatorType.FLATTEN_VALUE, FlattenRecordBatch.Metric.class); +register(CoreOperatorType.MERGE_JOIN_VALUE, MergeJoinBatch.Metric.class); } private static void register(final int operatorType, final Class metricDef) { http://git-wip-us.apache.org/repos/asf/drill/blob/da241134/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java -- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index a1f783f..aea415b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -45,7 +45,7 @@ import org.apache.drill.exec.physical.config.FlattenPOP; import org.apache.drill.exec.record.RecordBatchSizer; import org.apache.drill.exec.record.AbstractSingleRecordBatch; import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode; -import org.apache.drill.exec.record.AbstractRecordBatchMemoryManager; +import org.apache.drill.exec.record.RecordBatchMemoryManager; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.record.TransferPair; @@ -104,11 +104,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch { INPUT_BATCH_COUNT, AVG_INPUT_BATCH_BYTES, AVG_INPUT_ROW_BYTES, -TOTAL_INPUT_RECORDS, +INPUT_RECORD_COUNT, OUTPUT_BATCH_COUNT, AVG_OUTPUT_BATCH_BYTES, AVG_OUTPUT_ROW_BYTES, -TOTAL_OUTPUT_RECORDS; +OUTPUT_RECORD_COUNT; @Override public int metricId() { @@ -116,7 +116,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch { } } - private class FlattenMemoryManager extends AbstractRecordBatchMemoryManager { + private class FlattenMemoryManager extends RecordBatchMemoryManager { @Override public void update() { @@ -152,9 +152,10 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch { // i.e. all rows fit within memory budget. setOutputRowCount(Math.min(columnSize.getElementC
[08/11] drill git commit: DRILL-6230: Extend row set readers to handle hyper vectors
http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java index 352ab34..c402466 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderProtocol.java @@ -433,7 +433,7 @@ public class TestResultSetLoaderProtocol extends SubOperatorTest { assertEquals(5, schema.size()); assertEquals(4, schema.index("e")); assertEquals(4, schema.index("E")); -rootWriter.array(4).set("e1", "e2", "e3"); +rootWriter.array(4).setObject(strArray("e1", "e2", "e3")); rootWriter.save(); // Verify. No reason to expect problems, but might as well check. http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java index cc2a83e..07100ed 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/rowSet/impl/TestResultSetLoaderTorture.java @@ -29,7 +29,6 @@ import org.apache.drill.exec.record.metadata.TupleMetadata; import org.apache.drill.exec.vector.ValueVector; import org.apache.drill.exec.vector.accessor.ArrayReader; import org.apache.drill.exec.vector.accessor.ArrayWriter; -import org.apache.drill.exec.vector.accessor.ScalarElementReader; import org.apache.drill.exec.vector.accessor.ScalarReader; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.TupleReader; @@ -256,7 +255,8 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { ScalarReader n2Reader; ScalarReader s2Reader; ScalarReader n3Reader; -ScalarElementReader s3Reader; +ArrayReader s3Array; +ScalarReader s3Reader; ReadState readState; public BatchReader(TestSetup setup, RowSetReader reader, ReadState readState) { @@ -272,7 +272,8 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { s2Reader = m2Reader.scalar("s2"); TupleReader m3Reader = m2Reader.tuple("m3"); n3Reader = m3Reader.scalar("n3"); - s3Reader = m3Reader.array("s3").elements(); + s3Array = m3Reader.array("s3"); + s3Reader = s3Array.scalar(); } public void verify() { @@ -312,7 +313,7 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { private void verifyM2Array() { for (int i = 0; i < setup.m2Count; i++) { -a2Reader.setPosn(i); +assert(a2Reader.next()); // n2: usual int @@ -321,7 +322,7 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { if (readState.innerCount % setup.s2Cycle == 0) { // Skipped values should be null assertTrue( - String.format("Row %d, entry %d", rootReader.rowIndex(), i), + String.format("Row %d, entry %d", rootReader.offset(), i), s2Reader.isNull()); } else if (readState.innerCount % setup.s2Cycle % setup.nullCycle == 0) { assertTrue(s2Reader.isNull()); @@ -338,10 +339,11 @@ public class TestResultSetLoaderTorture extends SubOperatorTest { // s3: a repeated VarChar if (readState.innerCount % setup.s3Cycle == 0) { - assertEquals(0, s3Reader.size()); + assertEquals(0, s3Array.size()); } else { for (int j = 0; j < setup.s3Count; j++) { -assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count + j), s3Reader.getString(j)); +assertTrue(s3Array.next()); +assertEquals(setup.s3Value + (readState.innerCount * setup.s3Count + j), s3Reader.getString()); } } readState.innerCount++; http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java -- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestVectorContainer.java index a511a0a..0882a1c 100644 --- a/exec/java-exec