http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java index f7fd1d6..e5ba98f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java @@ -293,7 +293,8 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> { continue; } keyExprs[i] = expr; - final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType()); + final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(), + expr.getMajorType()); @SuppressWarnings("resource") final ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator()); keyOutputIds[i] = container.add(vector); @@ -309,7 +310,8 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> { continue; } - final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType()); + final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(), + expr.getMajorType()); @SuppressWarnings("resource") ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator()); TypedFieldId id = container.add(vector); @@ -425,17 +427,17 @@ public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> { case FOUR_BYTE: { JVar var = g.declareClassField("sv4_", g.getModel()._ref(SelectionVector4.class)); g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4")); - g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));; + g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex"))); return; } case NONE: { - g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));; + g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex")); return; } case TWO_BYTE: { JVar var = g.declareClassField("sv2_", g.getModel()._ref(SelectionVector2.class)); g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2")); - g.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));; + g.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex"))); return; }
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java index 436480e..387dad1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java @@ -196,7 +196,8 @@ public class ChainedHashTable { */ for (NamedExpression ne : htConfig.getKeyExprsBuild()) { LogicalExpression expr = keyExprsBuild[i]; - final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsUnescapedPath(), expr.getMajorType()); + final MaterializedField outputField = MaterializedField.create(ne.getRef().getLastSegment().getNameSegment().getPath(), + expr.getMajorType()); @SuppressWarnings("resource") ValueVector vv = TypeHelper.getNewVector(outputField, allocator); htKeyFieldIds[i] = htContainerOrig.add(vv); @@ -318,8 +319,7 @@ public class ChainedHashTable { * aggregate. For join we need to hash everything as double (both for distribution and for comparison) but * for aggregation we can avoid the penalty of casting to double */ - LogicalExpression hashExpression = HashPrelUtil.getHashExpression(Arrays.asList(keyExprs), - incomingProbe != null ? true : false); + LogicalExpression hashExpression = HashPrelUtil.getHashExpression(Arrays.asList(keyExprs), incomingProbe != null); final LogicalExpression materializedExpr = ExpressionTreeMaterializer.materializeAndCheckErrors(hashExpression, batch, context.getFunctionRegistry()); HoldingContainer hash = cg.addExpr(materializedExpr); cg.getEvalBlock()._return(hash.getValue()); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java index 8fd9441..e64e919 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java @@ -253,8 +253,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { } private FieldReference getRef(NamedExpression e) { - final FieldReference ref = e.getRef(); - return ref; + return e.getRef(); } /** @@ -333,8 +332,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { ClassifierResult result = new ClassifierResult(); - for (int i = 0; i < exprs.size(); i++) { - final NamedExpression namedExpression = exprs.get(i); + for (NamedExpression namedExpression : exprs) { result.clear(); String outputName = getRef(namedExpression).getRootSegment().getPath(); @@ -408,10 +406,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> { List<NamedExpression> exprs = Lists.newArrayList(); for (MaterializedField field : incoming.getSchema()) { - if (field.getPath().equals(popConfig.getColumn().getAsUnescapedPath())) { + String fieldName = field.getName(); + if (fieldName.equals(popConfig.getColumn().getRootSegmentPath())) { continue; } - exprs.add(new NamedExpression(SchemaPath.getSimplePath(field.getPath()), new FieldReference(field.getPath()))); + exprs.add(new NamedExpression(SchemaPath.getSimplePath(fieldName), new FieldReference(fieldName))); } return exprs; } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java index 72b8833..1f74ba1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java @@ -477,7 +477,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> { outputType = inputType; } - final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getPath(), outputType)); + final ValueVector v = container.addOrGet(MaterializedField.create(vv.getField().getName(), outputType)); if (v instanceof AbstractContainerVector) { vv.getValueVector().makeTransferPair(v); v.clear(); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java index c351517..e599702 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java @@ -400,7 +400,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> { } else { outputType = inputType; } - MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType); + MaterializedField newField = MaterializedField.create(w.getField().getName(), outputType); ValueVector v = container.addOrGet(newField); if (v instanceof AbstractContainerVector) { w.getValueVector().makeTransferPair(v); @@ -417,7 +417,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> { } else { outputType = inputType; } - MaterializedField newField = MaterializedField.create(w.getField().getPath(), outputType); + MaterializedField newField = MaterializedField.create(w.getField().getName(), outputType); ValueVector v = container.addOrGet(newField); if (v instanceof AbstractContainerVector) { w.getValueVector().makeTransferPair(v); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java index 8336e86..35cc710 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java @@ -317,7 +317,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP> outputType = inputType; } - MaterializedField newField = MaterializedField.create(field.getPath(), outputType); + MaterializedField newField = MaterializedField.create(field.getName(), outputType); container.addOrGet(newField); JVar inVV = nLJClassGenerator.declareVectorValueSetupAndMember("rightContainer", @@ -384,7 +384,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP> } else { outputType = inputType; } - MaterializedField newField = MaterializedField.create(vectorWrapper.getField().getPath(), outputType); + MaterializedField newField = MaterializedField.create(vectorWrapper.getField().getName(), outputType); ValueVector valueVector = container.addOrGet(newField); if (valueVector instanceof AbstractContainerVector) { vectorWrapper.getValueVector().makeTransferPair(valueVector); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java index fede487..63133d4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java @@ -431,11 +431,10 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart int i = 0; for (Ordering od : orderings) { final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), incoming, collector, context.getFunctionRegistry()); - SchemaPath schemaPath = SchemaPath.getSimplePath("f" + i++); TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().mergeFrom(expr.getMajorType()) .clearMode().setMode(TypeProtos.DataMode.REQUIRED); TypeProtos.MajorType newType = builder.build(); - MaterializedField outputField = MaterializedField.create(schemaPath.getAsUnescapedPath(), newType); + MaterializedField outputField = MaterializedField.create("f" + i++, newType); if (collector.hasErrors()) { throw new SchemaChangeException(String.format( "Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); @@ -533,7 +532,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart // if this now that all the batches on the queue are processed, we begin processing the incoming batches. For the // first one // we need to generate a new schema, even if the outcome is IterOutcome.OK After that we can reuse the schema. - if (this.startedUnsampledBatches == false) { + if (!this.startedUnsampledBatches) { this.startedUnsampledBatches = true; if (upstream == IterOutcome.OK) { upstream = IterOutcome.OK_NEW_SCHEMA; http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java index 589754f..3afa852 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/producer/ProducerConsumerBatch.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -99,9 +99,9 @@ public class ProducerConsumerBatch extends AbstractRecordBatch<ProducerConsumer> final MaterializedField field = schema.getColumn(i); final MajorType type = field.getType(); final ValueVector vOut = container.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()), - container.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()).getValueVector(); + container.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()).getValueVector(); final ValueVector vIn = newContainer.getValueAccessorById(TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()), - newContainer.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()).getValueVector(); + newContainer.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()).getValueVector(); final TransferPair tp = vIn.makeTransferPair(vOut); tp.transfer(); } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java index 6baf070..9a72fcb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java @@ -202,7 +202,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { this.recordCount = remainderIndex; } else { setValueCount(incomingRecordCount); - for(final VectorWrapper<?> v: incoming) { + for (final VectorWrapper<?> v: incoming) { v.clear(); } this.recordCount = outputRecords; @@ -331,8 +331,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { final ClassifierResult result = new ClassifierResult(); final boolean classify = isClassificationNeeded(exprs); - for (int i = 0; i < exprs.size(); i++) { - final NamedExpression namedExpression = exprs.get(i); + for (NamedExpression namedExpression : exprs) { result.clear(); if (classify && namedExpression.getExpr() instanceof SchemaPath) { @@ -341,15 +340,15 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { if (result.isStar) { // The value indicates which wildcard we are processing now final Integer value = result.prefixMap.get(result.prefix); - if (value != null && value.intValue() == 1) { + if (value != null && value == 1) { int k = 0; for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - if (k > result.outputNames.size()-1) { + if (k > result.outputNames.size() - 1) { assert false; } final String name = result.outputNames.get(k++); // get the renamed column names - if (name == EMPTY_STRING) { + if (name.isEmpty()) { continue; } @@ -362,16 +361,16 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { final TransferPair tp = vvIn.makeTransferPair(vvOut); transfers.add(tp); } - } else if (value != null && value.intValue() > 1) { // subsequent wildcards should do a copy of incoming valuevectors + } else if (value != null && value > 1) { // subsequent wildcards should do a copy of incoming valuevectors int k = 0; for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getPath()); - if (k > result.outputNames.size()-1) { + final SchemaPath originalPath = SchemaPath.getSimplePath(vvIn.getField().getName()); + if (k > result.outputNames.size() - 1) { assert false; } final String name = result.outputNames.get(k++); // get the renamed column names - if (name == EMPTY_STRING) { + if (name.isEmpty()) { continue; } @@ -379,7 +378,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { continue; } - final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry() ); + final LogicalExpression expr = ExpressionTreeMaterializer.materialize(originalPath, incoming, collector, context.getFunctionRegistry()); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); } @@ -387,7 +386,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { final MaterializedField outputField = MaterializedField.create(name, expr.getMajorType()); final ValueVector vv = container.addOrGet(outputField, callBack); allocationVectors.add(vv); - final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath())); + final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName())); final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true); final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND); } @@ -406,14 +405,14 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { if (result != null && result.outputNames != null && result.outputNames.size() > 0) { boolean isMatched = false; for (int j = 0; j < result.outputNames.size(); j++) { - if (!result.outputNames.get(j).equals(EMPTY_STRING)) { + if (!result.outputNames.get(j).isEmpty()) { outputName = result.outputNames.get(j); isMatched = true; break; } } - if(!isMatched) { + if (!isMatched) { continue; } } @@ -437,7 +436,8 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { Preconditions.checkNotNull(incoming); final FieldReference ref = getRef(namedExpression); - final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getAsUnescapedPath(), vectorRead.getMajorType()), callBack); + final ValueVector vvOut = container.addOrGet(MaterializedField.create(ref.getLastSegment().getNameSegment().getPath(), + vectorRead.getMajorType()), callBack); final TransferPair tp = vvIn.makeTransferPair(vvOut); transfers.add(tp); transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]); @@ -463,7 +463,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { // need to do evaluation. final ValueVector vector = container.addOrGet(outputField, callBack); allocationVectors.add(vector); - final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath())); + final TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName())); final boolean useSetSafe = !(vector instanceof FixedWidthVector); final ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe); final HoldingContainer hc = cg.addExpr(write, ClassGenerator.BlkCreateMode.TRUE_IF_BOUND); @@ -510,17 +510,19 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { final List<NamedExpression> exprs = Lists.newArrayList(); for (final MaterializedField field : incoming.getSchema()) { + String fieldName = field.getName(); if (Types.isComplex(field.getType()) || Types.isRepeated(field.getType())) { - final LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", SchemaPath.getSimplePath(field.getPath()), ExpressionPosition.UNKNOWN); + final LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON", + SchemaPath.getSimplePath(fieldName), ExpressionPosition.UNKNOWN); final String castFuncName = CastFunctions.getCastFunc(MinorType.VARCHAR); final List<LogicalExpression> castArgs = Lists.newArrayList(); castArgs.add(convertToJson); //input_expr // implicitly casting to varchar, since we don't know actual source length, cast to undefined length, which will preserve source length castArgs.add(new ValueExpressions.LongExpression(Types.MAX_VARCHAR_LENGTH, null)); final FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN); - exprs.add(new NamedExpression(castCall, new FieldReference(field.getPath()))); + exprs.add(new NamedExpression(castCall, new FieldReference(fieldName))); } else { - exprs.add(new NamedExpression(SchemaPath.getSimplePath(field.getPath()), new FieldReference(field.getPath()))); + exprs.add(new NamedExpression(SchemaPath.getSimplePath(fieldName), new FieldReference(fieldName))); } } return exprs; @@ -528,8 +530,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { private boolean isClassificationNeeded(final List<NamedExpression> exprs) { boolean needed = false; - for (int i = 0; i < exprs.size(); i++) { - final NamedExpression ex = exprs.get(i); + for (NamedExpression ex : exprs) { if (!(ex.getExpr() instanceof SchemaPath)) { continue; } @@ -570,7 +571,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { * @param origName the original input name of the column * @param result the data structure to keep track of the used names and decide what output name should be * to ensure uniqueness - * @Param allowDupsWithRename if the original name has been used, is renaming allowed to ensure output name unique + * @param allowDupsWithRename if the original name has been used, is renaming allowed to ensure output name unique */ private void addToResultMaps(final String origName, final ClassifierResult result, final boolean allowDupsWithRename) { String name = origName; @@ -610,7 +611,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { boolean exprIsFirstWildcard = false; if (exprContainsStar) { result.isStar = true; - final Integer value = (Integer) result.prefixMap.get(exprPrefix); + final Integer value = result.prefixMap.get(exprPrefix); if (value == null) { final Integer n = 1; result.prefixMap.put(exprPrefix, n); @@ -634,9 +635,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { assert(components.length == 2); final String prefix = components[0]; result.outputNames = Lists.newArrayList(); - for(final VectorWrapper<?> wrapper : incoming) { + for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - final String name = vvIn.getField().getPath(); + final String name = vvIn.getField().getName(); // add the prefix to the incoming column name final String newName = prefix + StarColumnHelper.PREFIX_DELIMITER + name; @@ -656,7 +657,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - final String incomingName = vvIn.getField().getPath(); + final String incomingName = vvIn.getField().getName(); // get the prefix of the name final String[] nameComponents = incomingName.split(StarColumnHelper.PREFIX_DELIMITER, 2); // if incoming valuevector does not have a prefix, ignore it since this expression is not referencing it @@ -666,10 +667,9 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { } final String namePrefix = nameComponents[0]; if (exprPrefix.equalsIgnoreCase(namePrefix)) { - final String newName = incomingName; - if (!result.outputMap.containsKey(newName)) { - result.outputNames.set(k, newName); - result.outputMap.put(newName, newName); + if (!result.outputMap.containsKey(incomingName)) { + result.outputNames.set(k, incomingName); + result.outputMap.put(incomingName, incomingName); } } k++; @@ -679,7 +679,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { if (exprContainsStar) { for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - final String incomingName = vvIn.getField().getPath(); + final String incomingName = vvIn.getField().getName(); if (refContainsStar) { addToResultMaps(incomingName, result, true); // allow dups since this is likely top-level project } else { @@ -698,11 +698,11 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { } // input is wildcard and it is not the first wildcard - else if(exprIsStar) { + else if (exprIsStar) { result.outputNames = Lists.newArrayList(); for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - final String incomingName = vvIn.getField().getPath(); + final String incomingName = vvIn.getField().getName(); addToResultMaps(incomingName, result, true); // allow dups since this is likely top-level project } } @@ -723,7 +723,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - final String name = vvIn.getField().getPath(); + final String name = vvIn.getField().getName(); final String[] components = name.split(StarColumnHelper.PREFIX_DELIMITER, 2); if (components.length <= 1) { k++; @@ -760,7 +760,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> { result.outputNames = Lists.newArrayList(); for (final VectorWrapper<?> wrapper : incoming) { final ValueVector vvIn = wrapper.getValueVector(); - final String incomingName = vvIn.getField().getPath(); + final String incomingName = vvIn.getField().getName(); if (expr.getPath().equalsIgnoreCase(incomingName)) { // case insensitive matching of field name. final String newName = ref.getPath(); addToResultMaps(newName, result, true); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java index e6a0dd4..5afe66b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/union/UnionAllRecordBatch.java @@ -103,7 +103,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { try { IterOutcome upstream = unionAllInput.nextBatch(); logger.debug("Upstream of Union-All: {}", upstream); - switch(upstream) { + switch (upstream) { case NONE: case OUT_OF_MEMORY: case STOP: @@ -114,7 +114,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { case OK: IterOutcome workOutcome = doWork(); - if(workOutcome != IterOutcome.OK) { + if (workOutcome != IterOutcome.OK) { return workOutcome; } else { return upstream; @@ -164,9 +164,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { transfers.clear(); // If both sides of Union-All are empty - if(unionAllInput.isBothSideEmpty()) { - for(int i = 0; i < outputFields.size(); ++i) { - final String colName = outputFields.get(i).getPath(); + if (unionAllInput.isBothSideEmpty()) { + for (MaterializedField materializedField : outputFields) { + final String colName = materializedField.getName(); final MajorType majorType = MajorType.newBuilder() .setMinorType(MinorType.INT) .setMode(DataMode.OPTIONAL) @@ -186,12 +186,12 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { // Uncomment out this line to debug the generated code. // cg.getCodeGenerator().saveCodeForDebugging(true); int index = 0; - for(VectorWrapper<?> vw : current) { + for (VectorWrapper<?> vw : current) { ValueVector vvIn = vw.getValueVector(); // get the original input column names - SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getPath()); + SchemaPath inputPath = SchemaPath.getSimplePath(vvIn.getField().getName()); // get the renamed column names - SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getPath()); + SchemaPath outputPath = SchemaPath.getSimplePath(outputFields.get(index).getName()); final ErrorCollector collector = new ErrorCollectorImpl(); // According to input data names, Minortypes, Datamodes, choose to @@ -202,19 +202,20 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { // Transfer column MajorType outputFieldType = outputFields.get(index).getType(); - MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), outputFieldType); + MaterializedField outputField = MaterializedField.create(outputPath.getLastSegment().getNameSegment().getPath(), + outputFieldType); /* todo: Fix if condition when DRILL-4824 is merged If condition should be changed to: - `if (outputFields.get(index).getPath().equals(inputPath.getAsUnescapedPath())) {` + `if (outputFields.get(index).getName().equals(inputPath.getRootSegmentPath())) {` DRILL-5419 has changed condition to correct one but this caused regression (DRILL-5521). Root cause is missing indication of child column in map types when it is null. DRILL-4824 is re-working json reader implementation, including map types and will fix this problem. Reverting condition to previous one to avoid regression till DRILL-4824 is merged. Unit test - TestJsonReader.testKvgenWithUnionAll(). */ - if (outputFields.get(index).getPath().equals(inputPath)) { + if (outputFields.get(index).getName().equals(inputPath)) { ValueVector vvOut = container.addOrGet(outputField); TransferPair tp = vvIn.makeTransferPair(vvOut); transfers.add(tp); @@ -227,7 +228,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { ValueVector vv = container.addOrGet(outputField, callBack); allocationVectors.add(vv); - TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath())); + TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getName())); ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true); cg.addExpr(write); } @@ -240,7 +241,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { // If the inputs' DataMode is required and the outputs' DataMode is not required // cast to the one with the least restriction - if(vvIn.getField().getType().getMode() == DataMode.REQUIRED + if (vvIn.getField().getType().getMode() == DataMode.REQUIRED && outputFields.get(index).getType().getMode() != DataMode.REQUIRED) { expr = ExpressionTreeMaterializer.convertToNullableType(expr, vvIn.getField().getType().getMinorType(), context.getFunctionRegistry(), collector); if (collector.hasErrors()) { @@ -250,17 +251,18 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { // If two inputs' MinorTypes are different, // Insert a cast before the Union operation - if(vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) { + if (vvIn.getField().getType().getMinorType() != outputFields.get(index).getType().getMinorType()) { expr = ExpressionTreeMaterializer.addCastExpression(expr, outputFields.get(index).getType(), context.getFunctionRegistry(), collector); if (collector.hasErrors()) { throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString())); } } - final MaterializedField outputField = MaterializedField.create(outputPath.getAsUnescapedPath(), expr.getMajorType()); + final MaterializedField outputField = MaterializedField.create(outputPath.getLastSegment().getNameSegment().getPath(), + expr.getMajorType()); ValueVector vector = container.addOrGet(outputField, callBack); allocationVectors.add(vector); - TypedFieldId fid = container.getValueVectorId(SchemaPath.getSimplePath(outputField.getPath())); + TypedFieldId fid = container.getValueVectorId(outputPath); boolean useSetSafe = !(vector instanceof FixedWidthVector); ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, useSetSafe); @@ -272,12 +274,12 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { unionall = context.getImplementationClass(cg.getCodeGenerator()); unionall.setup(context, current, this, transfers); - if(!schemaAvailable) { + if (!schemaAvailable) { container.buildSchema(BatchSchema.SelectionVectorMode.NONE); schemaAvailable = true; } - if(!doAlloc()) { + if (!doAlloc()) { return IterOutcome.OUT_OF_MEMORY; } @@ -298,7 +300,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { // This method is used by inner class to clear the current record batch private void clearCurrentRecordBatch() { - for(VectorWrapper<?> v: current) { + for (VectorWrapper<?> v: current) { v.clear(); } } @@ -333,9 +335,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { } public IterOutcome nextBatch() throws SchemaChangeException { - if(upstream == RecordBatch.IterOutcome.NOT_YET) { + if (upstream == RecordBatch.IterOutcome.NOT_YET) { IterOutcome iterLeft = leftSide.nextBatch(); - switch(iterLeft) { + switch (iterLeft) { case OK_NEW_SCHEMA: /* * If the first few record batches are all empty, @@ -345,7 +347,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { * Thus, while-loop is necessary to skip those empty batches. */ whileLoop: - while(leftSide.getRecordBatch().getRecordCount() == 0) { + while (leftSide.getRecordBatch().getRecordCount() == 0) { iterLeft = leftSide.nextBatch(); switch(iterLeft) { @@ -380,11 +382,11 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { } IterOutcome iterRight = rightSide.nextBatch(); - switch(iterRight) { + switch (iterRight) { case OK_NEW_SCHEMA: // Unless there is no record batch on the left side of the inputs, // always start processing from the left side. - if(leftIsFinish) { + if (leftIsFinish) { unionAllRecordBatch.setCurrentRecordBatch(rightSide.getRecordBatch()); } else { unionAllRecordBatch.setCurrentRecordBatch(leftSide.getRecordBatch()); @@ -402,9 +404,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { * Thus, while-loop is necessary to skip those empty batches. */ whileLoop: - while(rightSide.getRecordBatch().getRecordCount() == 0) { + while (rightSide.getRecordBatch().getRecordCount() == 0) { iterRight = rightSide.nextBatch(); - switch(iterRight) { + switch (iterRight) { case STOP: case OUT_OF_MEMORY: return iterRight; @@ -425,7 +427,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { } } - if(leftIsFinish && rightIsFinish) { + if (leftIsFinish && rightIsFinish) { setBothSideEmpty(true); } @@ -446,19 +448,19 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { upstream = IterOutcome.OK_NEW_SCHEMA; return upstream; } else { - if(isBothSideEmpty()) { + if (isBothSideEmpty()) { return IterOutcome.NONE; } unionAllRecordBatch.clearCurrentRecordBatch(); - if(leftIsFinish && rightIsFinish) { + if (leftIsFinish && rightIsFinish) { upstream = IterOutcome.NONE; return upstream; - } else if(leftIsFinish) { + } else if (leftIsFinish) { IterOutcome iterOutcome = rightSide.nextBatch(); - switch(iterOutcome) { + switch (iterOutcome) { case NONE: rightIsFinish = true; // fall through @@ -468,7 +470,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { return upstream; case OK_NEW_SCHEMA: - if(!rightSide.getRecordBatch().getSchema().equals(rightSchema)) { + if (!rightSide.getRecordBatch().getSchema().equals(rightSchema)) { throw new SchemaChangeException("Schema change detected in the right input of Union-All. This is not currently supported"); } iterOutcome = IterOutcome.OK; @@ -481,9 +483,9 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { default: throw new IllegalStateException(String.format("Unknown state %s.", upstream)); } - } else if(rightIsFinish) { + } else if (rightIsFinish) { IterOutcome iterOutcome = leftSide.nextBatch(); - switch(iterOutcome) { + switch (iterOutcome) { case STOP: case OUT_OF_MEMORY: case NONE: @@ -501,14 +503,14 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { } else { IterOutcome iterOutcome = leftSide.nextBatch(); - switch(iterOutcome) { + switch (iterOutcome) { case STOP: case OUT_OF_MEMORY: upstream = iterOutcome; return upstream; case OK_NEW_SCHEMA: - if(!leftSide.getRecordBatch().getSchema().equals(leftSchema)) { + if (!leftSide.getRecordBatch().getSchema().equals(leftSchema)) { throw new SchemaChangeException("Schema change detected in the left input of Union-All. This is not currently supported"); } @@ -549,10 +551,10 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { * types are nullable integer */ private void inferOutputFields() { - if(!leftIsFinish && !rightIsFinish) { + if (!leftIsFinish && !rightIsFinish) { // Both sides are non-empty inferOutputFieldsBothSide(); - } else if(!rightIsFinish) { + } else if (!rightIsFinish) { // Left side is non-empty // While use left side's column names as output column names, // use right side's column types as output column types. @@ -585,7 +587,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { if (hasSameTypeAndMode(leftField, rightField)) { MajorType.Builder builder = MajorType.newBuilder().setMinorType(leftField.getType().getMinorType()).setMode(leftField.getDataMode()); builder = Types.calculateTypePrecisionAndScale(leftField.getType(), rightField.getType(), builder); - outputFields.add(MaterializedField.create(leftField.getPath(), builder.build())); + outputFields.add(MaterializedField.create(leftField.getName(), builder.build())); } else { // If the output type is not the same, // cast the column of one of the table to a data type which is the Least Restrictive @@ -612,7 +614,7 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { dataModes.add(rightField.getType().getMode()); builder.setMode(TypeCastRules.getLeastRestrictiveDataMode(dataModes)); - outputFields.add(MaterializedField.create(leftField.getPath(), builder.build())); + outputFields.add(MaterializedField.create(leftField.getName(), builder.build())); } ++index; } @@ -624,20 +626,19 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { outputFields = Lists.newArrayList(); final List<String> outputColumnNames = Lists.newArrayList(); - final Iterator<MaterializedField> iterForNames = schemaForNames.iterator(); - while(iterForNames.hasNext()) { - outputColumnNames.add(iterForNames.next().getPath()); + for (MaterializedField materializedField : schemaForNames) { + outputColumnNames.add(materializedField.getName()); } final Iterator<MaterializedField> iterForTypes = schemaForTypes.iterator(); - for(int i = 0; iterForTypes.hasNext(); ++i) { + for (int i = 0; iterForTypes.hasNext(); ++i) { MaterializedField field = iterForTypes.next(); outputFields.add(MaterializedField.create(outputColumnNames.get(i), field.getType())); } } public List<MaterializedField> getOutputFields() { - if(outputFields == null) { + if (outputFields == null) { throw new NullPointerException("Output fields have not been inferred"); } @@ -670,11 +671,11 @@ public class UnionAllRecordBatch extends AbstractRecordBatch<UnionAll> { } public IterOutcome nextBatch() { - if(upstream == IterOutcome.NONE) { + if (upstream == IterOutcome.NONE) { throw new IllegalStateException(String.format("Unknown state %s.", upstream)); } - if(upstream == IterOutcome.NOT_YET) { + if (upstream == IterOutcome.NOT_YET) { upstream = unionAllRecordBatch.next(recordBatch); return upstream; http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java index 39c662f..e0cfc7a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java @@ -678,7 +678,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> { for (BatchGroup group : batchGroupList) { vectors[i++] = group.getValueAccessorById( field.getValueClass(), - group.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()) + group.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()) .getValueVector(); } cont.add(vectors); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java index 6ec8862..2d82cc1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/managed/PriorityQueueCopierWrapper.java @@ -310,7 +310,7 @@ public class PriorityQueueCopierWrapper extends BaseSortWrapper { for (BatchGroup group : batchGroupList) { vectors[i++] = group.getValueAccessorById( field.getValueClass(), - group.getValueVectorId(SchemaPath.getSimplePath(field.getPath())).getFieldIds()) + group.getValueVectorId(SchemaPath.getSimplePath(field.getName())).getFieldIds()) .getValueVector(); } cont.add(vectors); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java index 9a3ef96..837ddd8 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -160,7 +160,7 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule { metaContext = ((FormatSelection)selection).getSelection().getMetaContext(); } - RexNode condition = null; + RexNode condition; if (projectRel == null) { condition = filterRel.getCondition(); } else { @@ -244,7 +244,7 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule { for (int partitionColumnIndex : BitSets.toIter(partitionColumnBitSet)) { SchemaPath column = SchemaPath.getSimplePath(fieldNameMap.get(partitionColumnIndex)); MajorType type = descriptor.getVectorType(column, settings); - MaterializedField field = MaterializedField.create(column.getAsUnescapedPath(), type); + MaterializedField field = MaterializedField.create(column.getLastSegment().getNameSegment().getPath(), type); ValueVector v = TypeHelper.getNewVector(field, allocator); v.allocateNew(); vectors[partitionColumnIndex] = v; http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java index 3801cb5..9a8483b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -88,7 +88,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp final Map<String, ValueVector> oldFields = Maps.newHashMap(); for(final VectorWrapper<?> wrapper : container) { final ValueVector vector = wrapper.getValueVector(); - oldFields.put(vector.getField().getPath(), vector); + oldFields.put(vector.getField().getName(), vector); } final VectorContainer newVectors = new VectorContainer(); @@ -97,7 +97,7 @@ public class RecordBatchLoader implements VectorAccessible, Iterable<VectorWrapp int bufOffset = 0; for(final SerializedField field : fields) { final MaterializedField fieldDef = MaterializedField.create(field); - ValueVector vector = oldFields.remove(fieldDef.getPath()); + ValueVector vector = oldFields.remove(fieldDef.getName()); if (vector == null) { // Field did not exist previously--is schema change. http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java index 28f5bf2..e1a1031 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SchemaUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -53,7 +53,7 @@ public class SchemaUtil { for (BatchSchema s : schemas) { for (MaterializedField field : s) { - SchemaPath path = SchemaPath.getSimplePath(field.getPath()); + SchemaPath path = SchemaPath.getSimplePath(field.getName()); Set<MinorType> currentTypes = typeSetMap.get(path); if (currentTypes == null) { currentTypes = Sets.newHashSet(); @@ -64,9 +64,7 @@ public class SchemaUtil { throw new RuntimeException("Schema change not currently supported for schemas with complex types"); } if (newType == MinorType.UNION) { - for (MinorType subType : field.getType().getSubTypeList()) { - currentTypes.add(subType); - } + currentTypes.addAll(field.getType().getSubTypeList()); } else { currentTypes.add(newType); } @@ -82,10 +80,11 @@ public class SchemaUtil { for (MinorType t : types) { builder.addSubType(t); } - MaterializedField field = MaterializedField.create(path.getAsUnescapedPath(), builder.build()); + MaterializedField field = MaterializedField.create(path.getLastSegment().getNameSegment().getPath(), builder.build()); fields.add(field); } else { - MaterializedField field = MaterializedField.create(path.getAsUnescapedPath(), Types.optional(types.iterator().next())); + MaterializedField field = MaterializedField.create(path.getLastSegment().getNameSegment().getPath(), + Types.optional(types.iterator().next())); fields.add(field); } } @@ -162,12 +161,12 @@ public class SchemaUtil { if (w.isHyper()) { isHyper = true; final ValueVector[] vvs = w.getValueVectors(); - vectorMap.put(vvs[0].getField().getPath(), vvs); + vectorMap.put(vvs[0].getField().getName(), vvs); } else { assert !isHyper; @SuppressWarnings("resource") final ValueVector v = w.getValueVector(); - vectorMap.put(v.getField().getPath(), v); + vectorMap.put(v.getField().getName(), v); } } @@ -175,7 +174,7 @@ public class SchemaUtil { for (MaterializedField field : toSchema) { if (isHyper) { - final ValueVector[] vvs = (ValueVector[]) vectorMap.remove(field.getPath()); + final ValueVector[] vvs = (ValueVector[]) vectorMap.remove(field.getName()); final ValueVector[] vvsOut; if (vvs == null) { vvsOut = new ValueVector[1]; @@ -189,7 +188,7 @@ public class SchemaUtil { c.add(vvsOut); } else { @SuppressWarnings("resource") - final ValueVector v = (ValueVector) vectorMap.remove(field.getPath()); + final ValueVector v = (ValueVector) vectorMap.remove(field.getName()); c.add(coerceVector(v, c, field, recordCount, allocator)); } } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java index 99353ea..63cac7d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java @@ -136,7 +136,7 @@ public class VectorContainer implements VectorAccessible { @SuppressWarnings({ "resource", "unchecked" }) public <T extends ValueVector> T addOrGet(final MaterializedField field, final SchemaChangeCallBack callBack) { - final TypedFieldId id = getValueVectorId(SchemaPath.getSimplePath(field.getPath())); + final TypedFieldId id = getValueVectorId(SchemaPath.getSimplePath(field.getName())); final ValueVector vector; final Class<?> clazz = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getType().getMode()); if (id != null) { @@ -206,12 +206,12 @@ public class VectorContainer implements VectorAccessible { */ public static VectorContainer canonicalize(VectorContainer original) { VectorContainer vc = new VectorContainer(); - List<VectorWrapper<?>> canonicalWrappers = new ArrayList<VectorWrapper<?>>(original.wrappers); + List<VectorWrapper<?>> canonicalWrappers = new ArrayList<>(original.wrappers); // Sort list of VectorWrapper alphabetically based on SchemaPath. Collections.sort(canonicalWrappers, new Comparator<VectorWrapper<?>>() { @Override public int compare(VectorWrapper<?> v1, VectorWrapper<?> v2) { - return v1.getField().getPath().compareTo(v2.getField().getPath()); + return v1.getField().getName().compareTo(v2.getField().getName()); } }); @@ -255,9 +255,7 @@ public class VectorContainer implements VectorAccessible { schema = null; Class<?> clazz = hyperVector[0].getClass(); ValueVector[] c = (ValueVector[]) Array.newInstance(clazz, hyperVector.length); - for (int i = 0; i < hyperVector.length; i++) { - c[i] = hyperVector[i]; - } + System.arraycopy(hyperVector, 0, c, 0, hyperVector.length); // todo: work with a merged schema. wrappers.add(HyperVectorWrapper.create(hyperVector[0].getField(), c, releasable)); } @@ -283,7 +281,7 @@ public class VectorContainer implements VectorAccessible { for (VectorWrapper<?> w : wrappers){ if (!w.isHyper() && old == w.getValueVector()) { w.clear(); - wrappers.set(i, new SimpleVectorWrapper<ValueVector>(newVector)); + wrappers.set(i, new SimpleVectorWrapper<>(newVector)); return; } i++; http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java index 62c6efd..bcce9eb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java @@ -99,7 +99,7 @@ public class WebUserConnection extends AbstractDisposableUserClientConnection im // TODO: Clean: DRILL-2933: That load(...) no longer throws // SchemaChangeException, so check/clean catch clause below. for (int i = 0; i < loader.getSchema().getFieldCount(); ++i) { - columns.add(loader.getSchema().getColumn(i).getPath()); + columns.add(loader.getSchema().getColumn(i).getName()); } for (int i = 0; i < rows; ++i) { final Map<String, String> record = Maps.newHashMap(); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java index ccd622b..fa8121e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ColumnExplorer.java @@ -95,7 +95,7 @@ public class ColumnExplorer { */ public static boolean isPartitionColumn(OptionManager optionManager, SchemaPath column){ String partitionDesignator = optionManager.getOption(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL).string_val; - String path = column.getAsUnescapedPath(); + String path = column.getRootSegmentPath(); return isPartitionColumn(partitionDesignator, path); } @@ -187,7 +187,7 @@ public class ColumnExplorer { selectedImplicitColumns.putAll(allImplicitColumns); } else { for (SchemaPath column : columns) { - String path = column.getAsUnescapedPath(); + String path = column.getRootSegmentPath(); if (isPartitionColumn(partitionDesignator, path)) { selectedPartitionColumns.add(Integer.parseInt(path.substring(partitionDesignator.length()))); } else if (allImplicitColumns.get(path) != null) { http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java index 22fb48c..6262dce 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -138,7 +138,7 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void, public ExprNode visitCastExpression(CastExpression e, Void value) throws RuntimeException { if (e.getInput() instanceof FieldReference) { FieldReference fieldRef = (FieldReference) e.getInput(); - String field = fieldRef.getAsUnescapedPath().toUpperCase(); + String field = fieldRef.getRootSegmentPath().toUpperCase(); if (field.equals(CATS_COL_CATALOG_NAME) || field.equals(SCHS_COL_SCHEMA_NAME) || field.equals(SHRD_COL_TABLE_NAME) @@ -158,7 +158,7 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void, @Override public ExprNode visitSchemaPath(SchemaPath path, Void value) throws RuntimeException { - String field = path.getAsUnescapedPath().toUpperCase(); + String field = path.getRootSegmentPath().toUpperCase(); if (field.equals(CATS_COL_CATALOG_NAME) || field.equals(SCHS_COL_SCHEMA_NAME) || field.equals(SHRD_COL_TABLE_NAME) http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java index a33f46a..4a8c5f3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java @@ -80,10 +80,12 @@ import com.google.common.collect.Maps; import javax.annotation.Nullable; +import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.SUPPORTED_VERSIONS; import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V1; import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V2; import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3; import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3_1; +import static org.apache.drill.exec.store.parquet.MetadataVersion.Constants.V3_2; public class Metadata { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Metadata.class); @@ -234,7 +236,8 @@ public class Metadata { childFiles.add(file); } } - ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(V3_1, DrillVersionInfo.getVersion()); + ParquetTableMetadata_v3 parquetTableMetadata = new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(), + DrillVersionInfo.getVersion()); if (childFiles.size() > 0) { List<ParquetFileMetadata_v3 > childFilesMetadata = getParquetFileMetadata_v3(parquetTableMetadata, childFiles); @@ -308,7 +311,8 @@ public class Metadata { */ private ParquetTableMetadata_v3 getParquetTableMetadata(List<FileStatus> fileStatuses) throws IOException { - ParquetTableMetadata_v3 tableMetadata = new ParquetTableMetadata_v3(V3_1, DrillVersionInfo.getVersion()); + ParquetTableMetadata_v3 tableMetadata = new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(), + DrillVersionInfo.getVersion()); List<ParquetFileMetadata_v3> fileMetadataList = getParquetFileMetadata_v3(tableMetadata, fileStatuses); tableMetadata.files = fileMetadataList; tableMetadata.directories = new ArrayList<String>(); @@ -695,7 +699,8 @@ public class Metadata { @JsonSubTypes.Type(value = ParquetTableMetadata_v1.class, name = V1), @JsonSubTypes.Type(value = ParquetTableMetadata_v2.class, name = V2), @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3), - @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_1) + @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_1), + @JsonSubTypes.Type(value = ParquetTableMetadata_v3.class, name = V3_2) }) public static abstract class ParquetTableMetadataBase { @@ -1421,7 +1426,7 @@ public class Metadata { } - @JsonTypeName(V3_1) + @JsonTypeName(V3_2) public static class ParquetTableMetadata_v3 extends ParquetTableMetadataBase { @JsonProperty(value = "metadata_version", access = JsonProperty.Access.WRITE_ONLY) private String metadataVersion; /* @@ -1654,16 +1659,20 @@ public class Metadata { } private static class Key { - private String[] name; + private SchemaPath name; private int hashCode = 0; public Key(String[] name) { - this.name = name; + this.name = SchemaPath.getCompoundPath(name); + } + + public Key(SchemaPath name) { + this.name = new SchemaPath(name); } @Override public int hashCode() { if (hashCode == 0) { - hashCode = Arrays.hashCode(name); + hashCode = name.hashCode(); } return hashCode; } @@ -1676,20 +1685,11 @@ public class Metadata { return false; } final Key other = (Key) obj; - return Arrays.equals(this.name, other.name); + return this.name.equals(other.name); } @Override public String toString() { - String s = null; - for (String namePart : name) { - if (s != null) { - s += "."; - s += namePart; - } else { - s = namePart; - } - } - return s; + return name.toString(); } public static class DeSerializer extends KeyDeserializer { @@ -1701,6 +1701,10 @@ public class Metadata { @Override public Object deserializeKey(String key, com.fasterxml.jackson.databind.DeserializationContext ctxt) throws IOException, com.fasterxml.jackson.core.JsonProcessingException { + // key string should contain '`' char if the field was serialized as SchemaPath object + if (key.contains("`")) { + return new Key(SchemaPath.parseFromString(key)); + } return new Key(key.split("\\.")); } } @@ -1884,8 +1888,8 @@ public class Metadata { filesWithRelativePaths.add(new ParquetFileMetadata_v3( relativize(baseDir, file.getPath()), file.length, file.rowGroups)); } - return new ParquetTableMetadata_v3(V3_1, tableMetadataWithAbsolutePaths, filesWithRelativePaths, - directoriesWithRelativePaths, DrillVersionInfo.getVersion()); + return new ParquetTableMetadata_v3(SUPPORTED_VERSIONS.last().toString(), tableMetadataWithAbsolutePaths, + filesWithRelativePaths, directoriesWithRelativePaths, DrillVersionInfo.getVersion()); } /** http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java index bc6fd70..5ceadcd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/MetadataVersion.java @@ -134,6 +134,13 @@ public class MetadataVersion implements Comparable<MetadataVersion> { public static final String V3_1 = "3.1"; /** + * Version 3.2: An array with the components of the field name in + * {@link Metadata.ColumnTypeMetadata_v3.Key} class is replaced by the SchemaPath.<br> + * See DRILL-4264 + */ + public static final String V3_2 = "3.2"; + + /** * All historical versions of the Drill metadata cache files. In case of introducing a new parquet metadata version * please follow the {@link MetadataVersion#FORMAT}. */ @@ -141,7 +148,8 @@ public class MetadataVersion implements Comparable<MetadataVersion> { new MetadataVersion(V1), new MetadataVersion(V2), new MetadataVersion(V3), - new MetadataVersion(V3_1) + new MetadataVersion(V3_1), + new MetadataVersion(V3_2) ); /** http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java index 1d4d161..9b84e81 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordWriter.java @@ -199,7 +199,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter { private void newSchema() throws IOException { List<Type> types = Lists.newArrayList(); for (MaterializedField field : batchSchema) { - if (field.getPath().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) { + if (field.getName().equalsIgnoreCase(WriterPrel.PARTITION_COMPARATOR_FIELD)) { continue; } types.add(getType(field)); @@ -230,7 +230,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter { private PrimitiveType getPrimitiveType(MaterializedField field) { MinorType minorType = field.getType().getMinorType(); - String name = field.getLastName(); + String name = field.getName(); PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType); Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode()); OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType); @@ -248,7 +248,7 @@ public class ParquetRecordWriter extends ParquetOutputRecordWriter { for (MaterializedField childField : field.getChildren()) { types.add(getType(childField)); } - return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getLastName(), types); + return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getName(), types); case LIST: throw new UnsupportedOperationException("Unsupported type " + minorType); default: http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java index bbdf246..d49a416 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetColumnMetadata.java @@ -59,12 +59,12 @@ public class ParquetColumnMetadata { se = schemaElements.get(column.getPath()[0]); type = ParquetToDrillTypeConverter.toMajorType(column.getType(), se.getType_length(), getDataMode(column), se, options); - field = MaterializedField.create(toFieldName(column.getPath()), type); + field = MaterializedField.create(toFieldName(column.getPath()).getLastSegment().getNameSegment().getPath(), type); length = getDataTypeLength(); } - private String toFieldName(String[] paths) { - return SchemaPath.getCompoundPath(paths).getAsUnescapedPath(); + private SchemaPath toFieldName(String[] paths) { + return SchemaPath.getCompoundPath(paths); } private TypeProtos.DataMode getDataMode(ColumnDescriptor column) { http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java index ab4b1b8..9814b53 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetSchema.java @@ -108,7 +108,6 @@ public class ParquetSchema { * Build the schema for this read as a combination of the schema specified in * the Parquet footer and the list of columns selected in the query. * - * @param footer Parquet metadata * @param batchSize target size of the batch, in rows * @throws Exception if anything goes wrong */ @@ -188,7 +187,7 @@ public class ParquetSchema { /** * Determine if a Parquet field is selected for the query. It is selected * either if this is a star query (we want all columns), or the column - * appers in the select list. + * appears in the select list. * * @param field the Parquet column expressed as as Drill field. * @return true if the column is to be included in the scan, false @@ -205,7 +204,7 @@ public class ParquetSchema { int i = 0; for (SchemaPath expr : selectedCols) { - if ( field.getPath().equalsIgnoreCase(expr.getAsUnescapedPath())) { + if (field.getName().equalsIgnoreCase(expr.getRootSegmentPath())) { columnsFound[i] = true; return true; } @@ -243,8 +242,9 @@ public class ParquetSchema { */ private NullableIntVector createMissingColumn(SchemaPath col, OutputMutator output) throws SchemaChangeException { - MaterializedField field = MaterializedField.create(col.getAsUnescapedPath(), - Types.optional(TypeProtos.MinorType.INT)); + // col.toExpr() is used here as field name since we don't want to see these fields in the existing maps + MaterializedField field = MaterializedField.create(col.toExpr(), + Types.optional(TypeProtos.MinorType.INT)); return (NullableIntVector) output.addField(field, TypeHelper.getValueVectorClass(TypeProtos.MinorType.INT, DataMode.OPTIONAL)); } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java index 68d3bbb..9e66f6d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -208,27 +208,28 @@ public class DrillParquetReader extends AbstractRecordReader { try { this.operatorContext = context; schema = footer.getFileMetaData().getSchema(); - MessageType projection = null; + MessageType projection; if (isStarQuery()) { projection = schema; } else { - columnsNotFound=new ArrayList<SchemaPath>(); + columnsNotFound = new ArrayList<>(); projection = getProjection(schema, getColumns(), columnsNotFound); - if(projection == null){ + if (projection == null) { projection = schema; } - if(columnsNotFound!=null && columnsNotFound.size()>0) { + if (columnsNotFound != null && columnsNotFound.size() > 0) { nullFilledVectors = new ArrayList<>(); - for(SchemaPath col: columnsNotFound){ + for (SchemaPath col: columnsNotFound) { + // col.toExpr() is used here as field name since we don't want to see these fields in the existing maps nullFilledVectors.add( - (NullableIntVector)output.addField(MaterializedField.create(col.getAsUnescapedPath(), + (NullableIntVector) output.addField(MaterializedField.create(col.toExpr(), org.apache.drill.common.types.Types.optional(TypeProtos.MinorType.INT)), (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(TypeProtos.MinorType.INT, TypeProtos.DataMode.OPTIONAL))); } - if(columnsNotFound.size()==getColumns().size()){ - noColumnsFound=true; + if (columnsNotFound.size() == getColumns().size()) { + noColumnsFound = true; } } } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java index cb5183e..7e32a4d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -34,21 +34,18 @@ import com.google.common.collect.Lists; public class BatchPrinter { public static void printHyperBatch(VectorAccessible batch, SelectionVector4 sv4) { List<String> columns = Lists.newArrayList(); - List<ValueVector> vectors = Lists.newArrayList(); - int numBatches = 0; for (VectorWrapper vw : batch) { - columns.add(vw.getValueVectors()[0].getField().getPath()); - numBatches = vw.getValueVectors().length; + columns.add(vw.getValueVectors()[0].getField().getName()); } int width = columns.size(); for (int j = 0; j < sv4.getCount(); j++) { - if (j%50 == 0) { + if (j % 50 == 0) { System.out.println(StringUtils.repeat("-", width * 17 + 1)); for (String column : columns) { System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14)); } System.out.printf("|\n"); - System.out.println(StringUtils.repeat("-", width*17 + 1)); + System.out.println(StringUtils.repeat("-", width * 17 + 1)); } for (VectorWrapper vw : batch) { Object o = vw.getValueVectors()[sv4.get(j) >>> 16].getAccessor().getObject(sv4.get(j) & 65535); @@ -60,7 +57,7 @@ public class BatchPrinter { } else { value = o.toString(); } - System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14)); + System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14)); } System.out.printf("|\n"); } @@ -71,32 +68,31 @@ public class BatchPrinter { List<String> columns = Lists.newArrayList(); List<ValueVector> vectors = Lists.newArrayList(); for (VectorWrapper vw : batch) { - columns.add(vw.getValueVector().getField().getPath()); + columns.add(vw.getValueVector().getField().getName()); vectors.add(vw.getValueVector()); } int width = columns.size(); int rows = vectors.get(0).getMetadata().getValueCount(); for (int row = 0; row < rows; row++) { - if (row%50 == 0) { + if (row % 50 == 0) { System.out.println(StringUtils.repeat("-", width * 17 + 1)); for (String column : columns) { System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14)); } System.out.printf("|\n"); - System.out.println(StringUtils.repeat("-", width*17 + 1)); + System.out.println(StringUtils.repeat("-", width * 17 + 1)); } for (ValueVector vv : vectors) { Object o = vv.getAccessor().getObject(row); String value; if (o == null) { value = "null"; - } else - if (o instanceof byte[]) { + } else if (o instanceof byte[]) { value = new String((byte[]) o); } else { value = o.toString(); } - System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14)); + System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14)); } System.out.printf("|\n"); } @@ -106,19 +102,19 @@ public class BatchPrinter { List<String> columns = Lists.newArrayList(); List<ValueVector> vectors = Lists.newArrayList(); for (VectorWrapper vw : batch) { - columns.add(vw.getValueVector().getField().getPath()); + columns.add(vw.getValueVector().getField().getName()); vectors.add(vw.getValueVector()); } int width = columns.size(); int rows = vectors.get(0).getMetadata().getValueCount(); for (int i = 0; i < rows; i++) { - if (i%50 == 0) { + if (i % 50 == 0) { System.out.println(StringUtils.repeat("-", width * 17 + 1)); for (String column : columns) { System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14)); } System.out.printf("|\n"); - System.out.println(StringUtils.repeat("-", width*17 + 1)); + System.out.println(StringUtils.repeat("-", width * 17 + 1)); } int row = sv2.getIndex(i); for (ValueVector vv : vectors) { @@ -126,13 +122,12 @@ public class BatchPrinter { String value; if (o == null) { value = "null"; - } else - if (o instanceof byte[]) { + } else if (o instanceof byte[]) { value = new String((byte[]) o); } else { value = o.toString(); } - System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14)); + System.out.printf("| %-15s", value.length() <= 15 ? value : value.substring(0, 14)); } System.out.printf("|\n"); } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java index b150179..d836bfc 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -40,12 +40,12 @@ public class VectorUtil { System.out.println(rows + " row(s):"); List<String> columns = Lists.newArrayList(); for (VectorWrapper<?> vw : va) { - columns.add(vw.getValueVector().getField().getPath()); + columns.add(vw.getValueVector().getField().getName()); } int width = columns.size(); for (String column : columns) { - System.out.printf("%s%s",column, column == columns.get(width - 1) ? "\n" : delimiter); + System.out.printf("%s%s",column, column.equals(columns.get(width - 1)) ? "\n" : delimiter); } for (int row = 0; row < rows; row++) { int columnCounter = 0; @@ -54,8 +54,8 @@ public class VectorUtil { Object o ; try{ o = vw.getValueVector().getAccessor().getObject(row); - }catch(Exception e){ - throw new RuntimeException("failure while trying to read column " + vw.getField().getPath()); + } catch (Exception e) { + throw new RuntimeException("failure while trying to read column " + vw.getField().getName()); } if (o == null) { //null value @@ -83,7 +83,7 @@ public class VectorUtil { if (includeHeader) { List<String> columns = Lists.newArrayList(); for (VectorWrapper<?> vw : va) { - columns.add(vw.getValueVector().getField().getPath()); + columns.add(vw.getValueVector().getField().getName()); } formattedResults.append(Joiner.on(delimiter).join(columns)); @@ -134,7 +134,7 @@ public class VectorUtil { width += columnWidth + 2; formats.add("| %-" + columnWidth + "s"); MaterializedField field = vw.getValueVector().getField(); - columns.add(field.getPath() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">"); + columns.add(field.getName() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">"); columnIndex++; }