This is an automated email from the ASF dual-hosted git repository. progers pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit f5f557027cb6c809e46ac100e1013924d158581d Author: Paul Rogers <par0...@gmail.com> AuthorDate: Sun Jul 17 18:34:59 2022 -0700 Another wrong path --- .../accessor/writer/DummyUnionVectorShim.java | 98 ++++++++++++++ .../vector/accessor/writer/DummyUnionWriter.java | 140 ------------------- .../vector/accessor/writer/SimpleListShim.java | 4 +- .../exec/vector/accessor/writer/UnionShim.java | 91 +++++++++++++ .../vector/accessor/writer/UnionVectorShim.java | 38 +----- .../exec/vector/accessor/writer/UnionWriter.java | 149 --------------------- .../vector/accessor/writer/UnionWriterImpl.java | 124 ++++++++++++++++- 7 files changed, 316 insertions(+), 328 deletions(-) diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/DummyUnionVectorShim.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/DummyUnionVectorShim.java new file mode 100644 index 0000000000..b75b938c25 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/DummyUnionVectorShim.java @@ -0,0 +1,98 @@ +package org.apache.drill.exec.vector.accessor.writer; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; +import org.apache.drill.exec.vector.accessor.ObjectWriter; +import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; + +/** + * Shim for a non-existent (unprojected) union vector. + */ +public class DummyUnionVectorShim implements UnionShim { + + @Override + public void bindIndex(ColumnWriterIndex index) { } + + @Override + public void bindListener(ColumnWriterListener listener) { } + + @Override + public void startWrite() { } + + @Override + public void startRow() { } + + @Override + public void endArrayValue() { } + + @Override + public void restartRow() { } + + @Override + public void saveRow() { } + + @Override + public void endWrite() { } + + @Override + public void preRollover() { } + + @Override + public void postRollover() { } + + @Override + public void dump(HierarchicalFormatter format) { } + + @Override + public int writeIndex() { return 0; } + + @Override + public void bindWriter(UnionWriterImpl writer) { } + + @Override + public void setNull() { } + + @Override + public boolean hasType(MinorType type) { + // TODO Auto-generated method stub + return false; + } + + @Override + public ObjectWriter member(MinorType type) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setType(MinorType type) { + // TODO Auto-generated method stub + } + + @Override + public int lastWriteIndex() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public int rowStartIndex() { return 0; } + + @Override + public AbstractObjectWriter addMember(ColumnMetadata colSchema) { + // TODO Auto-generated method stub + return null; + } + + @Override + public AbstractObjectWriter addMember(MinorType type) { + // TODO Auto-generated method stub + return null; + } + + @Override + public void addMember(AbstractObjectWriter colWriter) { + // TODO Auto-generated method stub + } +} diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/DummyUnionWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/DummyUnionWriter.java deleted file mode 100644 index fd72d54006..0000000000 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/DummyUnionWriter.java +++ /dev/null @@ -1,140 +0,0 @@ -package org.apache.drill.exec.vector.accessor.writer; - -import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.vector.accessor.ArrayWriter; -import org.apache.drill.exec.vector.accessor.ColumnReader; -import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; -import org.apache.drill.exec.vector.accessor.ObjectWriter; -import org.apache.drill.exec.vector.accessor.ScalarWriter; -import org.apache.drill.exec.vector.accessor.TupleWriter; -import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; - -public class DummyUnionWriter extends UnionWriter { - - @Override - public ObjectWriter member(MinorType type) { - // TODO Auto-generated method stub - return null; - } - - @Override - public ScalarWriter scalar(MinorType type) { - // TODO Auto-generated method stub - return null; - } - - @Override - public TupleWriter tuple() { - // TODO Auto-generated method stub - return null; - } - - @Override - public ArrayWriter array() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isProjected() { - // TODO Auto-generated method stub - return false; - } - - @Override - public void copy(ColumnReader from) { - // TODO Auto-generated method stub - - } - - @Override - public void setObject(Object value) { - // TODO Auto-generated method stub - - } - - @Override - public void bindIndex(ColumnWriterIndex index) { - // TODO Auto-generated method stub - - } - - @Override - public void startWrite() { - // TODO Auto-generated method stub - - } - - @Override - public void startRow() { - // TODO Auto-generated method stub - - } - - @Override - public void endArrayValue() { - // TODO Auto-generated method stub - - } - - @Override - public void restartRow() { - // TODO Auto-generated method stub - - } - - @Override - public void saveRow() { - // TODO Auto-generated method stub - - } - - @Override - public void endWrite() { - // TODO Auto-generated method stub - - } - - @Override - public void preRollover() { - // TODO Auto-generated method stub - - } - - @Override - public void postRollover() { - // TODO Auto-generated method stub - - } - - @Override - public void dump(HierarchicalFormatter format) { - // TODO Auto-generated method stub - - } - - @Override - public int rowStartIndex() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public int lastWriteIndex() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public int writeIndex() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public void bindShim(UnionShim shim) { - // TODO Auto-generated method stub - - } - -} diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java index 23daef41a1..4721947f12 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/SimpleListShim.java @@ -35,7 +35,7 @@ import com.google.common.base.Preconditions; */ public class SimpleListShim implements UnionShim { - private UnionWriter writer; + private UnionWriterImpl writer; private AbstractObjectWriter colWriter; public SimpleListShim() { } @@ -45,7 +45,7 @@ public class SimpleListShim implements UnionShim { } @Override - public void bindWriter(UnionWriter writer) { + public void bindWriter(UnionWriterImpl writer) { this.writer = writer; } diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionShim.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionShim.java new file mode 100644 index 0000000000..4bfafa9ae4 --- /dev/null +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionShim.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.vector.accessor.writer; + +import org.apache.drill.common.types.TypeProtos.MinorType; +import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.vector.accessor.ObjectWriter; +import org.apache.drill.exec.vector.complex.UnionVector; + +/** + * Unions are overly complex. They can evolve from no type, to a single type, + * to multiple types. The kind of vector used in these cases differ. This + * shim acts as a facade between the writer and the underlying vector, allowing + * the writer to remain constant while the vector (and its shim) evolves. + */ +public interface UnionShim extends WriterEvents { + void bindWriter(UnionWriterImpl writer); + void setNull(); + boolean hasType(MinorType type); + + /** + * Return an existing writer for the given type, or create a new one + * if needed. + * + * @param type desired variant type + * @return a writer for that type + */ + ObjectWriter member(MinorType type); + void setType(MinorType type); + @Override + int lastWriteIndex(); + @Override + int rowStartIndex(); + AbstractObjectWriter addMember(ColumnMetadata colSchema); + AbstractObjectWriter addMember(MinorType type); + void addMember(AbstractObjectWriter colWriter); + + public abstract class AbstractUnionShim implements UnionShim { + + protected final AbstractObjectWriter variants[]; + + public AbstractUnionShim() { + variants = new AbstractObjectWriter[MinorType.values().length]; + } + + public AbstractUnionShim(AbstractObjectWriter variants[]) { + if (variants == null) { + this.variants = new AbstractObjectWriter[MinorType.values().length]; + } else { + this.variants = variants; + } + } + + @Override + public boolean hasType(MinorType type) { + return variants[type.ordinal()] != null; + } + + /** + * Performs just the work of adding a vector to the list of existing + * variants. Called when adding a type via the writer, but also when + * the result set loader promotes a list from single type to a union, + * and provides this shim with the writer from the single-list shim. + * In the latter case, the writer is already initialized and is already + * part of the metadata for this list; so we don't want to call the + * list's {@code addMember()} and repeat those operations. + * + * @param colWriter the column (type) writer to add + */ + public void addMemberWriter(AbstractObjectWriter colWriter) { + final MinorType type = colWriter.schema().type(); + assert variants[type.ordinal()] == null; + variants[type.ordinal()] = colWriter; + } + } +} \ No newline at end of file diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java index 566b7d46d0..c9133d207b 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionVectorShim.java @@ -25,7 +25,7 @@ import org.apache.drill.exec.vector.accessor.ObjectWriter; import org.apache.drill.exec.vector.accessor.VariantWriter.VariantWriterListener; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.accessor.writer.AbstractFixedWidthWriter.BaseFixedWidthWriter; -import org.apache.drill.exec.vector.accessor.writer.UnionWriter.UnionShim; +import org.apache.drill.exec.vector.accessor.writer.UnionShim.AbstractUnionShim; import org.apache.drill.exec.vector.complex.UnionVector; /** @@ -37,7 +37,7 @@ import org.apache.drill.exec.vector.complex.UnionVector; * list itself evolves from no type, to a single type and to * a union. */ -public class UnionVectorShim implements UnionShim { +public class UnionVectorShim extends AbstractUnionShim { static class DefaultListener implements VariantWriterListener { @@ -67,8 +67,7 @@ public class UnionVectorShim implements UnionShim { } private final UnionVector vector; - private final AbstractObjectWriter variants[]; - private UnionWriter writer; + private UnionWriterImpl writer; /** * Writer for the type vector associated with the union. The type vector @@ -80,22 +79,17 @@ public class UnionVectorShim implements UnionShim { public UnionVectorShim(UnionVector vector) { this.vector = vector; typeWriter = ColumnWriterFactory.newWriter(vector.getTypeVector()); - variants = new AbstractObjectWriter[MinorType.values().length]; } public UnionVectorShim(UnionVector vector, AbstractObjectWriter variants[]) { + super(variants); this.vector = vector; typeWriter = ColumnWriterFactory.newWriter(vector.getTypeVector()); - if (variants == null) { - this.variants = new AbstractObjectWriter[MinorType.values().length]; - } else { - this.variants = variants; - } } @Override - public void bindWriter(UnionWriter writer) { + public void bindWriter(UnionWriterImpl writer) { this.writer = writer; final ColumnWriterIndex index = writer.index(); if (index != null) { @@ -127,11 +121,6 @@ public class UnionVectorShim implements UnionShim { typeWriter.setInt(UnionVector.NULL_MARKER); } - @Override - public boolean hasType(MinorType type) { - return variants[type.ordinal()] != null; - } - @Override public ObjectWriter member(MinorType type) { final AbstractObjectWriter colWriter = variants[type.ordinal()]; @@ -176,23 +165,6 @@ public class UnionVectorShim implements UnionShim { writer.addMember(colWriter); } - /** - * Performs just the work of adding a vector to the list of existing - * variants. Called when adding a type via the writer, but also when - * the result set loader promotes a list from single type to a union, - * and provides this shim with the writer from the single-list shim. - * In the latter case, the writer is already initialized and is already - * part of the metadata for this list; so we don't want to call the - * list's {@code addMember()} and repeat those operations. - * - * @param colWriter the column (type) writer to add - */ - public void addMemberWriter(AbstractObjectWriter colWriter) { - final MinorType type = colWriter.schema().type(); - assert variants[type.ordinal()] == null; - variants[type.ordinal()] = colWriter; - } - @Override public void startWrite() { typeWriter.startWrite(); diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriter.java deleted file mode 100644 index 859f30f8a7..0000000000 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriter.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.vector.accessor.writer; - -import org.apache.drill.common.types.TypeProtos.MinorType; -import org.apache.drill.exec.record.metadata.ColumnMetadata; -import org.apache.drill.exec.record.metadata.VariantMetadata; -import org.apache.drill.exec.vector.accessor.ColumnWriter; -import org.apache.drill.exec.vector.accessor.ObjectType; -import org.apache.drill.exec.vector.accessor.ObjectWriter; -import org.apache.drill.exec.vector.accessor.VariantWriter; -import org.apache.drill.exec.vector.accessor.VariantWriter.VariantWriterListener; -import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; -import org.apache.drill.exec.vector.accessor.writer.UnionWriter.UnionShim; -import org.apache.drill.exec.vector.accessor.writer.WriterEvents.ColumnWriterListener; - -/** - * Writer to a union vector. - */ -public abstract class UnionWriter implements VariantWriter, WriterEvents { - - public interface UnionShim extends WriterEvents { - void bindWriter(UnionWriter writer); - void setNull(); - boolean hasType(MinorType type); - - /** - * Return an existing writer for the given type, or create a new one - * if needed. - * - * @param type desired variant type - * @return a writer for that type - */ - ObjectWriter member(MinorType type); - void setType(MinorType type); - @Override - int lastWriteIndex(); - @Override - int rowStartIndex(); - AbstractObjectWriter addMember(ColumnMetadata colSchema); - AbstractObjectWriter addMember(MinorType type); - void addMember(AbstractObjectWriter colWriter); - } - - public static class VariantObjectWriter extends AbstractObjectWriter { - - private final UnionWriterImpl writer; - - public VariantObjectWriter(UnionWriterImpl writer) { - this.writer = writer; - } - - @Override - public ColumnWriter writer() { return writer; } - - @Override - public VariantWriter variant() { return writer; } - - @Override - public WriterEvents events() { return writer; } - - @Override - public void dump(HierarchicalFormatter format) { - writer.dump(format); - } - } - - private final ColumnMetadata schema; - protected UnionShim shim; - private VariantWriterListener listener; - - public UnionWriter(ColumnMetadata schema) { - this.schema = schema; - } - - public abstract void bindShim(UnionShim shim); - public VariantWriterListener listener() { return listener; } - public UnionShim shim() { return shim; } - - public void bindListener(VariantWriterListener listener) { - this.listener = listener; - } - - // Unions are complex: listeners should bind to the components as they - // are created. - - @Override - public void bindListener(ColumnWriterListener listener) { } - - @Override - public ColumnMetadata schema() { return schema; } - - @Override - public VariantMetadata variantSchema() { return schema.variantSchema(); } - - @Override - public int size() { return variantSchema().size(); } - - @Override - public ObjectType type() { return ObjectType.VARIANT; } - - @Override - public boolean nullable() { return true; } - - @Override - public boolean hasType(MinorType type) { - return shim.hasType(type); - } - - @Override - public void setNull() { - shim.setNull(); - } - - @Override - public ObjectWriter memberWriter(MinorType type) { - return shim.member(type); - } - - @Override - public void setType(MinorType type) { - shim.setType(type); - } - - @Override - public ObjectWriter addMember(ColumnMetadata colSchema) { - return shim.addMember(colSchema); - } - - @Override - public ObjectWriter addMember(MinorType type) { - return shim.addMember(type); - } -} diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java index b6967c8a3f..7b5ef17804 100644 --- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java +++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/UnionWriterImpl.java @@ -1,25 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.drill.exec.vector.accessor.writer; import java.math.BigDecimal; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.exec.record.metadata.ColumnMetadata; +import org.apache.drill.exec.record.metadata.VariantMetadata; import org.apache.drill.exec.vector.accessor.ArrayWriter; import org.apache.drill.exec.vector.accessor.ColumnReader; +import org.apache.drill.exec.vector.accessor.ColumnWriter; import org.apache.drill.exec.vector.accessor.ColumnWriterIndex; +import org.apache.drill.exec.vector.accessor.ObjectType; import org.apache.drill.exec.vector.accessor.ObjectWriter; import org.apache.drill.exec.vector.accessor.ScalarWriter; import org.apache.drill.exec.vector.accessor.TupleWriter; import org.apache.drill.exec.vector.accessor.VariantReader; +import org.apache.drill.exec.vector.accessor.VariantWriter; import org.apache.drill.exec.vector.accessor.WriterPosition; import org.apache.drill.exec.vector.accessor.impl.HierarchicalFormatter; import org.apache.drill.exec.vector.complex.UnionVector; import org.joda.time.Period; /** - * Writer to a materialized union. + * Writer to a union vector. + * <p> + * A union vector has three attributes: null flag, type and value. + * The union vector holds the type: a bundle of other vectors hold + * the value. The type says which of the other vectors to consult + * to write the value. If a column is null, then we consult no other + * vectors. If all columns (thus far) are null, then there are no + * associated data vectors. + * <p> + * The protocol is to first set the type. Doing so creates the + * associated data vector, if it does not yet exist. This highlights the + * poor design of this vector: if we have even one value of a given type, + * we must have a vector that holds values for all rows, then we ignore + * the unwanted values. */ -public class UnionWriterImpl extends UnionWriter { +public class UnionWriterImpl implements VariantWriter, WriterEvents { + + public static class VariantObjectWriter extends AbstractObjectWriter { + + private final UnionWriterImpl writer; + + public VariantObjectWriter(UnionWriterImpl writer) { + this.writer = writer; + } + + @Override + public ColumnWriter writer() { return writer; } + + @Override + public VariantWriter variant() { return writer; } + + @Override + public WriterEvents events() { return writer; } + + @Override + public void dump(HierarchicalFormatter format) { + writer.dump(format); + } + } /** * The result set loader requires information about the child positions @@ -47,12 +104,15 @@ public class UnionWriterImpl extends UnionWriter { } } + private final ColumnMetadata schema; + private UnionShim shim; private ColumnWriterIndex index; private State state = State.IDLE; + private VariantWriterListener listener; private final WriterPosition elementPosition = new ElementPositions(); public UnionWriterImpl(ColumnMetadata schema) { - super(schema); + this.schema = schema; } public UnionWriterImpl(ColumnMetadata schema, UnionVector vector, @@ -67,13 +127,24 @@ public class UnionWriterImpl extends UnionWriter { shim.bindIndex(index); } + public void bindListener(VariantWriterListener listener) { + this.listener = listener; + } + + // Unions are complex: listeners should bind to the components as they + // are created. + + @Override + public void bindListener(ColumnWriterListener listener) { } + // The following are for coordinating with the shim. public State state() { return state; } public ColumnWriterIndex index() { return index; } + public VariantWriterListener listener() { return listener; } + public UnionShim shim() { return shim; } public WriterPosition elementPosition() { return elementPosition; } - @Override public void bindShim(UnionShim shim) { this.shim = shim; shim.bindWriter(this); @@ -85,6 +156,36 @@ public class UnionWriterImpl extends UnionWriter { } } + @Override + public ObjectType type() { return ObjectType.VARIANT; } + + @Override + public boolean nullable() { return true; } + + @Override + public ColumnMetadata schema() { return schema; } + + @Override + public VariantMetadata variantSchema() { return schema.variantSchema(); } + + @Override + public int size() { return variantSchema().size(); } + + @Override + public boolean hasType(MinorType type) { + return shim.hasType(type); + } + + @Override + public void setNull() { + shim.setNull(); + } + + @Override + public ObjectWriter memberWriter(MinorType type) { + return shim.member(type); + } + @Override public ObjectWriter member(MinorType type) { @@ -97,6 +198,21 @@ public class UnionWriterImpl extends UnionWriter { return writer; } + @Override + public void setType(MinorType type) { + shim.setType(type); + } + + @Override + public ObjectWriter addMember(ColumnMetadata colSchema) { + return shim.addMember(colSchema); + } + + @Override + public ObjectWriter addMember(MinorType type) { + return shim.addMember(type); + } + /** * Add a column writer to an existing union writer. Used for implementations * that support "live" schema evolution: column discovery while writing.