http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
deleted file mode 100644
index f32c101..0000000
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseElementReader.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.accessor.reader;
-
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
-import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
-import org.apache.drill.exec.vector.accessor.impl.AccessorUtilities;
-import org.joda.time.Period;
-
-public abstract class BaseElementReader implements ScalarElementReader {
-
-  public static class ScalarElementObjectReader extends AbstractObjectReader {
-
-    private BaseElementReader elementReader;
-
-    public ScalarElementObjectReader(BaseElementReader elementReader) {
-      this.elementReader = elementReader;
-    }
-
-    @Override
-    public void bindIndex(ColumnReaderIndex index) {
-      elementReader.bindIndex((ElementReaderIndex) index);
-    }
-
-    @Override
-    public ObjectType type() {
-      return ObjectType.SCALAR;
-    }
-
-    @Override
-    public ScalarElementReader elements() {
-      return elementReader;
-    }
-
-    @Override
-    public Object getObject() {
-      // Simple: return elements as an object list.
-      // If really needed, could return as a typed array, but that
-      // is a bit of a hassle.
-
-      List<Object> elements = new ArrayList<>();
-      for (int i = 0; i < elementReader.size(); i++) {
-        elements.add(elementReader.getObject(i));
-      }
-      return elements;
-    }
-
-    @Override
-    public String getAsString() {
-      StringBuilder buf = new StringBuilder();
-      buf.append("[");
-      for (int i = 0; i < elementReader.size(); i++) {
-        if (i > 0) {
-          buf.append( ", " );
-        }
-        buf.append(elementReader.getAsString(i));
-      }
-      buf.append("]");
-      return buf.toString();
-    }
-  }
-
-  protected ElementReaderIndex vectorIndex;
-  protected VectorAccessor vectorAccessor;
-
-  public abstract void bindVector(ValueVector vector);
-
-  public void bindVector(MajorType majorType, VectorAccessor va) {
-    vectorAccessor = va;
-  }
-
-  protected void bindIndex(ElementReaderIndex rowIndex) {
-    this.vectorIndex = rowIndex;
-  }
-
-  @Override
-  public int size() { return vectorIndex.size(); }
-
-  @Override
-  public Object getObject(int index) {
-    if (isNull(index)) {
-      return "null";
-    }
-    switch (valueType()) {
-    case BYTES:
-      return getBytes(index);
-    case DECIMAL:
-      return getDecimal(index);
-    case DOUBLE:
-      return getDouble(index);
-    case INTEGER:
-      return getInt(index);
-    case LONG:
-      return getLong(index);
-    case PERIOD:
-      return getPeriod(index);
-    case STRING:
-      return getString(index);
-    default:
-      throw new IllegalStateException("Unexpected type: " + valueType());
-    }
-  }
-
-  @Override
-  public String getAsString(int index) {
-    switch (valueType()) {
-    case BYTES:
-      return AccessorUtilities.bytesToString(getBytes(index));
-    case DOUBLE:
-      return Double.toString(getDouble(index));
-    case INTEGER:
-      return Integer.toString(getInt(index));
-    case LONG:
-      return Long.toString(getLong(index));
-    case STRING:
-      return "\"" + getString(index) + "\"";
-    case DECIMAL:
-      return getDecimal(index).toPlainString();
-    case PERIOD:
-      return getPeriod(index).normalizedStandard().toString();
-    default:
-      throw new IllegalArgumentException("Unsupported type " + valueType());
-    }
-  }
-
-  @Override
-  public boolean isNull(int index) {
-    return false;
-  }
-
-  @Override
-  public int getInt(int index) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public long getLong(int index) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public double getDouble(int index) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public String getString(int index) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public byte[] getBytes(int index) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public BigDecimal getDecimal(int index) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public Period getPeriod(int index) {
-    throw new UnsupportedOperationException();
-  }
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseScalarReader.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseScalarReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseScalarReader.java
index fb9a711..279fb58 100644
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseScalarReader.java
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/BaseScalarReader.java
@@ -17,15 +17,11 @@
  */
 package org.apache.drill.exec.vector.accessor.reader;
 
-import java.math.BigDecimal;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.vector.BaseDataValueVector;
 import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
-import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.accessor.ScalarReader;
-import org.apache.drill.exec.vector.accessor.impl.AccessorUtilities;
-import org.joda.time.Period;
+
+import io.netty.buffer.DrillBuf;
 
 /**
  * Column reader implementation that acts as the basis for the
@@ -34,156 +30,122 @@ import org.joda.time.Period;
  * method(s).
  */
 
-public abstract class BaseScalarReader implements ScalarReader {
+public abstract class BaseScalarReader extends AbstractScalarReader {
+
+  public abstract static class BaseFixedWidthReader extends BaseScalarReader {
 
-  public static class ScalarObjectReader extends AbstractObjectReader {
+    public abstract int width();
+  }
 
-    private BaseScalarReader scalarReader;
+  public abstract static class BaseVarWidthReader extends BaseScalarReader {
 
-    public ScalarObjectReader(BaseScalarReader scalarReader) {
-      this.scalarReader = scalarReader;
-    }
+    protected OffsetVectorReader offsetsReader;
 
     @Override
-    public void bindIndex(ColumnReaderIndex index) {
-      scalarReader.bindIndex(index);
+    public void bindVector(ColumnMetadata schema, VectorAccessor va) {
+      super.bindVector(schema, va);
+      offsetsReader = new OffsetVectorReader(
+          VectorAccessors.varWidthOffsetVectorAccessor(va));
     }
 
     @Override
-    public ObjectType type() {
-      return ObjectType.SCALAR;
+    public void bindIndex(ColumnReaderIndex index) {
+      super.bindIndex(index);
+      offsetsReader.bindIndex(index);
     }
+  }
 
-    @Override
-    public ScalarReader scalar() {
-      return scalarReader;
+  /**
+   * Provide access to the DrillBuf for the data vector.
+   */
+
+  public interface BufferAccessor {
+    DrillBuf buffer();
+  }
+
+  private static class SingleVectorBufferAccessor implements BufferAccessor {
+    private final DrillBuf buffer;
+
+    public SingleVectorBufferAccessor(VectorAccessor va) {
+      BaseDataValueVector vector = va.vector();
+      buffer = vector.getBuffer();
     }
 
     @Override
-    public Object getObject() {
-      return scalarReader.getObject();
+    public DrillBuf buffer() { return buffer; }
+  }
+
+  private static class HyperVectorBufferAccessor implements BufferAccessor {
+    private final VectorAccessor vectorAccessor;
+
+    public HyperVectorBufferAccessor(VectorAccessor va) {
+      vectorAccessor = va;
     }
 
     @Override
-    public String getAsString() {
-      return scalarReader.getAsString();
+    public DrillBuf buffer() {
+      BaseDataValueVector vector = vectorAccessor.vector();
+      return vector.getBuffer();
     }
   }
 
-  protected ColumnReaderIndex vectorIndex;
+  protected ColumnMetadata schema;
   protected VectorAccessor vectorAccessor;
+  protected BufferAccessor bufferAccessor;
 
-  public static ScalarObjectReader build(ValueVector vector, BaseScalarReader 
reader) {
-    reader.bindVector(vector);
-    return new ScalarObjectReader(reader);
-  }
+  public static ScalarObjectReader buildOptional(ColumnMetadata schema,
+      VectorAccessor va, BaseScalarReader reader) {
 
-  public static AbstractObjectReader build(MajorType majorType, VectorAccessor 
va,
-                                           BaseScalarReader reader) {
-    reader.bindVector(majorType, va);
-    return new ScalarObjectReader(reader);
-  }
+    // Reader is bound to the values vector inside the nullable vector.
 
-  public abstract void bindVector(ValueVector vector);
+    reader.bindVector(schema, VectorAccessors.nullableValuesAccessor(va));
 
-  protected void bindIndex(ColumnReaderIndex rowIndex) {
-    this.vectorIndex = rowIndex;
-    if (vectorAccessor != null) {
-      vectorAccessor.bind(rowIndex);
-    }
-  }
+    // The nullability of each value depends on the "bits" vector
+    // in the nullable vector.
 
-  public void bindVector(MajorType majorType, VectorAccessor va) {
-    vectorAccessor = va;
-  }
+    reader.bindNullState(new 
NullStateReaders.NullableIsSetVectorStateReader(va));
 
-  @Override
-  public Object getObject() {
-    if (isNull()) {
-      return null;
-    }
-    switch (valueType()) {
-    case BYTES:
-      return getBytes();
-    case DECIMAL:
-      return getDecimal();
-    case DOUBLE:
-      return getDouble();
-    case INTEGER:
-      return getInt();
-    case LONG:
-      return getLong();
-    case PERIOD:
-      return getPeriod();
-    case STRING:
-      return getString();
-    default:
-      throw new IllegalStateException("Unexpected type: " + valueType());
-    }
-  }
+    // Wrap the reader in an object reader.
 
-  @Override
-  public String getAsString() {
-    if (isNull()) {
-      return "null";
-    }
-    switch (valueType()) {
-    case BYTES:
-      return AccessorUtilities.bytesToString(getBytes());
-    case DOUBLE:
-      return Double.toString(getDouble());
-    case INTEGER:
-      return Integer.toString(getInt());
-    case LONG:
-      return Long.toString(getLong());
-    case STRING:
-      return "\"" + getString() + "\"";
-    case DECIMAL:
-      return getDecimal().toPlainString();
-    case PERIOD:
-      return getPeriod().normalizedStandard().toString();
-    default:
-      throw new IllegalArgumentException("Unsupported type " + valueType());
-    }
+    return new ScalarObjectReader(reader);
   }
 
-  @Override
-  public boolean isNull() {
-    return false;
-  }
+  public static ScalarObjectReader buildRequired(ColumnMetadata schema,
+      VectorAccessor va, BaseScalarReader reader) {
 
-  @Override
-  public int getInt() {
-    throw new UnsupportedOperationException();
-  }
+    // Reader is bound directly to the required vector.
 
-  @Override
-  public long getLong() {
-    throw new UnsupportedOperationException();
-  }
+    reader.bindVector(schema, va);
 
-  @Override
-  public double getDouble() {
-    throw new UnsupportedOperationException();
+    // The reader is required, values can't be null.
+
+    reader.bindNullState(NullStateReaders.REQUIRED_STATE_READER);
+
+    // Wrap the reader in an object reader.
+
+    return new ScalarObjectReader(reader);
   }
 
-  @Override
-  public String getString() {
-    throw new UnsupportedOperationException();
+  public void bindVector(ColumnMetadata schema, VectorAccessor va) {
+    this.schema = schema;
+    vectorAccessor = va;
+    bufferAccessor = bufferAccessor(va);
   }
 
-  @Override
-  public byte[] getBytes() {
-    throw new UnsupportedOperationException();
+  protected BufferAccessor bufferAccessor(VectorAccessor va) {
+    if (va.isHyper()) {
+      return new HyperVectorBufferAccessor(va);
+    } else {
+      return new SingleVectorBufferAccessor(va);
+    }
   }
 
   @Override
-  public BigDecimal getDecimal() {
-    throw new UnsupportedOperationException();
+  public void bindIndex(ColumnReaderIndex rowIndex) {
+    super.bindIndex(rowIndex);
+    vectorAccessor.bind(rowIndex);
   }
 
   @Override
-  public Period getPeriod() {
-    throw new UnsupportedOperationException();
-  }
+  public ColumnMetadata schema() { return schema; }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ColumnReaderFactory.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ColumnReaderFactory.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ColumnReaderFactory.java
index 0bcb6e2..ae15e5d 100644
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ColumnReaderFactory.java
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ColumnReaderFactory.java
@@ -17,12 +17,9 @@
  */
 package org.apache.drill.exec.vector.accessor.reader;
 
-import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.accessor.ColumnAccessors;
-import org.apache.drill.exec.vector.complex.RepeatedValueVector;
+import org.apache.drill.exec.vector.accessor.ColumnAccessorUtils;
 
 /**
  * Gather generated reader classes into a set of class tables to allow rapid
@@ -35,19 +32,14 @@ public class ColumnReaderFactory {
 
   private static final int typeCount = MinorType.values().length;
   private static final Class<? extends BaseScalarReader> requiredReaders[] = 
new Class[typeCount];
-  private static final Class<? extends BaseScalarReader> nullableReaders[] = 
new Class[typeCount];
-  private static final Class<? extends BaseElementReader> elementReaders[] = 
new Class[typeCount];
 
   static {
-    ColumnAccessors.defineRequiredReaders(requiredReaders);
-    ColumnAccessors.defineNullableReaders(nullableReaders);
-    ColumnAccessors.defineArrayReaders(elementReaders);
+    ColumnAccessorUtils.defineRequiredReaders(requiredReaders);
   }
 
-  public static AbstractObjectReader buildColumnReader(ValueVector vector) {
-    MajorType major = vector.getField().getType();
+  public static BaseScalarReader buildColumnReader(VectorAccessor va) {
+    MajorType major = va.type();
     MinorType type = major.getMinorType();
-    DataMode mode = major.getMode();
 
     switch (type) {
     case GENERIC_OBJECT:
@@ -57,41 +49,7 @@ public class ColumnReaderFactory {
     case MAP:
       throw new UnsupportedOperationException(type.toString());
     default:
-      switch (mode) {
-      case OPTIONAL:
-        return BaseScalarReader.build(vector, newAccessor(type, 
nullableReaders));
-      case REQUIRED:
-        return BaseScalarReader.build(vector, newAccessor(type, 
requiredReaders));
-      case REPEATED:
-        return ScalarArrayReader.build((RepeatedValueVector) vector, 
newAccessor(type, elementReaders));
-      default:
-        throw new UnsupportedOperationException(mode.toString());
-      }
-    }
-  }
-
-  public static AbstractObjectReader buildColumnReader(MajorType majorType, 
VectorAccessor va) {
-    MinorType type = majorType.getMinorType();
-    DataMode mode = majorType.getMode();
-
-    switch (type) {
-    case GENERIC_OBJECT:
-    case LATE:
-    case NULL:
-    case LIST:
-    case MAP:
-      throw new UnsupportedOperationException(type.toString());
-    default:
-      switch (mode) {
-      case OPTIONAL:
-        return BaseScalarReader.build(majorType, va, newAccessor(type, 
nullableReaders));
-      case REQUIRED:
-        return BaseScalarReader.build(majorType, va, newAccessor(type, 
requiredReaders));
-      case REPEATED:
-        return ScalarArrayReader.build(majorType, va, newAccessor(type, 
elementReaders));
-      default:
-        throw new UnsupportedOperationException(mode.toString());
-      }
+      return newAccessor(type, requiredReaders);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/FixedWidthElementReaderIndex.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/FixedWidthElementReaderIndex.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/FixedWidthElementReaderIndex.java
deleted file mode 100644
index 4f3aeeb..0000000
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/FixedWidthElementReaderIndex.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.accessor.reader;
-
-import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
-import 
org.apache.drill.exec.vector.accessor.reader.AbstractArrayReader.BaseElementIndex;
-
-/**
- * Index into the vector of elements for a repeated vector.
- * Keeps track of the current offset in terms of value positions.
- */
-
-public class FixedWidthElementReaderIndex extends BaseElementIndex implements 
ElementReaderIndex {
-
-  public FixedWidthElementReaderIndex(ColumnReaderIndex base) {
-    super(base);
-  }
-
-  @Override
-  public int vectorIndex(int posn) {
-    return elementIndex(posn);
-  }
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/MapReader.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/MapReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/MapReader.java
index 900e0a7..7329391 100644
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/MapReader.java
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/MapReader.java
@@ -20,6 +20,8 @@ package org.apache.drill.exec.vector.accessor.reader;
 import java.util.List;
 
 import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
+import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
 
 /**
  * Reader for a Drill Map type. Maps are actually tuples, just like rows.
@@ -27,17 +29,54 @@ import org.apache.drill.exec.record.metadata.ColumnMetadata;
 
 public class MapReader extends AbstractTupleReader {
 
+  protected final ColumnMetadata schema;
+
+  /**
+   * Accessor for the map vector. This class does not use the map vector
+   * directory. However, in the case of a map hyper-vector, we need to
+   * tell the vector which batch to use. (For an array, the array does
+   * this work and the map accessor is null.)
+   */
+
+  private final VectorAccessor mapAccessor;
+
   protected MapReader(ColumnMetadata schema, AbstractObjectReader readers[]) {
-    super(schema.mapSchema(), readers);
+    this(schema, null, readers);
+  }
+
+  protected MapReader(ColumnMetadata schema,
+      VectorAccessor mapAccessor, AbstractObjectReader readers[]) {
+    super(readers);
+    this.schema = schema;
+    this.mapAccessor = mapAccessor;
   }
 
-  public static TupleObjectReader build(ColumnMetadata schema, 
AbstractObjectReader readers[]) {
-    return new TupleObjectReader(new MapReader(schema, readers));
+  public static TupleObjectReader build(ColumnMetadata schema,
+      VectorAccessor mapAccessor,
+      AbstractObjectReader readers[]) {
+    MapReader mapReader = new MapReader(schema, mapAccessor, readers);
+    mapReader.bindNullState(NullStateReaders.REQUIRED_STATE_READER);
+    return new TupleObjectReader(mapReader);
   }
 
-  public static AbstractObjectReader build(ColumnMetadata metadata,
+  public static AbstractObjectReader build(ColumnMetadata schema,
+      VectorAccessor mapAccessor,
       List<AbstractObjectReader> readers) {
     AbstractObjectReader readerArray[] = new 
AbstractObjectReader[readers.size()];
-    return build(metadata, readers.toArray(readerArray));
+    return build(schema, mapAccessor, readers.toArray(readerArray));
   }
+
+  @Override
+  public void bindIndex(ColumnReaderIndex index) {
+    if (mapAccessor != null) {
+      mapAccessor.bind(index);
+    }
+    super.bindIndex(index);
+  }
+
+  @Override
+  public ColumnMetadata schema() { return schema; }
+
+  @Override
+  public TupleMetadata tupleSchema() { return schema.mapSchema(); }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReader.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReader.java
new file mode 100644
index 0000000..e52c0f5
--- /dev/null
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReader.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.reader;
+
+import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
+
+/**
+ * Internal mechanism to detect if a value is null. Handles the multiple ways
+ * that Drill represents nulls:
+ * <ul>
+ * <li>Required and repeated modes: value is never null.</li>
+ * <li>Optional mode: null state is carried by an associated "bits" 
vector.</li>
+ * <li>Union: null state is carried by <i>both</i> the bits state of
+ * the union itself, and the null state of the associated nullable vector.
+ * (The union states if the column value itself is null; the vector state is
+ * if that value is null, which will occur if either the column is null, or
+ * if the type of the column is something other than the type in 
question.)</li>
+ * <li>List, with a single data vector: null state is carried by the list 
vector
+ * and the associated nullable data vector. Presumably the list vector state
+ * takes precedence.</li>
+ * <li>List, with a union data vector (AKA variant array or union array): the
+ * null state is carried by all three of a) the list vector, b) the union
+ * vector, and c) the type vectors. Presumably, the list vector state has
+ * precedence.</li>
+ * </ul>
+ * <p>
+ * The interface here allows each reader to delegate the null logic to a
+ * separate component, keeping the data access portion itself simple.
+ * <p>
+ * As with all readers, this reader must handle both the single-batch and
+ * the hyper-batch cases.
+ */
+
+public interface NullStateReader {
+  void bindIndex(ColumnReaderIndex rowIndex);
+  boolean isNull();
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReaders.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReaders.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReaders.java
new file mode 100644
index 0000000..f1ea09a
--- /dev/null
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/NullStateReaders.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.reader;
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.vector.accessor.ColumnAccessors.UInt1ColumnReader;
+import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
+import org.apache.drill.exec.vector.complex.UnionVector;
+
+public class NullStateReaders {
+
+  public static final RequiredStateReader REQUIRED_STATE_READER = new 
RequiredStateReader();
+
+  private NullStateReaders() { }
+
+  /**
+   * Dummy implementation of a null state reader for cases in which the
+   * value is never null. Use the {@link 
NullStateReaders#REQUIRED_STATE_READER} instance
+   * for this case.
+   */
+
+  protected static class RequiredStateReader implements NullStateReader {
+
+    @Override
+    public void bindIndex(ColumnReaderIndex rowIndex) { }
+
+    @Override
+    public boolean isNull() { return false; }
+  }
+
+  /**
+   * Holder for the NullableVector wrapper around a bits vector and a
+   * data vector. Manages the bits vector to extract the nullability
+   * value.
+   * <p>
+   * This class allows the same reader to handle both the required and
+   * nullable cases; the only difference is how nulls are handled.
+   */
+
+  protected static class NullableIsSetVectorStateReader implements 
NullStateReader {
+
+    private final VectorAccessor nullableAccessor;
+    private final UInt1ColumnReader isSetReader;
+
+    public NullableIsSetVectorStateReader(VectorAccessor nullableAccessor) {
+      this.nullableAccessor = nullableAccessor;
+      isSetReader = new UInt1ColumnReader();
+      isSetReader.bindVector(null,
+          VectorAccessors.nullableBitsAccessor(nullableAccessor));
+      isSetReader.bindNullState(REQUIRED_STATE_READER);
+    }
+
+    @Override
+    public void bindIndex(ColumnReaderIndex rowIndex) {
+      nullableAccessor.bind(rowIndex);
+      isSetReader.bindIndex(rowIndex);
+    }
+
+    @Override
+    public boolean isNull() {
+      return isSetReader.getInt() == 0;
+    }
+  }
+
+  /**
+   * Holder for the NullableVector wrapper around a bits vector and a
+   * data vector. Manages the bits vector to extract the nullability
+   * value.
+   * <p>
+   * This class allows the same reader to handle both the required and
+   * nullable cases; the only difference is how nulls are handled.
+   */
+
+  protected static class ListIsSetVectorStateReader implements NullStateReader 
{
+
+    private final VectorAccessor bitsAccessor;
+    private final UInt1ColumnReader isSetReader;
+
+    public ListIsSetVectorStateReader(VectorAccessor bitsAccessor) {
+      this.bitsAccessor = bitsAccessor;
+      isSetReader = new UInt1ColumnReader();
+      isSetReader.bindVector(null, bitsAccessor);
+      isSetReader.bindNullState(REQUIRED_STATE_READER);
+    }
+
+    @Override
+    public void bindIndex(ColumnReaderIndex rowIndex) {
+      bitsAccessor.bind(rowIndex);
+      isSetReader.bindIndex(rowIndex);
+    }
+
+    @Override
+    public boolean isNull() {
+      return isSetReader.getInt() == 0;
+    }
+  }
+
+  /**
+   * Null state that handles the strange union semantics that both
+   * the union and the values can be null. A value is null if either
+   * the union or the value is null. (Though, presumably, in the normal
+   * case either the union is null or one of the associated values is
+   * null.)
+   */
+
+  protected static class MemberNullStateReader implements NullStateReader {
+
+    private final NullStateReader unionNullState;
+    private final NullStateReader memberNullState;
+
+    public MemberNullStateReader(NullStateReader unionNullState, 
NullStateReader memberNullState) {
+      this.unionNullState = unionNullState;
+      this.memberNullState = memberNullState;
+    }
+
+    @Override
+    public void bindIndex(ColumnReaderIndex rowIndex) {
+      memberNullState.bindIndex(rowIndex);
+    }
+
+    @Override
+    public boolean isNull() {
+      return unionNullState.isNull() || memberNullState.isNull();
+    }
+  }
+
+  /**
+   * Handle the awkward situation with complex types. They don't carry their 
own
+   * bits (null state) vector. Instead, we define them as null if the type of
+   * the union is other than the type of the map or list. (Since the same 
vector
+   * that holds state also holds the is-null value, this check includes the
+   * check if the entire union is null.)
+   */
+
+  protected static class ComplexMemberStateReader implements NullStateReader {
+
+    private UInt1ColumnReader typeReader;
+    private MinorType type;
+
+    public ComplexMemberStateReader(UInt1ColumnReader typeReader, MinorType 
type) {
+      this.typeReader = typeReader;
+      this.type = type;
+    }
+
+    @Override
+    public void bindIndex(ColumnReaderIndex rowIndex) { }
+
+    @Override
+    public boolean isNull() {
+      return typeReader.getInt() != type.getNumber();
+    }
+  }
+
+  /**
+   * Extract null state from the union vector's type vector. The union reader
+   * manages the type reader, so no binding is done here.
+   */
+
+  protected static class TypeVectorStateReader implements NullStateReader {
+
+    public final UInt1ColumnReader typeReader;
+
+    public TypeVectorStateReader(UInt1ColumnReader typeReader) {
+      this.typeReader = typeReader;
+    }
+
+    @Override
+    public void bindIndex(ColumnReaderIndex rowIndex) {
+      typeReader.bindIndex(rowIndex);
+    }
+
+    @Override
+    public boolean isNull() {
+      return typeReader.getInt() == UnionVector.NULL_MARKER;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ObjectArrayReader.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ObjectArrayReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ObjectArrayReader.java
deleted file mode 100644
index 9ed89f1..0000000
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ObjectArrayReader.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.accessor.reader;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
-import org.apache.drill.exec.vector.accessor.ObjectReader;
-import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.complex.RepeatedValueVector;
-
-/**
- * Reader for an array of either tuples or other arrays.
- */
-
-public class ObjectArrayReader extends AbstractArrayReader {
-
-  /**
-   * Index into the vector of elements for a repeated vector.
-   * Keeps track of the current offset in terms of value positions.
-   * This is a derived index. The base index points to an entry
-   * in the offset vector for the array. This inner index picks
-   * off elements within the range of offsets for that one entry.
-   * For example:<pre><code>
-   * [ ... 100 105 ...]
-   * </code></pre>In the above the value 100 might be at outer
-   * offset 5. The inner array will pick off the five values
-   * 100...104.
-   * <p>
-   * Because arrays allow random access on read, the inner offset
-   * is reset on each access to the array.
-   */
-
-  public static class ObjectElementReaderIndex extends BaseElementIndex 
implements ColumnReaderIndex {
-
-    private int posn;
-
-    public ObjectElementReaderIndex(ColumnReaderIndex base) {
-      super(base);
-    }
-
-    @Override
-    public int vectorIndex() {
-      return startOffset + posn;
-    }
-
-    public void set(int index) {
-      if (index < 0 ||  length <= index) {
-        throw new IndexOutOfBoundsException("Index = " + index + ", length = " 
+ length);
-      }
-      posn = index;
-    }
-
-    public int posn() { return posn; }
-  }
-
-  /**
-   * Reader for each element.
-   */
-
-  private final AbstractObjectReader elementReader;
-
-  /**
-   * Index used to access elements.
-   */
-
-  private ObjectElementReaderIndex objElementIndex;
-
-  private ObjectArrayReader(RepeatedValueVector vector, AbstractObjectReader 
elementReader) {
-    super(vector);
-    this.elementReader = elementReader;
-  }
-
-  private ObjectArrayReader(VectorAccessor vectorAccessor, 
AbstractObjectReader elementReader) {
-    super(vectorAccessor);
-    this.elementReader = elementReader;
-  }
-
-  public static ArrayObjectReader build(RepeatedValueVector vector,
-                                        AbstractObjectReader elementReader) {
-    return new ArrayObjectReader(
-        new ObjectArrayReader(vector, elementReader));
-  }
-
-  public static AbstractObjectReader build(VectorAccessor vectorAccessor,
-                                           AbstractObjectReader elementReader) 
{
-    return new ArrayObjectReader(
-        new ObjectArrayReader(vectorAccessor, elementReader));
-  }
-
-  @Override
-  public void bindIndex(ColumnReaderIndex index) {
-    super.bindIndex(index);
-    objElementIndex = new ObjectElementReaderIndex(baseIndex);
-    elementIndex = objElementIndex;
-    elementReader.bindIndex(objElementIndex);
-  }
-
-  @Override
-  public ObjectType entryType() {
-    return elementReader.type();
-  }
-
-  @Override
-  public void setPosn(int index) {
-    objElementIndex.set(index);
-    elementReader.reposition();
-  }
-
-  @Override
-  public ObjectReader entry() {
-    return elementReader;
-  }
-
-  @Override
-  public ObjectReader entry(int index) {
-    setPosn(index);
-    return entry();
-  }
-
-  @Override
-  public Object getObject() {
-    List<Object> array = new ArrayList<>();
-    for (int i = 0; i < objElementIndex.size(); i++) {
-      array.add(entry(i).getObject());
-    }
-    return array;
-  }
-
-  @Override
-  public String getAsString() {
-    StringBuilder buf = new StringBuilder();
-    buf.append("[");
-    for (int i = 0; i < size(); i++) {
-      if (i > 0) {
-        buf.append( ", " );
-      }
-      buf.append(entry(i).getAsString());
-    }
-    buf.append("]");
-    return buf.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/OffsetVectorReader.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/OffsetVectorReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/OffsetVectorReader.java
new file mode 100644
index 0000000..9d81638
--- /dev/null
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/OffsetVectorReader.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.reader;
+
+import org.apache.drill.exec.record.metadata.ColumnMetadata;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.accessor.ValueType;
+import 
org.apache.drill.exec.vector.accessor.reader.BaseScalarReader.BaseFixedWidthReader;
+import io.netty.buffer.DrillBuf;
+
+/**
+ * Reader for an offset vector.
+ */
+
+public class OffsetVectorReader extends BaseFixedWidthReader {
+
+  private static final int VALUE_WIDTH = UInt4Vector.VALUE_WIDTH;
+
+  public OffsetVectorReader(VectorAccessor offsetsAccessor) {
+    vectorAccessor = offsetsAccessor;
+    bufferAccessor = bufferAccessor(offsetsAccessor);
+    nullStateReader = NullStateReaders.REQUIRED_STATE_READER;
+  }
+
+  @Override
+  public ValueType valueType() {
+    return ValueType.INTEGER;
+  }
+
+  @Override public int width() { return VALUE_WIDTH; }
+
+  /**
+   * Return the offset and length of a value encoded as a long.
+   * The value is encoded to avoid the need to resolve the offset vector
+   * twice per value.
+   *
+   * @return a long with the format:<br>
+   * Upper 32 bits - offset: <tt>offset = (int) (entry >> 32)</tt><br>
+   * Lower 32 bits - length: <tt>length = (int) (entry & 0xFFFF_FFFF)</tt>
+   */
+
+  public long getEntry() {
+    final DrillBuf buf = bufferAccessor.buffer();
+    final int readOffset = vectorIndex.offset() * VALUE_WIDTH;
+    long start = buf.getInt(readOffset);
+    long end = buf.getInt(readOffset + VALUE_WIDTH);
+    return (start << 32) + (end - start);
+  }
+
+  @Override
+  public void reposition() { }
+
+  @Override
+  public ColumnMetadata schema() { return null; }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ReaderEvents.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ReaderEvents.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ReaderEvents.java
new file mode 100644
index 0000000..2f75946
--- /dev/null
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ReaderEvents.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.reader;
+
+import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
+
+/**
+ * Internal operations to wire up a set of readers.
+ */
+
+public interface ReaderEvents {
+  void bindIndex(ColumnReaderIndex rowIndex);
+  void bindNullState(NullStateReader nullStateReader);
+  NullStateReader nullStateReader();
+  void reposition();
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ScalarArrayReader.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ScalarArrayReader.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ScalarArrayReader.java
deleted file mode 100644
index d93e4a5..0000000
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/ScalarArrayReader.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.accessor.reader;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
-import org.apache.drill.exec.vector.accessor.ObjectType;
-import org.apache.drill.exec.vector.accessor.ScalarElementReader;
-import org.apache.drill.exec.vector.complex.RepeatedValueVector;
-
-public class ScalarArrayReader extends AbstractArrayReader {
-
-  private final BaseElementReader elementReader;
-
-  private ScalarArrayReader(RepeatedValueVector vector,
-                           BaseElementReader elementReader) {
-    super(vector);
-    this.elementReader = elementReader;
-  }
-
-  private ScalarArrayReader(VectorAccessor va,
-                            BaseElementReader elementReader) {
-    super(va);
-    this.elementReader = elementReader;
-  }
-
-  public static ArrayObjectReader build(RepeatedValueVector vector,
-                                        BaseElementReader elementReader) {
-    elementReader.bindVector(vector.getDataVector());
-    return new ArrayObjectReader(new ScalarArrayReader(vector, elementReader));
-  }
-
-  public static ArrayObjectReader build(MajorType majorType, VectorAccessor va,
-                                        BaseElementReader elementReader) {
-    elementReader.bindVector(majorType, va);
-    return new ArrayObjectReader(new ScalarArrayReader(va, elementReader));
-  }
-
-  @Override
-  public void bindIndex(ColumnReaderIndex index) {
-    super.bindIndex(index);
-    FixedWidthElementReaderIndex fwElementIndex = new 
FixedWidthElementReaderIndex(baseIndex);
-    elementIndex = fwElementIndex;
-    elementReader.bindIndex(fwElementIndex);
-  }
-
-  @Override
-  public ObjectType entryType() {
-    return ObjectType.SCALAR;
-  }
-
-  @Override
-  public ScalarElementReader elements() {
-    return elementReader;
-  }
-
-  @Override
-  public void setPosn(int index) {
-    throw new IllegalStateException("setPosn() not supported for scalar 
arrays");
-  }
-
-  @Override
-  public Object getObject() {
-    List<Object> elements = new ArrayList<>();
-    for (int i = 0; i < size(); i++) {
-      elements.add(elementReader.getObject(i));
-    }
-    return elements;
-  }
-
-  @Override
-  public String getAsString() {
-    StringBuilder buf = new StringBuilder();
-    buf.append("[");
-    for (int i = 0; i < size(); i++) {
-      if (i > 0) {
-        buf.append( ", " );
-      }
-      buf.append(elementReader.getAsString(i));
-    }
-    buf.append("]");
-    return buf.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessor.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessor.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessor.java
index 1cf2a19..050845b 100644
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessor.java
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessor.java
@@ -17,10 +17,13 @@
  */
 package org.apache.drill.exec.vector.accessor.reader;
 
+import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
 
 public interface VectorAccessor {
+  boolean isHyper();
+  MajorType type();
   void bind(ColumnReaderIndex index);
-  ValueVector vector();
+  <T extends ValueVector> T vector();
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessors.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessors.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessors.java
new file mode 100644
index 0000000..7d70d1d
--- /dev/null
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/VectorAccessors.java
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.accessor.reader;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.NullableVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
+import org.apache.drill.exec.vector.accessor.ColumnReaderIndex;
+import org.apache.drill.exec.vector.complex.AbstractMapVector;
+import org.apache.drill.exec.vector.complex.RepeatedValueVector;
+
+/**
+ * Collection of vector accessors. A single class handles the single-batch
+ * case. But, for hyper-vectors, we need a separate accessor for each
+ * (vector, sub-vector) combination to handle the
+ * indirections in the hyper-vector case.
+ * <p>
+ * For a required vector:<br>
+ * reader index --> hyper vector --> required vector
+ * <p>
+ * For a nullable vector:<br>
+ * reader index --> hyper vector --> nullable vector<br>
+ *   nullable vector --> bits vector<br>
+ *                   --> values vector
+ * <p>
+ * For a repeated vector:<br>
+ * reader index --> hyper vector --> repeated vector<br>
+ *   repeated vector --> offset vector<br>
+ *                   --> values vector
+ * <p>
+ * And so on. In each case, we must start with a top-level
+ * vector as indicated the row index, indirected through the
+ * SV4. That is done by the reader index. That points to a
+ * top-level vector in the hyper-vector.
+ * <p>
+ * Most of the vectors needed are nested. These inner vectors
+ * are not part of a hyper-vector list. Instead, we must get the
+ * top-level vector, then navigate down from that vector to the
+ * desired vector.
+ * <p>
+ * Sometimes the navigation is static (the "bits" vector for
+ * a nullable vector.) Other times, it is a bit more dynamic: a
+ * member of a map (given by index) or the member of a union
+ * (given by type.)
+ * <p>
+ * These accessors can be chained to handle deeply-nested
+ * structures such as an array of maps that contains a list of
+ * unions.
+ * <p>
+ * Because the navigation is required on every access, the use of hyper
+ * vectors is slow. Since hyper-vectors are seldom used, we
+ * optimize for the single-batch case by caching vectors at each
+ * stage. Thus, for the single-batch case, we use different accessor
+ * implementations. To keep the rest of the code simple, both the
+ * hyper and single batch cases use the same API, but they use
+ * entirely different implementations. The methods here choose
+ * the correct implementation for the single and hyper cases.
+ */
+
+public class VectorAccessors {
+
+  public static class NullVectorAccesor implements VectorAccessor {
+
+    private final MajorType type;
+
+    public NullVectorAccesor(MajorType type) {
+      this.type = type;
+    }
+
+    @Override
+    public boolean isHyper() { return false; }
+
+    @Override
+    public MajorType type() { return type; }
+
+    @Override
+    public void bind(ColumnReaderIndex index) { }
+
+    @Override
+    public <T extends ValueVector> T vector() {
+      throw new UnsupportedOperationException();
+    }
+  }
+
+  public static class SingleVectorAccessor implements VectorAccessor {
+
+    private final ValueVector vector;
+
+    public SingleVectorAccessor(ValueVector vector) {
+      this.vector = vector;
+    }
+
+    @Override
+    public boolean isHyper() { return false; }
+
+    @Override
+    public void bind(ColumnReaderIndex index) { }
+
+    @Override
+    public MajorType type() { return vector.getField().getType(); }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T extends ValueVector> T vector() { return (T) vector; }
+  }
+
+  /**
+   * Vector accessor used by the column accessors to obtain the vector for
+   * each column value. That is, position 0 might be batch 4, index 3,
+   * while position 1 might be batch 1, index 7, and so on.
+   */
+
+  public static abstract class BaseHyperVectorAccessor implements 
VectorAccessor {
+
+    protected final MajorType type;
+
+    public BaseHyperVectorAccessor(MajorType type) {
+      this.type = type;
+    }
+
+    @Override
+    public boolean isHyper() { return true; }
+
+    @Override
+    public void bind(ColumnReaderIndex index) { }
+
+    @Override
+    public MajorType type() { return type; }
+  }
+
+  /**
+   * Vector accessor for RepeatedVector &rarr; offsets vector
+   */
+
+  public static class ArrayOffsetHyperVectorAccessor extends 
BaseHyperVectorAccessor {
+
+    private VectorAccessor repeatedVectorAccessor;
+
+    public ArrayOffsetHyperVectorAccessor(VectorAccessor va) {
+      super(Types.required(MinorType.UINT4));
+      repeatedVectorAccessor = va;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T extends ValueVector> T vector() {
+      RepeatedValueVector vector = repeatedVectorAccessor.vector();
+      return (T) vector.getOffsetVector();
+    }
+  }
+
+  /**
+   * Vector accessor for RepeatedVector &rarr; data vector
+   */
+
+  public static class ArrayDataHyperVectorAccessor implements VectorAccessor {
+
+    private VectorAccessor repeatedVectorAccessor;
+
+    private ArrayDataHyperVectorAccessor(VectorAccessor va) {
+      repeatedVectorAccessor = va;
+    }
+
+    @Override
+    public boolean isHyper() { return true; }
+
+    @Override
+    public MajorType type() { return repeatedVectorAccessor.type(); }
+
+    @Override
+    public void bind(ColumnReaderIndex index) { }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T extends ValueVector> T vector() {
+      RepeatedValueVector vector = repeatedVectorAccessor.vector();
+      return (T) vector.getDataVector();
+    }
+  }
+
+  /**
+   * Vector accessor for VariableWidthVector &rarr; offsets vector
+   */
+
+  public static class VarWidthOffsetHyperVectorAccessor extends 
BaseHyperVectorAccessor {
+
+    private VectorAccessor varWidthVectorAccessor;
+
+    public VarWidthOffsetHyperVectorAccessor(VectorAccessor va) {
+      super(Types.required(MinorType.UINT4));
+      varWidthVectorAccessor = va;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T extends ValueVector> T vector() {
+      VariableWidthVector vector = varWidthVectorAccessor.vector();
+      return (T) vector.getOffsetVector();
+    }
+  }
+
+  /**
+   * Vector accessor for NullableVector &rarr; values vector
+   */
+
+  public static class NullableValuesHyperVectorAccessor implements 
VectorAccessor {
+
+    private VectorAccessor nullableAccessor;
+
+    private NullableValuesHyperVectorAccessor(VectorAccessor va) {
+      nullableAccessor = va;
+    }
+
+    @Override
+    public boolean isHyper() { return true; }
+
+    @Override
+    public MajorType type() { return nullableAccessor.type(); }
+
+    @Override
+    public void bind(ColumnReaderIndex index) { }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T extends ValueVector> T vector() {
+      NullableVector vector = nullableAccessor.vector();
+      return (T) vector.getValuesVector();
+    }
+  }
+
+  /**
+   * Vector accessor for NullableVector &rarr; bits vector
+   */
+
+  public static class NullableBitsHyperVectorStateReader extends 
BaseHyperVectorAccessor {
+
+    public final VectorAccessor nullableAccessor;
+
+    public NullableBitsHyperVectorStateReader(VectorAccessor nullableAccessor) 
{
+      super(Types.required(MinorType.UINT1));
+      this.nullableAccessor = nullableAccessor;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T extends ValueVector> T vector() {
+      NullableVector vector = nullableAccessor.vector();
+      return (T) vector.getBitsVector();
+    }
+  }
+
+  /**
+   * Vector accessor for AbstractMapVector &rarr; member vector
+   */
+
+  public static class MapMemberHyperVectorAccessor extends 
BaseHyperVectorAccessor {
+
+    private final VectorAccessor mapAccessor;
+    private final int index;
+
+    public MapMemberHyperVectorAccessor(VectorAccessor va, int index, 
MajorType type) {
+      super(type);
+      mapAccessor = va;
+      this.index = index;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T extends ValueVector> T vector() {
+      AbstractMapVector vector = mapAccessor.vector();
+      return (T) vector.getChildByOrdinal(index);
+    }
+  }
+
+  // Methods to create vector accessors for sub-vectors internal to various
+  // value vectors. These methods are called from the readers themselves rather
+  // than the reader builders.
+
+  public static VectorAccessor arrayOffsetVectorAccessor(VectorAccessor 
repeatedAccessor) {
+    if (repeatedAccessor.isHyper()) {
+      return new ArrayOffsetHyperVectorAccessor(repeatedAccessor);
+    } else {
+      RepeatedValueVector vector = repeatedAccessor.vector();
+      return new SingleVectorAccessor(vector.getOffsetVector());
+    }
+  }
+
+  public static VectorAccessor arrayDataAccessor(VectorAccessor 
repeatedAccessor) {
+    if (repeatedAccessor.isHyper()) {
+      return new ArrayDataHyperVectorAccessor(repeatedAccessor);
+    } else {
+      RepeatedValueVector vector = repeatedAccessor.vector();
+      return new SingleVectorAccessor(
+          vector.getDataVector());
+    }
+  }
+
+  public static VectorAccessor varWidthOffsetVectorAccessor(VectorAccessor 
varWidthAccessor) {
+    if (varWidthAccessor.isHyper()) {
+      return new VarWidthOffsetHyperVectorAccessor(varWidthAccessor);
+    } else {
+      VariableWidthVector vector = varWidthAccessor.vector();
+      return new SingleVectorAccessor(vector.getOffsetVector());
+    }
+  }
+
+  public static VectorAccessor nullableValuesAccessor(VectorAccessor 
nullableAccessor) {
+    if (nullableAccessor.isHyper()) {
+      return new NullableValuesHyperVectorAccessor(nullableAccessor);
+    } else {
+      NullableVector vector = nullableAccessor.vector();
+      return new SingleVectorAccessor(
+          vector.getValuesVector());
+    }
+  }
+
+  public static VectorAccessor nullableBitsAccessor(VectorAccessor 
nullableAccessor) {
+    if (nullableAccessor.isHyper()) {
+      return new NullableBitsHyperVectorStateReader(nullableAccessor);
+    } else {
+      NullableVector vector = nullableAccessor.vector();
+      return new SingleVectorAccessor(
+          vector.getBitsVector());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/package-info.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/package-info.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/package-info.java
index a94d2e8..823dd28 100644
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/package-info.java
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/package-info.java
@@ -17,10 +17,67 @@
  */
 /**
  * Provides the reader hierarchy as explained in the API package.
- * The only caveat is that a simplification is provided for arrays of
- * scalar values: rather than a scalar reader for each value, the
- * {#link ScalarElementReader} class provides access to the entire array
- * via indexed get methods.
+ *
+ * <h4>Structure</h4>
+ *
+ * The reader implementation divides into four parts:
+ * <ol>
+ * <li>The readers themselves which start with scalar readers to
+ * decode data from vectors, then build up to nullable, array,
+ * union and list readers. Readers are built up via composition,
+ * often using the (internal) offset vector reader.</li>
+ * <li>The column index abstraction that steps through items in a collection.
+ * At the top level, the index points to the current row. The top level
+ * may include an indirection (an SV2 or SV4) which is handled by the
+ * column index. Within arrays, the column index points to each element
+ * of the array.</li>
+ * <li>The vector accessor which provides a unified interface for both
+ * the single-batch and hyper-batch cases. The single-batch versions
+ * simply hold onto the vector itself. The hyper-batch versions either
+ * provide access to a specific vector within a hyper-vector (for
+ * top-level vectors), or navigate from a top-level vector down to an
+ * inner vector (for nested vectors.)</li>
+ * <li>The null state abstraction which provides a uniform way to
+ * detect nullability. For example, within the reader system, the
+ * reader for nullable and required vectors differ only in the associated
+ * null state reader. Unions and lists have complex null state
+ * logic: the nullability of a value depends on the nullability
+ * of the list, the union, and the value itself. The null state
+ * class implements this logic independent of the reader structure.
+ * </li>
+ * </ul>
+ *
+ * <h4>Composition</h4>
+ *
+ * The result is that reader structure makes heavy use of composition:
+ * readers are built up from each of the above components. The number of
+ * actual reader classes is small, but the methods to build the readers are
+ * complex. Most structure is built at build time. Indexes, however are
+ * provided at a later "bind" time at which a bind call traverses the
+ * reader tree to associate an index with each reader and vector accessor.
+ * When a reader is for an array, the bind step creates the index for the
+ * array elements.
+ *
+ * <h4>Construction</h4>
+ *
+ * Construction of readers is a multi-part process.
+ * <ul>
+ * <li>Start with a single or hyper-vector batch.</li>
+ * <li>The reader builders in another package parse the batch structure,
+ * create the required metadata, wrap the (single or hyper) vectors in
+ * a vector accessor, and call methods in this package.</li>
+ * <li>Methods here perform the final construction based on the specific
+ * type of the reader.</li>
+ * </ul>
+ * <p>
+ * The work divides into two main categories:
+ * <ul>
+ * <li>The work which is based on
+ * the vector structure and single/hyper-vector structure, which is done
+ * elsewhere.</li>
+ * <li>The work which is based on the structure of the readers (with
+ * vector cardinality factored out), which is done here.</li>
+ * </ul>
  */
 
 package org.apache.drill.exec.vector.accessor.reader;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java
index 30811bb..4a668c4 100644
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ColumnWriterFactory.java
@@ -28,7 +28,7 @@ import org.apache.drill.exec.record.metadata.ColumnMetadata;
 import org.apache.drill.exec.vector.NullableVector;
 import org.apache.drill.exec.vector.UInt4Vector;
 import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.accessor.ColumnAccessors;
+import org.apache.drill.exec.vector.accessor.ColumnAccessorUtils;
 import 
org.apache.drill.exec.vector.accessor.writer.AbstractArrayWriter.ArrayObjectWriter;
 import 
org.apache.drill.exec.vector.accessor.writer.AbstractScalarWriter.ScalarObjectWriter;
 import 
org.apache.drill.exec.vector.accessor.writer.AbstractTupleWriter.TupleObjectWriter;
@@ -56,7 +56,7 @@ public class ColumnWriterFactory {
   private static final Class<? extends BaseScalarWriter> requiredWriters[] = 
new Class[typeCount];
 
   static {
-    ColumnAccessors.defineRequiredWriters(requiredWriters);
+    ColumnAccessorUtils.defineRequiredWriters(requiredWriters);
   }
 
   public static AbstractObjectWriter buildColumnWriter(ColumnMetadata schema, 
ValueVector vector) {

http://git-wip-us.apache.org/repos/asf/drill/blob/4f2182e4/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
index 45d9160..49bd510 100644
--- 
a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
+++ 
b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
@@ -55,7 +55,7 @@ public class ListVector extends BaseRepeatedValueVector {
 
   public ListVector(MaterializedField field, BufferAllocator allocator, 
CallBack callBack) {
     super(field, allocator);
-    this.bits = new UInt1Vector(MaterializedField.create("$bits$", 
Types.required(MinorType.UINT1)), allocator);
+    this.bits = new UInt1Vector(MaterializedField.create(BITS_VECTOR_NAME, 
Types.required(MinorType.UINT1)), allocator);
     offsets = getOffsetVector();
     this.field.addChild(getDataVector().getField());
     this.writer = new UnionListWriter(this);

Reply via email to