lidavidm commented on a change in pull request #113: URL: https://github.com/apache/arrow-cookbook/pull/113#discussion_r782147857
########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. Review comment: What is the conceptual distinction between FieldVector and ValueVector? ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 Review comment: After looking at it rendered, I think the highlighting is not helpful here. It's only helpful once the example gets long enough. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 Review comment: We especially shouldn't do something like this where the entire code block is highlighted. ########## File path: java/source/data.rst ########## @@ -0,0 +1,608 @@ +================= +Data manipulation +================= + +Recipes related to compare, filtering or transforming data. + +.. contents:: + +We are going to use this util for data manipulation: + +.. code-block:: java + + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + class TestVectorValueComparator extends VectorValueComparator<VarCharVector> { + @Override + public int compareNotNull(int index1, int index2) { + byte b1 = vector1.get(index1)[0]; + byte b2 = vector2.get(index2)[0]; + return b1 - b2; + } + + @Override + public VectorValueComparator<VarCharVector> createNew() { + return new TestVectorValueComparator(); + } + } + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Compare Vectors for Field Equality +================================== + +.. code-block:: java + :emphasize-lines: 10 + + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.compare.TypeEqualsVisitor; + + IntVector right = new IntVector("int", rootAllocator); + IntVector left1 = new IntVector("int", rootAllocator); + IntVector left2 = new IntVector("int2", rootAllocator); + + setVector(right, 10,20,30); + + TypeEqualsVisitor visitor = new TypeEqualsVisitor(right); + +Comparing vector fields: + +.. code-block:: java + :emphasize-lines: 1-4 + + jshell> visitor.equals(left1); + true + jshell> visitor.equals(left2); + false + +Compare Values on the Array +=========================== + +.. code-block:: java + :emphasize-lines: 15-17 + + import org.apache.arrow.algorithm.sort.StableVectorComparator; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.VarCharVector; + + // compare two values at the given indices in the vectors. + // comparing org.apache.arrow.algorithm.sort.VectorValueComparator on algorithm + VarCharVector vec = new VarCharVector("valueindexcomparator", rootAllocator); + vec.allocateNew(100, 5); + vec.setValueCount(10); + vec.set(0, "ba".getBytes()); + vec.set(1, "abc".getBytes()); + vec.set(2, "aa".getBytes()); + vec.set(3, "abc".getBytes()); + vec.set(4, "a".getBytes()); + VectorValueComparator<VarCharVector> comparatorValues = new TestVectorValueComparator(); // less than, equal to, greater than + VectorValueComparator<VarCharVector> stableComparator = new StableVectorComparator<>(comparatorValues);//Stable comparator only supports comparing values from the same vector + stableComparator.attachVector(vec); + +Comparing two values at the given indices in the vectors: + +.. code-block:: java + :emphasize-lines: 1-12 + + jshell> stableComparator.compare(0, 1) > 0; + true + jshell> stableComparator.compare(1, 2) < 0; + true + jshell> stableComparator.compare(2, 3) < 0; + true + jshell> stableComparator.compare(1, 3) < 0; + true + jshell> stableComparator.compare(3, 1) > 0; + true + jshell> stableComparator.compare(3, 3) == 0; + true + +Search Values on the Array +========================== + +Linear Search - O(n) +******************** + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) Review comment: Instead of writing out the full name here, maybe just write `VectorSearcher#linearSearch` and link to Javadocs? ########## File path: java/source/data.rst ########## @@ -0,0 +1,608 @@ +================= +Data manipulation +================= + +Recipes related to compare, filtering or transforming data. + +.. contents:: + +We are going to use this util for data manipulation: + +.. code-block:: java + + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + class TestVectorValueComparator extends VectorValueComparator<VarCharVector> { + @Override + public int compareNotNull(int index1, int index2) { + byte b1 = vector1.get(index1)[0]; + byte b2 = vector2.get(index2)[0]; + return b1 - b2; + } + + @Override + public VectorValueComparator<VarCharVector> createNew() { + return new TestVectorValueComparator(); + } + } + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Compare Vectors for Field Equality Review comment: I realize I suggested this title but we need to be consistent with Array vs Vector. I still think we should only use Vector since that is what the library calls it. ########## File path: java/source/schema.rst ########## @@ -0,0 +1,329 @@ +=================== +Working with schema +=================== Review comment: ```suggestion =================== Working with Schema =================== ``` ########## File path: java/source/io.rst ########## @@ -0,0 +1,354 @@ +======================== +Reading and writing data +======================== + +Recipes related to reading and writing data from disk using +Apache Arrow. + +.. contents:: + +Writing array +============= + +It is possible to dump data in the raw arrow format which allows +direct memory mapping of data from disk. This format is called +the Arrow IPC format. There are two option: Random access format +& Streaming format. + +We are going to use this util for reading and writing data: + +.. code-block:: java + :name: Util + :emphasize-lines: 114 + + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.util.Arrays.asList; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + VectorSchemaRoot createVectorSchemaRoot(){ + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); + + Field age = new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null); + + FieldType intType = new FieldType(true, new ArrowType.Int(32, true), /*dictionary=*/null); + FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/null); + Field childField = new Field("intCol", intType, null); + List<Field> childFields = new ArrayList<>(); + childFields.add(childField); + Field points = new Field("points", listType, childFields); + + // create a definition + Schema schemaPerson = new Schema(asList(name, document, age, points)); + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schemaPerson, rootAllocator); + + // getting field vectors + VarCharVector nameVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("name"); //interface FieldVector + VarCharVector documentVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("document"); //interface FieldVector + IntVector ageVectorOption1 = (IntVector) vectorSchemaRoot.getVector("age"); + ListVector pointsVectorOption1 = (ListVector) vectorSchemaRoot.getVector("points"); + + // add values to the field vectors + setVector(nameVectorOption1, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + setVector(documentVectorOption1, "A".getBytes(), "B".getBytes(), "C".getBytes()); + setVector(ageVectorOption1, 10,20,30); + setVector(pointsVectorOption1, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + vectorSchemaRoot.setRowCount(3); + + return vectorSchemaRoot; + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + + VectorSchemaRoot vectorSchemaRoot = createVectorSchemaRoot(); + + +.. code-block:: java + :emphasize-lines: 1-6 + + jshell> System.out.println(vectorSchemaRoot.contentToTSVString()) + + name document age points + david A 10 [1,3,5,7,9] + gladis B 20 [2,4,6,8,10] + juan C 30 [1,2,3,5,8] + +Writing arrays with the IPC file format +*************************************** + +Write - Random access to file +----------------------------- + +.. code-block:: java + :emphasize-lines: 9 + + import org.apache.arrow.vector.ipc.*; + + import java.io.*; + + // random access format + // write - random access to file + File file = new File("randon_access.arrow"); + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter writer = new ArrowFileWriter(vectorSchemaRoot, null, fileOutputStream.getChannel()); Review comment: Even if we will have a separate tutorial, I don't think the cookbook should demonstrate incorrect code. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 + + + jshell> intVector + + intVector ==> [1, 2, 3] + +Array of Varchar +---------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.VarCharVector; + + VarCharVector varcharVector = new VarCharVector("varcharVector", rootAllocator); + setVector(varcharVector, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> varcharVector + + varcharVector ==> [david, gladis, juan] + +Array of List +------------- + +.. code-block:: java + :emphasize-lines: 6 + + import org.apache.arrow.vector.complex.ListVector; + + import static java.util.Arrays.asList; + + ListVector listVector = ListVector.empty("listVector", rootAllocator); + setVector(listVector, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listVector + + listVector ==> [[1,3,5,7,9], [2,4,6,8,10], [1,2,3,5,8]] + +Creating VectorSchemaRoot (Table) +================================= + +A `VectorSchemaRoot <https://arrow.apache.org/docs/java/vector_schema_root.html>`_ is a container that can hold batches, batches flow through VectorSchemaRoot as part of a pipeline. + +.. code-block:: java + :emphasize-lines: 21 + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.util.Arrays.asList; + + VectorSchemaRoot createVectorSchemaRoot(){ + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); + + Field age = new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null); + + FieldType intType = new FieldType(true, new ArrowType.Int(32, true), /*dictionary=*/null); + FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/null); + Field childField = new Field("intCol", intType, null); + List<Field> childFields = new ArrayList<>(); + childFields.add(childField); + Field points = new Field("points", listType, childFields); + + // create a definition + Schema schemaPerson = new Schema(asList(name, document, age, points)); + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation Review comment: The example is demonstrating a misuse of the Arrow Java library. This allocator needs to be created somewhere else - perhaps it should be an argument of this function. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. Review comment: Why is this using a line block (`| `)? ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 + + + jshell> intVector + + intVector ==> [1, 2, 3] + +Array of Varchar +---------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.VarCharVector; + + VarCharVector varcharVector = new VarCharVector("varcharVector", rootAllocator); + setVector(varcharVector, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> varcharVector + + varcharVector ==> [david, gladis, juan] + +Array of List +------------- + +.. code-block:: java + :emphasize-lines: 6 + + import org.apache.arrow.vector.complex.ListVector; + + import static java.util.Arrays.asList; + + ListVector listVector = ListVector.empty("listVector", rootAllocator); + setVector(listVector, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listVector + + listVector ==> [[1,3,5,7,9], [2,4,6,8,10], [1,2,3,5,8]] + +Creating VectorSchemaRoot (Table) +================================= + +A `VectorSchemaRoot <https://arrow.apache.org/docs/java/vector_schema_root.html>`_ is a container that can hold batches, batches flow through VectorSchemaRoot as part of a pipeline. Review comment: This means that a VectorSchemaRoot really isn't anything like a Table at all. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); Review comment: Frankly, the helpers make these examples rather pointless. The example isn't copy-pastable anymore and it lacks context; people have to scroll up and find the right overload. We should inline the helpers into the examples, and we need to provide more commentary on the steps. (For instance, setSafe will reallocate the vector, and we need to update the vector length manually.) ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 + + + jshell> intVector + + intVector ==> [1, 2, 3] + +Array of Varchar +---------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.VarCharVector; + + VarCharVector varcharVector = new VarCharVector("varcharVector", rootAllocator); + setVector(varcharVector, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> varcharVector + + varcharVector ==> [david, gladis, juan] + +Array of List +------------- + +.. code-block:: java + :emphasize-lines: 6 + + import org.apache.arrow.vector.complex.ListVector; + + import static java.util.Arrays.asList; + + ListVector listVector = ListVector.empty("listVector", rootAllocator); + setVector(listVector, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listVector + + listVector ==> [[1,3,5,7,9], [2,4,6,8,10], [1,2,3,5,8]] + +Creating VectorSchemaRoot (Table) +================================= + +A `VectorSchemaRoot <https://arrow.apache.org/docs/java/vector_schema_root.html>`_ is a container that can hold batches, batches flow through VectorSchemaRoot as part of a pipeline. + +.. code-block:: java + :emphasize-lines: 21 Review comment: Highlighting the function declaration doesn't really help. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 + + + jshell> intVector + + intVector ==> [1, 2, 3] + +Array of Varchar +---------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.VarCharVector; + + VarCharVector varcharVector = new VarCharVector("varcharVector", rootAllocator); + setVector(varcharVector, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> varcharVector + + varcharVector ==> [david, gladis, juan] + +Array of List +------------- + +.. code-block:: java + :emphasize-lines: 6 + + import org.apache.arrow.vector.complex.ListVector; + + import static java.util.Arrays.asList; + + ListVector listVector = ListVector.empty("listVector", rootAllocator); + setVector(listVector, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listVector + + listVector ==> [[1,3,5,7,9], [2,4,6,8,10], [1,2,3,5,8]] + +Creating VectorSchemaRoot (Table) +================================= + +A `VectorSchemaRoot <https://arrow.apache.org/docs/java/vector_schema_root.html>`_ is a container that can hold batches, batches flow through VectorSchemaRoot as part of a pipeline. + +.. code-block:: java + :emphasize-lines: 21 + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.util.Arrays.asList; + + VectorSchemaRoot createVectorSchemaRoot(){ + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); Review comment: The example is rather long. I think we should omit the metadata since it's not relevant here. We can put that in a separate recipe. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 + + + jshell> intVector + + intVector ==> [1, 2, 3] + +Array of Varchar +---------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.VarCharVector; + + VarCharVector varcharVector = new VarCharVector("varcharVector", rootAllocator); + setVector(varcharVector, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> varcharVector + + varcharVector ==> [david, gladis, juan] + +Array of List +------------- + +.. code-block:: java + :emphasize-lines: 6 + + import org.apache.arrow.vector.complex.ListVector; + + import static java.util.Arrays.asList; + + ListVector listVector = ListVector.empty("listVector", rootAllocator); + setVector(listVector, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listVector + + listVector ==> [[1,3,5,7,9], [2,4,6,8,10], [1,2,3,5,8]] + +Creating VectorSchemaRoot (Table) +================================= + +A `VectorSchemaRoot <https://arrow.apache.org/docs/java/vector_schema_root.html>`_ is a container that can hold batches, batches flow through VectorSchemaRoot as part of a pipeline. + +.. code-block:: java + :emphasize-lines: 21 + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.util.Arrays.asList; + + VectorSchemaRoot createVectorSchemaRoot(){ + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); + + Field age = new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null); + + FieldType intType = new FieldType(true, new ArrowType.Int(32, true), /*dictionary=*/null); + FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/null); + Field childField = new Field("intCol", intType, null); Review comment: Missing comment on the literal parameter. ########## File path: java/source/demo/pom.xml ########## @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> Review comment: Why do we have this Maven project? I don't see it referenced anywhere in the actual cookbook, and if it's for testing, then manually copy-pasting code between Maven and Sphinx will quickly get error-prone/forgotten. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 + + + jshell> intVector + + intVector ==> [1, 2, 3] + +Array of Varchar +---------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.VarCharVector; + + VarCharVector varcharVector = new VarCharVector("varcharVector", rootAllocator); + setVector(varcharVector, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> varcharVector + + varcharVector ==> [david, gladis, juan] + +Array of List +------------- + +.. code-block:: java + :emphasize-lines: 6 + + import org.apache.arrow.vector.complex.ListVector; + + import static java.util.Arrays.asList; + + ListVector listVector = ListVector.empty("listVector", rootAllocator); + setVector(listVector, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listVector + + listVector ==> [[1,3,5,7,9], [2,4,6,8,10], [1,2,3,5,8]] + +Creating VectorSchemaRoot (Table) +================================= + +A `VectorSchemaRoot <https://arrow.apache.org/docs/java/vector_schema_root.html>`_ is a container that can hold batches, batches flow through VectorSchemaRoot as part of a pipeline. + +.. code-block:: java + :emphasize-lines: 21 + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.util.Arrays.asList; + + VectorSchemaRoot createVectorSchemaRoot(){ + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); + + Field age = new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null); + + FieldType intType = new FieldType(true, new ArrowType.Int(32, true), /*dictionary=*/null); + FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/null); + Field childField = new Field("intCol", intType, null); + List<Field> childFields = new ArrayList<>(); + childFields.add(childField); + Field points = new Field("points", listType, childFields); + + // create a definition + Schema schemaPerson = new Schema(asList(name, document, age, points)); + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schemaPerson, rootAllocator); + + // getting field vectors + VarCharVector nameVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("name"); //interface FieldVector + VarCharVector documentVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("document"); //interface FieldVector + IntVector ageVectorOption1 = (IntVector) vectorSchemaRoot.getVector("age"); + ListVector pointsVectorOption1 = (ListVector) vectorSchemaRoot.getVector("points"); + + // add values to the field vectors + setVector(nameVectorOption1, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + setVector(documentVectorOption1, "A".getBytes(), "B".getBytes(), "C".getBytes()); + setVector(ageVectorOption1, 10,20,30); + setVector(pointsVectorOption1, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); Review comment: If it's too much code to inline the helpers, IMO we should just remove the list array from the example. The focus should be on VectorSchemaRoot anyways. ########## File path: java/source/data.rst ########## @@ -0,0 +1,608 @@ +================= +Data manipulation +================= + +Recipes related to compare, filtering or transforming data. + +.. contents:: + +We are going to use this util for data manipulation: + +.. code-block:: java + + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + class TestVectorValueComparator extends VectorValueComparator<VarCharVector> { + @Override + public int compareNotNull(int index1, int index2) { + byte b1 = vector1.get(index1)[0]; + byte b2 = vector2.get(index2)[0]; + return b1 - b2; + } + + @Override + public VectorValueComparator<VarCharVector> createNew() { + return new TestVectorValueComparator(); + } + } + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Compare Vectors for Field Equality +================================== + +.. code-block:: java + :emphasize-lines: 10 + + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.compare.TypeEqualsVisitor; + + IntVector right = new IntVector("int", rootAllocator); + IntVector left1 = new IntVector("int", rootAllocator); + IntVector left2 = new IntVector("int2", rootAllocator); + + setVector(right, 10,20,30); + + TypeEqualsVisitor visitor = new TypeEqualsVisitor(right); + +Comparing vector fields: + +.. code-block:: java + :emphasize-lines: 1-4 + + jshell> visitor.equals(left1); + true + jshell> visitor.equals(left2); + false + +Compare Values on the Array +=========================== + +.. code-block:: java + :emphasize-lines: 15-17 + + import org.apache.arrow.algorithm.sort.StableVectorComparator; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.VarCharVector; + + // compare two values at the given indices in the vectors. + // comparing org.apache.arrow.algorithm.sort.VectorValueComparator on algorithm + VarCharVector vec = new VarCharVector("valueindexcomparator", rootAllocator); + vec.allocateNew(100, 5); + vec.setValueCount(10); + vec.set(0, "ba".getBytes()); + vec.set(1, "abc".getBytes()); + vec.set(2, "aa".getBytes()); + vec.set(3, "abc".getBytes()); + vec.set(4, "a".getBytes()); + VectorValueComparator<VarCharVector> comparatorValues = new TestVectorValueComparator(); // less than, equal to, greater than + VectorValueComparator<VarCharVector> stableComparator = new StableVectorComparator<>(comparatorValues);//Stable comparator only supports comparing values from the same vector + stableComparator.attachVector(vec); + +Comparing two values at the given indices in the vectors: + +.. code-block:: java + :emphasize-lines: 1-12 + + jshell> stableComparator.compare(0, 1) > 0; + true + jshell> stableComparator.compare(1, 2) < 0; + true + jshell> stableComparator.compare(2, 3) < 0; + true + jshell> stableComparator.compare(1, 3) < 0; + true + jshell> stableComparator.compare(3, 1) > 0; + true + jshell> stableComparator.compare(3, 3) == 0; + true + +Search Values on the Array +========================== + +Linear Search - O(n) +******************** + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + +.. code-block:: java + :emphasize-lines: 27 + + import org.apache.arrow.algorithm.search.VectorSearcher; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // search values on the array + // linear search org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) Review comment: The comment just repeats the prose, I don't think it's useful. ########## File path: java/source/data.rst ########## @@ -0,0 +1,608 @@ +================= +Data manipulation +================= + +Recipes related to compare, filtering or transforming data. + +.. contents:: + +We are going to use this util for data manipulation: + +.. code-block:: java + + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + class TestVectorValueComparator extends VectorValueComparator<VarCharVector> { + @Override + public int compareNotNull(int index1, int index2) { + byte b1 = vector1.get(index1)[0]; + byte b2 = vector2.get(index2)[0]; + return b1 - b2; + } + + @Override + public VectorValueComparator<VarCharVector> createNew() { + return new TestVectorValueComparator(); + } + } + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Compare Vectors for Field Equality +================================== + +.. code-block:: java + :emphasize-lines: 10 + + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.compare.TypeEqualsVisitor; + + IntVector right = new IntVector("int", rootAllocator); + IntVector left1 = new IntVector("int", rootAllocator); + IntVector left2 = new IntVector("int2", rootAllocator); + + setVector(right, 10,20,30); + + TypeEqualsVisitor visitor = new TypeEqualsVisitor(right); + +Comparing vector fields: + +.. code-block:: java + :emphasize-lines: 1-4 + + jshell> visitor.equals(left1); + true + jshell> visitor.equals(left2); + false + +Compare Values on the Array +=========================== + +.. code-block:: java + :emphasize-lines: 15-17 + + import org.apache.arrow.algorithm.sort.StableVectorComparator; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.VarCharVector; + + // compare two values at the given indices in the vectors. + // comparing org.apache.arrow.algorithm.sort.VectorValueComparator on algorithm + VarCharVector vec = new VarCharVector("valueindexcomparator", rootAllocator); + vec.allocateNew(100, 5); + vec.setValueCount(10); + vec.set(0, "ba".getBytes()); + vec.set(1, "abc".getBytes()); + vec.set(2, "aa".getBytes()); + vec.set(3, "abc".getBytes()); + vec.set(4, "a".getBytes()); + VectorValueComparator<VarCharVector> comparatorValues = new TestVectorValueComparator(); // less than, equal to, greater than + VectorValueComparator<VarCharVector> stableComparator = new StableVectorComparator<>(comparatorValues);//Stable comparator only supports comparing values from the same vector + stableComparator.attachVector(vec); + +Comparing two values at the given indices in the vectors: + +.. code-block:: java + :emphasize-lines: 1-12 + + jshell> stableComparator.compare(0, 1) > 0; + true + jshell> stableComparator.compare(1, 2) < 0; + true + jshell> stableComparator.compare(2, 3) < 0; + true + jshell> stableComparator.compare(1, 3) < 0; + true + jshell> stableComparator.compare(3, 1) > 0; + true + jshell> stableComparator.compare(3, 3) == 0; + true + +Search Values on the Array +========================== + +Linear Search - O(n) +******************** + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + +.. code-block:: java + :emphasize-lines: 27 + + import org.apache.arrow.algorithm.search.VectorSearcher; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // search values on the array + // linear search org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + IntVector rawVector = new IntVector("", rootAllocator); + IntVector negVector = new IntVector("", rootAllocator); + rawVector.allocateNew(10); + rawVector.setValueCount(10); + negVector.allocateNew(1); + negVector.setValueCount(1); + for (int i = 0; i < 10; i++) { // prepare data in sorted order + if (i == 0) { + rawVector.setNull(i); + } else { + rawVector.set(i, i); + } + } + negVector.set(0, -333); + VectorValueComparator<IntVector> comparatorInt = DefaultVectorComparators.createDefaultComparator(rawVector); + + // do search + List<Integer> listResultLinearSearch = new ArrayList<Integer>(); + for (int i = 0; i < 10; i++) { + int result = VectorSearcher.linearSearch(rawVector, comparatorInt, rawVector, i); + listResultLinearSearch.add(result); + } + +Verify results: + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listResultLinearSearch + + listResultLinearSearch ==> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Binary Search - O(log(n)) +************************* + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#binarySearch - O(log(n)) + +.. code-block:: java + :emphasize-lines: 27 + + import org.apache.arrow.algorithm.search.VectorSearcher; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // search values on the array + // linear search org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + IntVector rawVector = new IntVector("", rootAllocator); + IntVector negVector = new IntVector("", rootAllocator); + rawVector.allocateNew(10); + rawVector.setValueCount(10); + negVector.allocateNew(1); + negVector.setValueCount(1); + for (int i = 0; i < 10; i++) { // prepare data in sorted order + if (i == 0) { Review comment: FWIW I think our Java code uses a 2-space indent convention, maybe we should try to be consistent? (It would be easier if the Java code could be linted/formatted separately from the reST.) ########## File path: java/source/io.rst ########## @@ -0,0 +1,348 @@ +.. _arrow-io: + +======================== +Reading and writing data +======================== + +Recipes related to reading and writing data from disk using Apache Arrow. + +Arrow defines two types of binary formats for serializing record batches `IPC <https://arrow.apache.org/docs/java/ipc.html>`_: Streaming format / File or Random Access format + +.. contents:: + +Writing Array +============= + +We are going to use this util for reading and writing data: + +.. code-block:: java + :name: Util + :emphasize-lines: 114 + + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.util.Arrays.asList; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + VectorSchemaRoot createVectorSchemaRoot(){ + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); + + Field age = new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null); + + FieldType intType = new FieldType(true, new ArrowType.Int(32, true), /*dictionary=*/null); + FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/null); + Field childField = new Field("intCol", intType, null); + List<Field> childFields = new ArrayList<>(); + childFields.add(childField); + Field points = new Field("points", listType, childFields); + + // create a definition + Schema schemaPerson = new Schema(asList(name, document, age, points)); + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schemaPerson, rootAllocator); + + // getting field vectors + VarCharVector nameVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("name"); //interface FieldVector + VarCharVector documentVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("document"); //interface FieldVector + IntVector ageVectorOption1 = (IntVector) vectorSchemaRoot.getVector("age"); + ListVector pointsVectorOption1 = (ListVector) vectorSchemaRoot.getVector("points"); + + // add values to the field vectors + setVector(nameVectorOption1, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + setVector(documentVectorOption1, "A".getBytes(), "B".getBytes(), "C".getBytes()); + setVector(ageVectorOption1, 10,20,30); + setVector(pointsVectorOption1, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + vectorSchemaRoot.setRowCount(3); + + return vectorSchemaRoot; + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + + VectorSchemaRoot vectorSchemaRoot = createVectorSchemaRoot(); + + +.. code-block:: java + :emphasize-lines: 1-6 + + jshell> System.out.println(vectorSchemaRoot.contentToTSVString()) + + name document age points + david A 10 [1,3,5,7,9] + gladis B 20 [2,4,6,8,10] + juan C 30 [1,2,3,5,8] + +Writing Arrays with the IPC File Format +*************************************** + +Write - Random Access to File +----------------------------- + +.. code-block:: java + :emphasize-lines: 9 + + import org.apache.arrow.vector.ipc.*; + + import java.io.*; + + // random access format + // write - random access to file + File file = new File("randon_access.arrow"); + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter writer = new ArrowFileWriter(vectorSchemaRoot, null, fileOutputStream.getChannel()); + writer.start(); + writer.writeBatch(); + writer.end(); + +Write - Random Access to Buffer +------------------------------- + +.. code-block:: java + :emphasize-lines: 8 + + import org.apache.arrow.vector.ipc.*; + + import java.io.*; + import java.nio.channels.Channels; + + // write - random access to buffer + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowFileWriter writerBuffer = new ArrowFileWriter(vectorSchemaRoot, null, Channels.newChannel(out)); + writerBuffer.start(); + writerBuffer.writeBatch(); + writerBuffer.end(); + + +Writing Arrays with the IPC Streamed Format Review comment: ```suggestion Writing Arrays with the IPC Stream Format ``` ########## File path: java/source/io.rst ########## @@ -0,0 +1,348 @@ +.. _arrow-io: + +======================== +Reading and writing data +======================== + +Recipes related to reading and writing data from disk using Apache Arrow. + +Arrow defines two types of binary formats for serializing record batches `IPC <https://arrow.apache.org/docs/java/ipc.html>`_: Streaming format / File or Random Access format + +.. contents:: + +Writing Array +============= + +We are going to use this util for reading and writing data: + +.. code-block:: java + :name: Util + :emphasize-lines: 114 + + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.VectorSchemaRoot; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + import org.apache.arrow.vector.types.pojo.Schema; + + import java.util.ArrayList; + import java.util.HashMap; + import java.util.List; + import java.util.Map; + + import static java.util.Arrays.asList; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + VectorSchemaRoot createVectorSchemaRoot(){ + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); + + Field age = new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null); + + FieldType intType = new FieldType(true, new ArrowType.Int(32, true), /*dictionary=*/null); + FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/null); + Field childField = new Field("intCol", intType, null); + List<Field> childFields = new ArrayList<>(); + childFields.add(childField); + Field points = new Field("points", listType, childFields); + + // create a definition + Schema schemaPerson = new Schema(asList(name, document, age, points)); + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schemaPerson, rootAllocator); + + // getting field vectors + VarCharVector nameVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("name"); //interface FieldVector + VarCharVector documentVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("document"); //interface FieldVector + IntVector ageVectorOption1 = (IntVector) vectorSchemaRoot.getVector("age"); + ListVector pointsVectorOption1 = (ListVector) vectorSchemaRoot.getVector("points"); + + // add values to the field vectors + setVector(nameVectorOption1, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + setVector(documentVectorOption1, "A".getBytes(), "B".getBytes(), "C".getBytes()); + setVector(ageVectorOption1, 10,20,30); + setVector(pointsVectorOption1, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + vectorSchemaRoot.setRowCount(3); + + return vectorSchemaRoot; + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + + VectorSchemaRoot vectorSchemaRoot = createVectorSchemaRoot(); + + +.. code-block:: java + :emphasize-lines: 1-6 + + jshell> System.out.println(vectorSchemaRoot.contentToTSVString()) + + name document age points + david A 10 [1,3,5,7,9] + gladis B 20 [2,4,6,8,10] + juan C 30 [1,2,3,5,8] + +Writing Arrays with the IPC File Format +*************************************** + +Write - Random Access to File +----------------------------- + +.. code-block:: java + :emphasize-lines: 9 + + import org.apache.arrow.vector.ipc.*; + + import java.io.*; + + // random access format + // write - random access to file + File file = new File("randon_access.arrow"); + FileOutputStream fileOutputStream = new FileOutputStream(file); + ArrowFileWriter writer = new ArrowFileWriter(vectorSchemaRoot, null, fileOutputStream.getChannel()); + writer.start(); + writer.writeBatch(); Review comment: It should be explicitly explained that `writeBatch` writes the current contents of the `vectorSchemaRoot`. ########## File path: java/source/flight.rst ########## @@ -0,0 +1,472 @@ +.. _arrow-flight: + +============ +Arrow Flight +============ + +Recipes related to leveraging Arrow Flight protocol + +.. contents:: + +Simple Service with Arrow Flight +================================ + +Common Classes +************** + +We are going to use this util for data manipulation: + +* InMemoryStore: A FlightProducer that hosts an in memory store of Arrow buffers. Used for integration testing. Review comment: I would rather we not just pull the integration test code 1:1. It's not relevant to people. I think we should omit the Flight example for now and we can add it later (I can work on that, and Tom and I have been working on some Java/Flight examples too). ########## File path: java/source/data.rst ########## @@ -0,0 +1,608 @@ +================= +Data manipulation +================= + +Recipes related to compare, filtering or transforming data. + +.. contents:: + +We are going to use this util for data manipulation: + +.. code-block:: java + + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + class TestVectorValueComparator extends VectorValueComparator<VarCharVector> { + @Override + public int compareNotNull(int index1, int index2) { + byte b1 = vector1.get(index1)[0]; + byte b2 = vector2.get(index2)[0]; + return b1 - b2; + } + + @Override + public VectorValueComparator<VarCharVector> createNew() { + return new TestVectorValueComparator(); + } + } + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Compare Vectors for Field Equality +================================== + +.. code-block:: java + :emphasize-lines: 10 + + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.compare.TypeEqualsVisitor; + + IntVector right = new IntVector("int", rootAllocator); + IntVector left1 = new IntVector("int", rootAllocator); + IntVector left2 = new IntVector("int2", rootAllocator); + + setVector(right, 10,20,30); + + TypeEqualsVisitor visitor = new TypeEqualsVisitor(right); + +Comparing vector fields: + +.. code-block:: java + :emphasize-lines: 1-4 + + jshell> visitor.equals(left1); + true + jshell> visitor.equals(left2); + false + +Compare Values on the Array +=========================== + +.. code-block:: java + :emphasize-lines: 15-17 + + import org.apache.arrow.algorithm.sort.StableVectorComparator; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.VarCharVector; + + // compare two values at the given indices in the vectors. + // comparing org.apache.arrow.algorithm.sort.VectorValueComparator on algorithm + VarCharVector vec = new VarCharVector("valueindexcomparator", rootAllocator); + vec.allocateNew(100, 5); + vec.setValueCount(10); + vec.set(0, "ba".getBytes()); + vec.set(1, "abc".getBytes()); + vec.set(2, "aa".getBytes()); + vec.set(3, "abc".getBytes()); + vec.set(4, "a".getBytes()); + VectorValueComparator<VarCharVector> comparatorValues = new TestVectorValueComparator(); // less than, equal to, greater than + VectorValueComparator<VarCharVector> stableComparator = new StableVectorComparator<>(comparatorValues);//Stable comparator only supports comparing values from the same vector + stableComparator.attachVector(vec); + +Comparing two values at the given indices in the vectors: + +.. code-block:: java + :emphasize-lines: 1-12 + + jshell> stableComparator.compare(0, 1) > 0; + true + jshell> stableComparator.compare(1, 2) < 0; + true + jshell> stableComparator.compare(2, 3) < 0; + true + jshell> stableComparator.compare(1, 3) < 0; + true + jshell> stableComparator.compare(3, 1) > 0; + true + jshell> stableComparator.compare(3, 3) == 0; + true + +Search Values on the Array +========================== + +Linear Search - O(n) +******************** + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + +.. code-block:: java + :emphasize-lines: 27 + + import org.apache.arrow.algorithm.search.VectorSearcher; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // search values on the array + // linear search org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + IntVector rawVector = new IntVector("", rootAllocator); + IntVector negVector = new IntVector("", rootAllocator); + rawVector.allocateNew(10); + rawVector.setValueCount(10); + negVector.allocateNew(1); + negVector.setValueCount(1); + for (int i = 0; i < 10; i++) { // prepare data in sorted order + if (i == 0) { + rawVector.setNull(i); + } else { + rawVector.set(i, i); + } + } + negVector.set(0, -333); + VectorValueComparator<IntVector> comparatorInt = DefaultVectorComparators.createDefaultComparator(rawVector); + + // do search + List<Integer> listResultLinearSearch = new ArrayList<Integer>(); + for (int i = 0; i < 10; i++) { + int result = VectorSearcher.linearSearch(rawVector, comparatorInt, rawVector, i); + listResultLinearSearch.add(result); + } + +Verify results: + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listResultLinearSearch + + listResultLinearSearch ==> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Binary Search - O(log(n)) +************************* + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#binarySearch - O(log(n)) + +.. code-block:: java + :emphasize-lines: 27 + + import org.apache.arrow.algorithm.search.VectorSearcher; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // search values on the array + // linear search org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + IntVector rawVector = new IntVector("", rootAllocator); + IntVector negVector = new IntVector("", rootAllocator); + rawVector.allocateNew(10); + rawVector.setValueCount(10); Review comment: We never explain this but effectively this zero-initializes the vector, right? Can we explain that in the vectors section of the cookbook? ########## File path: java/source/schema.rst ########## @@ -0,0 +1,329 @@ +=================== +Working with schema +=================== + +Common definition of table has an schema. Java arrow is columnar oriented and it also has an schema representation. +Consider that each name on the schema maps to a columns for a predefined data type Review comment: > Generally, tables have an associated schema. The Arrow Java library has classes for defining schemas. A schema consists of a list of Fields, where each Field has a name and a type for a particular column. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. Review comment: The mutability description is a little redundant. We can say something like `Vector in the Java library is a mutable container, unlike Arrays in many other Arrow implementations which are mutable.` ########## File path: java/source/data.rst ########## @@ -0,0 +1,608 @@ +================= +Data manipulation +================= + +Recipes related to compare, filtering or transforming data. + +.. contents:: + +We are going to use this util for data manipulation: + +.. code-block:: java + + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + class TestVectorValueComparator extends VectorValueComparator<VarCharVector> { + @Override + public int compareNotNull(int index1, int index2) { + byte b1 = vector1.get(index1)[0]; + byte b2 = vector2.get(index2)[0]; + return b1 - b2; + } + + @Override + public VectorValueComparator<VarCharVector> createNew() { + return new TestVectorValueComparator(); + } + } + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Compare Vectors for Field Equality +================================== + +.. code-block:: java + :emphasize-lines: 10 + + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.compare.TypeEqualsVisitor; + + IntVector right = new IntVector("int", rootAllocator); + IntVector left1 = new IntVector("int", rootAllocator); + IntVector left2 = new IntVector("int2", rootAllocator); + + setVector(right, 10,20,30); + + TypeEqualsVisitor visitor = new TypeEqualsVisitor(right); + +Comparing vector fields: + +.. code-block:: java + :emphasize-lines: 1-4 + + jshell> visitor.equals(left1); + true + jshell> visitor.equals(left2); + false + +Compare Values on the Array +=========================== + +.. code-block:: java + :emphasize-lines: 15-17 + + import org.apache.arrow.algorithm.sort.StableVectorComparator; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.VarCharVector; + + // compare two values at the given indices in the vectors. + // comparing org.apache.arrow.algorithm.sort.VectorValueComparator on algorithm + VarCharVector vec = new VarCharVector("valueindexcomparator", rootAllocator); + vec.allocateNew(100, 5); + vec.setValueCount(10); + vec.set(0, "ba".getBytes()); + vec.set(1, "abc".getBytes()); + vec.set(2, "aa".getBytes()); + vec.set(3, "abc".getBytes()); + vec.set(4, "a".getBytes()); + VectorValueComparator<VarCharVector> comparatorValues = new TestVectorValueComparator(); // less than, equal to, greater than + VectorValueComparator<VarCharVector> stableComparator = new StableVectorComparator<>(comparatorValues);//Stable comparator only supports comparing values from the same vector + stableComparator.attachVector(vec); + +Comparing two values at the given indices in the vectors: + +.. code-block:: java + :emphasize-lines: 1-12 + + jshell> stableComparator.compare(0, 1) > 0; + true + jshell> stableComparator.compare(1, 2) < 0; + true + jshell> stableComparator.compare(2, 3) < 0; + true + jshell> stableComparator.compare(1, 3) < 0; + true + jshell> stableComparator.compare(3, 1) > 0; + true + jshell> stableComparator.compare(3, 3) == 0; + true + +Search Values on the Array +========================== + +Linear Search - O(n) +******************** + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + +.. code-block:: java + :emphasize-lines: 27 + + import org.apache.arrow.algorithm.search.VectorSearcher; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // search values on the array + // linear search org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + IntVector rawVector = new IntVector("", rootAllocator); + IntVector negVector = new IntVector("", rootAllocator); + rawVector.allocateNew(10); + rawVector.setValueCount(10); + negVector.allocateNew(1); + negVector.setValueCount(1); + for (int i = 0; i < 10; i++) { // prepare data in sorted order + if (i == 0) { + rawVector.setNull(i); + } else { + rawVector.set(i, i); + } + } + negVector.set(0, -333); + VectorValueComparator<IntVector> comparatorInt = DefaultVectorComparators.createDefaultComparator(rawVector); + + // do search + List<Integer> listResultLinearSearch = new ArrayList<Integer>(); + for (int i = 0; i < 10; i++) { + int result = VectorSearcher.linearSearch(rawVector, comparatorInt, rawVector, i); + listResultLinearSearch.add(result); + } + +Verify results: + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listResultLinearSearch + + listResultLinearSearch ==> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Binary Search - O(log(n)) +************************* + +Algorithm: org.apache.arrow.algorithm.search.VectorSearcher#binarySearch - O(log(n)) + +.. code-block:: java + :emphasize-lines: 27 + + import org.apache.arrow.algorithm.search.VectorSearcher; + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // search values on the array + // linear search org.apache.arrow.algorithm.search.VectorSearcher#linearSearch - O(n) + IntVector rawVector = new IntVector("", rootAllocator); + IntVector negVector = new IntVector("", rootAllocator); + rawVector.allocateNew(10); + rawVector.setValueCount(10); + negVector.allocateNew(1); + negVector.setValueCount(1); + for (int i = 0; i < 10; i++) { // prepare data in sorted order + if (i == 0) { + rawVector.setNull(i); + } else { + rawVector.set(i, i); + } + } + negVector.set(0, -333); + VectorValueComparator<IntVector> comparatorInt = DefaultVectorComparators.createDefaultComparator(rawVector); + + // do search + List<Integer> listResultBinarySearch = new ArrayList<Integer>(); + for (int i = 0; i < 10; i++) { + int result = VectorSearcher.binarySearch(rawVector, comparatorInt, rawVector, i); + listResultBinarySearch.add(result); + } + +Verify results: + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listResultBinarySearch + + listResultBinarySearch ==> [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Sort Values on the Array +======================== + +In-place Sorter - O(nlog(n)) +**************************** + +Sorting by manipulating the original vector. +Algorithm: org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter - O(nlog(n)) + +.. code-block:: java + :emphasize-lines: 22-24 + + import org.apache.arrow.algorithm.sort.DefaultVectorComparators; + import org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter; + import org.apache.arrow.algorithm.sort.VectorValueComparator; + import org.apache.arrow.vector.IntVector; + + // Sort the vector - In-place sorter + IntVector vecToSort = new IntVector("in-place-sorter", rootAllocator); + vecToSort.allocateNew(10); + vecToSort.setValueCount(10); + // fill data to sort + vecToSort.set(0, 10); + vecToSort.set(1, 8); + vecToSort.setNull(2); + vecToSort.set(3, 10); + vecToSort.set(4, 12); + vecToSort.set(5, 17); + vecToSort.setNull(6); + vecToSort.set(7, 23); + vecToSort.set(8, 35); + vecToSort.set(9, 2); + // sort the vector + FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter(); + VectorValueComparator<IntVector> comparator = DefaultVectorComparators.createDefaultComparator(vecToSort); + sorter.sortInPlace(vecToSort, comparator); + +Verify results: + +.. code-block:: java + :emphasize-lines: 1-22 + + jshell> vecToSort.getValueCount()==10; + true + jshell> vecToSort.isNull(0); + true + jshell> vecToSort.isNull(1); + true + jshell> 2==vecToSort.get(2); + true + jshell> 8==vecToSort.get(3); + true + jshell> 10==vecToSort.get(4); + true + jshell> 10==vecToSort.get(5); + true + jshell> 12==vecToSort.get(6); + true + jshell> 17==vecToSort.get(7); + true + jshell> 23==vecToSort.get(8); + true + jshell> 35==vecToSort.get(9); + true + +Out-place Sorter - O(nlog(n)) +***************************** + +Sorting by copies vector elements to a new vector in sorted order - O(nlog(n)) +Algorithm: : org.apache.arrow.algorithm.sort.FixedWidthInPlaceVectorSorter. +FixedWidthOutOfPlaceVectorSorter & VariableWidthOutOfPlaceVectorSor Review comment: This looks truncated. ########## File path: java/source/create.rst ########## @@ -0,0 +1,134 @@ +====================== +Creating arrow objects +====================== + +A vector is the basic unit in the java arrow columnar format. +Vectors are provided by java arrow for the interface FieldVector that extends ValueVector. + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Array of int +============ + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + // create int vector + IntVector intVector = new IntVector("intVector", rootAllocator); Review comment: Or really, this goes to the question above: a FieldVector combines a schema field with a vector? ########## File path: java/source/schema.rst ########## @@ -0,0 +1,330 @@ +=================== +Working with schema +=================== + +Common definition of table has an schema. Java arrow is columnar oriented and it also has an schema representation. +Consider that each name on the schema maps to a columns for a predefined data type + + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Define data type +================ + +Definition of columnar fields for string (name), integer (age) and array (points): + +.. code-block:: java + :emphasize-lines: 6,8,12,15 + + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + + // create a column data type + Field name = new Field("name", FieldType.nullable(new ArrowType.Utf8()), null); + + Field age = new Field("age", FieldType.nullable(new ArrowType.Int(32, true)), null); + + FieldType intType = new FieldType(true, new ArrowType.Int(32, true), /*dictionary=*/null); + FieldType listType = new FieldType(true, new ArrowType.List(), /*dictionary=*/null); + Field childField = new Field("intCol", intType, null); + List<Field> childFields = new ArrayList<>(); + childFields.add(childField); + Field points = new Field("points", listType, childFields); + +.. code-block:: java + :emphasize-lines: 1-5 + + jshell> name; age; points; + + name ==> name: Utf8 + age ==> age: Int(32, true) + points ==> points: List<intCol: Int(32, true)> + +Define metadata +=============== + +In case we need to add metadata to our definition we could use: + +.. code-block:: java + :emphasize-lines: 10 + + import org.apache.arrow.vector.types.pojo.ArrowType; + import org.apache.arrow.vector.types.pojo.Field; + import org.apache.arrow.vector.types.pojo.FieldType; + + // create a column data type + metadata + Map<String, String> metadata = new HashMap<>(); + metadata.put("A", "Id card"); + metadata.put("B", "Passport"); + metadata.put("C", "Visa"); + Field document = new Field("document", new FieldType(true, new ArrowType.Utf8(), null, metadata), null); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> document + + document ==> document: Utf8 + +Create the schema +================= + +Tables detain multiple columns, each with its own name +and type. The union of types and names is what defines a schema. + +.. code-block:: java + :emphasize-lines: 5 + + import org.apache.arrow.vector.types.pojo.Schema; + import static java.util.Arrays.asList; + + // create a definition + Schema schemaPerson = new Schema(asList(name, document, age, points)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> schemaPerson + + schemaPerson ==> Schema<name: Utf8, document: Utf8, age: Int(32, true), points: List<intCol: Int(32, true)>> + +Populate data +============= + +.. code-block:: java + :emphasize-lines: 3,12-15 + + import org.apache.arrow.vector.*; + + VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schemaPerson, rootAllocator); + + // getting field vectors + VarCharVector nameVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("name"); //interface FieldVector + VarCharVector documentVectorOption1 = (VarCharVector) vectorSchemaRoot.getVector("document"); //interface FieldVector + IntVector ageVectorOption1 = (IntVector) vectorSchemaRoot.getVector("age"); + ListVector pointsVectorOption1 = (ListVector) vectorSchemaRoot.getVector("points"); + + // add values to the field vectors + setVector(nameVectorOption1, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + setVector(documentVectorOption1, "A".getBytes(), "B".getBytes(), "C".getBytes()); + setVector(ageVectorOption1, 10,20,30); + setVector(pointsVectorOption1, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + + vectorSchemaRoot.setRowCount(3); + +Render data & metadata: + +.. code-block:: java + + jshell> System.out.println(vectorSchemaRoot.contentToTSVString()); + + name document age points + david A 10 [1,3,5,7,9] + gladis B 20 [2,4,6,8,10] + juan C 30 [1,2,3,5,8] + + jshell> System.out.println(documentVectorOption1.getField().getMetadata()); + + {A=Id card, B=Passport, C=Visa} + +Create the schema from json +=========================== + +For this json definition: Review comment: Right, but this is not a "standard" Arrow representation. The Java library is just applying Jackson to the Java classes. It's not interoperable with Python or any of the other implementations, nor does it appear to have any compatibility or stability guarantees. I'm not sure if we want to tell people to use this, and potentially mislead them, vs. demonstrating how to serialize a schema to the IPC format instead. ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. Review comment: If we are going to mix the `Vector` and `Array` terminology as described below, then can we say something like > A Vector is the basic unit in the Arrow Java library. It's similar to Arrays in other Arrow implementations. And link the second sentence to the Terminology section of the Arrow docs: https://arrow.apache.org/docs/format/Columnar.html#terminology ########## File path: java/source/create.rst ########## @@ -0,0 +1,225 @@ +.. _arrow-create: + +====================== +Creating Arrow Objects +====================== + +| A vector is the basic unit in the Arrow Java library. Vector by definition is intended to be mutable, a Vector can be changed it is mutable. + +| Vectors are provided by java arrow for the interface `FieldVector <https://arrow.apache.org/docs/java/reference/org/apache/arrow/vector/FieldVector.html>`_ that extends `ValueVector <https://arrow.apache.org/docs/java/vector.html>`_. + +.. contents:: + +We are going to use this util for creating arrow objects: + +.. code-block:: java + + import org.apache.arrow.memory.RootAllocator; + import org.apache.arrow.vector.BitVectorHelper; + import org.apache.arrow.vector.IntVector; + import org.apache.arrow.vector.VarCharVector; + import org.apache.arrow.vector.complex.BaseRepeatedValueVector; + import org.apache.arrow.vector.complex.ListVector; + import org.apache.arrow.vector.types.Types; + import org.apache.arrow.vector.types.pojo.FieldType; + + import java.util.List; + + + void setVector(IntVector vector, Integer... values) { + final int length = values.length; + vector.allocateNew(length); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(VarCharVector vector, byte[]... values) { + final int length = values.length; + vector.allocateNewSafe(); + for (int i = 0; i < length; i++) { + if (values[i] != null) { + vector.set(i, values[i]); + } + } + vector.setValueCount(length); + } + + void setVector(ListVector vector, List<Integer>... values) { + vector.allocateNewSafe(); + Types.MinorType type = Types.MinorType.INT; + vector.addOrGetVector(FieldType.nullable(type.getType())); + + IntVector dataVector = (IntVector) vector.getDataVector(); + dataVector.allocateNew(); + + // set underlying vectors + int curPos = 0; + vector.getOffsetBuffer().setInt(0, curPos); + for (int i = 0; i < values.length; i++) { + if (values[i] == null) { + BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); + } else { + BitVectorHelper.setBit(vector.getValidityBuffer(), i); + for (int value : values[i]) { + dataVector.setSafe(curPos, value); + curPos += 1; + } + } + vector.getOffsetBuffer().setInt((i + 1) * BaseRepeatedValueVector.OFFSET_WIDTH, curPos); + } + dataVector.setValueCount(curPos); + vector.setLastSet(values.length - 1); + vector.setValueCount(values.length); + } + + RootAllocator rootAllocator = new RootAllocator(Long.MAX_VALUE); // deal with byte buffer allocation + +Creating Vectors (arrays) +========================= + +Array of Int (32-bit integer value) +----------------------------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.IntVector; + + IntVector intVector = new IntVector("intVector", rootAllocator); + setVector(intVector, 1,2,3); + +.. code-block:: java + :emphasize-lines: 1-3 + + + jshell> intVector + + intVector ==> [1, 2, 3] + +Array of Varchar +---------------- + +.. code-block:: java + :emphasize-lines: 4 + + import org.apache.arrow.vector.VarCharVector; + + VarCharVector varcharVector = new VarCharVector("varcharVector", rootAllocator); + setVector(varcharVector, "david".getBytes(), "gladis".getBytes(), "juan".getBytes()); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> varcharVector + + varcharVector ==> [david, gladis, juan] + +Array of List +------------- + +.. code-block:: java + :emphasize-lines: 6 + + import org.apache.arrow.vector.complex.ListVector; + + import static java.util.Arrays.asList; + + ListVector listVector = ListVector.empty("listVector", rootAllocator); + setVector(listVector, asList(1,3,5,7,9), asList(2,4,6,8,10), asList(1,2,3,5,8)); + +.. code-block:: java + :emphasize-lines: 1-3 + + jshell> listVector + + listVector ==> [[1,3,5,7,9], [2,4,6,8,10], [1,2,3,5,8]] + +Creating VectorSchemaRoot (Table) +================================= + +A `VectorSchemaRoot <https://arrow.apache.org/docs/java/vector_schema_root.html>`_ is a container that can hold batches, batches flow through VectorSchemaRoot as part of a pipeline. Review comment: I still don't think Java has something analogous to Table and it's confusing to treat them as such. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
