http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/reader/vector.ts ---------------------------------------------------------------------- diff --git a/js/src/reader/vector.ts b/js/src/reader/vector.ts new file mode 100644 index 0000000..a3cd798 --- /dev/null +++ b/js/src/reader/vector.ts @@ -0,0 +1,271 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { flatbuffers } from 'flatbuffers'; +import { MessageBatch } from './message'; +import { Vector } from '../vector/vector'; +import * as Schema_ from '../format/Schema_generated'; +import { StructVector } from '../vector/struct'; +import { IteratorState, Dictionaries } from './arrow'; +import { DictionaryVector } from '../vector/dictionary'; +import { Utf8Vector, ListVector, FixedSizeListVector } from '../vector/list'; +import { + TypedArray, TypedArrayCtor, IntArray, FloatArray, + Int8Vector, Int16Vector, Int32Vector, Int64Vector, + Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, + Float32Vector, Float64Vector, IndexVector, DateVector, +} from '../vector/typed'; + +import Int = Schema_.org.apache.arrow.flatbuf.Int; +import Type = Schema_.org.apache.arrow.flatbuf.Type; +import Field = Schema_.org.apache.arrow.flatbuf.Field; +import Precision = Schema_.org.apache.arrow.flatbuf.Precision; +import VectorType = Schema_.org.apache.arrow.flatbuf.VectorType; +import VectorLayout = Schema_.org.apache.arrow.flatbuf.VectorLayout; +import FixedSizeList = Schema_.org.apache.arrow.flatbuf.FixedSizeList; +import FloatingPoint = Schema_.org.apache.arrow.flatbuf.FloatingPoint; +import DictionaryEncoding = Schema_.org.apache.arrow.flatbuf.DictionaryEncoding; + +export function readVector(field: Field, batch: MessageBatch, state: IteratorState, dictionaries: Dictionaries) { + return readDictionaryVector(field, batch, state, dictionaries) || + readTypedVector(field, batch, state, dictionaries); +} + +function readTypedVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) { + let typeType = field.typeType(), readTyped = typedVectorReaders[typeType]; + if (!readTyped) { + throw new Error('Unrecognized vector name "' + Type[typeType] + '" type "' + typeType + '"'); + } + return readTyped(field, batch, iterator, dictionaries); +} + +function readDictionaryVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries) { + let encoding: DictionaryEncoding; + if (dictionaries && (encoding = field.dictionary())) { + let id = encoding.id().toFloat64().toString(); + let fieldType = encoding.indexType() || + /* a dictionary index defaults to signed 32 bit int if unspecified */ + { bitWidth: () => 32, isSigned: () => true }; + let indexField = createSyntheticDictionaryIndexField(field, fieldType); + let index = readIntVector(indexField, batch, iterator, null, fieldType); + return DictionaryVector.create(field, index.length, index, dictionaries[id]); + } +} + +const IntViews = [Int8Array, Int16Array, Int32Array, Int32Array ]; +const Int32Views = [Int32Array, Int32Array, Int32Array, Int32Array ]; +const UintViews = [Uint8Array, Uint16Array, Uint32Array, Uint32Array ]; +const Uint8Views = [Uint8Array, Uint8Array, Uint8Array, Uint8Array ]; +const Uint32Views = [Uint32Array, Uint32Array, Uint32Array, Uint32Array ]; +const FloatViews = [Int8Array, Int16Array, Float32Array, Float64Array]; + +const createIntDataViews = createDataView.bind(null, IntViews, null); +const createUintDataViews = createDataView.bind(null, UintViews, null); +const createDateDataViews = createDataView.bind(null, Uint32Views, null); +const createFloatDataViews = createDataView.bind(null, FloatViews, null); +const createNestedDataViews = createDataView.bind(null, Uint32Views, null); +const createValidityDataViews = createDataView.bind(null, Uint8Views, null); +const createUtf8DataViews = createDataView.bind(null, Uint8Views, Int32Views); + +const floatVectors = { + [Precision.SINGLE]: Float32Vector, + [Precision.DOUBLE]: Float64Vector +}; +const intVectors = [ + [/* unsigned */ Uint8Vector, /* signed */ Int8Vector ], + [/* unsigned */ Uint16Vector, /* signed */ Int16Vector], + [/* unsigned */ Uint32Vector, /* signed */ Int32Vector], + [/* unsigned */ Uint64Vector, /* signed */ Int64Vector] +]; + +function readIntVector(field: Field, batch: MessageBatch, iterator: IteratorState, dictionaries: Dictionaries, fieldType?: FieldType) { + let type = (fieldType || field.type(new Int())); + return type.isSigned() ? + read_IntVector(field, batch, iterator, dictionaries, type) : + readUintVector(field, batch, iterator, dictionaries, type); +} + +const read_IntVector = readVectorLayout<number, IntArray>(createIntDataViews, createIntVector); +const readUintVector = readVectorLayout<number, IntArray>(createUintDataViews, createIntVector); +function createIntVector(field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) { + let type = fieldType || field.type(new Int()), bitWidth = type.bitWidth(); + let Vector = valueForBitWidth(bitWidth, intVectors)[+type.isSigned()]; + return Vector.create(field, length, validity, data || offsets); + // ---------------------- so this is kinda strange ð: + // The dictionary encoded vectors I generated from sample mapd-core queries have the indicies' data buffers + // tagged as VectorType.OFFSET (0) in the field metadata. The current TS impl ignores buffers' layout type, + // and assumes the second buffer is the data for a NullableIntVector. Since we've been stricter about enforcing + // the Arrow spec while parsing, the IntVector's data buffer reads empty in this case. If so, fallback to using + // the offsets buffer as the data, since IntVectors don't have offsets. +} + +const readFloatVector = readVectorLayout<number, FloatArray>( + createFloatDataViews, + (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { + let type = field.type(new FloatingPoint()); + let Vector = floatVectors[type.precision()]; + return Vector.create(field, length, validity, data); + } +); + +const readDateVector = readVectorLayout<Date, Uint32Array>( + createDateDataViews, + (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { + return DateVector.create(field, length, validity, data); + } +); + +const readUtf8Vector = readVectorLayout<string, Uint8Array>( + createUtf8DataViews, + (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { + let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1); + return Utf8Vector.create( + field, length, validity, + Uint8Vector.create(field, data.length, null, data), + IndexVector.create(field, length + 1, null, offsetsAdjusted) + ); + } +); + +const readListVector = readVectorLayout<any[], Uint32Array>( + createNestedDataViews, + (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { + let offsetsAdjusted = new Int32Array(offsets.buffer, offsets.byteOffset, length + 1); + return ListVector.create( + field, length, validity, + readVector(field.children(0), batch, iterator, dictionaries), + IndexVector.create(field, length + 1, null, offsetsAdjusted) + ); + } +); + +const readFixedSizeListVector = readVectorLayout<any[], Uint32Array>( + createNestedDataViews, + (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { + let size = field.type(new FixedSizeList()).listSize(); + return FixedSizeListVector.create( + field, length, size, validity, + readVector(field.children(0), batch, iterator, dictionaries) + ); + } +); + +const readStructVector = readVectorLayout<any[], ArrayLike<any>>( + createNestedDataViews, + (field, length, data, validity, offsets, fieldType, batch, iterator, dictionaries) => { + let vectors: Vector<any>[] = []; + for (let i = -1, n = field.childrenLength(); ++i < n;) { + vectors[i] = readVector(field.children(i), batch, iterator, dictionaries); + } + return StructVector.create(field, length, validity, ...vectors); + } +); + +const typedVectorReaders = { + [Type.Int]: readIntVector, + [Type.Date]: readDateVector, + [Type.List]: readListVector, + [Type.Utf8]: readUtf8Vector, + [Type.Struct_]: readStructVector, + [Type.FloatingPoint]: readFloatVector, + [Type.FixedSizeList]: readFixedSizeListVector, +}; + +type FieldType = { bitWidth(): number; isSigned(): boolean }; +type dataViewFactory<V = TypedArray> = (batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number) => V; +type vectorFactory<TList, V = Vector<any>> = (field: Field, + length: number, + data: TList, + nulls: Uint8Array, + offsets: TypedArray, + fieldType: FieldType, + chunk: MessageBatch, + iterable: IteratorState, + dictionaries: Dictionaries) => V; + +function readVectorLayout<T, TList>(createDataView: dataViewFactory<TList>, createVector: vectorFactory<TList, Vector<T>>) { + return function readLayout( + field: Field, + chunk: MessageBatch, + iterator: IteratorState, + dictionaries: Dictionaries, + integerFieldType?: FieldType + ) { + let batch = chunk.data; + let layoutLength = field.layoutLength(); + let node = batch.nodes(iterator.nodeIndex++); + let data: TList, offsets: any, validity: Uint8Array; + let type, bitWidth, bufferLength, nodeLength = node.length().low; + for (let i = -1; ++i < layoutLength;) { + let layout = field.layout(i); + let buffer = batch.buffers(iterator.bufferIndex++); + if ((type = layout.type()) === VectorType.TYPE || + (bufferLength = buffer.length().low) <= 0 || + (bitWidth = layout.bitWidth()) <= 0) { + continue; + } else if (type === VectorType.DATA) { + data = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength); + } else if (type === VectorType.OFFSET) { + offsets = createDataView(chunk, type, bitWidth, buffer.offset().low, bufferLength); + } else if (node.nullCount().low > 0) { + validity = createValidityDataViews(chunk, type, bitWidth, buffer.offset().low, nodeLength); + } + } + return createVector(field, nodeLength, data, validity, offsets, integerFieldType, chunk, iterator, dictionaries); + }; +} + +function createDataView( + dataViews: TypedArrayCtor<any>[], offsetViews: TypedArrayCtor<any>[] | null, + batch: MessageBatch, type: VectorType, bitWidth: number, offset: number, length: number +) { + const buffer = batch.bytes.buffer; + const byteLength = buffer.byteLength; + const byteOffset = batch.offset + offset; + const DataViewType = valueForBitWidth(bitWidth, type === VectorType.OFFSET && offsetViews || dataViews); + const dataViewLength = ((byteOffset + length) <= byteLength + ? length + : byteLength - byteOffset + ) / DataViewType['BYTES_PER_ELEMENT']; + return new DataViewType(buffer, byteOffset, dataViewLength); +} + +function valueForBitWidth(bitWidth: number, values: any[]) { + return values[bitWidth >> 4] || values[3]; +} + +function createSyntheticDictionaryIndexField(field: Field, type: FieldType) { + let layouts = []; + let builder = new flatbuffers.Builder(); + if (field.nullable()) { + VectorLayout.startVectorLayout(builder); + VectorLayout.addBitWidth(builder, 8); + VectorLayout.addType(builder, VectorType.VALIDITY); + builder.finish(VectorLayout.endVectorLayout(builder)); + layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer())); + builder = new flatbuffers.Builder(); + } + VectorLayout.startVectorLayout(builder); + VectorLayout.addBitWidth(builder, type.bitWidth()); + VectorLayout.addType(builder, VectorType.DATA); + builder.finish(VectorLayout.endVectorLayout(builder)); + layouts.push(VectorLayout.getRootAsVectorLayout(builder.dataBuffer())); + return Object.create(field, { + layout: { value(i) { return layouts[i]; } }, + layoutLength: { value() { return layouts.length; } } + }); +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/table.ts ---------------------------------------------------------------------- diff --git a/js/src/table.ts b/js/src/table.ts new file mode 100644 index 0000000..999bb24 --- /dev/null +++ b/js/src/table.ts @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { readBuffers } from './reader/arrow'; +import { StructVector } from './vector/struct'; +import { Vector, sliceToRangeArgs } from './vector/vector'; + +export class Table implements Iterable<Map<string, any>> { + public length: number; + protected _columns: Vector<any>[]; + protected _columnsMap: { [k: string]: Vector<any> }; + static from(...bytes: Array<Uint8Array | Buffer | string>) { + let columns: Vector<any>[]; + for (let vectors of readBuffers(...bytes)) { + columns = !columns ? vectors : columns.map((v, i) => v.concat(vectors[i])); + } + return new Table(columns); + } + static fromStruct(vector: StructVector) { + return new Table((<any> vector).vectors); + } + constructor(columns: Vector<any>[]) { + this._columns = columns || []; + this.length = Math.max(...this._columns.map((v) => v.length)); + this._columnsMap = this._columns.reduce((map, vec) => { + return (map[vec.name] = vec) && map || map; + }, <any> {}); + } + *[Symbol.iterator]() { + for (let cols = this._columns, i = -1, n = this.length; ++i < n;) { + yield rowAsMap(i, cols); + } + } + *rows(startRow?: number | boolean, endRow?: number | boolean, compact?: boolean) { + let start = startRow as number, end = endRow as number; + if (typeof startRow === 'boolean') { + compact = startRow; + start = end; + end = undefined; + } else if (typeof endRow === 'boolean') { + compact = endRow; + end = undefined; + } + let rowIndex = -1, { length } = this; + const [rowOffset, rowsTotal] = sliceToRangeArgs(length, start, end); + while (++rowIndex < rowsTotal) { + yield this.getRow((rowIndex + rowOffset) % length, compact); + } + } + *cols(startCol?: number, endCol?: number) { + for (const column of this._columns.slice(startCol, endCol)) { + yield column; + } + } + getRow(rowIndex: number, compact?: boolean) { + return (compact && rowAsArray || rowAsObject)(rowIndex, this._columns); + } + getCell(columnName: string, rowIndex: number) { + return this.getColumn(columnName).get(rowIndex); + } + getCellAt(columnIndex: number, rowIndex: number) { + return this.getColumnAt(columnIndex).get(rowIndex); + } + getColumn<T = any>(columnName: string) { + return this._columnsMap[columnName] as Vector<T>; + } + getColumnAt<T = any>(columnIndex: number) { + return this._columns[columnIndex] as Vector<T>; + } + toString({ index = false } = {}) { + const { length } = this; + if (length <= 0) { return ''; } + const maxColumnWidths = []; + const rows = new Array(length + 1); + rows[0] = this._columns.map((c) => c.name); + index && rows[0].unshift('Index'); + for (let i = -1, n = rows.length - 1; ++i < n;) { + rows[i + 1] = this.getRow(i, true); + index && rows[i + 1].unshift(i); + } + // Pass one to convert to strings and count max column widths + for (let i = -1, n = rows.length; ++i < n;) { + const row = rows[i]; + for (let j = -1, k = row.length; ++j < k;) { + const val = row[j] = `${row[j]}`; + maxColumnWidths[j] = !maxColumnWidths[j] + ? val.length + : Math.max(maxColumnWidths[j], val.length); + } + } + // Pass two to pad each one to max column width + for (let i = -1, n = rows.length; ++i < n;) { + const row = rows[i]; + for (let j = -1, k = row.length; ++j < k;) { + row[j] = leftPad(row[j], ' ', maxColumnWidths[j]); + } + rows[i] = row.join(', '); + } + return rows.join('\n'); + } +} + +Table.prototype.length = 0; + +function leftPad(str, fill, n) { + return (new Array(n + 1).join(fill) + str).slice(-1 * n); +} + +function rowAsMap(row: number, columns: Vector<any>[]) { + return columns.reduce((map, vector) => map.set(vector.name, vector.get(row)), new Map()); +} + +function rowAsObject(rowIndex: number, columns: Vector<any>[]) { + return columns.reduce((row, vector) => (row[vector.name] = vector.get(rowIndex)) && row || row, Object.create(null)); +} + +function rowAsArray(rowIndex: number, columns: Vector<any>[]) { + return columns.reduce((row, vector, columnIndex) => (row[columnIndex] = vector.get(rowIndex)) && row || row, new Array(columns.length)); +} http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/types.ts ---------------------------------------------------------------------- diff --git a/js/src/types.ts b/js/src/types.ts deleted file mode 100644 index c541098..0000000 --- a/js/src/types.ts +++ /dev/null @@ -1,597 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import { org } from "./Arrow_generated"; -import { BitArray } from "./bitarray"; - -import { TextDecoder } from "text-encoding"; - -const Type = org.apache.arrow.flatbuf.Type; - -interface IArrayView { - slice(start: number, end: number): IArrayView; - toString(): string; -} - -interface IViewConstructor<T extends IArrayView> { - BYTES_PER_ELEMENT: number; - new(buffer: any, offset: number, length: number): T; -} - -export abstract class Vector { - /** - * Helper function for loading a VALIDITY buffer (for Nullable types) - * bb: flatbuffers.ByteBuffer - * buffer: org.apache.arrow.flatbuf.Buffer - */ - public static loadValidityBuffer(bb, buffer): BitArray { - const arrayBuffer = bb.bytes_.buffer; - const offset = bb.bytes_.byteOffset + buffer.offset; - return new BitArray(arrayBuffer, offset, buffer.length * 8); - } - - /** - * Helper function for loading an OFFSET buffer - * buffer: org.apache.arrow.flatbuf.Buffer - */ - public static loadOffsetBuffer(bb, buffer): Int32Array { - const arrayBuffer = bb.bytes_.buffer; - const offset = bb.bytes_.byteOffset + buffer.offset; - const length = buffer.length / Int32Array.BYTES_PER_ELEMENT; - return new Int32Array(arrayBuffer, offset, length); - } - - public field: any; - public name: string; - public length: number; - public nullCount: number; - - constructor(field) { - this.field = field; - this.name = field.name(); - } - - /* Access datum at index i */ - public abstract get(i); - /* Return array representing data in the range [start, end) */ - public abstract slice(start: number, end: number); - /* Return array of child vectors, for container types */ - public getChildVectors() { - return []; - } - - /** - * Use recordBatch fieldNodes and Buffers to construct this Vector - * bb: flatbuffers.ByteBuffer - * node: org.apache.arrow.flatbuf.FieldNode - * buffers: { offset: number, length: number }[] - */ - public loadData(bb, node, buffers) { - this.length = node.length().low; - this.nullCount = node.nullCount().low; - this.loadBuffers(bb, node, buffers); - } - - protected abstract loadBuffers(bb, node, buffers); -} - -class SimpleVector<T extends IArrayView> extends Vector { - protected dataView: T; - private TypedArray: IViewConstructor<T>; - - constructor(field, TypedArray: IViewConstructor<T>) { - super(field); - this.TypedArray = TypedArray; - } - - public get(i) { - return this.dataView[i]; - } - - public getDataView() { - return this.dataView; - } - - public toString() { - return this.dataView.toString(); - } - - public slice(start, end) { - return this.dataView.slice(start, end); - } - - protected loadBuffers(bb, node, buffers) { - this.loadDataBuffer(bb, buffers[0]); - } - - /* - * buffer: org.apache.arrow.flatbuf.Buffer - */ - protected loadDataBuffer(bb, buffer) { - const arrayBuffer = bb.bytes_.buffer; - const offset = bb.bytes_.byteOffset + buffer.offset; - const length = buffer.length / this.TypedArray.BYTES_PER_ELEMENT; - this.dataView = new this.TypedArray(arrayBuffer, offset, length); - } - -} - -class NullableSimpleVector<T extends IArrayView> extends SimpleVector<T> { - protected validityView: BitArray; - - public get(i: number) { - if (this.validityView.get(i)) { - return this.dataView[i]; - } else { - return null; - } - } - - public getValidityVector() { - return this.validityView; - } - - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - this.loadDataBuffer(bb, buffers[1]); - } -} - -/* tslint:disable max-line-length */ -class Uint8Vector extends SimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } } -class Uint16Vector extends SimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } } -class Uint32Vector extends SimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } } -class Int8Vector extends SimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } } -class Int16Vector extends SimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } } -class Int32Vector extends SimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } } -class Float32Vector extends SimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } } -class Float64Vector extends SimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } } - -class NullableUint8Vector extends NullableSimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } } -class NullableUint16Vector extends NullableSimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } } -class NullableUint32Vector extends NullableSimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } } -class NullableInt8Vector extends NullableSimpleVector<Uint8Array> { constructor(field) { super(field, Uint8Array); } } -class NullableInt16Vector extends NullableSimpleVector<Uint16Array> { constructor(field) { super(field, Uint16Array); } } -class NullableInt32Vector extends NullableSimpleVector<Uint32Array> { constructor(field) { super(field, Uint32Array); } } -class NullableFloat32Vector extends NullableSimpleVector<Float32Array> { constructor(field) { super(field, Float32Array); } } -class NullableFloat64Vector extends NullableSimpleVector<Float64Array> { constructor(field) { super(field, Float64Array); } } -/* tslint:enable max-line-length */ - -class Uint64Vector extends SimpleVector<Uint32Array> { - constructor(field) { - super(field, Uint32Array); - } - - public get(i: number) { - return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; - } -} - -class NullableUint64Vector extends NullableSimpleVector<Uint32Array> { - constructor(field) { - super(field, Uint32Array); - } - - public get(i: number) { - if (this.validityView.get(i)) { - return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; - } else { - return null; - } - } -} - -class Int64Vector extends NullableSimpleVector<Uint32Array> { - constructor(field) { - super(field, Uint32Array); - } - - public get(i: number) { - return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; - } -} - -class NullableInt64Vector extends NullableSimpleVector<Uint32Array> { - constructor(field) { - super(field, Uint32Array); - } - - public get(i: number) { - if (this.validityView.get(i)) { - return { low: this.dataView[i * 2], high: this.dataView[(i * 2) + 1] }; - } else { - return null; - } - } -} - -class DateVector extends SimpleVector<Uint32Array> { - constructor(field) { - super(field, Uint32Array); - } - - public get(i) { - return new Date(super.get(2 * i + 1) * Math.pow(2, 32) + super.get(2 * i)); - } -} - -class NullableDateVector extends DateVector { - private validityView: BitArray; - - public get(i) { - if (this.validityView.get(i)) { - return super.get(i); - } else { - return null; - } - } - - public getValidityVector() { - return this.validityView; - } - - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - this.loadDataBuffer(bb, buffers[1]); - } -} - -class Utf8Vector extends SimpleVector<Uint8Array> { - private static decoder: TextDecoder = new TextDecoder("utf8"); - - protected offsetView: Int32Array; - - constructor(field) { - super(field, Uint8Array); - } - - public get(i) { - return Utf8Vector.decoder.decode(this.dataView.slice(this.offsetView[i], this.offsetView[i + 1])); - } - - public slice(start: number, end: number) { - const result: string[] = []; - for (let i: number = start; i < end; i++) { - result.push(this.get(i)); - } - return result; - } - - public getOffsetView() { - return this.offsetView; - } - - protected loadBuffers(bb, node, buffers) { - this.offsetView = Vector.loadOffsetBuffer(bb, buffers[0]); - this.loadDataBuffer(bb, buffers[1]); - } -} - -class NullableUtf8Vector extends Utf8Vector { - private validityView: BitArray; - - public get(i) { - if (this.validityView.get(i)) { - return super.get(i); - } else { - return null; - } - } - - public getValidityVector() { - return this.validityView; - } - - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - this.offsetView = Vector.loadOffsetBuffer(bb, buffers[1]); - this.loadDataBuffer(bb, buffers[2]); - } -} - -// Nested Types -class ListVector extends Uint32Vector { - private dataVector: Vector; - - constructor(field, dataVector: Vector) { - super(field); - this.dataVector = dataVector; - } - - public getChildVectors() { - return [this.dataVector]; - } - - public get(i) { - const offset = super.get(i); - if (offset === null) { - return null; - } - const nextOffset = super.get(i + 1); - return this.dataVector.slice(offset, nextOffset); - } - - public toString() { - return "length: " + (this.length); - } - - public slice(start: number, end: number) { - const result = []; - for (let i = start; i < end; i++) { - result.push(this.get(i)); - } - return result; - } - - protected loadBuffers(bb, node, buffers) { - super.loadBuffers(bb, node, buffers); - this.length -= 1; - } -} - -class NullableListVector extends ListVector { - private validityView: BitArray; - - public get(i) { - if (this.validityView.get(i)) { - return super.get(i); - } else { - return null; - } - } - - public getValidityVector() { - return this.validityView; - } - - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - this.loadDataBuffer(bb, buffers[1]); - this.length -= 1; - } -} - -class FixedSizeListVector extends Vector { - public size: number; - private dataVector: Vector; - - constructor(field, size: number, dataVector: Vector) { - super(field); - this.size = size; - this.dataVector = dataVector; - } - - public getChildVectors() { - return [this.dataVector]; - } - - public get(i: number) { - return this.dataVector.slice(i * this.size, (i + 1) * this.size); - } - - public slice(start: number, end: number) { - const result = []; - for (let i = start; i < end; i++) { - result.push(this.get(i)); - } - return result; - } - - public getListSize() { - return this.size; - } - - protected loadBuffers(bb, node, buffers) { - // no buffers to load - } -} - -class NullableFixedSizeListVector extends FixedSizeListVector { - private validityView: BitArray; - - public get(i: number) { - if (this.validityView.get(i)) { - return super.get(i); - } else { - return null; - } - } - - public getValidityVector() { - return this.validityView; - } - - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - } -} - -class StructVector extends Vector { - private validityView: BitArray; - private vectors: Vector[]; - - constructor(field, vectors: Vector[]) { - super(field); - this.vectors = vectors; - } - - public getChildVectors() { - return this.vectors; - } - - public get(i: number) { - if (this.validityView.get(i)) { - return this.vectors.map((v: Vector) => v.get(i)); - } else { - return null; - } - } - - public slice(start: number, end: number) { - const result = []; - for (let i = start; i < end; i++) { - result.push(this.get(i)); - } - return result; - } - - public getValidityVector() { - return this.validityView; - } - - protected loadBuffers(bb, node, buffers) { - this.validityView = Vector.loadValidityBuffer(bb, buffers[0]); - } -} - -class DictionaryVector extends Vector { - private indices: Vector; - private dictionary: Vector; - - constructor(field, indices: Vector, dictionary: Vector) { - super(field); - this.indices = indices; - this.dictionary = dictionary; - } - - public get(i) { - const encoded = this.indices.get(i); - if (encoded == null) { - return null; - } else { - return this.dictionary.get(encoded); - } - } - - /** Get the dictionary encoded value */ - public getEncoded(i) { - return this.indices.get(i); - } - - public slice(start, end) { - return this.indices.slice(start, end); // TODO decode - } - - public getChildVectors() { - return this.indices.getChildVectors(); - } - - /** Get the index (encoded) vector */ - public getIndexVector() { - return this.indices; - } - - /** Get the dictionary vector */ - public getDictionaryVector() { - return this.dictionary; - } - - public toString() { - return this.indices.toString(); - } - - protected loadBuffers(bb, node, buffers) { - this.indices.loadData(bb, node, buffers); - } -} - -export function vectorFromField(field, dictionaries): Vector { - const dictionary = field.dictionary(); - const nullable = field.nullable(); - if (dictionary == null) { - const typeType = field.typeType(); - if (typeType === Type.List) { - const dataVector = vectorFromField(field.children(0), dictionaries); - return nullable ? new NullableListVector(field, dataVector) : new ListVector(field, dataVector); - } else if (typeType === Type.FixedSizeList) { - const dataVector = vectorFromField(field.children(0), dictionaries); - const size = field.type(new org.apache.arrow.flatbuf.FixedSizeList()).listSize(); - if (nullable) { - return new NullableFixedSizeListVector(field, size, dataVector); - } else { - return new FixedSizeListVector(field, size, dataVector); - } - } else if (typeType === Type.Struct_) { - const vectors: Vector[] = []; - for (let i: number = 0; i < field.childrenLength(); i++) { - vectors.push(vectorFromField(field.children(i), dictionaries)); - } - return new StructVector(field, vectors); - } else { - if (typeType === Type.Int) { - const type = field.type(new org.apache.arrow.flatbuf.Int()); - return _createIntVector(field, type.bitWidth(), type.isSigned(), nullable); - } else if (typeType === Type.FloatingPoint) { - const precision = field.type(new org.apache.arrow.flatbuf.FloatingPoint()).precision(); - if (precision === org.apache.arrow.flatbuf.Precision.SINGLE) { - return nullable ? new NullableFloat32Vector(field) : new Float32Vector(field); - } else if (precision === org.apache.arrow.flatbuf.Precision.DOUBLE) { - return nullable ? new NullableFloat64Vector(field) : new Float64Vector(field); - } else { - throw new Error("Unimplemented FloatingPoint precision " + precision); - } - } else if (typeType === Type.Utf8) { - return nullable ? new NullableUtf8Vector(field) : new Utf8Vector(field); - } else if (typeType === Type.Date) { - return nullable ? new NullableDateVector(field) : new DateVector(field); - } else { - throw new Error("Unimplemented type " + typeType); - } - } - } else { - // determine arrow type - default is signed 32 bit int - const type = dictionary.indexType(); - let bitWidth = 32; - let signed = true; - if (type != null) { - bitWidth = type.bitWidth(); - signed = type.isSigned(); - } - const indices = _createIntVector(field, bitWidth, signed, nullable); - return new DictionaryVector(field, indices, dictionaries[dictionary.id().toFloat64().toString()]); - } -} - -function _createIntVector(field, bitWidth, signed, nullable) { - if (bitWidth === 64) { - if (signed) { - return nullable ? new NullableInt64Vector(field) : new Int64Vector(field); - } else { - return nullable ? new NullableUint64Vector(field) : new Uint64Vector(field); - } - } else if (bitWidth === 32) { - if (signed) { - return nullable ? new NullableInt32Vector(field) : new Int32Vector(field); - } else { - return nullable ? new NullableUint32Vector(field) : new Uint32Vector(field); - } - } else if (bitWidth === 16) { - if (signed) { - return nullable ? new NullableInt16Vector(field) : new Int16Vector(field); - } else { - return nullable ? new NullableUint16Vector(field) : new Uint16Vector(field); - } - } else if (bitWidth === 8) { - if (signed) { - return nullable ? new NullableInt8Vector(field) : new Int8Vector(field); - } else { - return nullable ? new NullableUint8Vector(field) : new Uint8Vector(field); - } - } else { - throw new Error("Unimplemented Int bit width " + bitWidth); - } -} http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/vector/dictionary.ts ---------------------------------------------------------------------- diff --git a/js/src/vector/dictionary.ts b/js/src/vector/dictionary.ts new file mode 100644 index 0000000..de811ea --- /dev/null +++ b/js/src/vector/dictionary.ts @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; + +export class DictionaryVector<T> extends Vector<T> { + protected data: Vector<T>; + protected keys: Vector<number>; + constructor(index: Vector<number>, dictionary: Vector<T>) { + super(); + this.keys = index; + this.data = dictionary; + this.length = index && index.length || 0; + } + index(index: number) { + return this.keys.get(index); + } + value(index: number) { + return this.data.get(index); + } + get(index: number) { + return this.value(this.index(index)); + } + concat(vector: DictionaryVector<T>) { + return DictionaryVector.from(this, + this.length + vector.length, + this.keys.concat(vector.keys), + this.data + ); + } + *[Symbol.iterator]() { + let { data } = this; + for (const loc of this.keys) { + yield data.get(loc); + } + } +} http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/vector/list.ts ---------------------------------------------------------------------- diff --git a/js/src/vector/list.ts b/js/src/vector/list.ts new file mode 100644 index 0000000..7360d96 --- /dev/null +++ b/js/src/vector/list.ts @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { TextDecoder } from 'text-encoding'; +import { IndexVector, BitVector, ValidityArgs } from './typed'; + +export class ListVectorBase<T> extends Vector<T> { + protected values: Vector<T>; + protected offsets: IndexVector; + constructor(validity: ValidityArgs, values: Vector<any>, offsets: IndexVector) { + super(); + this.values = values; + this.offsets = offsets; + validity && (this.validity = BitVector.from(validity)); + } + get(index: number) { + let batch, from, to, { offsets } = this; + if (!this.validity.get(index) || + /* return null if `to` is null */ + ((to = offsets.get(index + 1)) === null) || !( + /* + return null if `batch` is less than than 0. this check is placed + second to avoid creating the [from, batch] tuple if `to` is null + */ + ([from, batch] = offsets.get(index, true) as number[]) && batch > -1)) { + return null; + } + return this.values.slice(from, to, batch) as any; + } + concat(vector: ListVectorBase<T>) { + return (this.constructor as typeof ListVectorBase).from(this, + this.length + vector.length, + this.validity.concat(vector.validity), + this.values.concat(vector.values), + this.offsets.concat(vector.offsets) + ); + } + *[Symbol.iterator]() { + let v, r1, r2, { values } = this; + let it = this.offsets[Symbol.iterator](); + let iv = this.validity[Symbol.iterator](); + while (!(v = iv.next()).done && !(r1 = it.next()).done && !(r2 = it.next()).done) { + yield !v.value ? null : values.slice(r1.value[0], r2.value, r1.value[1]) as any; + } + } +} + +export class ListVector<T> extends ListVectorBase<T[]> {} +export class Utf8Vector extends ListVectorBase<string> { + protected static decoder = new TextDecoder(`utf-8`); + get(index: number) { + let chars = super.get(index) as any; + return chars ? Utf8Vector.decoder.decode(chars) : null; + } + *[Symbol.iterator]() { + let decoder = Utf8Vector.decoder; + for (const chars of super[Symbol.iterator]()) { + yield !chars ? null : decoder.decode(chars); + } + } +} + +export class FixedSizeListVector<T> extends Vector<T[]> { + protected size: number; + protected values: Vector<T>; + constructor(size: number, validity: ValidityArgs, values: Vector<T>) { + super(); + this.values = values; + this.size = Math.abs(size | 0) || 1; + validity && (this.validity = BitVector.from(validity)); + } + get(index: number) { + return !this.validity.get(index) ? null : this.values.slice( + this.size * index, this.size * (index + 1) + ) as T[]; + } + concat(vector: FixedSizeListVector<T>) { + return FixedSizeListVector.from(this, + this.length + vector.length, + this.size, + this.validity.concat(vector.validity), + this.values.concat(vector.values) + ); + } + *[Symbol.iterator]() { + let v, i = -1; + let { size, length, values } = this; + let iv = this.validity[Symbol.iterator](); + while (!(v = iv.next()).done && ++i < length) { + yield !v.value ? null : values.slice(size * i, size * (i + 1)) as T[]; + } + } +} http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/vector/struct.ts ---------------------------------------------------------------------- diff --git a/js/src/vector/struct.ts b/js/src/vector/struct.ts new file mode 100644 index 0000000..e59ac91 --- /dev/null +++ b/js/src/vector/struct.ts @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { BitVector, ValidityArgs } from './typed'; + +export class StructVector extends Vector<any[]> { + protected vectors: Vector<any>[]; + constructor(validity: ValidityArgs, ...vectors: Vector<any>[]) { + super(); + this.vectors = vectors; + this.length = Math.max(0, ...vectors.map((v) => v.length)); + validity && (this.validity = BitVector.from(validity)); + } + get(index: number) { + return this.validity.get(index) ? this.vectors.map((v) => v.get(index)) : null; + } + concat(vector: StructVector) { + return StructVector.from(this, + this.length + vector.length, + this.validity.concat(vector.validity), + ...this.vectors.map((v, i) => v.concat(vector.vectors[i])) + ); + } +} http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/vector/typed.ts ---------------------------------------------------------------------- diff --git a/js/src/vector/typed.ts b/js/src/vector/typed.ts new file mode 100644 index 0000000..b38812e --- /dev/null +++ b/js/src/vector/typed.ts @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { flatbuffers } from 'flatbuffers'; + +import Long = flatbuffers.Long; + +export type VArray<T = any> = { + [k: number]: T; length: number; + constructor: VArrayCtor<VArray<T>>; +}; + +export type VArrayCtor<VArray> = { + readonly prototype: VArray; + BYTES_PER_ELEMENT?: number; + new(...args: any[]): VArray; +}; + +export class VirtualVector<T, TArrayType = VArray<T>> extends Vector<T> { + protected lists: TArrayType[]; + protected _arrayType: VArrayCtor<TArrayType>; + public get arrayType() { return this._arrayType; } + constructor(...lists: TArrayType[]) { + super(); + this.lists = lists.filter(Boolean); + } + get(index: number): T { + /* inlined `findVirtual` impl */ + let rows, length, lists = this.lists; + for (let batch = -1; + (rows = lists[++batch]) && + (length = rows.length) <= index && + 0 <= (index -= length);) {} + return rows && -1 < index ? rows[index] : null; + } + protected range(from: number, total: number, batch?: number) { + /* inlined `findVirtual` impl */ + let rows, local = from, length; + let { lists, _arrayType } = this; + for (batch = (batch || 0) - 1; + (rows = lists[++batch]) && + (length = rows.length) <= local && + 0 <= (local -= length);) {} + if (rows && local > -1) { + let index = 0, listsLength = lists.length; + let set: any = Array.isArray(rows) ? arraySet : typedArraySet; + let slice = _arrayType['prototype']['subarray'] || _arrayType['prototype']['slice']; + let source = slice.call(rows, local, local + total), target = source; + // Perf optimization: if the first slice contains all the values we're looking for, + // we don't have to copy values to a target Array. If we're slicing a TypedArray, + // this is a significant improvement as we avoid the memcpy ð + if (source.length < total) { + target = new _arrayType(total); + while ((index = set(source, target, index)) < total) { + rows = lists[batch = ((batch + 1) % listsLength)]; + source = slice.call(rows, 0, Math.min(rows.length, total - index)); + } + } + return target as any; + } + return new _arrayType(0); + } + *[Symbol.iterator]() { + let index = -1, { lists, length } = this; + for (let outer = -1, n = lists.length; ++outer < n;) { + let list = lists[outer] as any; + for (let inner = -1, k = list.length; ++index < length && ++inner < k;) { + yield list[inner]; + } + } + } +} + +export type ValidityArgs = Vector<boolean> | Uint8Array; +export class BitVector extends VirtualVector<boolean, Uint8Array> { + static constant: Vector<boolean> = new (class ValidVector extends Vector<boolean> { + get() { return true; } + *[Symbol.iterator]() { + do { yield true; } while (true); + } + })(); + static from(src: any) { + return src instanceof BitVector ? src + : src === BitVector.constant ? src + : src instanceof Uint8Array ? new BitVector(src) + : src instanceof Array ? new BitVector(BitVector.pack(src)) + : src instanceof Vector ? new BitVector(BitVector.pack(src)) + : BitVector.constant as Vector<any>; + } + static pack(values: Iterable<any>) { + let xs = [], n, i = 0; + let bit = 0, byte = 0; + for (const value of values) { + value && (byte |= 1 << bit); + if (++bit === 8) { + xs[i++] = byte; + byte = bit = 0; + } + } + if (i === 0 || bit > 0) { xs[i++] = byte; } + if (i % 8 && (n = n = i + 8 - i % 8)) { + do { xs[i] = 0; } while (++i < n); + } + return new Uint8Array(xs); + } + constructor(...lists: Uint8Array[]) { + super(...lists); + this.length = this.lists.reduce((l, xs) => l + xs['length'], 0); + } + get(index: number) { + /* inlined `findVirtual` impl */ + let rows, length, lists = this.lists; + for (let batch = -1; + (rows = lists[++batch]) && + (length = rows.length) <= index && + 0 <= (index -= length);) {} + return !(!rows || index < 0 || (rows[index >> 3 | 0] & 1 << index % 8) === 0); + } + set(index: number, value: boolean) { + /* inlined `findVirtual` impl */ + let rows, length, lists = this.lists; + for (let batch = -1; + (rows = lists[++batch]) && + (length = rows.length) <= index && + 0 <= (index -= length);) {} + if (rows && index > -1) { + value + ? (rows[index >> 3 | 0] |= (1 << (index % 8))) + : (rows[index >> 3 | 0] &= ~(1 << (index % 8))); + } + } + concat(vector: BitVector) { + return new BitVector(...this.lists, ...vector.lists); + } + *[Symbol.iterator]() { + for (const byte of super[Symbol.iterator]()) { + for (let i = -1; ++i < 8;) { + yield (byte & 1 << i) !== 0; + } + } + } +} + +export class TypedVector<T, TArrayType> extends VirtualVector<T, TArrayType> { + constructor(validity: ValidityArgs, ...lists: TArrayType[]) { + super(...lists); + validity && (this.validity = BitVector.from(validity)); + } + concat(vector: TypedVector<T, TArrayType>) { + return (this.constructor as typeof TypedVector).from(this, + this.length + vector.length, + this.validity.concat(vector.validity), + ...this.lists, ...vector.lists + ); + } +} + +export class DateVector extends TypedVector<Date, Uint32Array> { + get(index: number) { + return !this.validity.get(index) ? null : new Date( + Math.pow(2, 32) * + <any> super.get(2 * index + 1) + + <any> super.get(2 * index) + ); + } + *[Symbol.iterator]() { + let v, low, high; + let it = super[Symbol.iterator](); + let iv = this.validity[Symbol.iterator](); + while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) { + yield !v.value ? null : new Date(Math.pow(2, 32) * high.value + low.value); + } + } +} + +export class IndexVector extends TypedVector<number | number[], Int32Array> { + get(index: number, returnWithBatchIndex = false) { + /* inlined `findVirtual` impl */ + let rows, length, batch = -1, lists = this.lists; + for (; + (rows = lists[++batch]) && + (length = rows.length) <= index && + 0 <= (index -= length);) {} + return !returnWithBatchIndex + ? (rows && -1 < index ? rows[index + batch] : null) as number + : (rows && -1 < index ? [rows[index + batch], batch] : [0, -1]) as number[]; + } + *[Symbol.iterator]() { + // Alternate between iterating a tuple of [from, batch], and to. The from + // and to values are relative to the record batch they're defined in, so + // `ListVectorBase` needs to know the right batch to read. + let xs = new Int32Array(2), { lists } = this; + for (let i = -1, n = lists.length; ++i < n;) { + let list = lists[i] as any; + for (let j = -1, k = list.length - 1; ++j < k;) { + xs[1] = i; + xs[0] = list[j]; + yield xs; + yield list[j + 1]; + } + } + } +} + +export class ByteVector<TList> extends TypedVector<number, TList> { + get(index: number) { + return this.validity.get(index) ? super.get(index) : null; + } + *[Symbol.iterator]() { + let v, r, { validity } = this; + let it = super[Symbol.iterator](); + // fast path the case of no nulls + if (validity === BitVector.constant) { + yield* it; + } else { + let iv = validity[Symbol.iterator](); + while (!(v = iv.next()).done && !(r = it.next()).done) { + yield !v.value ? null : r.value; + } + } + } +} + +export class LongVector<TList> extends TypedVector<Long, TList> { + get(index: number) { + return !this.validity.get(index) ? null : new Long( + <any> super.get(index * 2), /* low */ + <any> super.get(index * 2 + 1) /* high */ + ); + } + *[Symbol.iterator]() { + let v, low, high; + let it = super[Symbol.iterator](); + let iv = this.validity[Symbol.iterator](); + while (!(v = iv.next()).done && !(low = it.next()).done && !(high = it.next()).done) { + yield !v.value ? null : new Long(low.value, high.value); + } + } +} + +export class Int8Vector extends ByteVector<Int8Array> {} +export class Int16Vector extends ByteVector<Int16Array> {} +export class Int32Vector extends ByteVector<Int32Array> {} +export class Int64Vector extends LongVector<Int32Array> {} +export class Uint8Vector extends ByteVector<Uint8Array> {} +export class Uint16Vector extends ByteVector<Uint16Array> {} +export class Uint32Vector extends ByteVector<Uint32Array> {} +export class Uint64Vector extends LongVector<Uint32Array> {} +export class Float32Vector extends ByteVector<Float32Array> {} +export class Float64Vector extends ByteVector<Float64Array> {} + +LongVector.prototype.stride = 2; +(Vector.prototype as any).lists = []; +(Vector.prototype as any).validity = BitVector.constant; +(VirtualVector.prototype as any)._arrayType = Array; +(BitVector.prototype as any)._arrayType = Uint8Array; +(Int8Vector.prototype as any)._arrayType = Int8Array; +(Int16Vector.prototype as any)._arrayType = Int16Array; +(Int32Vector.prototype as any)._arrayType = Int32Array; +(Int64Vector.prototype as any)._arrayType = Int32Array; +(Uint8Vector.prototype as any)._arrayType = Uint8Array; +(Uint16Vector.prototype as any)._arrayType = Uint16Array; +(Uint32Vector.prototype as any)._arrayType = Uint32Array; +(Uint64Vector.prototype as any)._arrayType = Uint32Array; +(DateVector.prototype as any)._arrayType = Uint32Array; +(IndexVector.prototype as any)._arrayType = Int32Array; +(Float32Vector.prototype as any)._arrayType = Float32Array; +(Float64Vector.prototype as any)._arrayType = Float64Array; + +function arraySet<T>(source: Array<T>, target: Array<T>, index: number) { + for (let i = 0, n = source.length; i < n;) { + target[index++] = source[i++]; + } + return index; +} + +function typedArraySet(source: TypedArray, target: TypedArray, index: number) { + return target.set(source, index) || index + source.length; +} + +// Rather than eat the iterator cost, we've inlined this function into the relevant functions +// function* findVirtual<TList>(index: number, lists: TList[], batch?: number) { +// let rows, length; +// for (batch = (batch || 0) - 1; +// (rows = lists[++batch]) && +// (length = rows.length) <= index && +// 0 <= (index -= length);) {} +// return rows && -1 < index ? yield [rows, index, batch] : null; +// } + +export type TypedArrayCtor<T extends TypedArray> = { + readonly prototype: T; + readonly BYTES_PER_ELEMENT: number; + new(length: number): T; + new(array: ArrayLike<number>): T; + new(buffer: ArrayBufferLike, byteOffset?: number, length?: number): T; +}; + +export type FloatArray = Float32Array | Float64Array; +export type IntArray = Int8Array | Int16Array | Int32Array | Uint8ClampedArray | Uint8Array | Uint16Array | Uint32Array; + +export type TypedArray = ( + Int8Array | + Uint8Array | + Int16Array | + Int32Array | + Uint16Array | + Uint32Array | + Float32Array | + Float64Array | + Uint8ClampedArray); http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/src/vector/vector.ts ---------------------------------------------------------------------- diff --git a/js/src/vector/vector.ts b/js/src/vector/vector.ts new file mode 100644 index 0000000..1f39f87 --- /dev/null +++ b/js/src/vector/vector.ts @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Schema_ from '../format/Schema_generated'; +export import Type = Schema_.org.apache.arrow.flatbuf.Type; +export import Field = Schema_.org.apache.arrow.flatbuf.Field; + +export function sliceToRangeArgs(length: number, start: number, end?: number) { + let total = length, from = start || 0; + let to = end === end && typeof end == 'number' ? end : total; + if (to < 0) { to = total + to; } + if (from < 0) { from = total - (from * -1) % total; } + if (to < from) { from = to; to = start; } + total = !isFinite(total = (to - from)) || total < 0 ? 0 : total; + return [from, total]; +} + +export class Vector<T> implements Iterable<T> { + static defaultName = ''; + static defaultProps = new Map(); + static defaultType = Type[Type.NONE]; + static create<T = any>(field: Field, length: number, ...args: any[]) { + let vector = new this<T>(...args), m; + vector.length = length; + vector.name = field.name(); + vector.type = Type[field.typeType()]; + if ((m = field.customMetadataLength()) > 0) { + let entry, i = 0, data = vector.props = new Map(); + do { + entry = field.customMetadata(i); + data[entry.key()] = entry.value(); + } while (++i < m); + } + return vector; + } + static from<T = any>(source: Vector<T>, length: number, ...args: any[]) { + let vector = new this<T>(...args); + vector.length = length; + source.name !== Vector.defaultName && (vector.name = source.name); + source.type !== Vector.defaultType && (vector.type = source.type); + source.props !== Vector.defaultProps && (vector.props = source.props); + return vector; + } + public name: string; + public type: string; + public length: number; + public stride: number; + public props: Map<PropertyKey, any>; + protected validity: Vector<boolean>; + get(index: number): T { return null; } + concat(vector: Vector<T>) { return vector; } + slice<R = T>(start?: number, end?: number, batch?: number) { + const { stride } = this; + const [offset, length] = sliceToRangeArgs( + stride * this.length, stride * (start || 0), stride * end + ); + return this.range<R>(offset, length, batch); + } + protected range<R = T>(index: number, length: number, batch?: number) { + const result = new Array<R>(length); + for (let i = -1, n = this.length; ++i < length;) { + result[i] = this.get((i + index) % n) as any; + } + return result as Iterable<R>; + } + *[Symbol.iterator]() { + for (let i = -1, n = this.length; ++i < n;) { + yield this.get(i); + } + } +} + +Vector.prototype.length = 0; +Vector.prototype.stride = 1; +Vector.prototype.name = Vector.defaultName; +Vector.prototype.type = Vector.defaultType; +Vector.prototype.props = Vector.defaultProps; http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/test/Arrow.ts ---------------------------------------------------------------------- diff --git a/js/test/Arrow.ts b/js/test/Arrow.ts new file mode 100644 index 0000000..a9ab2b7 --- /dev/null +++ b/js/test/Arrow.ts @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* tslint:disable */ +// Dynamically load an Ix target build based on command line arguments + +const target = process.env.TEST_TARGET; +const format = process.env.TEST_MODULE; +const resolve = require('path').resolve; + +// these are duplicated in the gulpfile :< +const targets = [`es5`, `es2015`, `esnext`]; +const formats = [`cjs`, `esm`, `cls`, `umd`]; + +function throwInvalidImportError(name: string, value: string, values: string[]) { + throw new Error('Unrecognized ' + name + ' \'' + value + '\'. Please run tests with \'--' + name + ' <any of ' + values.join(', ') + '>\''); +} + +if (!~targets.indexOf(target)) throwInvalidImportError('target', target, targets); +if (!~formats.indexOf(format)) throwInvalidImportError('module', format, formats); + +let Arrow: any = require(resolve(`./targets/${target}/${format}/Arrow.js`)); +let ArrowInternal: any = require(resolve(`./targets/${target}/${format}/Arrow.internal.js`)); + +import { vectors as vectors_ } from '../src/Arrow.internal'; +import { Table as Table_, readBuffers as readBuffers_ } from '../src/Arrow'; + +export let Table = Arrow.Table as typeof Table_; +export let readBuffers = Arrow.readBuffers as typeof readBuffers_; + +export let vectors: typeof vectors_ = ArrowInternal.vectors; +export namespace vectors { + export type Vector<T> = vectors_.Vector<T>; + export type BitVector = vectors_.BitVector; + export type ListVector<T> = vectors_.ListVector<T>; + export type Utf8Vector = vectors_.Utf8Vector; + export type DateVector = vectors_.DateVector; + export type IndexVector = vectors_.IndexVector; + export type Int8Vector = vectors_.Int8Vector; + export type Int16Vector = vectors_.Int16Vector; + export type Int32Vector = vectors_.Int32Vector; + export type Int64Vector = vectors_.Int64Vector; + export type Uint8Vector = vectors_.Uint8Vector; + export type Uint16Vector = vectors_.Uint16Vector; + export type Uint32Vector = vectors_.Uint32Vector; + export type Uint64Vector = vectors_.Uint64Vector; + export type Float32Vector = vectors_.Float32Vector; + export type Float64Vector = vectors_.Float64Vector; + export type StructVector = vectors_.StructVector; + export type DictionaryVector<T> = vectors_.DictionaryVector<T>; + export type FixedSizeListVector<T> = vectors_.FixedSizeListVector<T>; +}; + http://git-wip-us.apache.org/repos/asf/arrow/blob/0c8853f9/js/test/__snapshots__/reader-tests.ts.snap ---------------------------------------------------------------------- diff --git a/js/test/__snapshots__/reader-tests.ts.snap b/js/test/__snapshots__/reader-tests.ts.snap new file mode 100644 index 0000000..961ce87 --- /dev/null +++ b/js/test/__snapshots__/reader-tests.ts.snap @@ -0,0 +1,497 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` +Array [ + "Hermione", + 25, + Float32Array [ + -53.235599517822266, + 40.231998443603516, + ], +] +`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` +Array [ + "Severus", + 30, + Float32Array [ + -62.22999954223633, + 3, + ], +] +`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`; + +exports[`dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` +Array [ + "Harry", + 20, + Float32Array [ + 23, + -100.23652648925781, + ], +] +`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"example-csv"`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` +Array [ + "Hermione", + 25, + Float32Array [ + -53.235599517822266, + 40.231998443603516, + ], +] +`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` +Array [ + "Severus", + 30, + Float32Array [ + -62.22999954223633, + 3, + ], +] +`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"example-csv"`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"Struct_"`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `1`; + +exports[`dictionary stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` +Array [ + "Harry", + 20, + Float32Array [ + 23, + -100.23652648925781, + ], +] +`; + +exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`; + +exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; + +exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; + +exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` +Array [ + "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", + "Airbus", + 1502880750, + Float32Array [ + 32.45663833618164, + 1.8712350130081177, + ], +] +`; + +exports[`dictionary2 file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` +Array [ + "50fb46f4-fefa-42c1-919c-0121974cdd00", + "Boeing", + 1502880750, + Float32Array [ + 38.766666412353516, + -4.181231498718262, + ], +] +`; + +exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct"`; + +exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; + +exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `2`; + +exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = ` +Array [ + "a0fb47f9-f8fb-4403-a64a-786d7611f8ef", + "12345", + "Airbus", + 1502880750, + Float32Array [ + 32.45663833618164, + 1.8712350130081177, + ], +] +`; + +exports[`multi_dictionary file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` +Array [ + "50fb46f4-fefa-42c1-919c-0121974cdd00", + "67890", + "Boeing", + 1502880750, + Float32Array [ + 38.766666412353516, + -4.181231498718262, + ], +] +`; + +exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"row_count"`; + +exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`; + +exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `1`; + +exports[`multipart count Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `10000`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_lat"`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"FloatingPoint"`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `35.393089294433594`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `35.393089294433594`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `35.393089294433594`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `29.533695220947266`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `29.533695220947266`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"origin_lon"`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `-97.6007308959961`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `-97.6007308959961`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `-97.6007308959961`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `-98.46977996826172`; + +exports[`multipart latlong Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `-98.46977996826172`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"origin_city"`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Utf8"`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `"Oklahoma City"`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `"Oklahoma City"`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `"Oklahoma City"`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `"San Antonio"`; + +exports[`multipart origins Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `"San Antonio"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`; + +exports[`simple file Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"foo"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Int"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `5`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `1`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = `null`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = `3`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = `4`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `5`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = `"bar"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `"FloatingPoint"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `5`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `1`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `null`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = `4`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = `5`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `"baz"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = `"Utf8"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `5`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `"aa"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = `null`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = `null`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = `"bbb"`; + +exports[`simple stream Arrow readBuffers enumerates each batch as an Array of Vectors 24`] = `"cccc"`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` +Array [ + null, + "MhRNxD4", +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = ` +Array [ + 137773603, + "3F9HBxK", +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = ` +Array [ + 410361374, + "aVd88fp", +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` +Array [ + null, + "3loZrRf", +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = ` +Array [ + null, + null, +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = ` +Array [ + null, + null, +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = ` +Array [ + null, + "78SLiRw", +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = ` +Array [ + null, + "0ilsf82", +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = ` +Array [ + null, + "LjS9MbU", +] +`; + +exports[`struct file Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = ` +Array [ + null, + null, +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 1`] = `"struct_nullable"`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 2`] = `"Struct_"`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 3`] = `7`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 4`] = `null`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 5`] = ` +Array [ + null, + "MhRNxD4", +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 6`] = ` +Array [ + 137773603, + "3F9HBxK", +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 7`] = ` +Array [ + 410361374, + "aVd88fp", +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 8`] = `null`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 9`] = ` +Array [ + null, + "3loZrRf", +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 10`] = `null`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 11`] = `"struct_nullable"`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 12`] = `"Struct_"`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 13`] = `10`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 14`] = `null`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 15`] = ` +Array [ + null, + null, +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 16`] = ` +Array [ + null, + null, +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 17`] = `null`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 18`] = ` +Array [ + null, + "78SLiRw", +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 19`] = `null`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 20`] = `null`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 21`] = ` +Array [ + null, + "0ilsf82", +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 22`] = ` +Array [ + null, + "LjS9MbU", +] +`; + +exports[`struct stream Arrow readBuffers enumerates each batch as an Array of Vectors 23`] = ` +Array [ + null, + null, +] +`;
