ARROW-1479: [JS] Expand JavaScript implementation
Finally PR'ing https://github.com/graphistry/arrow to the official Arrow
project ð
Primary motivations:
- Multi-part ArrowReader to support async and streaming
- ArrowReader and related types as native JS iterators (and soon, async
iterators)
- Unify vectors across RecordBatches into a single logical virtual instance
- Fast vector subarray/slice to dense typed Arrays (handy for GPU compute)
Example:
```js
import { readFileSync } from 'fs';
import { Table } from 'apache-arrow';
const table = Table.from(...[
'latlong/schema.arrow',
'latlong/records.arrow'
].map((file) => readFileSync(file)));
const column = table.getColumn('origin_lat');
const typed = column.slice();
assert(typed instanceof Float32Array);
for (let i = -1, n = column.length; ++i < n;) {
assert(column.get(i) === typed[i]);
}
console.log(table.toString());
/*
origin_lat, origin_lon
35.393089294433594, -97.6007308959961
35.393089294433594, -97.6007308959961
35.393089294433594, -97.6007308959961
29.533695220947266, -98.46977996826172
29.533695220947266, -98.46977996826172
*/
```
Todos:
- ArrowWriter
- Map type?
-
[`readBuffersAsync`/`readStreamAsync`](https://github.com/graphistry/arrow/issues/8)
- fix prepublish script for Arrow project CI/deploy strategy
Author: Paul Taylor <paul.e.tay...@me.com>
Closes #1062 from trxcllnt/new-arrow-js and squashes the following commits:
4f71e769 [Paul Taylor] add more rat excludes and delete changelog
fc01b80a [Paul Taylor] Update license/docs per Wes's feedback
5f66c097 [Paul Taylor] add .npmrc to rat exlusions
927ce614 [Paul Taylor] Merge branch 'master' into new-arrow-js
f6da81eb [Paul Taylor] Fix rat exlusions, add header to prepublish script
aff1fdb9 [Paul Taylor] Tweak the JS CI scripts, fix case-sensitive file name in
git
8b9208a1 [Paul Taylor] Merge branch 'master' into new-arrow-js
4a74d856 [Paul Taylor] Update CI scripts for new Arrow JS lib
28261892 [Paul Taylor] chore(js): Move graphistry/arrow into apache/arrow
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/9cab3a2f
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/9cab3a2f
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/9cab3a2f
Branch: refs/heads/master
Commit: 9cab3a2f5f11e5d928f7dda961ac922158cb239a
Parents: 8911a34
Author: Paul Taylor <paul.e.tay...@me.com>
Authored: Fri Sep 8 18:42:08 2017 -0400
Committer: Wes McKinney <wes.mckin...@twosigma.com>
Committed: Fri Sep 8 18:42:08 2017 -0400
----------------------------------------------------------------------
ci/travis_before_script_js.sh | 10 -
ci/travis_script_js.sh | 6 +-
dev/release/rat_exclude_files.txt | 5 +
js/.gitignore | 96 +-
js/.npmignore | 7 -
js/.npmrc | 1 +
js/DEVELOP.md | 140 ++
js/LICENSE | 39 +
js/README.md | 208 +-
js/bin/arrow2csv.js | 92 +-
js/bin/arrow_schema.js | 29 -
js/bower.json | 17 -
js/closure-compiler-scripts/File_generated.js | 264 ++
.../Message_generated.js | 486 ++++
js/closure-compiler-scripts/Schema_generated.js | 2231 +++++++++++++++++
js/closure-compiler-scripts/flatbuffers.js | 1204 ++++++++++
js/closure-compiler-scripts/text-encoding.js | 648 +++++
js/closure-compiler-scripts/tslib.js | 151 ++
js/examples/read_file.html | 39 +-
js/flatbuffers.sh | 30 -
js/gulpfile.js | 285 +++
js/lerna.json | 9 +
js/package.json | 153 +-
js/perf/arrows/file/dictionary.arrow | Bin 0 -> 2522 bytes
js/perf/arrows/file/simple.arrow | Bin 0 -> 1642 bytes
js/perf/arrows/file/struct.arrow | Bin 0 -> 2354 bytes
js/perf/arrows/multi/count/records.arrow | Bin 0 -> 224 bytes
js/perf/arrows/multi/count/schema.arrow | Bin 0 -> 184 bytes
js/perf/arrows/multi/edges/records.arrow | Bin 0 -> 12284640 bytes
js/perf/arrows/multi/edges/schema.arrow | Bin 0 -> 256 bytes
js/perf/arrows/multi/latlong/records.arrow | Bin 0 -> 352 bytes
js/perf/arrows/multi/latlong/schema.arrow | Bin 0 -> 264 bytes
js/perf/arrows/multi/origins/records.arrow | Bin 0 -> 224 bytes
js/perf/arrows/multi/origins/schema.arrow | Bin 0 -> 1604 bytes
js/perf/arrows/stream/dictionary.arrow | Bin 0 -> 1776 bytes
js/perf/arrows/stream/simple.arrow | Bin 0 -> 1188 bytes
js/perf/arrows/stream/struct.arrow | Bin 0 -> 1884 bytes
js/perf/config.js | 38 +
js/perf/index.js | 113 +
js/prepublish.sh | 26 +
js/spec/arrow.js | 179 --
js/spec/dictionary-stream.arrow | Bin 1776 -> 0 bytes
js/spec/dictionary.arrow | Bin 2522 -> 0 bytes
js/spec/simple-stream.arrow | Bin 1188 -> 0 bytes
js/spec/simple.arrow | Bin 1642 -> 0 bytes
js/spec/struct_example-stream.arrow | Bin 1884 -> 0 bytes
js/spec/struct_example.arrow | Bin 2354 -> 0 bytes
js/src/Arrow.externs.ts | 67 +
js/src/Arrow.internal.ts | 105 +
js/src/Arrow.ts | 31 +
js/src/Arrow_generated.d.ts | 5 -
js/src/arrow.ts | 515 ----
js/src/bitarray.ts | 42 -
js/src/format/File_generated.ts | 240 ++
js/src/format/Message_generated.ts | 469 ++++
js/src/format/Schema_generated.ts | 2254 ++++++++++++++++++
js/src/reader/arrow.ts | 78 +
js/src/reader/dictionary.ts | 43 +
js/src/reader/file.ts | 79 +
js/src/reader/message.ts | 63 +
js/src/reader/stream.ts | 43 +
js/src/reader/vector.ts | 271 +++
js/src/table.ts | 133 ++
js/src/types.ts | 597 -----
js/src/vector/dictionary.ts | 51 +
js/src/vector/list.ts | 108 +
js/src/vector/struct.ts | 39 +
js/src/vector/typed.ts | 326 +++
js/src/vector/vector.ts | 91 +
js/test/Arrow.ts | 67 +
js/test/__snapshots__/reader-tests.ts.snap | 497 ++++
js/test/__snapshots__/table-tests.ts.snap | 1815 ++++++++++++++
js/test/arrows/file/dictionary.arrow | Bin 0 -> 2522 bytes
js/test/arrows/file/dictionary2.arrow | Bin 0 -> 2762 bytes
js/test/arrows/file/multi_dictionary.arrow | Bin 0 -> 3482 bytes
js/test/arrows/file/simple.arrow | Bin 0 -> 1642 bytes
js/test/arrows/file/struct.arrow | Bin 0 -> 2354 bytes
js/test/arrows/multi/count/records.arrow | Bin 0 -> 224 bytes
js/test/arrows/multi/count/schema.arrow | Bin 0 -> 184 bytes
js/test/arrows/multi/latlong/records.arrow | Bin 0 -> 352 bytes
js/test/arrows/multi/latlong/schema.arrow | Bin 0 -> 264 bytes
js/test/arrows/multi/origins/records.arrow | Bin 0 -> 224 bytes
js/test/arrows/multi/origins/schema.arrow | Bin 0 -> 1604 bytes
js/test/arrows/stream/dictionary.arrow | Bin 0 -> 1776 bytes
js/test/arrows/stream/simple.arrow | Bin 0 -> 1188 bytes
js/test/arrows/stream/struct.arrow | Bin 0 -> 1884 bytes
js/test/reader-tests.ts | 50 +
js/test/table-tests.ts | 88 +
js/test/test-config.ts | 42 +
js/test/tsconfig.json | 11 +
js/test/vector-tests.ts | 226 ++
js/tsconfig.json | 19 +-
js/tsconfig/tsconfig.base.json | 25 +
js/tsconfig/tsconfig.es2015.cjs.json | 8 +
js/tsconfig/tsconfig.es2015.cls.json | 10 +
js/tsconfig/tsconfig.es2015.esm.json | 8 +
js/tsconfig/tsconfig.es5.cjs.json | 8 +
js/tsconfig/tsconfig.es5.cls.json | 10 +
js/tsconfig/tsconfig.es5.esm.json | 8 +
js/tsconfig/tsconfig.esnext.cjs.json | 8 +
js/tsconfig/tsconfig.esnext.cls.json | 10 +
js/tsconfig/tsconfig.esnext.esm.json | 8 +
js/tslint.json | 49 +-
js/webpack.config.js | 54 -
104 files changed, 13567 insertions(+), 1610 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/ci/travis_before_script_js.sh
----------------------------------------------------------------------
diff --git a/ci/travis_before_script_js.sh b/ci/travis_before_script_js.sh
index b72accc..1d6bece 100755
--- a/ci/travis_before_script_js.sh
+++ b/ci/travis_before_script_js.sh
@@ -21,19 +21,9 @@
set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-
-# Download flatbuffers
-export FLATBUFFERS_HOME=$TRAVIS_BUILD_DIR/flatbuffers
-conda create -y -q -p $FLATBUFFERS_HOME python=2.7 flatbuffers
-export PATH="$FLATBUFFERS_HOME/bin:$PATH"
-
-npm install -g typescript
-npm install -g webpack
pushd $ARROW_JS_DIR
npm install
-npm run build
popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/ci/travis_script_js.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_js.sh b/ci/travis_script_js.sh
index cb1e9e1..9f77dec 100755
--- a/ci/travis_script_js.sh
+++ b/ci/travis_script_js.sh
@@ -23,6 +23,10 @@ JS_DIR=${TRAVIS_BUILD_DIR}/js
pushd $JS_DIR
-npm test
+npm run validate
+
+# Uncomment to use coveralls
+# npm run test:coverage
+# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/dev/release/rat_exclude_files.txt
----------------------------------------------------------------------
diff --git a/dev/release/rat_exclude_files.txt
b/dev/release/rat_exclude_files.txt
index 15a3893..83c9f19 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -1,6 +1,10 @@
+*.npmrc
*.gitignore
*_generated.h
+*_generated.js
+*_generated.ts
*.json
+*.snap
cpp/src/arrow/io/mman.h
cpp/src/arrow/util/random.h
cpp/src/arrow/status.cc
@@ -29,6 +33,7 @@ cpp/src/plasma/thirdparty/xxhash.cc
cpp/src/plasma/thirdparty/xxhash.h
dev/release/rat_exclude_files.txt
js/.npmignore
+js/closure-compiler-scripts/*
python/cmake_modules/BuildUtils.cmake
python/cmake_modules/FindPythonLibsNew.cmake
python/cmake_modules/FindNumPy.cmake
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/.gitignore
----------------------------------------------------------------------
diff --git a/js/.gitignore b/js/.gitignore
index ea5514f..b48f35b 100644
--- a/js/.gitignore
+++ b/js/.gitignore
@@ -1,7 +1,89 @@
-src/Arrow_generated.js
-lib
-lib-esm
-_bundles
-node_modules
-.idea
-*.iml
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+.vscode
+yarn.lock
+package-lock.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage
(http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Typescript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# JS package manager files
+yarn.lock
+package-lock.json
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# compilation targets
+dist
+targets/es5
+targets/es2015
+targets/esnext
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/.npmignore
----------------------------------------------------------------------
diff --git a/js/.npmignore b/js/.npmignore
deleted file mode 100644
index 333aeec..0000000
--- a/js/.npmignore
+++ /dev/null
@@ -1,7 +0,0 @@
-.gitignore
-.npmignore
-src/
-spec/
-tsconfig.json
-webpack.config.js
-flatbuffers.sh
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/.npmrc
----------------------------------------------------------------------
diff --git a/js/.npmrc b/js/.npmrc
new file mode 100644
index 0000000..43c97e7
--- /dev/null
+++ b/js/.npmrc
@@ -0,0 +1 @@
+package-lock=false
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/DEVELOP.md
----------------------------------------------------------------------
diff --git a/js/DEVELOP.md b/js/DEVELOP.md
new file mode 100644
index 0000000..5b4ac14
--- /dev/null
+++ b/js/DEVELOP.md
@@ -0,0 +1,140 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# The npm scripts
+
+* `npm run clean` - cleans targets
+* `npm run build` - cleans and compiles all targets
+* `npm test` - executes tests against built targets
+
+These npm scripts accept argument lists of targets à modules:
+
+* Available `targets` are `es5`, `es2015`, `esnext`, and `all` (default: `all`)
+* Available `modules` are `cjs`, `esm`, `umd`, and `all` (default: `all`)
+
+Examples:
+
+* `npm run build` -- builds all ES targets in all module formats
+* `npm run build -- -t es5 -m all` -- builds the ES5 target in all module
formats
+* `npm run build -- -t all -m cjs` -- builds all ES targets in the CommonJS
module format
+* `npm run build -- --targets es5 es2015 -m all` -- builds the ES5 and ES2015
targets in all module formats
+* `npm run build -- -t es5 --modules cjs esm` -- builds the ES5 target in
CommonJS and ESModules module formats
+
+This argument configuration also applies to `clean` and `test` scripts.
+
+* `npm run deploy`
+
+Uses [learna](https://github.com/lerna/lerna) to publish each build target to
npm with [conventional](https://conventionalcommits.org/)
[changelogs](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-cli).
+
+# Updating the Arrow format flatbuffers generated code
+
+Once generated, the flatbuffers format code needs to be adjusted for our TS
and JS build environments.
+
+## TypeScript
+
+1. Generate the flatbuffers TypeScript source from the Arrow project root
directory:
+ ```sh
+ flatc --ts -o ./js/src/format ./format/*.fbs
+ ```
+1. Change all the `flatbuffers` imports to
+ ```ts
+ import { flatbuffers } from "flatbuffers"
+ ```
+1. Delete `Tensor_generated.ts` (remove this step once we support Tensors)
+1. Remove Tensor import from `Schema_generated.ts`
+1. Add `/* tslint:disable:class-name */` to the top of `Schema_generated.ts`
+
+## JavaScript (for Google Closure Compiler builds)
+
+1. Generate the flatbuffers JS source from the Arrow project root directory
+ ```sh
+ flatc --js -o ./js/closure-compiler-scripts ./format/*.fbs
+ ```
+1. Delete `Tensor_generated.js` (remove this step once we support Tensors)
+1. Add `goog.module` declarations to the top of each generated file
+
+ Each file starts with a header that looks like this:
+ ```js
+ // automatically generated by the FlatBuffers compiler, do not modify
+
+ /**
+ * @const
+ * @namespace
+ */
+ var org = org || {};
+ ```
+
+ Update the header of each file to explicitly declare its module.
+
+ `Schema_generated.js`:
+ ```js
+ // automatically generated by the FlatBuffers compiler, do not modify
+ goog.module("module$targets$es5$cls$format$Schema_generated");
+ goog.module.declareLegacyNamespace();
+
+ /**
+ * @const
+ * @namespace
+ */
+ var org = org || {};
+ ```
+
+ `File_generated.js`:
+
+ ```js
+ // automatically generated by the FlatBuffers compiler, do not modify
+ goog.module("module$targets$es5$cls$format$File_generated");
+ goog.module.declareLegacyNamespace();
+ var Schema_ =
goog.require("module$targets$es5$cls$format$Schema_generated");
+ /**
+ * @const
+ * @namespace
+ */
+ var org = Schema_.org;
+ ```
+
+ `Message_generated.js`:
+
+ ```js
+ // automatically generated by the FlatBuffers compiler, do not modify
+ goog.module("module$targets$es5$cls$format$Message_generated");
+ goog.module.declareLegacyNamespace();
+ var Schema_ =
goog.require("module$targets$es5$cls$format$Schema_generated");
+ /**
+ * @const
+ * @namespace
+ */
+ var org = Schema_.org;
+ ```
+
+1. Replace the last line's export declaration
+
+ The last line of each file is:
+
+ ```js
+ // Exports for Node.js and RequireJS
+ this.org = org;
+ ```
+
+ This should instead read:
+
+ ```js
+ // Exports for Node.js and RequireJS
+ exports.org = org;
+ ```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/LICENSE
----------------------------------------------------------------------
diff --git a/js/LICENSE b/js/LICENSE
new file mode 100644
index 0000000..02e7948
--- /dev/null
+++ b/js/LICENSE
@@ -0,0 +1,39 @@
+## 3rd-party licenses for code that has been adapted for the Arrow JavaScript
+ library
+
+--------------------------------------------------------------------------------
+
+This project includes code from the FlatBuffers project
+
+Copyright 2014 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the tslib project
+
+Copyright 2015 Microsoft Corporation. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/README.md
----------------------------------------------------------------------
diff --git a/js/README.md b/js/README.md
index 38e8faf..bee3a9c 100644
--- a/js/README.md
+++ b/js/README.md
@@ -17,50 +17,194 @@
under the License.
-->
-### Installation
+# [Apache Arrow](https://github.com/apache/arrow) in JS
-From this directory, run:
+[](https://travis-ci.org/apache/arrow)
+[](https://coveralls.io/github/apache/arrow)
-``` bash
-$ npm install # pull dependencies
-$ npm run lint -- <filename> # run tslint
-$ npm run build # build typescript (run tsc and webpack)
-$ npm run test # run the unit tests (node.js only)
+Arrow is a set of technologies that enable big-data systems to process and
move data fast.
+
+## install [apache-arrow from npm](https://www.npmjs.com/package/apache-arrow)
+
+`npm install apache-arrow`
+
+(read about how we [package apache-arrow](#packaging) below)
+
+# Powering Columnar In-Memory Analytics
+
+Apache Arrow is a columnar memory layout specification for encoding vectors
and table-like containers of flat and nested data. The Arrow spec aligns
columnar data in memory to minimize cache misses and take advantage of the
latest SIMD (Single input multiple data) and GPU operations on modern
processors.
+
+Apache Arrow is the emerging standard for large in-memory columnar data
([Spark](https://spark.apache.org/),
[Pandas](http://wesmckinney.com/blog/pandas-and-apache-arrow/),
[Drill](https://drill.apache.org/), ...). By standardizing on a common binary
interchange format, big data systems can reduce the costs and friction
associated with cross-system communication.
+
+# Related Projects
+
+* [GoAI](http://gpuopenanalytics.com/) -- Arrow-powered GPU analytics
+* [rxjs-mapd](https://github.com/graphistry/rxjs-mapd) -- A MapD Core
node-driver that returns query results as Arrow columns
+
+# Usage
+
+## Get a table from an Arrow file on disk
+
+```es6
+import { readFileSync } from 'fs';
+import { Table } from 'apache-arrow';
+
+const arrow = readFileSync('simple.arrow');
+const table = Table.from(arrow);
+
+console.log(table.toString());
+
+/*
+ foo, bar, baz
+ 1, 1, aa
+null, null, null
+ 3, null, null
+ 4, 4, bbb
+ 5, 5, cccc
+*/
```
-### Usage
-The library is designed to be used with node.js or in the browser, this
repository contains examples of both.
+## Create a Table when the Arrow file is split across buffers
+
+```es6
+import { readFileSync } from 'fs';
+import { Table } from 'apache-arrow';
-#### Node
-Import the arrow module:
+const table = Table.from(...[
+ 'latlong/schema.arrow',
+ 'latlong/records.arrow'
+].map((file) => readFileSync(file)));
-``` js
-var arrow = require("arrow");
+console.log(table.toString());
+
+/*
+ origin_lat, origin_lon
+35.393089294433594, -97.6007308959961
+35.393089294433594, -97.6007308959961
+35.393089294433594, -97.6007308959961
+29.533695220947266, -98.46977996826172
+29.533695220947266, -98.46977996826172
+*/
```
-See [bin/arrow_schema.js](bin/arrow_schema.js) and
[bin/arrow2csv.js](bin/arrow2csv.js) for usage examples.
+## Columns are what you'd expect
+
+```es6
+import { readFileSync } from 'fs';
+import { Table } from 'apache-arrow';
+
+const table = Table.from(...[
+ 'latlong/schema.arrow',
+ 'latlong/records.arrow'
+].map(readFileSync));
+
+const column = table.getColumn('origin_lat');
+const typed = column.slice();
+
+assert(typed instanceof Float32Array);
-#### Browser
-Include `_bundles/arrow.js` in a `<script />` tag:
-``` html
-<script src="_bundles/arrow.js"/>
+for (let i = -1, n = column.length; ++i < n;) {
+ assert(column.get(i) === typed[i]);
+}
```
-See [examples/read_file.html](examples/read_file.html) for a usage example.
-### API
-##### `arrow.getReader(buffer)`
-Returns an `ArrowReader` object representing the Arrow file or stream
contained in
-the `buffer`.
+## Usage with MapD Core
+
+```es6
+import MapD from 'rxjs-mapd';
+import { Table } from 'apache-arrow';
+
+const port = 9091;
+const host = `localhost`;
+const db = `mapd`;
+const user = `mapd`;
+const password = `HyperInteractive`;
+
+MapD.open(host, port)
+ .connect(db, user, password)
+ .flatMap((session) =>
+ // queryDF returns Arrow buffers
+ session.queryDF(`
+ SELECT origin_city
+ FROM flights
+ WHERE dest_city ILIKE 'dallas'
+ LIMIT 5`
+ ).disconnect()
+ )
+ .map(([schema, records]) =>
+ // Create Arrow Table from results
+ Table.from(schema, records))
+ .map((table) =>
+ // Stringify the table to CSV
+ table.toString({ index: true }))
+ .subscribe((csvStr) =>
+ console.log(csvStr));
+/*
+Index, origin_city
+ 0, Oklahoma City
+ 1, Oklahoma City
+ 2, Oklahoma City
+ 3, San Antonio
+ 4, San Antonio
+*/
+```
+
+# Getting involved
+
+See [develop.md](https://github.com/apache/arrow/blob/master/develop.md)
+
+Even if you do not plan to contribute to Apache Arrow itself or Arrow
+integrations in other projects, we'd be happy to have you involved:
+
+* Join the mailing list: send an email to
+ [dev-subscr...@arrow.apache.org][1]. Share your ideas and use cases for the
+ project.
+* [Follow our activity on JIRA][3]
+* [Learn the format][2]
+* Contribute code to one of the reference implementations
+
+We prefer to receive contributions in the form of GitHub pull requests. Please
send pull requests against the [github.com/apache/arrow][4] repository.
+
+If you are looking for some ideas on what to contribute, check out the [JIRA
+issues][3] for the Apache Arrow project. Comment on the issue and/or contact
+[d...@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+with your questions and ideas.
+
+If youâd like to report a bug but donât have time to fix it, you can still
post
+it on JIRA, or email the mailing list
+[d...@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/)
+
+## Packaging
+
+`apache-arrow` is written in TypeScript, but the project is compiled to
multiple JS versions and common module formats.
+
+The base `apache-arrow` package includes all the compilation targets for
convenience, but if you're conscientious about your `node_modules` footprint,
we got you.
+
+The targets are also published under the `@apache-arrow` namespace:
+
+```sh
+npm install @apache-arrow/es5-cjs # ES5 CommonJS target
+npm install @apache-arrow/es5-esm # ES5 ESModules target
+npm install @apache-arrow/es5-umd # ES5 UMD target
+npm install @apache-arrow/es2015-cjs # ES2015 CommonJS target
+npm install @apache-arrow/es2015-esm # ES2015 ESModules target
+npm install @apache-arrow/es2015-umd # ES2015 UMD target
+npm install @apache-arrow/esnext-esm # ESNext CommonJS target
+npm install @apache-arrow/esnext-esm # ESNext ESModules target
+npm install @apache-arrow/esnext-umd # ESNext UMD target
+```
+
+### Why we package like this
+
+The JS community is a diverse group with a varied list of target environments
and tool chains. Publishing multiple packages accommodates projects of all
stripes.
-##### `ArrowReader.loadNextBatch()`
-Loads the next record batch and returns it's length.
+If you think we missed a compilation target and it's a blocker for adoption,
please open an issue. We're here for you â¤ï¸.
-##### `ArrowReader.getSchema()`
-Returns a JSON representation of the file's Arrow schema.
+# License
-##### `ArrowReader.getVectors()`
-Returns a list of `Vector` objects, one for each column.
-Vector objects have, at minimum, a `get(i)` method and a `length` attribute.
+[Apache 2.0](https://github.com/apache/arrow/blob/master/LICENSE)
-##### `ArrowReader.getVector(name: String)`
-Return a Vector object for column `name`
+[1]: mailto:dev-subscr...@arrow.apache.org
+[2]: https://github.com/apache/arrow/tree/master/format
+[3]: https://issues.apache.org/jira/browse/ARROW
+[4]: https://github.com/apache/arrow
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/bin/arrow2csv.js
----------------------------------------------------------------------
diff --git a/js/bin/arrow2csv.js b/js/bin/arrow2csv.js
index c1801f7..f316b84 100755
--- a/js/bin/arrow2csv.js
+++ b/js/bin/arrow2csv.js
@@ -17,37 +17,73 @@
// specific language governing permissions and limitations
// under the License.
-var fs = require('fs')
-var process = require('process');
-var arrow = require('../lib/arrow.js');
-var program = require('commander');
+var fs = require('fs');
+var Table = require('../dist/Arrow.js').Table;
+var optionList = [
+ {
+ type: String,
+ name: 'schema',
+ alias: 's', multiple: true,
+ typeLabel: '[underline]{columns}',
+ description: 'A space-delimited list of column names'
+ },
+ {
+ type: String,
+ name: 'file', alias: 'f',
+ description: 'The Arrow file to read'
+ }
+];
-function list (val) {
- return val.split(',');
-}
+var argv = require(`command-line-args`)(optionList, { partial: true });
+var files = [argv.file, ...(argv._unknown || [])].filter(Boolean);
-program
- .version('0.1.0')
- .usage('[options] <file>')
- .option('-s --schema <list>', 'A comma-separated list of column names', list)
- .parse(process.argv);
+// console.log(JSON.stringify(argv));
-if (!program.schema) {
- program.outputHelp();
+if (!argv.schema || !files.length) {
+ console.log(require('command-line-usage')([
+ {
+ header: 'arrow2csv',
+ content: 'Print a CSV from an Arrow file'
+ },
+ {
+ header: 'Synopsis',
+ content: [
+ '$ arrow2csv [underline]{file.arrow} [[bold]{--schema}
column_name ...]',
+ '$ arrow2csv [[bold]{--schema} column_name ...]
[[bold]{--file} [underline]{file.arrow}]',
+ '$ arrow2csv [bold]{-s} column_1 [bold]{-s} column_2
[[bold]{-f} [underline]{file.arrow}]',
+ '$ arrow2csv [[bold]{--help}]'
+ ]
+ },
+ {
+ header: 'Options',
+ optionList: [
+ ...optionList,
+ {
+ name: 'help',
+ description: 'Print this usage guide.'
+ }
+ ]
+ },
+ {
+ header: 'Example',
+ content: [
+ '$ arrow2csv --schema foo baz -f simple.arrow',
+ '> foo, baz',
+ '> 1, aa',
+ '> null, null',
+ '> 3, null',
+ '> 4, bbb',
+ '> 5, cccc',
+ ]
+ }
+ ]));
process.exit(1);
}
-var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-var reader = arrow.getReader(buf);
-var nrecords
-
-nrecords = reader.loadNextBatch();
-while (nrecords > 0) {
- for (var i = 0; i < nrecords; i += 1|0) {
- console.log(program.schema.map(function (field) {
- return '' + reader.getVector(field).get(i);
- }).join(','));
- }
- nrecords = reader.loadNextBatch();
- if (nrecords > 0) console.log('---');
-}
+files.forEach((source) => {
+ var allColumns = Table.from(fs.readFileSync(source));
+ var selectedColumns = new Table(argv.schema.map((columnName) => {
+ return allColumns.getColumn(columnName);
+ }));
+ console.log(selectedColumns.toString());
+});
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/bin/arrow_schema.js
----------------------------------------------------------------------
diff --git a/js/bin/arrow_schema.js b/js/bin/arrow_schema.js
deleted file mode 100755
index 4917628..0000000
--- a/js/bin/arrow_schema.js
+++ /dev/null
@@ -1,29 +0,0 @@
-#! /usr/bin/env node
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-var fs = require('fs');
-var process = require('process');
-var arrow = require('../lib/arrow.js');
-
-var buf = fs.readFileSync(process.argv[process.argv.length - 1]);
-var reader = arrow.getReader(buf);
-console.log(JSON.stringify(reader.getSchema(), null, '\t'));
-//console.log(JSON.stringify(reader.getVectors(), null, '\t'));
-console.log('block count: ' + reader.getBatchCount());
-
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/bower.json
----------------------------------------------------------------------
diff --git a/js/bower.json b/js/bower.json
deleted file mode 100644
index c2099f8..0000000
--- a/js/bower.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "name": "arrow",
- "description": "",
- "main": "_bundles/arrow.js",
- "authors": [],
- "license": "Apache-2.0",
- "homepage": "http://arrow.apache.org";,
- "ignore": [
- ".gitignore",
- ".npmignore",
- "src/",
- "spec/",
- "tsconfig.json",
- "webpack.config.js",
- "flatbuffers.sh"
- ]
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/closure-compiler-scripts/File_generated.js
----------------------------------------------------------------------
diff --git a/js/closure-compiler-scripts/File_generated.js
b/js/closure-compiler-scripts/File_generated.js
new file mode 100644
index 0000000..bb82cc4
--- /dev/null
+++ b/js/closure-compiler-scripts/File_generated.js
@@ -0,0 +1,264 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+goog.module("module$targets$es5$cls$format$File_generated");
+goog.module.declareLegacyNamespace();
+var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
+/**
+ * @const
+ * @namespace
+ */
+var org = Schema_.org;
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache = org.apache || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow = org.apache.arrow || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
+
+/**
+ * ----------------------------------------------------------------------
+ * Arrow File metadata
+ *
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Footer = function() {
+ /**
+ * @type {flatbuffers.ByteBuffer}
+ */
+ this.bb = null;
+
+ /**
+ * @type {number}
+ */
+ this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Footer}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.__init = function(i, bb) {
+ this.bb_pos = i;
+ this.bb = bb;
+ return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Footer=} obj
+ * @returns {org.apache.arrow.flatbuf.Footer}
+ */
+org.apache.arrow.flatbuf.Footer.getRootAsFooter = function(bb, obj) {
+ return (obj || new
org.apache.arrow.flatbuf.Footer).__init(bb.readInt32(bb.position()) +
bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.MetadataVersion}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.version = function() {
+ var offset = this.bb.__offset(this.bb_pos, 4);
+ return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */
(this.bb.readInt16(this.bb_pos + offset)) :
org.apache.arrow.flatbuf.MetadataVersion.V1;
+};
+
+/**
+ * @param {org.apache.arrow.flatbuf.Schema=} obj
+ * @returns {org.apache.arrow.flatbuf.Schema|null}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.schema = function(obj) {
+ var offset = this.bb.__offset(this.bb_pos, 6);
+ return offset ? (obj || new
org.apache.arrow.flatbuf.Schema).__init(this.bb.__indirect(this.bb_pos +
offset), this.bb) : null;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Block=} obj
+ * @returns {org.apache.arrow.flatbuf.Block}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.dictionaries = function(index, obj) {
+ var offset = this.bb.__offset(this.bb_pos, 8);
+ return offset ? (obj || new
org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) +
index * 24, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.dictionariesLength = function() {
+ var offset = this.bb.__offset(this.bb_pos, 8);
+ return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Block=} obj
+ * @returns {org.apache.arrow.flatbuf.Block}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.recordBatches = function(index, obj)
{
+ var offset = this.bb.__offset(this.bb_pos, 10);
+ return offset ? (obj || new
org.apache.arrow.flatbuf.Block).__init(this.bb.__vector(this.bb_pos + offset) +
index * 24, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Footer.prototype.recordBatchesLength = function() {
+ var offset = this.bb.__offset(this.bb_pos, 10);
+ return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Footer.startFooter = function(builder) {
+ builder.startObject(4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.MetadataVersion} version
+ */
+org.apache.arrow.flatbuf.Footer.addVersion = function(builder, version) {
+ builder.addFieldInt16(0, version,
org.apache.arrow.flatbuf.MetadataVersion.V1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} schemaOffset
+ */
+org.apache.arrow.flatbuf.Footer.addSchema = function(builder, schemaOffset) {
+ builder.addFieldOffset(1, schemaOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dictionariesOffset
+ */
+org.apache.arrow.flatbuf.Footer.addDictionaries = function(builder,
dictionariesOffset) {
+ builder.addFieldOffset(2, dictionariesOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Footer.startDictionariesVector = function(builder,
numElems) {
+ builder.startVector(24, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} recordBatchesOffset
+ */
+org.apache.arrow.flatbuf.Footer.addRecordBatches = function(builder,
recordBatchesOffset) {
+ builder.addFieldOffset(3, recordBatchesOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.Footer.startRecordBatchesVector = function(builder,
numElems) {
+ builder.startVector(24, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Footer.endFooter = function(builder) {
+ var offset = builder.endObject();
+ return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+org.apache.arrow.flatbuf.Footer.finishFooterBuffer = function(builder, offset)
{
+ builder.finish(offset);
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Block = function() {
+ /**
+ * @type {flatbuffers.ByteBuffer}
+ */
+ this.bb = null;
+
+ /**
+ * @type {number}
+ */
+ this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Block}
+ */
+org.apache.arrow.flatbuf.Block.prototype.__init = function(i, bb) {
+ this.bb_pos = i;
+ this.bb = bb;
+ return this;
+};
+
+/**
+ * Index to the start of the RecordBlock (note this is past the Message header)
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Block.prototype.offset = function() {
+ return this.bb.readInt64(this.bb_pos);
+};
+
+/**
+ * Length of the metadata
+ *
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.Block.prototype.metaDataLength = function() {
+ return this.bb.readInt32(this.bb_pos + 8);
+};
+
+/**
+ * Length of the data (this is aligned so there can be a gap between this and
+ * the metatdata).
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Block.prototype.bodyLength = function() {
+ return this.bb.readInt64(this.bb_pos + 16);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} offset
+ * @param {number} metaDataLength
+ * @param {flatbuffers.Long} bodyLength
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Block.createBlock = function(builder, offset,
metaDataLength, bodyLength) {
+ builder.prep(8, 24);
+ builder.writeInt64(bodyLength);
+ builder.pad(4);
+ builder.writeInt32(metaDataLength);
+ builder.writeInt64(offset);
+ return builder.offset();
+};
+
+// Exports for Node.js and RequireJS
+exports.org = org;
http://git-wip-us.apache.org/repos/asf/arrow/blob/9cab3a2f/js/closure-compiler-scripts/Message_generated.js
----------------------------------------------------------------------
diff --git a/js/closure-compiler-scripts/Message_generated.js
b/js/closure-compiler-scripts/Message_generated.js
new file mode 100644
index 0000000..0c1a1a9
--- /dev/null
+++ b/js/closure-compiler-scripts/Message_generated.js
@@ -0,0 +1,486 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+goog.module("module$targets$es5$cls$format$Message_generated");
+goog.module.declareLegacyNamespace();
+var Schema_ = goog.require("module$targets$es5$cls$format$Schema_generated");
+/**
+ * @const
+ * @namespace
+ */
+var org = Schema_.org;
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache = org.apache || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow = org.apache.arrow || {};
+
+/**
+ * @const
+ * @namespace
+ */
+org.apache.arrow.flatbuf = org.apache.arrow.flatbuf || {};
+
+/**
+ * ----------------------------------------------------------------------
+ * The root Message type
+ * This union enables us to easily send different message types without
+ * redundant storage, and in the future we can easily add new message types.
+ *
+ * Arrow implementations do not need to implement all of the message types,
+ * which may include experimental metadata types. For maximum compatibility,
+ * it is best to send data using RecordBatch
+ *
+ * @enum
+ */
+org.apache.arrow.flatbuf.MessageHeader = {
+ NONE: 0, 0: 'NONE',
+ Schema: 1, 1: 'Schema',
+ DictionaryBatch: 2, 2: 'DictionaryBatch',
+ RecordBatch: 3, 3: 'RecordBatch',
+ Tensor: 4, 4: 'Tensor',
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * Data structures for describing a table row batch (a collection of
+ * equal-length Arrow arrays)
+ * Metadata about a field at some level of a nested type tree (but not
+ * its children).
+ *
+ * For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
+ * would have {length: 5, null_count: 2} for its List node, and {length: 6,
+ * null_count: 0} for its Int16 node, as separate FieldNode structs
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.FieldNode = function() {
+ /**
+ * @type {flatbuffers.ByteBuffer}
+ */
+ this.bb = null;
+
+ /**
+ * @type {number}
+ */
+ this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.FieldNode}
+ */
+org.apache.arrow.flatbuf.FieldNode.prototype.__init = function(i, bb) {
+ this.bb_pos = i;
+ this.bb = bb;
+ return this;
+};
+
+/**
+ * The number of value slots in the Arrow array at this level of a nested
+ * tree
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.FieldNode.prototype.length = function() {
+ return this.bb.readInt64(this.bb_pos);
+};
+
+/**
+ * The number of observed nulls. Fields with null_count == 0 may choose not
+ * to write their physical validity bitmap out as a materialized buffer,
+ * instead setting the length of the bitmap buffer to 0.
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.FieldNode.prototype.nullCount = function() {
+ return this.bb.readInt64(this.bb_pos + 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} length
+ * @param {flatbuffers.Long} null_count
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.FieldNode.createFieldNode = function(builder, length,
null_count) {
+ builder.prep(8, 16);
+ builder.writeInt64(null_count);
+ builder.writeInt64(length);
+ return builder.offset();
+};
+
+/**
+ * A data header describing the shared memory layout of a "record" or "row"
+ * batch. Some systems call this a "row batch" internally and others a "record
+ * batch".
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.RecordBatch = function() {
+ /**
+ * @type {flatbuffers.ByteBuffer}
+ */
+ this.bb = null;
+
+ /**
+ * @type {number}
+ */
+ this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.RecordBatch}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.__init = function(i, bb) {
+ this.bb_pos = i;
+ this.bb = bb;
+ return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.RecordBatch=} obj
+ * @returns {org.apache.arrow.flatbuf.RecordBatch}
+ */
+org.apache.arrow.flatbuf.RecordBatch.getRootAsRecordBatch = function(bb, obj) {
+ return (obj || new
org.apache.arrow.flatbuf.RecordBatch).__init(bb.readInt32(bb.position()) +
bb.position(), bb);
+};
+
+/**
+ * number of records / rows. The arrays in the batch should all have this
+ * length
+ *
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.length = function() {
+ var offset = this.bb.__offset(this.bb_pos, 4);
+ return offset ? this.bb.readInt64(this.bb_pos + offset) :
this.bb.createLong(0, 0);
+};
+
+/**
+ * Nodes correspond to the pre-ordered flattened logical schema
+ *
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.FieldNode=} obj
+ * @returns {org.apache.arrow.flatbuf.FieldNode}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.nodes = function(index, obj) {
+ var offset = this.bb.__offset(this.bb_pos, 6);
+ return offset ? (obj || new
org.apache.arrow.flatbuf.FieldNode).__init(this.bb.__vector(this.bb_pos +
offset) + index * 16, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.nodesLength = function() {
+ var offset = this.bb.__offset(this.bb_pos, 6);
+ return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * Buffers correspond to the pre-ordered flattened buffer tree
+ *
+ * The number of buffers appended to this list depends on the schema. For
+ * example, most primitive arrays will have 2 buffers, 1 for the validity
+ * bitmap and 1 for the values. For struct arrays, there will only be a
+ * single buffer for the validity (nulls) bitmap
+ *
+ * @param {number} index
+ * @param {org.apache.arrow.flatbuf.Buffer=} obj
+ * @returns {org.apache.arrow.flatbuf.Buffer}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.buffers = function(index, obj) {
+ var offset = this.bb.__offset(this.bb_pos, 8);
+ return offset ? (obj || new
org.apache.arrow.flatbuf.Buffer).__init(this.bb.__vector(this.bb_pos + offset)
+ index * 24, this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+org.apache.arrow.flatbuf.RecordBatch.prototype.buffersLength = function() {
+ var offset = this.bb.__offset(this.bb_pos, 8);
+ return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.RecordBatch.startRecordBatch = function(builder) {
+ builder.startObject(3);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} length
+ */
+org.apache.arrow.flatbuf.RecordBatch.addLength = function(builder, length) {
+ builder.addFieldInt64(0, length, builder.createLong(0, 0));
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} nodesOffset
+ */
+org.apache.arrow.flatbuf.RecordBatch.addNodes = function(builder, nodesOffset)
{
+ builder.addFieldOffset(1, nodesOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.RecordBatch.startNodesVector = function(builder,
numElems) {
+ builder.startVector(16, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} buffersOffset
+ */
+org.apache.arrow.flatbuf.RecordBatch.addBuffers = function(builder,
buffersOffset) {
+ builder.addFieldOffset(2, buffersOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+org.apache.arrow.flatbuf.RecordBatch.startBuffersVector = function(builder,
numElems) {
+ builder.startVector(24, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.RecordBatch.endRecordBatch = function(builder) {
+ var offset = builder.endObject();
+ return offset;
+};
+
+/**
+ * ----------------------------------------------------------------------
+ * For sending dictionary encoding information. Any Field can be
+ * dictionary-encoded, but in this case none of its children may be
+ * dictionary-encoded.
+ * There is one vector / column per dictionary
+ *
+ *
+ * @constructor
+ */
+org.apache.arrow.flatbuf.DictionaryBatch = function() {
+ /**
+ * @type {flatbuffers.ByteBuffer}
+ */
+ this.bb = null;
+
+ /**
+ * @type {number}
+ */
+ this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.DictionaryBatch}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.prototype.__init = function(i, bb) {
+ this.bb_pos = i;
+ this.bb = bb;
+ return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.DictionaryBatch=} obj
+ * @returns {org.apache.arrow.flatbuf.DictionaryBatch}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.getRootAsDictionaryBatch =
function(bb, obj) {
+ return (obj || new
org.apache.arrow.flatbuf.DictionaryBatch).__init(bb.readInt32(bb.position()) +
bb.position(), bb);
+};
+
+/**
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.prototype.id = function() {
+ var offset = this.bb.__offset(this.bb_pos, 4);
+ return offset ? this.bb.readInt64(this.bb_pos + offset) :
this.bb.createLong(0, 0);
+};
+
+/**
+ * @param {org.apache.arrow.flatbuf.RecordBatch=} obj
+ * @returns {org.apache.arrow.flatbuf.RecordBatch|null}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.prototype.data = function(obj) {
+ var offset = this.bb.__offset(this.bb_pos, 6);
+ return offset ? (obj || new
org.apache.arrow.flatbuf.RecordBatch).__init(this.bb.__indirect(this.bb_pos +
offset), this.bb) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.startDictionaryBatch =
function(builder) {
+ builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} id
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.addId = function(builder, id) {
+ builder.addFieldInt64(0, id, builder.createLong(0, 0));
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.addData = function(builder,
dataOffset) {
+ builder.addFieldOffset(1, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.DictionaryBatch.endDictionaryBatch =
function(builder) {
+ var offset = builder.endObject();
+ return offset;
+};
+
+/**
+ * @constructor
+ */
+org.apache.arrow.flatbuf.Message = function() {
+ /**
+ * @type {flatbuffers.ByteBuffer}
+ */
+ this.bb = null;
+
+ /**
+ * @type {number}
+ */
+ this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {org.apache.arrow.flatbuf.Message}
+ */
+org.apache.arrow.flatbuf.Message.prototype.__init = function(i, bb) {
+ this.bb_pos = i;
+ this.bb = bb;
+ return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {org.apache.arrow.flatbuf.Message=} obj
+ * @returns {org.apache.arrow.flatbuf.Message}
+ */
+org.apache.arrow.flatbuf.Message.getRootAsMessage = function(bb, obj) {
+ return (obj || new
org.apache.arrow.flatbuf.Message).__init(bb.readInt32(bb.position()) +
bb.position(), bb);
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.MetadataVersion}
+ */
+org.apache.arrow.flatbuf.Message.prototype.version = function() {
+ var offset = this.bb.__offset(this.bb_pos, 4);
+ return offset ? /** @type {org.apache.arrow.flatbuf.MetadataVersion} */
(this.bb.readInt16(this.bb_pos + offset)) :
org.apache.arrow.flatbuf.MetadataVersion.V1;
+};
+
+/**
+ * @returns {org.apache.arrow.flatbuf.MessageHeader}
+ */
+org.apache.arrow.flatbuf.Message.prototype.headerType = function() {
+ var offset = this.bb.__offset(this.bb_pos, 6);
+ return offset ? /** @type {org.apache.arrow.flatbuf.MessageHeader} */
(this.bb.readUint8(this.bb_pos + offset)) :
org.apache.arrow.flatbuf.MessageHeader.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+org.apache.arrow.flatbuf.Message.prototype.header = function(obj) {
+ var offset = this.bb.__offset(this.bb_pos, 8);
+ return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @returns {flatbuffers.Long}
+ */
+org.apache.arrow.flatbuf.Message.prototype.bodyLength = function() {
+ var offset = this.bb.__offset(this.bb_pos, 10);
+ return offset ? this.bb.readInt64(this.bb_pos + offset) :
this.bb.createLong(0, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+org.apache.arrow.flatbuf.Message.startMessage = function(builder) {
+ builder.startObject(4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.MetadataVersion} version
+ */
+org.apache.arrow.flatbuf.Message.addVersion = function(builder, version) {
+ builder.addFieldInt16(0, version,
org.apache.arrow.flatbuf.MetadataVersion.V1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {org.apache.arrow.flatbuf.MessageHeader} headerType
+ */
+org.apache.arrow.flatbuf.Message.addHeaderType = function(builder, headerType)
{
+ builder.addFieldInt8(1, headerType,
org.apache.arrow.flatbuf.MessageHeader.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} headerOffset
+ */
+org.apache.arrow.flatbuf.Message.addHeader = function(builder, headerOffset) {
+ builder.addFieldOffset(2, headerOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Long} bodyLength
+ */
+org.apache.arrow.flatbuf.Message.addBodyLength = function(builder, bodyLength)
{
+ builder.addFieldInt64(3, bodyLength, builder.createLong(0, 0));
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+org.apache.arrow.flatbuf.Message.endMessage = function(builder) {
+ var offset = builder.endObject();
+ return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+org.apache.arrow.flatbuf.Message.finishMessageBuffer = function(builder,
offset) {
+ builder.finish(offset);
+};
+
+// Exports for Node.js and RequireJS
+exports.org = org;
