This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new cde18a6 ARROW-2234: [JS] Read timestamp low bits as Uint32s
cde18a6 is described below
commit cde18a6ba525d090d11ac6f3563370dce60d03a1
Author: Paul Taylor <paul.e.tay...@me.com>
AuthorDate: Tue Mar 6 14:22:07 2018 -0500
ARROW-2234: [JS] Read timestamp low bits as Uint32s
I ran into this with the epoch millisecond timestamp `1394015437000`. We
need all 32 bits of precision for the low bits, but still need the underlying
TypedArray to be an Int32Array so the high bits can be signed. Example:
```sh
> mult = Math.pow(2, 32)
4294967296
> hi = (1394015437000 / mult) | 0
324
> lo = (1394015437000 - (mult * hi) | 0)
-1848934200
> mult * hi + lo
1389720469704 # wrong
> mult * hi + (new Uint32Array([lo])[0])
1394015437000 # right
```
Author: Paul Taylor <paul.e.tay...@me.com>
Closes #1678 from trxcllnt/js-fix-timestamps and squashes the following
commits:
bd34db52 <Paul Taylor> add more public methods to the closure-compiler
mangler whitelist
22210f89 <Paul Taylor> add initial DateVector tests to validate low bits
are read unsigned
56d5c297 <Paul Taylor> fix jest deprecation warning
ddb1b444 <Paul Taylor> read timestamp low bits as Uint32s
---
js/package.json | 1 -
js/src/Arrow.externs.js | 18 ++++++
js/src/vector/flat.ts | 6 +-
js/test/unit/date-vector-tests.ts | 120 ++++++++++++++++++++++++++++++++++++++
4 files changed, 141 insertions(+), 4 deletions(-)
diff --git a/js/package.json b/js/package.json
index 507babf..af3c97f 100644
--- a/js/package.json
+++ b/js/package.json
@@ -125,7 +125,6 @@
"ts",
"tsx"
],
- "mapCoverage": true,
"coverageReporters": [
"lcov"
],
diff --git a/js/src/Arrow.externs.js b/js/src/Arrow.externs.js
index cf4db91..a14f959 100644
--- a/js/src/Arrow.externs.js
+++ b/js/src/Arrow.externs.js
@@ -510,24 +510,42 @@ var FloatVector = function() {};
FloatVector.from = function() {};
var DateVector = function() {};
+/** @type {?} */
+DateVector.prototype.asEpochMilliseconds;
var DecimalVector = function() {};
var TimeVector = function() {};
var TimestampVector = function() {};
+/** @type {?} */
+TimestampVector.prototype.asEpochMilliseconds;
var IntervalVector = function() {};
var BinaryVector = function() {};
+/** @type {?} */
+BinaryVector.prototype.asUtf8;
var FixedSizeBinaryVector = function() {};
var Utf8Vector = function() {};
+/** @type {?} */
+Utf8Vector.prototype.asBinary;
var ListVector = function() {};
var FixedSizeListVector = function() {};
var MapVector = function() {};
+/** @type {?} */
+MapVector.prototype.asStruct;
var StructVector = function() {};
+/** @type {?} */
+StructVector.prototype.asMap;
var UnionVector = function() {};
var DictionaryVector = function() {};
/** @type {?} */
+DictionaryVector.prototype.indices;
+/** @type {?} */
+DictionaryVector.prototype.dictionary;
+/** @type {?} */
DictionaryVector.prototype.getKey;
/** @type {?} */
DictionaryVector.prototype.getValue;
+/** @type {?} */
+DictionaryVector.prototype.reverseLookup;
var FlatView = function() {};
/** @type {?} */
diff --git a/js/src/vector/flat.ts b/js/src/vector/flat.ts
index acc2f1a..06189c4 100644
--- a/js/src/vector/flat.ts
+++ b/js/src/vector/flat.ts
@@ -326,9 +326,9 @@ export class IntervalMonthView extends PrimitiveView<Int32>
{
export function epochSecondsToMs(data: Int32Array, index: number) { return
1000 * data[index]; }
export function epochDaysToMs(data: Int32Array, index: number) { return
86400000 * data[index]; }
-export function epochMillisecondsLongToMs(data: Int32Array, index: number) {
return 4294967296 * (data[index + 1]) + data[index]; }
-export function epochMicrosecondsLongToMs(data: Int32Array, index: number) {
return 4294967296 * (data[index + 1] / 1000) + (data[index] / 1000); }
-export function epochNanosecondsLongToMs(data: Int32Array, index: number) {
return 4294967296 * (data[index + 1] / 1000000) + (data[index] / 1000000); }
+export function epochMillisecondsLongToMs(data: Int32Array, index: number) {
return 4294967296 * (data[index + 1]) + (data[index] >>> 0); }
+export function epochMicrosecondsLongToMs(data: Int32Array, index: number) {
return 4294967296 * (data[index + 1] / 1000) + ((data[index] >>> 0) / 1000); }
+export function epochNanosecondsLongToMs(data: Int32Array, index: number) {
return 4294967296 * (data[index + 1] / 1000000) + ((data[index] >>> 0) /
1000000); }
export function epochMillisecondsToDate(epochMs: number) { return new
Date(epochMs); }
export function epochDaysToDate(data: Int32Array, index: number) { return
epochMillisecondsToDate(epochDaysToMs(data, index)); }
diff --git a/js/test/unit/date-vector-tests.ts
b/js/test/unit/date-vector-tests.ts
new file mode 100644
index 0000000..b30d049
--- /dev/null
+++ b/js/test/unit/date-vector-tests.ts
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Arrow from '../Arrow';
+import { DateVector } from '../../src/vector';
+const { Table } = Arrow;
+
+describe(`DateVector`, () => {
+ it('returns days since the epoch as correct JS Dates', () => {
+ const table = Table.from(test_data);
+ const date32 = table.getColumnAt(0) as DateVector;
+ const expectedMillis = expectedMillis32();
+ for (const date of date32) {
+ const millis = expectedMillis.shift();
+ expect(date).toEqual(millis === null ? null : new Date(millis!));
+ }
+ });
+ it('returns millisecond longs since the epoch as correct JS Dates', () => {
+ const table = Table.from(test_data);
+ const date64 = table.getColumnAt(1) as DateVector;
+ const expectedMillis = expectedMillis64();
+ for (const date of date64) {
+ const millis = expectedMillis.shift();
+ expect(date).toEqual(millis === null ? null : new Date(millis!));
+ }
+ });
+ it('converts days since the epoch to milliseconds', () => {
+ const table = Table.from(test_data);
+ const date32 = table.getColumnAt(0) as DateVector;
+ const expectedMillis = expectedMillis32();
+ for (const timestamp of date32.asEpochMilliseconds()) {
+ expect(timestamp).toEqual(expectedMillis.shift());
+ }
+ });
+ it('converts millisecond longs since the epoch to millisecond ints', () =>
{
+ const table = Table.from(test_data);
+ const date64 = table.getColumnAt(1) as DateVector;
+ const expectedMillis = expectedMillis64();
+ for (const timestamp of date64.asEpochMilliseconds()) {
+ expect(timestamp).toEqual(expectedMillis.shift());
+ }
+ });
+});
+
+const expectedMillis32 = () => [
+ 165247430400000, 34582809600000, 232604524800000, null,
+ 199808812800000, 165646771200000, 209557238400000, null
+];
+
+const expectedMillis64 = () => [
+ 27990830234011, -41278585914325, 12694624797111,
+ null, null, 10761360520213, null, 1394015437000
+];
+
+const test_data = {
+ 'schema': {
+ 'fields': [
+ {
+ 'name': 'f0',
+ 'type': {
+ 'name': 'date',
+ 'unit': 'DAY'
+ },
+ 'nullable': true,
+ 'children': []
+ },
+ {
+ 'name': 'f1',
+ 'type': {
+ 'name': 'date',
+ 'unit': 'MILLISECOND'
+ },
+ 'nullable': true,
+ 'children': []
+ }
+ ]
+ },
+ 'batches': [
+ {
+ 'count': 8,
+ 'columns': [
+ {
+ 'name': 'f0',
+ 'count': 8,
+ 'VALIDITY': [1, 1, 1, 0, 1, 1, 1, 0],
+ 'DATA': [1912586, 400264, 2692182, 2163746, 2312602,
1917208, 2425431]
+ },
+ {
+ 'name': 'f1',
+ 'count': 8,
+ 'VALIDITY': [1, 1, 1, 0, 0, 1, 0, 1],
+ 'DATA': [
+ 27990830234011,
+ -41278585914325,
+ 12694624797111,
+ -38604948562547,
+ -37802308043516,
+ 10761360520213,
+ -25129181633384,
+ 1394015437000 // <-- the tricky one
+ ]
+ }
+ ]
+ }
+ ]
+};
--
To stop receiving notification emails like this one, please contact
w...@apache.org.
