This is an automated email from the ASF dual-hosted git repository. raulcd pushed a commit to branch maint-16.x.x in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 1c430be8b0cdf9621766d73610b90a6e908e2626 Author: Dominik Moritz <[email protected]> AuthorDate: Tue Apr 16 17:01:13 2024 -0400 GH-40959: [JS] Store Timestamps in 64 bits (#40960) Merge after #40892. This pull request also changes Dates to return timestamps instead of Date instances (similar to Timestamps and for the same reason. * GitHub Issue: #40959 --- js/src/type.ts | 26 +++++++++++----- js/src/util/bigint.ts | 13 ++++++++ js/src/visitor/get.ts | 21 +++++-------- js/src/visitor/iterator.ts | 9 +++--- js/src/visitor/set.ts | 25 +++------------ js/test/generate-test-data.ts | 53 +++++++++++++++----------------- js/test/unit/builders/date-tests.ts | 20 ++++++------ js/test/unit/builders/utils.ts | 20 +++--------- js/test/unit/vector/date-vector-tests.ts | 36 +++++++++++++++++++--- js/test/unit/vector/vector-tests.ts | 17 ++-------- 10 files changed, 119 insertions(+), 121 deletions(-) diff --git a/js/src/type.ts b/js/src/type.ts index a42552d65a..88aadb864e 100644 --- a/js/src/type.ts +++ b/js/src/type.ts @@ -333,16 +333,28 @@ export class Decimal extends DataType<Type.Decimal> { /** @ignore */ export type Dates = Type.Date | Type.DateDay | Type.DateMillisecond; /** @ignore */ -export interface Date_<T extends Dates = Dates> extends DataType<T> { TArray: Int32Array; TValue: Date; ArrayType: TypedArrayConstructor<Int32Array> } +type DateType = { + [Type.Date]: { TArray: Int32Array | BigInt64Array }; + [Type.DateDay]: { TArray: Int32Array }; + [Type.DateMillisecond]: { TArray: BigInt64Array }; +}; +/** @ignore */ +export interface Date_<T extends Dates = Dates> extends DataType<T> { + TArray: DateType[T]['TArray']; + TValue: number; +} /** @ignore */ export class Date_<T extends Dates = Dates> extends DataType<T> { constructor(public readonly unit: DateUnit) { super(Type.Date as T); } public toString() { return `Date${(this.unit + 1) * 32}<${DateUnit[this.unit]}>`; } + + public get ArrayType() { + return this.unit === DateUnit.DAY ? Int32Array : BigInt64Array; + } protected static [Symbol.toStringTag] = ((proto: Date_) => { (<any>proto).unit = null; - (<any>proto).ArrayType = Int32Array; return proto[Symbol.toStringTag] = 'Date'; })(Date_.prototype); } @@ -417,9 +429,9 @@ export class TimeNanosecond extends Time_<Type.TimeNanosecond> { constructor() { type Timestamps = Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond; /** @ignore */ interface Timestamp_<T extends Timestamps = Timestamps> extends DataType<T> { - TArray: Int32Array; + TArray: BigInt64Array; TValue: number; - ArrayType: TypedArrayConstructor<Int32Array>; + ArrayType: BigIntArrayConstructor<BigInt64Array>; } /** @ignore */ @@ -432,7 +444,7 @@ class Timestamp_<T extends Timestamps = Timestamps> extends DataType<T> { protected static [Symbol.toStringTag] = ((proto: Timestamp_) => { (<any>proto).unit = null; (<any>proto).timezone = null; - (<any>proto).ArrayType = Int32Array; + (<any>proto).ArrayType = BigInt64Array; return proto[Symbol.toStringTag] = 'Timestamp'; })(Timestamp_.prototype); } @@ -483,7 +495,7 @@ type Durations = Type.Duration | Type.DurationSecond | Type.DurationMillisecond export interface Duration<T extends Durations = Durations> extends DataType<T> { TArray: BigInt64Array; TValue: bigint; - ArrayType: BigInt64Array; + ArrayType: BigIntArrayConstructor<BigInt64Array>; } /** @ignore */ @@ -737,8 +749,6 @@ export function strideForType(type: DataType) { const t: any = type; switch (type.typeId) { case Type.Decimal: return (type as Decimal).bitWidth / 32; - case Type.Timestamp: return 2; - case Type.Date: return 1 + (t as Date_).unit; case Type.Interval: return 1 + (t as Interval_).unit; // case Type.Int: return 1 + +((t as Int_).bitWidth > 32); // case Type.Time: return 1 + +((t as Time_).bitWidth > 32); diff --git a/js/src/util/bigint.ts b/js/src/util/bigint.ts index 5af2f7f052..470b83f5fb 100644 --- a/js/src/util/bigint.ts +++ b/js/src/util/bigint.ts @@ -24,3 +24,16 @@ export function bigIntToNumber(number: bigint | number): number { } return Number(number); } + +/** + * Duivides the bigint number by the divisor and returns the result as a number. + * Dividing bigints always results in bigints so we don't get the remainder. + * This function gives us the remainder but assumes that the result fits into a number. + * + * @param number The number to divide. + * @param divisor The divisor. + * @returns The result of the division as a number. + */ +export function divideBigInts(number: bigint, divisor: bigint): number { + return bigIntToNumber(number / divisor) + bigIntToNumber(number % divisor) / bigIntToNumber(divisor); +} diff --git a/js/src/visitor/get.ts b/js/src/visitor/get.ts index 3ab3bcb68c..ddfc04884f 100644 --- a/js/src/visitor/get.ts +++ b/js/src/visitor/get.ts @@ -21,7 +21,7 @@ import { Vector } from '../vector.js'; import { Visitor } from '../visitor.js'; import { MapRow } from '../row/map.js'; import { StructRow, StructRowProxy } from '../row/struct.js'; -import { bigIntToNumber } from '../util/bigint.js'; +import { bigIntToNumber, divideBigInts } from '../util/bigint.js'; import { decodeUtf8 } from '../util/utf8.js'; import { TypeToDataType } from '../interfaces.js'; import { uint16ToFloat64 } from '../util/math.js'; @@ -106,13 +106,6 @@ function wrapGet<T extends DataType>(fn: (data: Data<T>, _1: any) => any) { } /** @ignore */const epochDaysToMs = (data: Int32Array, index: number) => 86400000 * data[index]; -/** @ignore */const epochMillisecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1]) + (data[index] >>> 0); -/** @ignore */const epochMicrosecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1] / 1000) + ((data[index] >>> 0) / 1000); -/** @ignore */const epochNanosecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1] / 1000000) + ((data[index] >>> 0) / 1000000); - -/** @ignore */const epochMillisecondsToDate = (epochMs: number) => new Date(epochMs); -/** @ignore */const epochDaysToDate = (data: Int32Array, index: number) => epochMillisecondsToDate(epochDaysToMs(data, index)); -/** @ignore */const epochMillisecondsLongToDate = (data: Int32Array, index: number) => epochMillisecondsToDate(epochMillisecondsLongToMs(data, index)); /** @ignore */ const getNull = <T extends Null>(_data: Data<T>, _index: number): T['TValue'] => null; @@ -139,9 +132,9 @@ type Numeric1X = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32 | Float32 | Floa type Numeric2X = Int64 | Uint64; /** @ignore */ -const getDateDay = <T extends DateDay>({ values }: Data<T>, index: number): T['TValue'] => epochDaysToDate(values, index); +const getDateDay = <T extends DateDay>({ values }: Data<T>, index: number): T['TValue'] => epochDaysToMs(values, index); /** @ignore */ -const getDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number): T['TValue'] => epochMillisecondsLongToDate(values, index * 2); +const getDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number): T['TValue'] => bigIntToNumber(values[index]); /** @ignore */ const getNumeric = <T extends Numeric1X>({ stride, values }: Data<T>, index: number): T['TValue'] => values[stride * index]; /** @ignore */ @@ -178,13 +171,13 @@ const getDate = <T extends Date_>(data: Data<T>, index: number): T['TValue'] => ); /** @ignore */ -const getTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number): T['TValue'] => 1000 * epochMillisecondsLongToMs(values, index * 2); +const getTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number): T['TValue'] => 1000 * bigIntToNumber(values[index]); /** @ignore */ -const getTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number): T['TValue'] => epochMillisecondsLongToMs(values, index * 2); +const getTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number): T['TValue'] => bigIntToNumber(values[index]); /** @ignore */ -const getTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number): T['TValue'] => epochMicrosecondsLongToMs(values, index * 2); +const getTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number): T['TValue'] => divideBigInts(values[index], BigInt(1000)); /** @ignore */ -const getTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number): T['TValue'] => epochNanosecondsLongToMs(values, index * 2); +const getTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number): T['TValue'] => divideBigInts(values[index], BigInt(1000000)); /* istanbul ignore next */ /** @ignore */ const getTimestamp = <T extends Timestamp>(data: Data<T>, index: number): T['TValue'] => { diff --git a/js/src/visitor/iterator.ts b/js/src/visitor/iterator.ts index bf7e9d1591..2a55f74367 100644 --- a/js/src/visitor/iterator.ts +++ b/js/src/visitor/iterator.ts @@ -101,10 +101,11 @@ function vectorIterator<T extends DataType>(vector: Vector<T>): IterableIterator // Fast case, defer to native iterators if possible if (vector.nullCount === 0 && vector.stride === 1 && ( - (type.typeId === Type.Timestamp) || - (type instanceof Int && (type as Int).bitWidth !== 64) || - (type instanceof Time && (type as Time).bitWidth !== 64) || - (type instanceof Float && (type as Float).precision !== Precision.HALF) + // Don't defer to native iterator for timestamps since Numbers are expected + // (DataType.isTimestamp(type)) && type.unit === TimeUnit.MILLISECOND || + (DataType.isInt(type) && type.bitWidth !== 64) || + (DataType.isTime(type) && type.bitWidth !== 64) || + (DataType.isFloat(type) && type.precision !== Precision.HALF) )) { return new ChunkedIterator(vector.data.length, (chunkIndex) => { const data = vector.data[chunkIndex]; diff --git a/js/src/visitor/set.ts b/js/src/visitor/set.ts index 5dc42283c3..5f2a3af4df 100644 --- a/js/src/visitor/set.ts +++ b/js/src/visitor/set.ts @@ -109,21 +109,6 @@ function wrapSet<T extends DataType>(fn: (data: Data<T>, _1: any, _2: any) => vo /** @ignore */ export const setEpochMsToDays = (data: Int32Array, index: number, epochMs: number) => { data[index] = Math.floor(epochMs / 86400000); }; -/** @ignore */ -export const setEpochMsToMillisecondsLong = (data: Int32Array, index: number, epochMs: number) => { - data[index] = Math.floor(epochMs % 4294967296); - data[index + 1] = Math.floor(epochMs / 4294967296); -}; -/** @ignore */ -export const setEpochMsToMicrosecondsLong = (data: Int32Array, index: number, epochMs: number) => { - data[index] = Math.floor((epochMs * 1000) % 4294967296); - data[index + 1] = Math.floor((epochMs * 1000) / 4294967296); -}; -/** @ignore */ -export const setEpochMsToNanosecondsLong = (data: Int32Array, index: number, epochMs: number) => { - data[index] = Math.floor((epochMs * 1000000) % 4294967296); - data[index + 1] = Math.floor((epochMs * 1000000) / 4294967296); -}; /** @ignore */ export const setVariableWidthBytes = <T extends Int32Array | BigInt64Array>(values: Uint8Array, valueOffsets: T, index: number, value: Uint8Array) => { @@ -161,7 +146,7 @@ export const setAnyFloat = <T extends Float>(data: Data<T>, index: number, value /** @ignore */ export const setDateDay = <T extends DateDay>({ values }: Data<T>, index: number, value: T['TValue']): void => { setEpochMsToDays(values, index, value.valueOf()); }; /** @ignore */ -export const setDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { setEpochMsToMillisecondsLong(values, index * 2, value.valueOf()); }; +export const setDateMillisecond = <T extends DateMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value); }; /** @ignore */ export const setFixedSizeBinary = <T extends FixedSizeBinary>({ stride, values }: Data<T>, index: number, value: T['TValue']): void => { values.set(value.subarray(0, stride), stride * index); }; @@ -178,13 +163,13 @@ export const setDate = <T extends Date_>(data: Data<T>, index: number, value: T[ }; /** @ignore */ -export const setTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToMillisecondsLong(values, index * 2, value / 1000); +export const setTimestampSecond = <T extends TimestampSecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value / 1000); }; /** @ignore */ -export const setTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToMillisecondsLong(values, index * 2, value); +export const setTimestampMillisecond = <T extends TimestampMillisecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value); }; /** @ignore */ -export const setTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToMicrosecondsLong(values, index * 2, value); +export const setTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value * 1000); }; /** @ignore */ -export const setTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => setEpochMsToNanosecondsLong(values, index * 2, value); +export const setTimestampNanosecond = <T extends TimestampNanosecond>({ values }: Data<T>, index: number, value: T['TValue']): void => { values[index] = BigInt(value * 1000000); }; /* istanbul ignore next */ /** @ignore */ export const setTimestamp = <T extends Timestamp>(data: Data<T>, index: number, value: T['TValue']): void => { diff --git a/js/test/generate-test-data.ts b/js/test/generate-test-data.ts index 8e6e47de83..65719f875c 100644 --- a/js/test/generate-test-data.ts +++ b/js/test/generate-test-data.ts @@ -402,10 +402,7 @@ function generateDate<T extends Date_>(this: TestDataVectorGenerator, type: T, l const data = type.unit === DateUnit.DAY ? createDate32(length, nullBitmap, values) : createDate64(length, nullBitmap, values); - return { - values: () => values.map((x) => x == null ? null : new Date(x)), - vector: new Vector([makeData({ type, length, nullCount, nullBitmap, data })]) - }; + return { values: () => values, vector: new Vector([makeData({ type, length, nullCount, nullBitmap, data })]) }; } function generateTimestamp<T extends Timestamp>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = Math.trunc(length * 0.2)): GeneratedVector<T> { @@ -649,6 +646,7 @@ type TypedArrayConstructor = const rand = Math.random.bind(Math); +const randSign = () => rand() > 0.5 ? -1 : 1; const randomBytes = (length: number) => fillRandom(Uint8Array, length); const memoize = (fn: () => any) => ((x?: any) => () => x || (x = fn()))(); @@ -661,7 +659,7 @@ function fillRandom<T extends TypedArrayConstructor>(ArrayType: T, length: numbe const BPE = ArrayType.BYTES_PER_ELEMENT; const array = new ArrayType(length); const max = (2 ** (8 * BPE)) - 1; - for (let i = -1; ++i < length; array[i] = rand() * max * (rand() > 0.5 ? -1 : 1)); + for (let i = -1; ++i < length; array[i] = rand() * max * randSign()); return array as InstanceType<T>; } @@ -669,7 +667,7 @@ function fillRandomBigInt<T extends (typeof BigInt64Array) | (typeof BigUint64Ar const BPE = ArrayType.BYTES_PER_ELEMENT; const array = new ArrayType(length); const max = (2 ** (8 * BPE)) - 1; - for (let i = -1; ++i < length; array[i] = BigInt(rand() * max * (rand() > 0.5 ? -1 : 1))); + for (let i = -1; ++i < length; array[i] = BigInt(rand() * max * randSign())); return array as InstanceType<T>; } @@ -735,6 +733,9 @@ function createVariableWidthBytes(length: number, nullBitmap: Uint8Array, offset return bytes; } +/** + * Creates timestamps with the accuracy of days (86400000 millisecond). + */ function createDate32(length: number, nullBitmap: Uint8Array, values: (number | null)[] = []) { const data = new Int32Array(length).fill(Math.trunc(Date.now() / 86400000)); iterateBitmap(length, nullBitmap, (i, valid) => { @@ -742,7 +743,7 @@ function createDate32(length: number, nullBitmap: Uint8Array, values: (number | data[i] = 0; values[i] = null; } else { - data[i] = Math.trunc(data[i] + (rand() * 10000 * (rand() > 0.5 ? -1 : 1))); + data[i] = Math.trunc(data[i] + (rand() * 10000 * randSign())); values[i] = data[i] * 86400000; } }); @@ -750,32 +751,26 @@ function createDate32(length: number, nullBitmap: Uint8Array, values: (number | } function createDate64(length: number, nullBitmap: Uint8Array, values: (number | null)[] = []) { - const data = new Int32Array(length * 2).fill(0); const data32 = createDate32(length, nullBitmap, values); - iterateBitmap(length, nullBitmap, (i, valid) => { - if (valid) { - const value = data32[i] * 86400000; - const hi = Math.trunc(value / 4294967296); - const lo = Math.trunc(value - 4294967296 * hi); - values[i] = value; - data[i * 2 + 0] = lo; - data[i * 2 + 1] = hi; - } - }); - return data; + return BigInt64Array.from(data32, x => BigInt(x * 86400000)); +} + +function divideBigInts(number: bigint, divisor: bigint): number { + return Number(number / divisor) + Number(number % divisor) / Number(divisor); } function createTimestamp(length: number, nullBitmap: Uint8Array, multiple: number, values: (number | null)[] = []) { - const mult = 86400 * multiple; - const data = new Int32Array(length * 2).fill(0); - const data32 = createDate32(length, nullBitmap, values); + const data = new BigInt64Array(length).fill(0n); + const tenYears = 10 * 365 * 24 * 60 * 60 * multiple; + const now = Math.trunc(Date.now() / 1000 * multiple); iterateBitmap(length, nullBitmap, (i, valid) => { - if (valid) { - const value = data32[i] * mult; - const hi = Math.trunc(value / 4294967296); - const lo = Math.trunc(value - 4294967296 * hi); - data[i * 2 + 0] = lo; - data[i * 2 + 1] = hi; + if (!valid) { + data[i] = 0n; + values[i] = null; + } else { + const value = BigInt(now + Math.trunc(rand() * randSign() * tenYears)); + data[i] = value; + values[i] = divideBigInts(value * 1000n, BigInt(multiple)); } }); return data; @@ -788,7 +783,7 @@ function createTime32(length: number, nullBitmap: Uint8Array, multiple: number, data[i] = 0; values[i] = null; } else { - values[i] = data[i] = ((1000 * rand()) | 0 * multiple) * (rand() > 0.5 ? -1 : 1); + values[i] = data[i] = ((1000 * rand()) | 0 * multiple) * randSign(); } }); return data; diff --git a/js/test/unit/builders/date-tests.ts b/js/test/unit/builders/date-tests.ts index 318bb7dfc4..3fd3f7832f 100644 --- a/js/test/unit/builders/date-tests.ts +++ b/js/test/unit/builders/date-tests.ts @@ -17,10 +17,8 @@ import 'web-streams-polyfill'; import { - date32sNoNulls, - date32sWithNulls, - date64sNoNulls, - date64sWithNulls, + dateNoNulls, + dateWithNulls, encodeAll, encodeEach, encodeEachDOM, @@ -41,14 +39,14 @@ describe('DateDayBuilder', () => { testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new DateDay(), 25)); testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new DateDay(), 25)); - function runTestsWithEncoder(name: string, encode: (vals: (Date | null)[], nullVals?: any[]) => Promise<Vector<DateDay>>) { + function runTestsWithEncoder(name: string, encode: (vals: (number | null)[], nullVals?: any[]) => Promise<Vector<DateDay>>) { describe(`${encode.name} ${name}`, () => { it(`encodes dates no nulls`, async () => { - const vals = date32sNoNulls(20); + const vals = dateNoNulls(20); validateVector(vals, await encode(vals, []), []); }); it(`encodes dates with nulls`, async () => { - const vals = date32sWithNulls(20); + const vals = dateWithNulls(20); validateVector(vals, await encode(vals, [null]), [null]); }); }); @@ -63,14 +61,14 @@ describe('DateMillisecondBuilder', () => { testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new DateMillisecond(), 25)); testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new DateMillisecond(), 25)); - function runTestsWithEncoder(name: string, encode: (vals: (Date | null)[], nullVals?: any[]) => Promise<Vector<DateMillisecond>>) { + function runTestsWithEncoder(name: string, encode: (vals: (number | null)[], nullVals?: any[]) => Promise<Vector<DateMillisecond>>) { describe(`${encode.name} ${name}`, () => { it(`encodes dates no nulls`, async () => { - const vals = date64sNoNulls(20); + const vals = dateNoNulls(20); validateVector(vals, await encode(vals, []), []); }); it(`encodes dates with nulls`, async () => { - const vals = date64sWithNulls(20); + const vals = dateWithNulls(20); validateVector(vals, await encode(vals, [null]), [null]); }); }); @@ -100,7 +98,7 @@ describe('DateMillisecondBuilder with nulls', () => { '2019-03-10T21:15:32.237Z', '2019-03-21T07:25:34.864Z', null - ].map((x) => x === null ? x : new Date(x)); + ].map((x) => x === null ? x : new Date(x).getTime()); it(`encodes dates with nulls`, async () => { const vals = dates.slice(); validateVector(vals, await encode(vals, [null]), [null]); diff --git a/js/test/unit/builders/utils.ts b/js/test/unit/builders/utils.ts index db4e80d002..1d0707a6ca 100644 --- a/js/test/unit/builders/utils.ts +++ b/js/test/unit/builders/utils.ts @@ -32,22 +32,11 @@ const randnulls = <T, TNull = null>(values: T[], n: TNull = <any>null) => values export const randomBytes = (length: number) => fillRandom(Uint8Array, length); export const stringsNoNulls = (length = 20) => Array.from({ length }, (_) => randomString(1 + (Math.trunc(Math.random() * 19)))); -export const timestamp32sNoNulls = (length = 20, now = Math.trunc(Date.now() / 86400000)) => - Array.from({ length }, (_) => (Math.trunc(now + (rand() * 10000 * (rand() > 0.5 ? -1 : 1)))) * 86400000); - -export const timestamp64sNoNulls = (length = 20, now = Date.now()) => Array.from({ length }, (_) => { - const ms = now + (Math.trunc(rand() * 31557600000 * (rand() > 0.5 ? -1 : 1))); - return new Int32Array([Math.trunc(ms % 4294967296), Math.trunc(ms / 4294967296)]); -}); - -export const timestamp32sWithNulls = (length = 20) => randnulls(timestamp32sNoNulls(length), null); -export const timestamp64sWithNulls = (length = 20) => randnulls(timestamp64sNoNulls(length), null); -export const timestamp32sWithMaxInts = (length = 20) => randnulls(timestamp32sNoNulls(length), 0x7FFFFFFF); -export const timestamp64sWithMaxInts = (length = 20) => randnulls(timestamp64sNoNulls(length), 9223372034707292159n); export const boolsNoNulls = (length = 20) => Array.from({ length }, () => rand() > 0.5); -export const date32sNoNulls = (length = 20) => timestamp32sNoNulls(length).map((x) => new Date(x)); -export const date64sNoNulls = (length = 20) => timestamp64sNoNulls(length).map((x) => new Date(4294967296 * x[1] + (x[0] >>> 0))); + +export const dateNoNulls = (length = 20, now = Math.trunc(Date.now() / 86400000)) => + Array.from({ length }, (_) => (Math.trunc(now + (rand() * 100000 * (rand() > 0.5 ? -1 : 1)))) * 86400000); export const int8sNoNulls = (length = 20) => Array.from(new Int8Array(randomBytes(length * Int8Array.BYTES_PER_ELEMENT).buffer)); export const int16sNoNulls = (length = 20) => Array.from(new Int16Array(randomBytes(length * Int16Array.BYTES_PER_ELEMENT).buffer)); export const int32sNoNulls = (length = 20) => Array.from(new Int32Array(randomBytes(length * Int32Array.BYTES_PER_ELEMENT).buffer)); @@ -66,8 +55,7 @@ export const stringsWithNulls = (length = 20) => randnulls(stringsNoNulls(length export const stringsWithEmpties = (length = 20) => randnulls(stringsNoNulls(length), '\0'); export const boolsWithNulls = (length = 20) => randnulls(boolsNoNulls(length), null); -export const date32sWithNulls = (length = 20) => randnulls(date32sNoNulls(length), null); -export const date64sWithNulls = (length = 20) => randnulls(date64sNoNulls(length), null); +export const dateWithNulls = (length = 20) => randnulls(dateNoNulls(length), null); export const int8sWithNulls = (length = 20) => randnulls(int8sNoNulls(length), null); export const int16sWithNulls = (length = 20) => randnulls(int16sNoNulls(length), null); export const int32sWithNulls = (length = 20) => randnulls(int32sNoNulls(length), null); diff --git a/js/test/unit/vector/date-vector-tests.ts b/js/test/unit/vector/date-vector-tests.ts index e5cd49933e..4a6ffc1a82 100644 --- a/js/test/unit/vector/date-vector-tests.ts +++ b/js/test/unit/vector/date-vector-tests.ts @@ -15,14 +15,40 @@ // specific language governing permissions and limitations // under the License. -import { DateDay, DateMillisecond, TimestampMillisecond, RecordBatchReader, Table, vectorFromArray } from 'apache-arrow'; +import { + DateDay, TimestampSecond, DateMillisecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, RecordBatchReader, + Table, vectorFromArray +} from 'apache-arrow'; describe(`TimestampVector`, () => { test(`Dates are stored in TimestampMillisecond`, () => { const date = new Date('2023-02-01T12:34:56Z'); const vec = vectorFromArray([date]); expect(vec.type).toBeInstanceOf(TimestampMillisecond); - expect(vec.get(0)).toBe(date.valueOf()); + expect(vec.get(0)).toBe(date.getTime()); + }); + + test(`Correctly get back TimestampSecond from Date`, () => { + const date = new Date('2023-02-01T12:34:56Z'); + const vec = vectorFromArray([date], new TimestampSecond); + expect(vec.type).toBeInstanceOf(TimestampSecond); + expect(vec.get(0)).toBe(date.getTime()); + }); + + test(`Correctly get back TimestampMicrosecond from Date`, () => { + const date = new Date('2023-02-01T12:34:56Z'); + const vec = vectorFromArray([date, 0.5], new TimestampMicrosecond); + expect(vec.type).toBeInstanceOf(TimestampMicrosecond); + expect(vec.get(0)).toBe(date.getTime()); + expect(vec.get(1)).toBe(0.5); + }); + + test(`Correctly get back TimestampNanosecond from Date`, () => { + const date = new Date('2023-02-01T12:34:56Z'); + const vec = vectorFromArray([date, 0.5], new TimestampNanosecond); + expect(vec.type).toBeInstanceOf(TimestampNanosecond); + expect(vec.get(0)).toBe(date.getTime()); + expect(vec.get(1)).toBe(0.5); }); }); @@ -33,7 +59,7 @@ describe(`DateVector`, () => { const date32 = table.getChildAt<DateDay>(0)!; for (const date of date32) { const millis = expectedMillis.shift(); - expect(date).toEqual(millis === null ? null : new Date(millis!)); + expect(date).toEqual(millis); } }); @@ -43,7 +69,7 @@ describe(`DateVector`, () => { const date64 = table.getChildAt<DateMillisecond>(1)!; for (const date of date64) { const millis = expectedMillis.shift(); - expect(date).toEqual(millis === null ? null : new Date(millis!)); + expect(date).toEqual(millis); } }); @@ -51,7 +77,7 @@ describe(`DateVector`, () => { const dates = [new Date(1950, 1, 0)]; const vec = vectorFromArray(dates, new DateMillisecond()); for (const date of vec) { - expect(date).toEqual(dates.shift()); + expect(date).toEqual(dates.shift()?.getTime()); } }); }); diff --git a/js/test/unit/vector/vector-tests.ts b/js/test/unit/vector/vector-tests.ts index 881bf987b5..c05838d7a9 100644 --- a/js/test/unit/vector/vector-tests.ts +++ b/js/test/unit/vector/vector-tests.ts @@ -16,7 +16,7 @@ // under the License. import { - Bool, DateDay, DateMillisecond, Dictionary, Float64, Int32, List, makeVector, Struct, Timestamp, TimeUnit, Utf8, LargeUtf8, util, Vector, vectorFromArray, makeData + Bool, DateDay, DateMillisecond, Dictionary, Float64, Int32, List, makeVector, Struct, Utf8, LargeUtf8, util, Vector, vectorFromArray, makeData } from 'apache-arrow'; describe(`makeVectorFromArray`, () => { @@ -130,7 +130,7 @@ describe(`DateVector`, () => { new Date(1988, 3, 25, 4, 5, 6), new Date(1987, 2, 24, 7, 8, 9), new Date(2018, 4, 12, 17, 30, 0) - ]; + ].map(v => v.getTime()); const vector = vectorFromArray(values, new DateMillisecond); basicVectorTests(vector, values, extras); }); @@ -141,7 +141,7 @@ describe(`DateVector`, () => { new Date(Date.UTC(1988, 3, 25)), new Date(Date.UTC(1987, 2, 24)), new Date(Date.UTC(2018, 4, 12)) - ]; + ].map(v => v.getTime()); const vector = vectorFromArray(values, new DateDay); basicVectorTests(vector, values, extras); @@ -288,17 +288,6 @@ describe(`toArray()`, () => { const array = vector.toArray(); expect(array).toHaveLength(26); }); - - test(`when stride is 2`, () => { - let d1 = vectorFromArray([0, 1, 2], new Timestamp(TimeUnit.MILLISECOND)).data[0]; - let d2 = vectorFromArray([3, 4, 5], new Timestamp(TimeUnit.MILLISECOND)).data[0]; - - const vector = new Vector([d1, d2]); - - let array = Array.from(vector.toArray()); - expect(array).toHaveLength(6 * 2); - expect(Array.from(array)).toMatchObject([0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0]); - }); }); // Creates some basic tests for the given vector.
