This is an automated email from the ASF dual-hosted git repository. CritasWang pushed a commit to branch fix/date-encoding-yyyymmdd in repository https://gitbox.apache.org/repos/asf/iotdb-client-nodejs.git
commit 40928b873f2a25824e825035473ef507fc740a2d Author: CritasWang <[email protected]> AuthorDate: Mon Jul 13 10:36:01 2026 +0800 fix: encode DATE as INT32 yyyyMMdd instead of days-since-epoch IoTDB's DATE wire format is an INT32 encoded as year*10000 + month*100 + day (e.g. 2026-07-13 -> 20260713), matching the Java client's DateUtils.parseDateExpressionToInt and the C# client. The Node.js client was writing and reading days-since-epoch, so DATE values written via tablets were corrupted and unreadable by other clients (and vice versa). Adjudicated against a live IoTDB 2.0.6 server: a row inserted via SQL literal '2026-07-13' reads back as INT32 20260713 on the wire. Changes: - Add shared parseDateToInt / parseIntToDate utils in src/utils/DataTypes.ts (UTC calendar components, matching Java LocalDate semantics) and export them - Session.serializeColumn case 9: write yyyyMMdd (was days-since-epoch) - FastSerializer.serializeDateColumn: same fix on the fast path - Session.deserializeColumn case 9 (TSQueryDataSet): decode yyyyMMdd -> Date - ColumnDecoder Int32ArrayColumnDecoder: decode DATE (type 9) columns to Date - Session.parseTsBlock: DATE columns arrive typed as INT32 (1) in the TsBlock header; convert them to Date using the metadata dataTypeList - Unit tests: yyyyMMdd round-trip, exact wire bytes for 2026-07-13 (0x01352769), TsBlock decode; updated existing DATE serialization tests Verified: tablet-written Date and SQL-literal date now read back identical against a live IoTDB 2.0.6 (both 2026-07-13T00:00:00.000Z). --- src/client/ColumnDecoder.ts | 13 ++++- src/client/Session.ts | 40 ++++++++------ src/index.ts | 7 ++- src/utils/DataTypes.ts | 49 ++++++++++++++++- src/utils/FastSerializer.ts | 18 +++---- tests/unit/DataTypes.test.ts | 98 ++++++++++++++++++++++++++++++++++ tests/unit/FastSerializer.test.ts | 16 ++++-- tests/unit/TabletSerialization.test.ts | 18 +++++-- 8 files changed, 218 insertions(+), 41 deletions(-) diff --git a/src/client/ColumnDecoder.ts b/src/client/ColumnDecoder.ts index 52701cc..25d5c13 100644 --- a/src/client/ColumnDecoder.ts +++ b/src/client/ColumnDecoder.ts @@ -18,6 +18,7 @@ */ import { logger } from "../utils/Logger"; +import { parseIntToDate } from "../utils/DataTypes"; /** * Column encoding types matching Apache IoTDB ColumnEncoding enum @@ -152,7 +153,6 @@ class Int32ArrayColumnDecoder implements ColumnDecoder { switch (dataType) { case 1: // INT32 - case 9: // DATE for (let i = 0; i < positionCount; i++) { if (nullIndicators && nullIndicators[i]) { values[i] = null; @@ -163,6 +163,17 @@ class Int32ArrayColumnDecoder implements ColumnDecoder { } break; + case 9: // DATE (INT32 yyyyMMdd encoding -> Date object) + for (let i = 0; i < positionCount; i++) { + if (nullIndicators && nullIndicators[i]) { + values[i] = null; + continue; + } + values[i] = parseIntToDate(buffer.readInt32BE(currentOffset)); + currentOffset += 4; + } + break; + case 3: // FLOAT for (let i = 0; i < positionCount; i++) { if (nullIndicators && nullIndicators[i]) { diff --git a/src/client/Session.ts b/src/client/Session.ts index 879f62e..3df82e4 100644 --- a/src/client/Session.ts +++ b/src/client/Session.ts @@ -35,6 +35,7 @@ import { serializeTimestamps } from "../utils/FastSerializer"; import { globalBufferPool } from "../utils/BufferPool"; +import { parseDateToInt, parseIntToDate } from "../utils/DataTypes"; const ttypes = require("../thrift/generated/client_types"); @@ -741,18 +742,11 @@ export class Session { return buffer; } case 9: { - // DATE (stored as INT32 - days since epoch) - Use big-endian + // DATE (stored as INT32 - yyyyMMdd, e.g. 20240101) - Use big-endian const buffer = Buffer.alloc(values.length * 4); values.forEach((v, i) => { - let days = 0; - if (v !== null && v !== undefined) { - if (v instanceof Date) { - days = Math.floor(v.getTime() / (24 * 60 * 60 * 1000)); - } else { - days = v; - } - } - buffer.writeInt32BE(days, i * 4); + const encoded = v === null || v === undefined ? 0 : parseDateToInt(v); + buffer.writeInt32BE(encoded, i * 4); }); return buffer; } @@ -1021,6 +1015,23 @@ export class Session { ); } + // DATE columns arrive in TsBlock with wire type INT32 (1); the real DATE + // type is only present in the query metadata. Convert those columns' + // yyyyMMdd integers (e.g. 20240101) to Date objects here. + for (let i = 0; i < valueColumns.length; i++) { + if ( + valueColumnTypes[i] === 1 && + this.getDataTypeCode(dataTypes[i]) === 9 + ) { + const colValues = valueColumns[i].values; + for (let j = 0; j < colValues.length; j++) { + if (colValues[j] !== null) { + colValues[j] = parseIntToDate(colValues[j]); + } + } + } + } + // Build rows from columns const rows: any[][] = []; for (let rowIndex = 0; rowIndex < positionCount; rowIndex++) { @@ -1313,15 +1324,14 @@ export class Session { break; } case 9: { - // DATE (stored as INT32 - days since epoch) - TSQueryDataSet uses BIG ENDIAN + // DATE (stored as INT32 - yyyyMMdd) - TSQueryDataSet uses BIG ENDIAN for (let i = 0; i < rowCount; i++) { if (this.isNull(bitmap, i)) { values.push(null); } else { - // Convert days since epoch to Date object - const days = buffer.readInt32BE(i * 4); - const date = new Date(days * 24 * 60 * 60 * 1000); - values.push(date); + // Convert yyyyMMdd integer to Date object + const encoded = buffer.readInt32BE(i * 4); + values.push(parseIntToDate(encoded)); } } break; diff --git a/src/index.ts b/src/index.ts index 54fdee4..76a2965 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,7 +39,12 @@ export { parseNodeUrls, } from "./utils/Config"; export { logger, LogLevel } from "./utils/Logger"; -export { TSDataType, getDataTypeName } from "./utils/DataTypes"; +export { + TSDataType, + getDataTypeName, + parseDateToInt, + parseIntToDate, +} from "./utils/DataTypes"; export { RedirectException, TSStatusCode } from "./utils/Errors"; export { RedirectCache } from "./client/RedirectCache"; export { enableGlobalCleanup } from "./utils/ProcessCleanup"; diff --git a/src/utils/DataTypes.ts b/src/utils/DataTypes.ts index a497e8b..166d192 100644 --- a/src/utils/DataTypes.ts +++ b/src/utils/DataTypes.ts @@ -77,8 +77,8 @@ export enum TSDataType { /** * Date with day precision (no time component) - * JavaScript type: Date or number (days since epoch) - * Storage size: 4 bytes (stored as INT32) + * JavaScript type: Date or number (yyyyMMdd integer, e.g. 20240101) + * Storage size: 4 bytes (stored as INT32, encoded as year*10000 + month*100 + day) */ DATE = 9, @@ -99,6 +99,51 @@ export enum TSDataType { // OBJECT = 12, // Reserved - not yet implemented } +/** + * Convert a JavaScript Date (or an already-encoded yyyyMMdd number) to the + * IoTDB DATE wire format: an INT32 encoded as year*10000 + month*100 + day + * (e.g. 2024-01-01 -> 20240101). + * + * This matches the Java client's DateUtils.parseDateExpressionToInt and the + * C# client. The calendar date is taken from the Date's UTC components, + * consistent with `new Date("2024-01-01")` which parses as UTC midnight. + * + * @param value - Date object or a yyyyMMdd integer (passed through unchanged) + * @returns The yyyyMMdd integer encoding + */ +export function parseDateToInt(value: Date | number): number { + if (value instanceof Date) { + return ( + value.getUTCFullYear() * 10000 + + (value.getUTCMonth() + 1) * 100 + + value.getUTCDate() + ); + } + return value; +} + +/** + * Convert an IoTDB DATE wire value (INT32, year*10000 + month*100 + day) + * back to a JavaScript Date at UTC midnight of that calendar date. + * + * Inverse of {@link parseDateToInt}; matches the Java client's + * DateUtils.parseIntToDate. + * + * @param value - The yyyyMMdd integer (e.g. 20240101) + * @returns Date at UTC midnight of the encoded calendar date + */ +export function parseIntToDate(value: number): Date { + const year = Math.trunc(value / 10000); + const month = Math.trunc(value / 100) % 100; + const day = value % 100; + const date = new Date(Date.UTC(year, month - 1, day)); + // Date.UTC maps years 0-99 to 1900-1999; correct that explicitly + if (year >= 0 && year < 100) { + date.setUTCFullYear(year); + } + return date; +} + /** * Get the name of a TSDataType from its numeric code * @param typeCode - The numeric type code diff --git a/src/utils/FastSerializer.ts b/src/utils/FastSerializer.ts index a87f9b3..76ae9ed 100644 --- a/src/utils/FastSerializer.ts +++ b/src/utils/FastSerializer.ts @@ -18,6 +18,7 @@ */ import { globalBufferPool } from "./BufferPool"; +import { parseDateToInt } from "./DataTypes"; /** * Fast serialization utilities for IoTDB data types @@ -167,26 +168,19 @@ export function serializeTimestampColumn(values: any[]): Buffer { } /** - * Serialize DATE column (4 bytes per value, days since epoch) + * Serialize DATE column (4 bytes per value, INT32 yyyyMMdd encoding) * Optimized: Single buffer allocation with direct writes */ export function serializeDateColumn(values: any[]): Buffer { const size = values.length * 4; const buffer = size >= 1024 ? globalBufferPool.acquire(size) : Buffer.allocUnsafe(size); - + for (let i = 0; i < values.length; i++) { const v = values[i]; - let days = 0; - if (v !== null && v !== undefined) { - if (v instanceof Date) { - days = Math.floor(v.getTime() / (24 * 60 * 60 * 1000)); - } else { - days = v; - } - } - buffer.writeInt32BE(days, i * 4); + const encoded = v === null || v === undefined ? 0 : parseDateToInt(v); + buffer.writeInt32BE(encoded, i * 4); } - + return buffer.subarray(0, size); } diff --git a/tests/unit/DataTypes.test.ts b/tests/unit/DataTypes.test.ts new file mode 100644 index 0000000..6ab66a9 --- /dev/null +++ b/tests/unit/DataTypes.test.ts @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + parseDateToInt, + parseIntToDate, +} from "../../src/utils/DataTypes"; +import { + BaseColumnDecoder, + ColumnEncoding, +} from "../../src/client/ColumnDecoder"; + +describe("DATE yyyyMMdd conversion", () => { + describe("parseDateToInt", () => { + it("should encode a Date as INT32 yyyyMMdd (year*10000 + month*100 + day)", () => { + expect(parseDateToInt(new Date("2026-07-13"))).toBe(20260713); + expect(parseDateToInt(new Date("2024-01-01"))).toBe(20240101); + expect(parseDateToInt(new Date("2024-12-31"))).toBe(20241231); + // Leap day + expect(parseDateToInt(new Date("2024-02-29"))).toBe(20240229); + }); + + it("should pass through already-encoded numbers unchanged", () => { + expect(parseDateToInt(20260713)).toBe(20260713); + expect(parseDateToInt(0)).toBe(0); + }); + }); + + describe("parseIntToDate", () => { + it("should decode INT32 yyyyMMdd to a Date at UTC midnight", () => { + const date = parseIntToDate(20260713); + expect(date.getUTCFullYear()).toBe(2026); + expect(date.getUTCMonth()).toBe(6); // July (0-based) + expect(date.getUTCDate()).toBe(13); + expect(date.getUTCHours()).toBe(0); + expect(date.getUTCMinutes()).toBe(0); + }); + }); + + describe("round-trip", () => { + it("should round-trip Date -> yyyyMMdd -> Date", () => { + const dates = [ + new Date("1970-01-01"), + new Date("2000-02-29"), + new Date("2024-01-01"), + new Date("2026-07-13"), + new Date("9999-12-31"), + ]; + for (const original of dates) { + const encoded = parseDateToInt(original); + const decoded = parseIntToDate(encoded); + expect(decoded.getTime()).toBe(original.getTime()); + // And the integer round-trips too + expect(parseDateToInt(decoded)).toBe(encoded); + } + }); + }); + + describe("TsBlock column decode (Int32ArrayColumnDecoder)", () => { + it("should decode a DATE column value as a Date from yyyyMMdd wire bytes", () => { + // Column layout: 1 byte null flag (0 = no nulls) + INT32 BE values + // 20260713 = 0x01352769 + const buffer = Buffer.from([0x00, 0x01, 0x35, 0x27, 0x69]); + const decoder = BaseColumnDecoder.getDecoder(ColumnEncoding.Int32Array); + const { column, bytesRead } = decoder.readColumn(buffer, 0, 9, 1); + + expect(bytesRead).toBe(5); + const value = column.values[0]; + expect(value).toBeInstanceOf(Date); + expect(value.getUTCFullYear()).toBe(2026); + expect(value.getUTCMonth()).toBe(6); + expect(value.getUTCDate()).toBe(13); + }); + + it("should still decode plain INT32 columns as numbers", () => { + const buffer = Buffer.from([0x00, 0x01, 0x35, 0x27, 0x69]); + const decoder = BaseColumnDecoder.getDecoder(ColumnEncoding.Int32Array); + const { column } = decoder.readColumn(buffer, 0, 1, 1); + expect(column.values[0]).toBe(20260713); + }); + }); +}); diff --git a/tests/unit/FastSerializer.test.ts b/tests/unit/FastSerializer.test.ts index 4f2b7ba..eb6d686 100644 --- a/tests/unit/FastSerializer.test.ts +++ b/tests/unit/FastSerializer.test.ts @@ -155,18 +155,24 @@ describe("FastSerializer", () => { }); describe("DATE Serialization", () => { - it("should serialize DATE values correctly", () => { + it("should serialize DATE values as INT32 yyyyMMdd", () => { const date1 = new Date("2024-01-01"); - const days1 = Math.floor(date1.getTime() / (24 * 60 * 60 * 1000)); - const values = [date1, 100, null, undefined]; + const values = [date1, 20241231, null, undefined]; const buffer = serializeDateColumn(values); expect(buffer.length).toBe(16); // 4 * 4 bytes - expect(buffer.readInt32BE(0)).toBe(days1); - expect(buffer.readInt32BE(4)).toBe(100); + expect(buffer.readInt32BE(0)).toBe(20240101); // yyyyMMdd, NOT days since epoch + expect(buffer.readInt32BE(4)).toBe(20241231); // numbers pass through unchanged expect(buffer.readInt32BE(8)).toBe(0); // null -> 0 expect(buffer.readInt32BE(12)).toBe(0); // undefined -> 0 }); + + it("should produce exact wire bytes for 2026-07-13", () => { + // IoTDB DATE wire format: INT32 yyyyMMdd, big-endian + // 20260713 = 0x01352769 + const buffer = serializeDateColumn([new Date("2026-07-13")]); + expect(Array.from(buffer)).toEqual([0x01, 0x35, 0x27, 0x69]); + }); }); describe("BLOB Serialization", () => { diff --git a/tests/unit/TabletSerialization.test.ts b/tests/unit/TabletSerialization.test.ts index fb86706..65278cd 100644 --- a/tests/unit/TabletSerialization.test.ts +++ b/tests/unit/TabletSerialization.test.ts @@ -342,20 +342,28 @@ describe('Tablet Serialization', () => { expect(buffer.readDoubleBE(16)).toBe(0.0); }); - test('should serialize DATE column', () => { + test('should serialize DATE column as INT32 yyyyMMdd', () => { const date1 = new Date('2024-01-01'); const date2 = new Date('2024-12-31'); const values = [date1, date2, 0]; const buffer = (session as any).serializeColumn(values, TSDataType.DATE); expect(buffer.length).toBe(12); // 3 values * 4 bytes - const days1 = Math.floor(date1.getTime() / (24 * 60 * 60 * 1000)); - const days2 = Math.floor(date2.getTime() / (24 * 60 * 60 * 1000)); - expect(buffer.readInt32BE(0)).toBe(days1); - expect(buffer.readInt32BE(4)).toBe(days2); + expect(buffer.readInt32BE(0)).toBe(20240101); // yyyyMMdd, NOT days since epoch + expect(buffer.readInt32BE(4)).toBe(20241231); expect(buffer.readInt32BE(8)).toBe(0); }); + test('should serialize DATE 2026-07-13 with exact wire bytes', () => { + // IoTDB DATE wire format: INT32 yyyyMMdd, big-endian + // 20260713 = 0x01352769 + const buffer = (session as any).serializeColumn( + [new Date('2026-07-13')], + TSDataType.DATE, + ); + expect(Array.from(buffer)).toEqual([0x01, 0x35, 0x27, 0x69]); + }); + test('should serialize TIMESTAMP column', () => { const ts1 = Date.now(); const ts2 = ts1 + 1000;
