This is an automated email from the ASF dual-hosted git repository.
piotr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 3f69440a feat(js): return message id as bigint instead of uuid string
(#1902)
3f69440a is described below
commit 3f69440a7e8b62667ccb0820086c91364f263131
Author: T1B0 <[email protected]>
AuthorDate: Mon Jun 23 08:42:21 2025 +0200
feat(js): return message id as bigint instead of uuid string (#1902)
return polled messages id as Bigint instead of uuid string
---
foreign/node/src/bdd/message.ts | 11 +++++++----
foreign/node/src/tcp.sm.utils.ts | 17 +++++++++++++++--
foreign/node/src/wire/message/iggy-header.utils.ts | 10 +++++++---
foreign/node/src/wire/message/message.utils.ts | 19 +++++--------------
.../src/wire/message/send-messages.command.test.ts | 20 ++++++++++++++------
foreign/node/src/wire/number.utils.ts | 14 ++++++++++++--
6 files changed, 60 insertions(+), 31 deletions(-)
diff --git a/foreign/node/src/bdd/message.ts b/foreign/node/src/bdd/message.ts
index 892e8b52..f26d1c00 100644
--- a/foreign/node/src/bdd/message.ts
+++ b/foreign/node/src/bdd/message.ts
@@ -88,8 +88,8 @@ Then(
Then(
'the messages should have sequential offsets from {int} to {int}',
- function (this: TestWorld,from: number, to: number) {
- for(let i = from; i < to; i++) {
+ function (this: TestWorld, from: number, to: number) {
+ for (let i = from; i < to; i++) {
assert.equal(BigInt(i), this.polledMessages[i].headers.offset)
}
}
@@ -100,8 +100,10 @@ Then(
function (this: TestWorld) {
this.sendMessages.forEach((msg, i) => {
assert.deepEqual(msg.payload.toString(),
this.polledMessages[i].payload.toString());
+ assert.equal(BigInt(msg.id || 0), this.polledMessages[i].headers.id)
})
-});
+ }
+);
Then(
'the last polled message should match the last sent message',
@@ -110,4 +112,5 @@ Then(
const lastPoll = this.polledMessages[this.polledMessages.length - 1];
assert.deepEqual(lastSent.payload.toString(), lastPoll.payload.toString());
assert.deepEqual(lastSent.headers || {}, lastPoll.userHeaders);
-});
+ }
+);
diff --git a/foreign/node/src/tcp.sm.utils.ts b/foreign/node/src/tcp.sm.utils.ts
index 7eefb842..c68eda45 100644
--- a/foreign/node/src/tcp.sm.utils.ts
+++ b/foreign/node/src/tcp.sm.utils.ts
@@ -20,7 +20,7 @@
import assert from 'node:assert/strict';
import { v7 } from './wire/uuid.utils.js';
-import { sendMessages, type Partitioning, HeaderValue } from './wire/index.js';
+import { sendMessages, type Partitioning, HeaderValue, type Message } from
'./wire/index.js';
import type { ClientProvider } from './client/client.type.js';
import type { Id } from './wire/identifier.utils.js';
@@ -48,7 +48,6 @@ export const generateMessages = (count = 1) => {
return [...Array(count)].map(() => ({ id: v7(), ...someMessageContent() }));
}
-
export const sendSomeMessages = (s: ClientProvider) =>
async (streamId: Id, topicId: Id, partition: Partitioning) => {
const rSend = await sendMessages(s)({
@@ -57,3 +56,17 @@ export const sendSomeMessages = (s: ClientProvider) =>
assert.ok(rSend);
return rSend;
};
+
+
+export const formatPolledMessages = (msgs: Message[]) =>
+ msgs.map(m => {
+ const { headers: { id, offset, timestamp, checksum }, payload, userHeaders
} = m;
+ return {
+ id,
+ offset,
+ headers: userHeaders,
+ payload: payload.toString(),
+ timestamp,
+ checksum
+ };
+ });
diff --git a/foreign/node/src/wire/message/iggy-header.utils.ts
b/foreign/node/src/wire/message/iggy-header.utils.ts
index 2f43e8e2..e5f3335e 100644
--- a/foreign/node/src/wire/message/iggy-header.utils.ts
+++ b/foreign/node/src/wire/message/iggy-header.utils.ts
@@ -17,11 +17,13 @@
* under the License.
*/
-import { deserializeUUID, toDate } from "../serialize.utils.js";
+import { toDate } from "../serialize.utils.js";
+import { u128LEBufToBigint } from "../number.utils.js";
+
export type IggyMessageHeader = {
checksum: bigint,
- id: string | 0 | 0n,
+ id: string | BigInt,
offset: bigint,
timestamp: Date,
originTimestamp: Date,
@@ -48,6 +50,8 @@ export const serializeIggyMessageHeader = (
return b;
};
+export const deserialiseMessageId = (b: Buffer) => u128LEBufToBigint(b);
+
export const deserializeIggyMessageHeaders = (b: Buffer) => {
if(b.length !== IGGY_MESSAGE_HEADER_SIZE)
throw new Error(
@@ -56,7 +60,7 @@ export const deserializeIggyMessageHeaders = (b: Buffer) => {
);
const headers: IggyMessageHeader = {
checksum: b.readBigUInt64LE(0),
- id: deserializeUUID(b.subarray(8, 24)),
+ id: deserialiseMessageId(b.subarray(8, 24)),
offset: b.readBigUInt64LE(24),
timestamp: toDate(b.readBigUInt64LE(32)),
originTimestamp: toDate(b.readBigUInt64LE(40)),
diff --git a/foreign/node/src/wire/message/message.utils.ts
b/foreign/node/src/wire/message/message.utils.ts
index 71382246..d2e4da6a 100644
--- a/foreign/node/src/wire/message/message.utils.ts
+++ b/foreign/node/src/wire/message/message.utils.ts
@@ -19,7 +19,7 @@
import Debug from 'debug';
-import { uint32ToBuf } from '../number.utils.js';
+import { uint32ToBuf, u128ToBuf } from '../number.utils.js';
import { serializeHeaders, type Headers } from './header.utils.js';
import { serializeIdentifier, type Id } from '../identifier.utils.js';
import { serializePartitioning, type Partitioning } from
'./partitioning.utils.js';
@@ -31,7 +31,7 @@ const debug = Debug('iggy:client');
/** index size per messages in bit */
const INDEX_SIZE = 16;
-export type MessageIdKind = 0 | 0n | number | bigint | string;
+export type MessageIdKind = number | bigint | string;
export type CreateMessage = {
id?: MessageIdKind,
@@ -40,7 +40,7 @@ export type CreateMessage = {
};
export const isValidMessageId = (x?: unknown): x is MessageIdKind =>
- x === undefined || x === 0 || x === 0n ||
+ x === undefined ||
'string' === typeof x ||
'bigint' === typeof x ||
'number' === typeof x;
@@ -57,17 +57,8 @@ export const serializeMessageId = (id?: unknown) => {
if (id < 0)
throw new Error(`invalid message id: '${id}' (numeric id must be >= 0)`)
- if ('number' === typeof id) {
- const b = Buffer.alloc(16, 0);
- b.writeUInt32LE(id)
- return b; // id_u32 + 0u96
- }
-
- if ('bigint' === typeof id) {
- const b = Buffer.alloc(16, 0);
- b.writeBigUInt64LE(id)
- return b; // id_u64 + 0u64
- }
+ const idValue = 'number' === typeof id ? BigInt(id) : id;
+ return u128ToBuf(idValue);
}
try {
diff --git a/foreign/node/src/wire/message/send-messages.command.test.ts
b/foreign/node/src/wire/message/send-messages.command.test.ts
index 46329b63..e4945b65 100644
--- a/foreign/node/src/wire/message/send-messages.command.test.ts
+++ b/foreign/node/src/wire/message/send-messages.command.test.ts
@@ -22,7 +22,6 @@ import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { uuidv7, uuidv4 } from 'uuidv7'
import { SEND_MESSAGES, type SendMessages } from './send-messages.command.js';
-import type { MessageIdKind } from './message.utils.js';
import { HeaderValue } from './header.utils.js';
describe('SendMessages', () => {
@@ -34,8 +33,10 @@ describe('SendMessages', () => {
topicId: 213,
messages: [
{ payload: 'a' },
- { id: 0 as const, payload: 'b' },
- { id: 0n as const, payload: 'c' },
+ { id: 0, payload: 'b' },
+ { id: 123, payload: 'X' },
+ { id: 0n, payload: 'c' },
+ { id: 1236234534554n, payload: 'X' },
{ id: uuidv4(), payload: 'd' },
{ id: uuidv7(), payload: 'e' },
],
@@ -44,7 +45,7 @@ describe('SendMessages', () => {
it('serialize SendMessages into a buffer', () => {
assert.deepEqual(
SEND_MESSAGES.serialize(t1).length,
- 387
+ 533
);
});
@@ -56,14 +57,21 @@ describe('SendMessages', () => {
it('does not throw on number message id', () => {
- const t = { ...t1, messages: [{ id: 42 as MessageIdKind, payload: 'm' }]
};
+ const t = { ...t1, messages: [{ id: 42, payload: 'm' }] };
assert.doesNotThrow(
() => SEND_MESSAGES.serialize(t)
);
});
it('does not throw on bigint message id', () => {
- const t = { ...t1, messages: [{ id: 123n as MessageIdKind, payload: 'm'
}] };
+ const t = { ...t1, messages: [{ id: 123n, payload: 'm' }] };
+ assert.doesNotThrow(
+ () => SEND_MESSAGES.serialize(t)
+ );
+ });
+
+ it('does not throw on uuid message id', () => {
+ const t = { ...t1, messages: [{ id: uuidv4(), payload: 'uuid' }] };
assert.doesNotThrow(
() => SEND_MESSAGES.serialize(t)
);
diff --git a/foreign/node/src/wire/number.utils.ts
b/foreign/node/src/wire/number.utils.ts
index ad390b71..7ab9d790 100644
--- a/foreign/node/src/wire/number.utils.ts
+++ b/foreign/node/src/wire/number.utils.ts
@@ -84,6 +84,16 @@ export const doubleToBuf = (v: number) => {
return b;
}
+// bigint => u128 LE
+export function u128ToBuf(num: bigint, width = 16): Buffer {
+ const hex = num.toString(16);
+ const b = Buffer.from(hex.padStart(width * 2, '0').slice(0, width * 2),
'hex');
+ return b.reverse();
+}
+
+// u128 LE => Bigint
+export function u128LEBufToBigint(b: Buffer): bigint {
+ const hex = b.reverse().toString('hex');
+ return hex.length === 0 ? BigInt(0) : BigInt(`0x${hex}`);
+}
-// no js support ... use buffer or dataview or arraybuffer ?
-// const uint128ToBuf = (v: Buffer) => { }
