This is an automated email from the ASF dual-hosted git repository.

guan404ming pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git


The following commit(s) were added to refs/heads/main by this push:
     new 46e6e596c8 [Web] Add ordered asynchronous sampled-token readback rings 
(#21006)
46e6e596c8 is described below

commit 46e6e596c8603e2d90b5d4d0d72fd4a39495e067
Author: Akaash Parthasarathy <[email protected]>
AuthorDate: Wed Nov 18 23:36:28 2026 -0800

    [Web] Add ordered asynchronous sampled-token readback rings (#21006)
    
    Add an asynchronous readback ring for int32 token outputs so callers can
    continue submitting GPU work while earlier results are mapped. Deliver
    results in submission order and defer buffer destruction until pending
    maps finish.
---
 web/src/index.ts                      |   8 +-
 web/src/runtime.ts                    | 167 +++++++++++++-
 web/src/types.ts                      |  38 +++
 web/src/webgpu.ts                     | 422 +++++++++++++++++++++++++++++++++-
 web/tests/node/test_webgpu.js         | 257 ++++++++++++++++++++-
 web/tests/node/test_webgpu_runtime.js |  40 +++-
 6 files changed, 923 insertions(+), 9 deletions(-)

diff --git a/web/src/index.ts b/web/src/index.ts
index 7cd05e7621..453508b582 100644
--- a/web/src/index.ts
+++ b/web/src/index.ts
@@ -20,6 +20,7 @@
 export {
   Scalar, DLDevice, DLDataType,
   PackedFunc, Module, Tensor,
+  SampledTokenReadbackRing,
   TVMArray, TVMObject, VirtualMachine,
   InitProgressCallback, InitProgressReport, FetchTensorCacheOptions,
   Instance, instantiate
@@ -37,7 +38,12 @@ export {
   hasTensorInCache,
   deleteTensorCache
 } from "./artifact_cache";
-export { Disposable, LibraryProvider } from "./types";
+export {
+  Disposable,
+  LibraryProvider,
+  SampledTokenReadbackBatch,
+  SampledTokenReadbackRingOptions,
+} from "./types";
 export { RPCServer } from "./rpc_server";
 export { assert, wasmPath, LinearCongruentialGenerator } from "./support";
 export type { RNGState } from "./support";
diff --git a/web/src/runtime.ts b/web/src/runtime.ts
index e57e9ddbcf..6a4f0625bb 100644
--- a/web/src/runtime.ts
+++ b/web/src/runtime.ts
@@ -21,7 +21,11 @@
  * TVM JS Wasm Runtime library.
  */
 import { Pointer, PtrOffset, SizeOf, TypeIndex } from "./ctypes";
-import { Disposable } from "./types";
+import {
+  Disposable,
+  SampledTokenReadbackBatch,
+  SampledTokenReadbackRingOptions,
+} from "./types";
 import { Memory, CachedCallStack, WasmByteArraySource } from "./memory";
 import {
   assert,
@@ -349,6 +353,147 @@ const DeviceStrToEnum: Record<string, number> = {
   webgpu: 15
 };
 
+/**
+ * Handle to a sampled-token readback ring created on a WebGPU device.
+ */
+export class SampledTokenReadbackRing implements Disposable {
+  private lib: FFILibrary;
+  private ringId: number;
+  private disposed = false;
+
+  constructor(lib: FFILibrary, ringId: number) {
+    this.lib = lib;
+    this.ringId = ringId;
+  }
+
+  /**
+   * Submit sampled-token readback from a GPU tensor.
+   *
+   * @param from GPU tensor containing int32 token ids.
+   * @param tokenCount Number of token ids to copy.
+   * @param fromTokenOffset Token offset (in token units) from tensor start.
+   * @param into Host array to copy the token ids into when the batch maps,
+   *   instead of a new array; the batch's tokens then view its first
+   *   tokenCount elements. Reusing one array for large batches avoids an
+   *   allocation per batch. The caller must not reuse it before it has
+   *   consumed the batch; a ring disposed before the batch maps leaves it
+   *   untouched.
+   * @returns Monotonic ring-local batch sequence id.
+   */
+  submit(
+    from: Tensor,
+    tokenCount: number,
+    fromTokenOffset = 0,
+    into?: Int32Array,
+  ): number {
+    this.checkSourceTensor(from, fromTokenOffset, tokenCount);
+    return this.submitFromPointer(
+      from.getDataPtr(),
+      tokenCount,
+      from.byteOffset / 4 + fromTokenOffset,
+      into,
+    );
+  }
+
+  /**
+   * Submit one sampled token from each GPU tensor as one contiguous readback 
batch.
+   *
+   * This avoids a GPU packing kernel when speculative decode produces K 
separate
+   * one-token tensors.
+   *
+   * @param from GPU tensors containing one int32 token id each.
+   * @returns Monotonic ring-local batch sequence id.
+   */
+  submitTokens(from: Array<Tensor>): number {
+    const ctx = this.requireActiveWebGPUContext();
+    from.forEach((tensor) => this.checkSourceTensor(tensor, 0, 1));
+    return ctx.submitSampledTokenReadbackRingSources(
+      this.ringId,
+      from.map((tensor) => ({
+        from: tensor.getDataPtr(),
+        fromTokenOffset: tensor.byteOffset / 4,
+        tokenCount: 1,
+      })),
+    );
+  }
+
+  private checkSourceTensor(from: Tensor, offset: number, count: number): void 
{
+    if (from.device.deviceType !== DeviceStrToEnum.webgpu || from.dtype !== 
"int32") {
+      throw new Error("Sampled-token readback requires an int32 WebGPU 
tensor.");
+    }
+    if (
+      from.byteOffset % 4 !== 0 ||
+      !Number.isSafeInteger(offset) || offset < 0 ||
+      !Number.isSafeInteger(count) || count <= 0 ||
+      offset + count > from.shape.reduce((size, dim) => size * dim, 1)
+    ) {
+      throw new Error("Sampled-token readback exceeds the tensor's token 
range.");
+    }
+  }
+
+  /**
+   * Read raw 32-bit words from a WebGPU allocation without dtype conversion.
+   * This also supports floating-point probabilities and checkpoint bytes.
+   *
+   * Offsets and counts are in 32-bit words. The allocation bounds are checked.
+   * Callers must include any tensor byte offset and stay within its view.
+   * The destination array follows the same ownership rules as submit().
+   */
+  submitFromPointer(
+    fromPtr: Pointer,
+    tokenCount: number,
+    fromTokenOffset = 0,
+    into?: Int32Array,
+  ): number {
+    const ctx = this.requireActiveWebGPUContext();
+    return ctx.submitSampledTokenReadbackRing(
+      this.ringId,
+      fromPtr,
+      fromTokenOffset,
+      tokenCount,
+      into,
+    );
+  }
+
+  /**
+   * Poll ready batches in sequence order.
+   */
+  pollReady(): Array<SampledTokenReadbackBatch> {
+    const ctx = this.requireActiveWebGPUContext();
+    return ctx.pollSampledTokenReadbackRing(this.ringId);
+  }
+
+  /**
+   * Wait until at least one batch becomes ready, then poll contiguous ready 
batches.
+   */
+  async waitReady(): Promise<Array<SampledTokenReadbackBatch>> {
+    const ctx = this.requireActiveWebGPUContext();
+    return await ctx.waitSampledTokenReadbackRing(this.ringId);
+  }
+
+  /**
+   * Dispose ring resources.
+   */
+  dispose(): void {
+    if (this.disposed) {
+      return;
+    }
+    this.disposed = true;
+    this.lib.webGPUContext?.disposeSampledTokenReadbackRing(this.ringId);
+  }
+
+  private requireActiveWebGPUContext(): WebGPUContext {
+    if (this.disposed) {
+      throw new Error("SampledTokenReadbackRing has already been disposed");
+    }
+    const ctx = this.lib.webGPUContext;
+    if (ctx === undefined) {
+      throw new Error("WebGPU context is not available");
+    }
+    return ctx;
+  }
+}
+
 /**
  * Represent a runtime context where a Tensor can reside.
  */
@@ -401,6 +546,23 @@ export class DLDevice {
     return this.lib.webGPUContext.submitPendingCommands();
   }
 
+  /**
+   * Create a sampled-token readback ring on this device.
+   *
+   * @param options Ring configuration.
+   * @returns A sampled-token readback ring handle.
+   */
+  createSampledTokenReadbackRing(
+    options: SampledTokenReadbackRingOptions,
+  ): SampledTokenReadbackRing {
+    if (this.deviceType !== DeviceStrToEnum.webgpu) {
+      throw new Error("Sampled-token readback ring is only available on WebGPU 
devices.");
+    }
+    assert(this.lib.webGPUContext !== undefined);
+    const ringId = 
this.lib.webGPUContext.createSampledTokenReadbackRing(options);
+    return new SampledTokenReadbackRing(this.lib, ringId);
+  }
+
   toString(): string {
     return (
       DeviceEnumToStr[this.deviceType] + ":" + this.deviceId.toString()
@@ -548,7 +710,8 @@ export class Tensor extends TVMObject {
   device: DLDevice;
   /** Whether it is a temporary view that can become invalid after the call. */
   isView: boolean;
-  private byteOffset: number;
+  /** Byte offset from the start of the tensor's data allocation. */
+  readonly byteOffset: number;
   private dltensor: Pointer;
   private dataPtr: Pointer;
   private dlDataType: DLDataType;
diff --git a/web/src/types.ts b/web/src/types.ts
index 621375a23f..8263855363 100644
--- a/web/src/types.ts
+++ b/web/src/types.ts
@@ -51,3 +51,41 @@ export interface Disposable {
    */
   dispose: () => void;
 }
+
+/**
+ * Options for creating a sampled-token readback ring.
+ *
+ * The ring is intended for batched GPU->CPU token-id readback where each
+ * token is represented as int32.
+ */
+export interface SampledTokenReadbackRingOptions {
+  /**
+   * Number of staging slots in the ring. Defaults to 3 when unspecified.
+   */
+  slotCount?: number;
+
+  /**
+   * Maximum number of token ids (int32) per submitted batch.
+   */
+  maxTokensPerBatch: number;
+}
+
+/**
+ * One completed sampled-token readback batch.
+ */
+export interface SampledTokenReadbackBatch {
+  /**
+   * Monotonic ring-local batch sequence number.
+   */
+  batchSeq: number;
+
+  /**
+   * Number of valid token ids in `tokens`.
+   */
+  tokenCount: number;
+
+  /**
+   * Contiguous token-id vector (int32) copied from GPU staging slot.
+   */
+  tokens: Int32Array;
+}
diff --git a/web/src/webgpu.ts b/web/src/webgpu.ts
index 2f1e8de755..7d8886725b 100644
--- a/web/src/webgpu.ts
+++ b/web/src/webgpu.ts
@@ -19,11 +19,53 @@
 import { assert } from "./support";
 import { Pointer } from "./ctypes";
 import { Memory } from "./memory";
-import { Disposable } from "./types";
+import {
+  Disposable,
+  SampledTokenReadbackBatch,
+  SampledTokenReadbackRingOptions,
+} from "./types";
 
 /** A pointer to points to the raw address space. */
 export type GPUPointer = number;
 
+type ReadbackRingSlotState = "free" | "pending" | "ready";
+
+interface InternalSampledTokenReadbackSlot {
+  buffer: GPUBuffer;
+  state: ReadbackRingSlotState;
+  batchSeq: number;
+  tokenCount: number;
+  tokens?: Int32Array;
+  pendingPromise?: Promise<void>;
+  destroyAfterComplete: boolean;
+}
+
+interface InternalReadbackRingWaiter {
+  resolve: () => void;
+  reject: (reason: unknown) => void;
+}
+
+interface InternalSampledTokenReadbackRing {
+  id: number;
+  slotCount: number;
+  maxTokensPerBatch: number;
+  slotBytes: number;
+  slots: InternalSampledTokenReadbackSlot[];
+  nextSubmitCursor: number;
+  nextBatchSeq: number;
+  nextPollSeq: number;
+  readyByBatchSeq: Map<number, number>;
+  waiters: InternalReadbackRingWaiter[];
+  disposed: boolean;
+  fatalError?: Error;
+}
+
+interface SampledTokenReadbackSource {
+  from: GPUPointer;
+  fromTokenOffset: number;
+  tokenCount: number;
+}
+
 export interface GPUDeviceDetectOutput {
   adapter: GPUAdapter;
   adapterInfo: GPUAdapterInfo;
@@ -440,6 +482,12 @@ export class WebGPUContext {
   private pendingGPUToCPUCopy: Promise<void> | null = null;
   // Whether a pending GPU→CPU copy is still the last queue operation.
   private pendingGPUToCPUCopyIsQueueTail = false;
+  // Pending read promises from sampled-token readback rings.
+  // These are tracked separately from pendingGPUToCPUCopy because ring
+  // submissions can overlap and should be awaited collectively in sync().
+  private pendingSampledTokenReadbackReads: Set<Promise<void>> = new Set();
+  private sampledTokenReadbackRings: Map<number, 
InternalSampledTokenReadbackRing> = new Map();
+  private nextSampledTokenReadbackRingId = 1;
   // Batched command encoding: accumulate compute passes and GPU copies in a
   // single encoder, and submit only on flush to reduce JS-native transition 
overhead.
   private pendingEncoder: GPUCommandEncoder | null = null;
@@ -507,6 +555,9 @@ export class WebGPUContext {
    */
   dispose() {
     this.flushCommands();
+    for (const ringId of Array.from(this.sampledTokenReadbackRings.keys())) {
+      this.disposeSampledTokenReadbackRing(ringId);
+    }
     this.canvasRenderManager?.dispose();
     this.bufferTableFreeId = [];
     while (this.bufferTable.length != 0) {
@@ -545,6 +596,9 @@ export class WebGPUContext {
         await queueDone;
       }
     }
+    if (this.pendingSampledTokenReadbackReads.size > 0) {
+      await Promise.all(Array.from(this.pendingSampledTokenReadbackReads));
+    }
   }
 
   /**
@@ -557,6 +611,270 @@ export class WebGPUContext {
     return info;
   }
 
+  /**
+   * Create a sampled-token readback ring.
+   *
+   * Each submission copies a contiguous int32 token vector from a GPU buffer 
to a
+   * per-slot MAP_READ staging buffer and starts slot-local mapAsync.
+   */
+  createSampledTokenReadbackRing(options: SampledTokenReadbackRingOptions): 
number {
+    const slotCount = options.slotCount ?? 3;
+    const maxTokensPerBatch = options.maxTokensPerBatch;
+    if (!Number.isSafeInteger(slotCount) || slotCount <= 0) {
+      throw new Error(`slotCount must be a positive integer. Got 
${slotCount}.`);
+    }
+    if (!Number.isSafeInteger(maxTokensPerBatch) || maxTokensPerBatch <= 0) {
+      throw new Error(
+        `maxTokensPerBatch must be a positive integer. Got 
${maxTokensPerBatch}.`,
+      );
+    }
+    const slotBytes = maxTokensPerBatch * 4; // assume int32 tokens
+    const slots: InternalSampledTokenReadbackSlot[] = [];
+    for (let i = 0; i < slotCount; ++i) {
+      slots.push({
+        buffer: tryCreateBuffer(this.device, {
+          size: slotBytes,
+          usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,
+        }),
+        state: "free",
+        batchSeq: 0,
+        tokenCount: 0,
+        destroyAfterComplete: false,
+      });
+    }
+    const ringId = this.nextSampledTokenReadbackRingId++;
+    this.sampledTokenReadbackRings.set(ringId, {
+      id: ringId,
+      slotCount,
+      maxTokensPerBatch,
+      slotBytes,
+      slots,
+      nextSubmitCursor: 0,
+      nextBatchSeq: 1,
+      nextPollSeq: 1,
+      readyByBatchSeq: new Map<number, number>(),
+      waiters: [],
+      disposed: false,
+    });
+    return ringId;
+  }
+
+  /**
+   * Dispose a previously created sampled-token readback ring.
+   */
+  disposeSampledTokenReadbackRing(ringId: number): void {
+    const ring = this.sampledTokenReadbackRings.get(ringId);
+    if (ring === undefined) {
+      return;
+    }
+    ring.disposed = true;
+    this.sampledTokenReadbackRings.delete(ringId);
+    for (const slot of ring.slots) {
+      slot.tokens = undefined;
+      ring.readyByBatchSeq.delete(slot.batchSeq);
+      if (slot.state === "pending") {
+        slot.destroyAfterComplete = true;
+      } else {
+        slot.buffer.destroy();
+        this.resetSampledTokenReadbackSlot(slot);
+      }
+    }
+    this.notifySampledTokenReadbackRingWaiters(ring);
+  }
+
+  /**
+   * Submit a sampled-token readback batch from a GPU buffer.
+   *
+   * @param ringId Ring handle returned by createSampledTokenReadbackRing.
+   * @param from Source GPU pointer containing int32 tokens.
+   * @param fromTokenOffset Source token offset (in token units, not bytes).
+   * @param tokenCount Number of int32 token ids to copy.
+   * @param into Host array to copy the batch into instead of a new one; see
+   *   SampledTokenReadbackRing.submit.
+   * @returns Monotonic batch sequence id.
+   */
+  submitSampledTokenReadbackRing(
+    ringId: number,
+    from: GPUPointer,
+    fromTokenOffset: number,
+    tokenCount: number,
+    into?: Int32Array,
+  ): number {
+    return this.submitSampledTokenReadbackRingSources(
+      ringId,
+      [{ from, fromTokenOffset, tokenCount }],
+      into,
+    );
+  }
+
+  /** Submit token ranges, including offsets into their allocations. */
+  submitSampledTokenReadbackRingSources(
+    ringId: number,
+    sources: Array<SampledTokenReadbackSource>,
+    into?: Int32Array,
+  ): number {
+    const ring = this.requireSampledTokenReadbackRing(ringId);
+    this.throwIfSampledTokenReadbackRingFailed(ring);
+    if (sources.length === 0) {
+      throw new Error("At least one sampled-token source is required.");
+    }
+    let tokenCount = 0;
+    for (const source of sources) {
+      if (!Number.isSafeInteger(source.fromTokenOffset) || 
source.fromTokenOffset < 0) {
+        throw new Error(
+          `fromTokenOffset must be a non-negative integer. Got 
${source.fromTokenOffset}.`,
+        );
+      }
+      if (!Number.isSafeInteger(source.tokenCount) || source.tokenCount <= 0) {
+        throw new Error(
+          `tokenCount must be a positive integer. Got ${source.tokenCount}.`,
+        );
+      }
+      tokenCount += source.tokenCount;
+    }
+    if (tokenCount > ring.maxTokensPerBatch) {
+      throw new Error(
+        `tokenCount ${tokenCount} exceeds maxTokensPerBatch 
${ring.maxTokensPerBatch}.`,
+      );
+    }
+    if (into !== undefined && into.length < tokenCount) {
+      throw new Error(
+        `tokenCount ${tokenCount} exceeds the destination length 
${into.length}.`,
+      );
+    }
+    // Resolve every source before a slot or batch sequence is consumed, so a
+    // bad pointer cannot leave a sequence number that never becomes ready.
+    const sourceBuffers = sources.map((source) => {
+      const buffer = this.gpuBufferFromPtr(source.from);
+      if (source.fromTokenOffset > buffer.size / 4 - source.tokenCount) {
+        throw new Error("Sampled-token readback exceeds the source buffer 
size.");
+      }
+      return buffer;
+    });
+    const slotIdx = this.findFreeSampledTokenReadbackSlot(ring);
+    if (slotIdx === -1) {
+      throw new Error(
+        `Sampled-token readback ring ${ring.id} is full. Drain or wait before 
submitting more batches.`,
+      );
+    }
+    const slot = ring.slots[slotIdx];
+    const nbytes = tokenCount * 4;
+
+    // A synchronous encoding/submission failure must also wake existing 
waiters.
+    try {
+      if (!this.pendingEncoder) {
+        this.pendingEncoder = this.device.createCommandEncoder();
+      }
+      let toByteOffset = 0;
+      for (const [i, source] of sources.entries()) {
+        const sourceBytes = source.tokenCount * 4;
+        this.pendingEncoder.copyBufferToBuffer(
+          sourceBuffers[i],
+          source.fromTokenOffset * 4,
+          slot.buffer,
+          toByteOffset,
+          sourceBytes,
+        );
+        toByteOffset += sourceBytes;
+      }
+      this.flushCommands();
+    } catch (err) {
+      const error = err instanceof Error ? err : new Error(String(err));
+      ring.fatalError = error;
+      this.notifySampledTokenReadbackRingWaiters(ring, error);
+      throw error;
+    }
+
+    const batchSeq = ring.nextBatchSeq++;
+    slot.state = "pending";
+    slot.batchSeq = batchSeq;
+    slot.tokenCount = tokenCount;
+    slot.tokens = undefined;
+
+    const readPromise = slot.buffer
+      .mapAsync(GPUMapMode.READ, 0, nbytes)
+      .then(() => {
+        // The owner of a disposed ring may already be reusing `into`.
+        if (ring.disposed) {
+          slot.buffer.unmap();
+          if (slot.destroyAfterComplete) {
+            slot.buffer.destroy();
+            slot.destroyAfterComplete = false;
+          }
+          this.resetSampledTokenReadbackSlot(slot);
+          return;
+        }
+        const tokens = (into ?? new Int32Array(tokenCount)).subarray(0, 
tokenCount);
+        tokens.set(new Int32Array(slot.buffer.getMappedRange(0, nbytes)));
+        slot.buffer.unmap();
+
+        slot.tokens = tokens;
+        slot.state = "ready";
+        ring.readyByBatchSeq.set(batchSeq, slotIdx);
+        this.notifySampledTokenReadbackRingWaiters(ring);
+      })
+      .catch((err) => {
+        slot.buffer.unmap();
+        if (ring.disposed) {
+          if (slot.destroyAfterComplete) {
+            slot.buffer.destroy();
+            slot.destroyAfterComplete = false;
+          }
+          this.resetSampledTokenReadbackSlot(slot);
+          return;
+        }
+        // The error is delivered to waiters and to the next ring call through
+        // fatalError. Rethrowing here would only make it an unhandled 
rejection.
+        const error = err instanceof Error ? err : new Error(String(err));
+        this.resetSampledTokenReadbackSlot(slot);
+        ring.fatalError = error;
+        this.notifySampledTokenReadbackRingWaiters(ring, error);
+      })
+      .finally(() => {
+        slot.pendingPromise = undefined;
+      });
+    slot.pendingPromise = readPromise;
+    this.trackPendingSampledTokenReadbackRead(readPromise);
+    return batchSeq;
+  }
+
+  /**
+   * Poll ready batches in sequence order.
+   */
+  pollSampledTokenReadbackRing(ringId: number): 
Array<SampledTokenReadbackBatch> {
+    const ring = this.requireSampledTokenReadbackRing(ringId);
+    this.throwIfSampledTokenReadbackRingFailed(ring);
+    return this.drainReadySampledTokenReadbackBatches(ring);
+  }
+
+  /**
+   * Wait until at least one batch becomes ready, then return all currently
+   * contiguous ready batches.
+   */
+  async waitSampledTokenReadbackRing(
+    ringId: number,
+  ): Promise<Array<SampledTokenReadbackBatch>> {
+    const ring = this.requireSampledTokenReadbackRing(ringId);
+    this.throwIfSampledTokenReadbackRingFailed(ring);
+    const readyNow = this.drainReadySampledTokenReadbackBatches(ring);
+    if (readyNow.length > 0) {
+      return readyNow;
+    }
+    // Slots can finish mapping out of submission order, so a wake-up for a
+    // later batch is not enough: wait until the next batch in sequence is 
ready.
+    while (!ring.disposed && this.sampledTokenReadbackRingHasPending(ring)) {
+      await new Promise<void>((resolve, reject) => {
+        ring.waiters.push({ resolve, reject });
+      });
+      this.throwIfSampledTokenReadbackRingFailed(ring);
+      const ready = this.drainReadySampledTokenReadbackBatches(ring);
+      if (ready.length > 0) {
+        return ready;
+      }
+    }
+    return [];
+  }
+
   /**
    * Draw image from data in storage buffer.
    * @param ptr The GPU ptr
@@ -942,6 +1260,108 @@ export class WebGPUContext {
     }
   }
 
+  private requireSampledTokenReadbackRing(
+    ringId: number,
+  ): InternalSampledTokenReadbackRing {
+    const ring = this.sampledTokenReadbackRings.get(ringId);
+    if (ring === undefined || ring.disposed) {
+      throw new Error(`Cannot find sampled-token readback ring ${ringId}.`);
+    }
+    return ring;
+  }
+
+  private throwIfSampledTokenReadbackRingFailed(
+    ring: InternalSampledTokenReadbackRing,
+  ): void {
+    if (ring.fatalError !== undefined) {
+      throw ring.fatalError;
+    }
+  }
+
+  private findFreeSampledTokenReadbackSlot(
+    ring: InternalSampledTokenReadbackRing,
+  ): number {
+    for (let i = 0; i < ring.slotCount; ++i) {
+      const idx = (ring.nextSubmitCursor + i) % ring.slotCount;
+      if (ring.slots[idx].state === "free") {
+        ring.nextSubmitCursor = (idx + 1) % ring.slotCount;
+        return idx;
+      }
+    }
+    return -1;
+  }
+
+  private resetSampledTokenReadbackSlot(
+    slot: InternalSampledTokenReadbackSlot,
+  ): void {
+    slot.state = "free";
+    slot.batchSeq = 0;
+    slot.tokenCount = 0;
+    slot.tokens = undefined;
+    slot.pendingPromise = undefined;
+  }
+
+  private sampledTokenReadbackRingHasPending(
+    ring: InternalSampledTokenReadbackRing,
+  ): boolean {
+    for (const slot of ring.slots) {
+      if (slot.state === "pending") {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private drainReadySampledTokenReadbackBatches(
+    ring: InternalSampledTokenReadbackRing,
+  ): Array<SampledTokenReadbackBatch> {
+    const ret: Array<SampledTokenReadbackBatch> = [];
+    while (true) {
+      const slotIdx = ring.readyByBatchSeq.get(ring.nextPollSeq);
+      if (slotIdx === undefined) {
+        break;
+      }
+      const slot = ring.slots[slotIdx];
+      if (slot.state !== "ready" || slot.tokens === undefined) {
+        break;
+      }
+      ret.push({
+        batchSeq: slot.batchSeq,
+        tokenCount: slot.tokenCount,
+        tokens: slot.tokens,
+      });
+      ring.readyByBatchSeq.delete(ring.nextPollSeq);
+      ring.nextPollSeq += 1;
+      this.resetSampledTokenReadbackSlot(slot);
+    }
+    return ret;
+  }
+
+  private notifySampledTokenReadbackRingWaiters(
+    ring: InternalSampledTokenReadbackRing,
+    error?: Error,
+  ): void {
+    if (ring.waiters.length === 0) {
+      return;
+    }
+    const waiters = ring.waiters.splice(0, ring.waiters.length);
+    for (const waiter of waiters) {
+      if (error !== undefined) {
+        waiter.reject(error);
+      } else {
+        waiter.resolve();
+      }
+    }
+  }
+
+  private trackPendingSampledTokenReadbackRead(readPromise: Promise<void>): 
void {
+    let trackedPromise: Promise<void>;
+    trackedPromise = readPromise.finally(() => {
+      this.pendingSampledTokenReadbackReads.delete(trackedPromise);
+    });
+    this.pendingSampledTokenReadbackReads.add(trackedPromise);
+  }
+
   // DeviceAPI
   private deviceAllocDataSpace(nbytes: number): GPUPointer {
     // WebGPU buffer copies and queue writes operate in four-byte units.
diff --git a/web/tests/node/test_webgpu.js b/web/tests/node/test_webgpu.js
index 6f15d1be37..e340757d88 100644
--- a/web/tests/node/test_webgpu.js
+++ b/web/tests/node/test_webgpu.js
@@ -48,6 +48,7 @@ function createMockDevice({
 } = {}) {
   const events = [];
   const encoders = [];
+  const buffers = [];
 
   const queue = {
     submit: jest.fn(() => events.push("submit")),
@@ -71,7 +72,10 @@ function createMockDevice({
           }),
           end: jest.fn(),
         })),
-        copyBufferToBuffer: jest.fn(() => {
+        copyBufferToBuffer: jest.fn((src, srcOffset, dst, dstOffset, size) => {
+          if (src.data && dst.data) {
+            new Uint8Array(dst.data, dstOffset, size).set(new 
Uint8Array(src.data, srcOffset, size));
+          }
           commands.push("copy");
           events.push("copy");
         }),
@@ -86,13 +90,17 @@ function createMockDevice({
     }),
     createBuffer: jest.fn((descriptor) => {
       const mappedData = new ArrayBuffer(descriptor.size);
-      return {
+      const buffer = {
+        data: mappedData,
         size: descriptor.size,
         destroy: jest.fn(() => events.push("destroy")),
         mapAsync: jest.fn(mapAsync),
-        getMappedRange: jest.fn(() => mappedData),
+        getMappedRange: jest.fn((offset = 0, size = descriptor.size - offset) 
=>
+          mappedData.slice(offset, offset + size)),
         unmap: jest.fn(),
       };
+      buffers.push(buffer);
+      return buffer;
     }),
     createBindGroupLayout: jest.fn(() => ({})),
     createPipelineLayout: jest.fn(() => ({})),
@@ -104,7 +112,7 @@ function createMockDevice({
     destroy: jest.fn(),
   };
 
-  return { device, queue, events, encoders };
+  return { device, queue, events, encoders, buffers };
 }
 
 function createContext(deviceOptions) {
@@ -181,6 +189,247 @@ test("pending WebGPU commands can be submitted without 
waiting", () => {
   expect(queue.submit).toHaveBeenCalledTimes(1);
 });
 
+test("sampled-token ring copy shares the pending compute submission", async () 
=> {
+  const { context, device, queue, encoders, source } = createContext();
+  const shader = context.createShader(
+    {
+      name: "main",
+      arg_types: [],
+      launch_param_tags: [],
+    },
+    "@compute @workgroup_size(1) fn main() {}"
+  );
+  const ringId = context.createSampledTokenReadbackRing({
+    slotCount: 2,
+    maxTokensPerBatch: 1,
+  });
+
+  shader();
+  const batchSeq = context.submitSampledTokenReadbackRing(ringId, source, 0, 
1);
+
+  expect(batchSeq).toBe(1);
+  expect(device.createCommandEncoder).toHaveBeenCalledTimes(1);
+  expect(encoders[0].commands).toEqual(["compute", "copy"]);
+  expect(queue.submit).toHaveBeenCalledTimes(1);
+  expect(queue.submit.mock.calls[0][0]).toEqual([
+    {
+      encoderId: 0,
+      commands: encoders[0].commands,
+    },
+  ]);
+
+  const batches = await context.waitSampledTokenReadbackRing(ringId);
+  expect(batches).toHaveLength(1);
+  expect(batches[0].batchSeq).toBe(1);
+  expect(batches[0].tokenCount).toBe(1);
+  context.disposeSampledTokenReadbackRing(ringId);
+});
+
+test("a readback ring submit with an unknown source leaves the ring usable", 
async () => {
+  const { context, source } = createContext();
+  const ringId = context.createSampledTokenReadbackRing({
+    slotCount: 1,
+    maxTokensPerBatch: 1,
+  });
+
+  expect(() =>
+    context.submitSampledTokenReadbackRing(ringId, 0x7fff0000, 0, 1),
+  ).toThrow();
+
+  // The failed submit consumed neither the slot nor a batch sequence.
+  expect(context.submitSampledTokenReadbackRing(ringId, source, 0, 1)).toBe(1);
+  const batches = await context.waitSampledTokenReadbackRing(ringId);
+  expect(batches.map((batch) => batch.batchSeq)).toEqual([1]);
+  context.disposeSampledTokenReadbackRing(ringId);
+});
+
+test("invalid readback ranges leave slots and sequence numbers available", 
async () => {
+  const { context, source } = createContext();
+  const ringId = context.createSampledTokenReadbackRing({ slotCount: 1, 
maxTokensPerBatch: 2 });
+  for (const [offset, count] of [[-1, 1], [0, 0], [0.5, 1], [16, 1], [15, 2], 
[0, 3]]) {
+    expect(() => context.submitSampledTokenReadbackRing(ringId, source, 
offset, count)).toThrow();
+  }
+  expect(() => context.submitSampledTokenReadbackRing(
+    ringId, source, 0, 2, new Int32Array(1),
+  )).toThrow();
+  expect(context.submitSampledTokenReadbackRing(ringId, source, 15, 
1)).toBe(1);
+  expect((await 
context.waitSampledTokenReadbackRing(ringId))[0].batchSeq).toBe(1);
+  context.disposeSampledTokenReadbackRing(ringId);
+});
+
+test("a full ring becomes reusable after draining its oldest batch", async () 
=> {
+  const { context, source } = createContext();
+  const ringId = context.createSampledTokenReadbackRing({ slotCount: 1, 
maxTokensPerBatch: 1 });
+  expect(context.submitSampledTokenReadbackRing(ringId, source, 0, 1)).toBe(1);
+  expect(() => context.submitSampledTokenReadbackRing(ringId, source, 0, 
1)).toThrow("full");
+  await context.waitSampledTokenReadbackRing(ringId);
+  expect(context.submitSampledTokenReadbackRing(ringId, source, 0, 1)).toBe(2);
+  await context.waitSampledTokenReadbackRing(ringId);
+  context.disposeSampledTokenReadbackRing(ringId);
+});
+
+test("disposal wakes a waiter and defers buffer destruction until mapping 
settles", async () => {
+  const mapped = createDeferred();
+  const { context, source, buffers } = createContext({ mapAsync: () => 
mapped.promise });
+  const ringId = context.createSampledTokenReadbackRing({ slotCount: 1, 
maxTokensPerBatch: 1 });
+  context.submitSampledTokenReadbackRing(ringId, source, 0, 1);
+  const waiting = context.waitSampledTokenReadbackRing(ringId);
+  context.disposeSampledTokenReadbackRing(ringId);
+  expect(buffers[2].destroy).not.toHaveBeenCalled();
+  await expect(waiting).resolves.toEqual([]);
+  mapped.resolve();
+  await context.sync();
+  expect(buffers[2].unmap).toHaveBeenCalledTimes(1);
+  expect(buffers[2].destroy).toHaveBeenCalledTimes(1);
+});
+
+test("submission failure rejects an existing ring waiter", async () => {
+  const mapped = createDeferred();
+  const { context, source, queue } = createContext({ mapAsync: () => 
mapped.promise });
+  const ringId = context.createSampledTokenReadbackRing({ slotCount: 2, 
maxTokensPerBatch: 1 });
+  context.submitSampledTokenReadbackRing(ringId, source, 0, 1);
+  const waiting = context.waitSampledTokenReadbackRing(ringId);
+  queue.submit.mockImplementationOnce(() => { throw new Error("submission 
failed"); });
+  expect(() => context.submitSampledTokenReadbackRing(ringId, source, 0, 1))
+    .toThrow("submission failed");
+  await expect(waiting).rejects.toThrow("submission failed");
+  expect(() => 
context.pollSampledTokenReadbackRing(ringId)).toThrow("submission failed");
+  context.disposeSampledTokenReadbackRing(ringId);
+  mapped.resolve();
+  await context.sync();
+});
+
+test("a readback ring map failure is reported through the ring", async () => {
+  const unhandled = [];
+  const onUnhandled = (reason) => unhandled.push(reason);
+  process.on("unhandledRejection", onUnhandled);
+  try {
+    const { context, source } = createContext({
+      mapAsync: () => Promise.reject(new Error("map failed")),
+    });
+    const ringId = context.createSampledTokenReadbackRing({
+      slotCount: 1,
+      maxTokensPerBatch: 1,
+    });
+    context.submitSampledTokenReadbackRing(ringId, source, 0, 1);
+
+    await expect(context.waitSampledTokenReadbackRing(ringId)).rejects.toThrow(
+      "map failed",
+    );
+    expect(() => context.pollSampledTokenReadbackRing(ringId)).toThrow("map 
failed");
+    await context.sync();
+    await new Promise((resolve) => setImmediate(resolve));
+    expect(unhandled).toEqual([]);
+    context.disposeSampledTokenReadbackRing(ringId);
+  } finally {
+    process.off("unhandledRejection", onUnhandled);
+  }
+});
+
+test.each([1, 4, 16])(
+  "sampled-token ring concatenates K=%i token buffers in one submission",
+  async (batchSize) => {
+    const { context, device, queue, encoders, buffers } = createContext();
+    const allocate = context.getDeviceAPI("deviceAllocDataSpace");
+    const tokenSources = Array.from({ length: batchSize }, () => allocate(64));
+    const ringId = context.createSampledTokenReadbackRing({
+      slotCount: 2,
+      maxTokensPerBatch: batchSize,
+    });
+
+    const batchSeq = context.submitSampledTokenReadbackRingSources(
+      ringId,
+      tokenSources.map((from) => ({ from, fromTokenOffset: 0, tokenCount: 1 
})),
+    );
+
+    expect(batchSeq).toBe(1);
+    expect(device.createCommandEncoder).toHaveBeenCalledTimes(1);
+    expect(encoders[0].commands).toEqual(Array(batchSize).fill("copy"));
+    expect(encoders[0].copyBufferToBuffer.mock.calls).toEqual(
+      Array.from({ length: batchSize }, (_, index) => [
+        buffers[2 + index],
+        0,
+        buffers[2 + batchSize],
+        index * 4,
+        4,
+      ]),
+    );
+    expect(queue.submit).toHaveBeenCalledTimes(1);
+
+    const batches = await context.waitSampledTokenReadbackRing(ringId);
+    expect(batches).toHaveLength(1);
+    expect(batches[0].tokenCount).toBe(batchSize);
+    expect(Array.from(batches[0].tokens)).toEqual(Array(batchSize).fill(0));
+    context.disposeSampledTokenReadbackRing(ringId);
+  },
+);
+
+test("sampled-token ring rejects an empty token-buffer batch", () => {
+  const { context } = createContext();
+  const ringId = context.createSampledTokenReadbackRing({
+    slotCount: 2,
+    maxTokensPerBatch: 3,
+  });
+
+  expect(() =>
+    context.submitSampledTokenReadbackRingSources(ringId, [])
+  ).toThrow("At least one sampled-token source is required.");
+  context.disposeSampledTokenReadbackRing(ringId);
+});
+
+// Guards out-of-order map completion (seen in Firefox): waiting must not
+// return an empty batch list while the next batch in sequence is pending.
+test("ring wait returns batches in sequence when maps resolve out of order", 
async () => {
+  const resolvers = [];
+  const { context, source } = createContext({
+    mapAsync: () => new Promise((resolve) => resolvers.push(resolve)),
+  });
+  const ringId = context.createSampledTokenReadbackRing({
+    slotCount: 2,
+    maxTokensPerBatch: 1,
+  });
+  context.submitSampledTokenReadbackRing(ringId, source, 0, 1);
+  context.submitSampledTokenReadbackRing(ringId, source, 0, 1);
+  const waiting = context.waitSampledTokenReadbackRing(ringId);
+  resolvers[1]();
+  await new Promise((resolve) => setTimeout(resolve, 0));
+  resolvers[0]();
+  const batches = await waiting;
+  expect(batches.map((batch) => batch.batchSeq)).toEqual([1, 2]);
+  context.disposeSampledTokenReadbackRing(ringId);
+});
+
+// Guards readback into a caller's array, which a caller reuses across
+// batches: a batch is copied into it, and a ring disposed while a slot maps
+// must not write into an array its owner may already have reused.
+test("ring copies a batch into the caller's array only while the ring lives", 
async () => {
+  const resolvers = [];
+  const { context, source, buffers } = createContext({
+    mapAsync: () => new Promise((resolve) => resolvers.push(resolve)),
+  });
+  const ringId = context.createSampledTokenReadbackRing({
+    slotCount: 2,
+    maxTokensPerBatch: 2,
+  });
+  const into = new Int32Array(4).fill(-1);
+  expect(() =>
+    context.submitSampledTokenReadbackRing(ringId, source, 0, 2, new 
Int32Array(1)),
+  ).toThrow("destination length");
+  context.submitSampledTokenReadbackRing(ringId, source, 0, 2, into);
+  new Int32Array(buffers[2].data).set([7, 8]);
+  resolvers[0]();
+  const [batch] = await context.waitSampledTokenReadbackRing(ringId);
+  expect(batch.tokens.buffer).toBe(into.buffer);
+  expect(Array.from(into)).toEqual([7, 8, -1, -1]);
+
+  into.fill(-1);
+  context.submitSampledTokenReadbackRing(ringId, source, 0, 2, into);
+  context.disposeSampledTokenReadbackRing(ringId);
+  resolvers[1]();
+  await context.sync();
+  expect(Array.from(into)).toEqual([-1, -1, -1, -1]);
+});
+
 test("a host write flushes pending GPU copies before writeBuffer", () => {
   const { context, queue, events, source, destination } = createContext();
   const copyWithinGPU = context.getDeviceAPI("deviceCopyWithinGPU");
diff --git a/web/tests/node/test_webgpu_runtime.js 
b/web/tests/node/test_webgpu_runtime.js
index eb4e838493..36475276a4 100644
--- a/web/tests/node/test_webgpu_runtime.js
+++ b/web/tests/node/test_webgpu_runtime.js
@@ -17,7 +17,45 @@
  * under the License.
  */
 jest.mock("../../src/compact", () => ({}));
-const { DLDevice } = require("../../src/runtime");
+const { DLDevice, Tensor } = require("../../src/runtime");
+
+function tokenTensor(byteOffset = 0) {
+  return Object.assign(Object.create(Tensor.prototype), {
+    handle: 1, dataPtr: 10, byteOffset, dtype: "int32", shape: [2],
+    device: { deviceType: 15 },
+  });
+}
+
+test("tensor readback includes view offsets and checks tensor bounds", () => {
+  const context = {
+    createSampledTokenReadbackRing: jest.fn(() => 1),
+    submitSampledTokenReadbackRing: jest.fn(() => 1),
+    submitSampledTokenReadbackRingSources: jest.fn(() => 2),
+    disposeSampledTokenReadbackRing: jest.fn(),
+  };
+  const ring = new DLDevice("webgpu", 0, { webGPUContext: context })
+    .createSampledTokenReadbackRing({ maxTokensPerBatch: 2 });
+  const from = tokenTensor(8);
+  expect(ring.submit(from, 1, 1)).toBe(1);
+  expect(context.submitSampledTokenReadbackRing).toHaveBeenCalledWith(1, 10, 
3, 1, undefined);
+  expect(ring.submitTokens([from, tokenTensor(4)])).toBe(2);
+  
expect(context.submitSampledTokenReadbackRingSources).toHaveBeenCalledWith(1, [
+    { from: 10, fromTokenOffset: 2, tokenCount: 1 },
+    { from: 10, fromTokenOffset: 1, tokenCount: 1 },
+  ]);
+  expect(() => ring.submit(from, 2, 1)).toThrow("token range");
+  from.dtype = "float32";
+  expect(() => ring.submit(from, 1)).toThrow("int32 WebGPU tensor");
+  ring.dispose();
+  ring.dispose();
+  expect(context.disposeSampledTokenReadbackRing).toHaveBeenCalledTimes(1);
+  expect(() => ring.submitTokens([])).toThrow("disposed");
+});
+
+test("readback rings reject CPU devices", () => {
+  expect(() => new DLDevice("cpu", 0, 
{}).createSampledTokenReadbackRing({maxTokensPerBatch: 1}))
+    .toThrow("only available on WebGPU");
+});
 
 test("public nonblocking submission delegates only for WebGPU devices", () => {
   const context = { submitPendingCommands: 
jest.fn().mockReturnValueOnce(true).mockReturnValueOnce(false) };

Reply via email to