This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new c8515e1ddf [Web] Replace string with TVMFFIByteArray* to avoid memory
issues (#18467)
c8515e1ddf is described below
commit c8515e1ddfaf4d1afff916c484e68e1513631dd6
Author: Akaash Parthasarathy <[email protected]>
AuthorDate: Thu Nov 20 13:41:15 2025 -0500
[Web] Replace string with TVMFFIByteArray* to avoid memory issues (#18467)
Passing in a string to `ArrayDecodeStorage` via the packed function
definition led to memory issues for larger models (such as
`gemma-2-9b-it-q4f32_1-MLC`). Replacing string with TVMFFIByteArray*
fixes this issue and also alleviates the stack pollution issue discussed
in an earlier PR (https://github.com/apache/tvm/pull/18415). Note that
this does not completely fix generation for q0f32 models.
---
web/emcc/wasm_runtime.cc | 23 ++++++++------------
web/package-lock.json | 56 ++++++++++++++++++++++++------------------------
2 files changed, 37 insertions(+), 42 deletions(-)
diff --git a/web/emcc/wasm_runtime.cc b/web/emcc/wasm_runtime.cc
index c5541392d9..c1839947ba 100644
--- a/web/emcc/wasm_runtime.cc
+++ b/web/emcc/wasm_runtime.cc
@@ -125,24 +125,25 @@ TVM_FFI_STATIC_INIT_BLOCK() {
});
}
-void ArrayDecodeStorage(Tensor cpu_arr, std::string bytes, std::string format,
std::string dtype) {
+void ArrayDecodeStorage(Tensor cpu_arr, TVMFFIByteArray* bytes, const
std::string& format,
+ const std::string& dtype) {
+ ICHECK_NE(bytes, nullptr);
+ const char* byte_data = bytes->data;
+ const size_t byte_size = bytes->size;
if (format == "f32-to-bf16" && dtype == "float32") {
- std::vector<uint16_t> buffer(bytes.length() / 2);
- std::memcpy(buffer.data(), bytes.data(), buffer.size() * 2);
- // decode bf16 to f32
- const uint16_t* bf16 = reinterpret_cast<const uint16_t*>(buffer.data());
+ const uint16_t* bf16 = reinterpret_cast<const uint16_t*>(byte_data);
uint32_t* data = static_cast<uint32_t*>(cpu_arr->data);
ICHECK(cpu_arr.IsContiguous());
size_t size = 1;
for (int i = 0; i < cpu_arr->ndim; ++i) {
size *= cpu_arr->shape[i];
}
- ICHECK_EQ(size, bytes.length() / 2);
+ ICHECK_EQ(size, byte_size / 2);
for (size_t i = 0; i < size; ++i) {
data[i] = static_cast<uint32_t>(bf16[i]) << 16;
}
} else {
- cpu_arr.CopyFromBytes(bytes.data(), bytes.length());
+ cpu_arr.CopyFromBytes(byte_data, byte_size);
}
}
@@ -151,16 +152,10 @@ TVM_FFI_STATIC_INIT_BLOCK() {
refl::GlobalDef().def_packed(
"tvmjs.array.decode_storage", [](ffi::PackedArgs args, ffi::Any* ret) {
Tensor cpu_arr = args[0].cast<Tensor>();
- auto bytes = args[1].cast<ffi::Bytes>();
+ TVMFFIByteArray* bytes = args[1].cast<TVMFFIByteArray*>();
std::string format = args[2].cast<ffi::String>().operator
std::string();
std::string dtype = args[3].cast<ffi::String>().operator std::string();
ArrayDecodeStorage(cpu_arr, bytes, format, dtype);
- if (ret != nullptr) {
- auto* ret_data = reinterpret_cast<TVMFFIAny*>(ret);
- ret_data->type_index = TVMFFITypeIndex::kTVMFFINone;
- ret_data->zero_padding = 0;
- ret_data->v_int64 = 0;
- }
});
}
diff --git a/web/package-lock.json b/web/package-lock.json
index 50a4ca2831..3287cd00b8 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -634,9 +634,9 @@
}
},
"node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": {
- "version": "3.14.1",
- "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz";,
- "integrity":
"sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+ "version": "3.14.2",
+ "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz";,
+ "integrity":
"sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==",
"dev": true,
"dependencies": {
"argparse": "^1.0.7",
@@ -1188,9 +1188,9 @@
"dev": true
},
"node_modules/@types/node": {
- "version": "20.19.24",
- "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.24.tgz";,
- "integrity":
"sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA==",
+ "version": "20.19.25",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz";,
+ "integrity":
"sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
"dev": true,
"dependencies": {
"undici-types": "~6.21.0"
@@ -1230,9 +1230,9 @@
"dev": true
},
"node_modules/@types/yargs": {
- "version": "15.0.19",
- "resolved":
"https://registry.npmjs.org/@types/yargs/-/yargs-15.0.19.tgz";,
- "integrity":
"sha512-2XUaGVmyQjgyAZldf0D0c14vvo/yv0MhQBSTJcejMMaitsn3nxCB6TmH4G0ZQf+uxROOa9mpanoSm8h6SG/1ZA==",
+ "version": "15.0.20",
+ "resolved":
"https://registry.npmjs.org/@types/yargs/-/yargs-15.0.20.tgz";,
+ "integrity":
"sha512-KIkX+/GgfFitlASYCGoSF+T4XRXhOubJLhkLVtSfsRTe9jWMmuM2g28zQ41BtPTG7TRBb2xHW+LCNVE9QR/vsg==",
"dev": true,
"dependencies": {
"@types/yargs-parser": "*"
@@ -1844,9 +1844,9 @@
}
},
"node_modules/baseline-browser-mapping": {
- "version": "2.8.25",
- "resolved":
"https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.25.tgz";,
- "integrity":
"sha512-2NovHVesVF5TXefsGX1yzx1xgr7+m9JQenvz6FQY3qd+YXkKkYiv+vTCc7OriP9mcDZpTC5mAOYN4ocd29+erA==",
+ "version": "2.8.29",
+ "resolved":
"https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.29.tgz";,
+ "integrity":
"sha512-sXdt2elaVnhpDNRDz+1BDx1JQoJRuNk7oVlAlbGiFkLikHCAQiccexF/9e91zVi6RCgqspl04aP+6Cnl9zRLrA==",
"dev": true,
"bin": {
"baseline-browser-mapping": "dist/cli.js"
@@ -1881,9 +1881,9 @@
"dev": true
},
"node_modules/browserslist": {
- "version": "4.27.0",
- "resolved":
"https://registry.npmjs.org/browserslist/-/browserslist-4.27.0.tgz";,
- "integrity":
"sha512-AXVQwdhot1eqLihwasPElhX2tAZiBjWdJ9i/Zcj2S6QYIjkx62OKSfnobkriB81C3l4w0rVy3Nt4jaTBltYEpw==",
+ "version": "4.28.0",
+ "resolved":
"https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz";,
+ "integrity":
"sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==",
"dev": true,
"funding": [
{
@@ -1900,10 +1900,10 @@
}
],
"dependencies": {
- "baseline-browser-mapping": "^2.8.19",
- "caniuse-lite": "^1.0.30001751",
- "electron-to-chromium": "^1.5.238",
- "node-releases": "^2.0.26",
+ "baseline-browser-mapping": "^2.8.25",
+ "caniuse-lite": "^1.0.30001754",
+ "electron-to-chromium": "^1.5.249",
+ "node-releases": "^2.0.27",
"update-browserslist-db": "^1.1.4"
},
"bin": {
@@ -1992,9 +1992,9 @@
}
},
"node_modules/caniuse-lite": {
- "version": "1.0.30001754",
- "resolved":
"https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001754.tgz";,
- "integrity":
"sha512-x6OeBXueoAceOmotzx3PO4Zpt4rzpeIFsSr6AAePTZxSkXiYDUmpypEl7e2+8NCd9bD7bXjqyef8CJYPC1jfxg==",
+ "version": "1.0.30001756",
+ "resolved":
"https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001756.tgz";,
+ "integrity":
"sha512-4HnCNKbMLkLdhJz3TToeVWHSnfJvPaq6vu/eRP0Ahub/07n484XHhBF5AJoSGHdVrS8tKFauUQz8Bp9P7LVx7A==",
"dev": true,
"funding": [
{
@@ -2415,9 +2415,9 @@
}
},
"node_modules/electron-to-chromium": {
- "version": "1.5.249",
- "resolved":
"https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.249.tgz";,
- "integrity":
"sha512-5vcfL3BBe++qZ5kuFhD/p8WOM1N9m3nwvJPULJx+4xf2usSlZFJ0qoNYO2fOX4hi3ocuDcmDobtA+5SFr4OmBg==",
+ "version": "1.5.258",
+ "resolved":
"https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.258.tgz";,
+ "integrity":
"sha512-rHUggNV5jKQ0sSdWwlaRDkFc3/rRJIVnOSe9yR4zrR07m3ZxhP4N27Hlg8VeJGGYgFTxK5NqDmWI4DSH72vIJg==",
"dev": true
},
"node_modules/emittery": {
@@ -4550,9 +4550,9 @@
"dev": true
},
"node_modules/js-yaml": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz";,
- "integrity":
"sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz";,
+ "integrity":
"sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
"dev": true,
"dependencies": {
"argparse": "^2.0.1"
