This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-ffi.git
The following commit(s) were added to refs/heads/main by this push:
new 5818c66b [FIX] Make JSON numeric conversion independent of locale
(#948)
5818c66b is described below
commit 5818c66b43ef83f0137a2aa956dd2a70c295951d
Author: Tianqi Chen <[email protected]>
AuthorDate: Fri Nov 6 17:44:17 2026 -0500
[FIX] Make JSON numeric conversion independent of locale (#948)
JSON numbers use a decimal point regardless of the process locale. Parse
and print integers with standard character conversion. Use
floating-point character conversion with MSVC and libstdc++ 11 or newer,
and explicit C-locale conversion with libc++ and older libstdc++
libraries.
Keep the conversion logic in the existing parser and writer source
files, with no build-time feature probes or new headers. Preserve
floating-point precision, integer-like float formatting, and range-error
behavior while avoiding temporary numeric-token copies on the standard
conversion path.
---
src/ffi/extra/json_parser.cc | 82 +++++++++++++++++++++++++------------
src/ffi/extra/json_writer.cc | 52 +++++++++++++++++++----
tests/cpp/extra/test_json_parser.cc | 17 ++++++++
3 files changed, 117 insertions(+), 34 deletions(-)
diff --git a/src/ffi/extra/json_parser.cc b/src/ffi/extra/json_parser.cc
index 117ef541..8c7a14de 100644
--- a/src/ffi/extra/json_parser.cc
+++ b/src/ffi/extra/json_parser.cc
@@ -29,10 +29,18 @@
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ffi/string.h>
-#include <cinttypes>
+#include <cerrno>
+#include <charconv>
+#include <clocale>
+#include <cmath>
+#include <cstdlib>
#include <limits>
#include <utility>
+#ifdef __APPLE__
+#include <xlocale.h>
+#endif
+
namespace tvm {
namespace ffi {
namespace json {
@@ -162,11 +170,9 @@ class JSONParserContext {
// e = %x65 / %x45 ; e E
// exp = e [ minus / plus ] 1*DIGIT
// frac = decimal-point 1*DIGIT
- std::string temp_buffer;
bool maybe_int = true;
// parse [minus], cross check for Infinity/NaN/-Infinity
if (*cur_ == '-') {
- temp_buffer.push_back('-');
++cur_;
if (cur_ != end_ && *cur_ == 'I') {
if (this->MatchLiteral("Infinity", 8)) {
@@ -202,7 +208,6 @@ class JSONParserContext {
char next_char = *cur_;
if ((next_char >= '0' && next_char <= '9') || next_char == 'e' ||
next_char == 'E' ||
next_char == '+' || next_char == '-' || next_char == '.') {
- temp_buffer.push_back(next_char);
if (next_char == '.' || next_char == 'e' || next_char == 'E') {
maybe_int = false;
}
@@ -211,37 +216,60 @@ class JSONParserContext {
break;
}
}
- if (temp_buffer.empty()) {
+ if (start_pos == cur_) {
this->SetErrorExpectingValue();
return false;
}
- // parse from temp_buffer_
if (maybe_int) {
- // now try to parse the number as int64
- char* end_ptr;
- errno = 0;
- intmax_t int_val = strtoimax(temp_buffer.data(), &end_ptr, 10);
- if (errno == 0 && int_val >= std::numeric_limits<int64_t>::min() &&
- int_val <= std::numeric_limits<int64_t>::max() &&
- end_ptr == temp_buffer.data() + temp_buffer.size()) {
- *out = static_cast<int64_t>(int_val);
+ int64_t int_value;
+ auto result = std::from_chars(start_pos, cur_, int_value);
+ if (result.ec == std::errc{} && result.ptr == cur_) {
+ *out = int_value;
return true;
}
}
- {
- // now try to parse number as double
- char* end_ptr;
- errno = 0;
- double double_val = strtod(temp_buffer.data(), &end_ptr);
- if (errno == 0 && end_ptr == temp_buffer.data() + temp_buffer.size()) {
- *out = double_val;
- return true;
- } else {
- this->SetCurrentPosForBetterErrorMsg(start_pos);
- this->SetErrorExpectingValue();
- return false;
- }
+ double double_value;
+// libc++ and older libstdc++ use the C-locale fallback.
+#if defined(_MSC_VER) || (defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE >= 11)
+ auto result = std::from_chars(start_pos, cur_, double_value);
+ if (result.ec != std::errc{} || result.ptr != cur_) {
+ this->SetCurrentPosForBetterErrorMsg(start_pos);
+ this->SetErrorExpectingValue();
+ return false;
}
+ // Keep strtod's underflow behavior for subnormal results.
+ if (double_value == 0 || std::fabs(double_value) >
std::numeric_limits<double>::min()) {
+ *out = double_value;
+ return true;
+ }
+#endif
+ // Subnormals fall through to strtod_l to preserve underflow reporting
(ERANGE).
+ std::string temp_buffer(start_pos, cur_);
+ char* end_ptr;
+ // Create "C" numeric locale once to ignore process locale; twin:
json_writer.cc.
+ static const struct NumberLocale {
+#ifdef _WIN32
+ _locale_t value = _create_locale(LC_NUMERIC, "C");
+ ~NumberLocale() { _free_locale(value); }
+#else
+ locale_t value = newlocale(LC_NUMERIC_MASK, "C", nullptr);
+ ~NumberLocale() { freelocale(value); }
+#endif
+ NumberLocale() { TVM_FFI_ICHECK(value != nullptr) << "Cannot create JSON
numeric locale"; }
+ } locale;
+ errno = 0;
+#ifdef _WIN32
+ double_value = _strtod_l(temp_buffer.c_str(), &end_ptr, locale.value);
+#else
+ double_value = strtod_l(temp_buffer.c_str(), &end_ptr, locale.value);
+#endif
+ if (errno == 0 && end_ptr == temp_buffer.c_str() + temp_buffer.size()) {
+ *out = double_value;
+ return true;
+ }
+ this->SetCurrentPosForBetterErrorMsg(start_pos);
+ this->SetErrorExpectingValue();
+ return false;
}
/*!
diff --git a/src/ffi/extra/json_writer.cc b/src/ffi/extra/json_writer.cc
index 2f62feab..a4108bb7 100644
--- a/src/ffi/extra/json_writer.cc
+++ b/src/ffi/extra/json_writer.cc
@@ -31,13 +31,19 @@
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ffi/string.h>
-#include <cinttypes>
+#include <charconv>
+#include <clocale>
#include <cmath>
#include <cstdint>
+#include <cstdio>
#include <string>
#include <unordered_set>
#include <utility>
+#ifdef __APPLE__
+#include <xlocale.h>
+#endif
+
namespace tvm {
namespace ffi {
namespace json {
@@ -150,8 +156,40 @@ class JSONWriter {
void WriteInt(int64_t value) {
// the biggest possible string representation of -INT64_MIN
char buffer[sizeof("-9223372036854775808") + 1];
- int size = TVM_FFI_SNPRINTF(buffer, sizeof(buffer), "%" PRId64, value);
- WriteLiteral(buffer, size);
+ auto result = std::to_chars(buffer, buffer + sizeof(buffer), value);
+ result_.append(buffer, static_cast<size_t>(result.ptr - buffer));
+ }
+
+ static int WriteFloatToBuffer(char* buffer, size_t size, double value, bool
fixed) {
+// libc++ and older libstdc++ use the C-locale fallback.
+#if defined(_MSC_VER) || (defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE >= 11)
+ auto result = std::to_chars(buffer, buffer + size, value,
+ fixed ? std::chars_format::fixed :
std::chars_format::general,
+ fixed ? 1 : 17);
+ return static_cast<int>(result.ptr - buffer);
+#else
+ // Create "C" numeric locale once to ignore process locale; twin:
json_parser.cc.
+ static const struct NumberLocale {
+#ifdef _WIN32
+ _locale_t value = _create_locale(LC_NUMERIC, "C");
+ ~NumberLocale() { _free_locale(value); }
+#else
+ locale_t value = newlocale(LC_NUMERIC_MASK, "C", nullptr);
+ ~NumberLocale() { freelocale(value); }
+#endif
+ NumberLocale() { TVM_FFI_ICHECK(value != nullptr) << "Cannot create JSON
numeric locale"; }
+ } locale;
+#ifdef _WIN32
+ return _snprintf_l(buffer, size, fixed ? "%.1f" : "%.17g", locale.value,
value);
+#elif defined(__APPLE__)
+ return snprintf_l(buffer, size, locale.value, fixed ? "%.1f" : "%.17g",
value);
+#else
+ locale_t previous_locale = uselocale(locale.value);
+ int count = snprintf(buffer, size, fixed ? "%.1f" : "%.17g", value);
+ uselocale(previous_locale);
+ return count;
+#endif
+#endif
}
void WriteFloat(double value) {
@@ -173,13 +211,13 @@ class JSONWriter {
// always print an extra .0 for integer so integer numbers are printed
as floats
// this helps us to distinguish between integer and float, which is
not necessary
// but helps to ensure roundtrip property of the parser/printer in
terms of int/float types
- int size = TVM_FFI_SNPRINTF(buffer, sizeof(buffer), "%.1f", int_part);
- WriteLiteral(buffer, size);
+ int size = WriteFloatToBuffer(buffer, sizeof(buffer), int_part, true);
+ result_.append(buffer, size);
} else {
// Save 17 decimal digits to avoid loss during loading JSON
// this is the maximum precision that can be represented in a double
- int size = TVM_FFI_SNPRINTF(buffer, sizeof(buffer), "%.17g", value);
- WriteLiteral(buffer, size);
+ int size = WriteFloatToBuffer(buffer, sizeof(buffer), value, false);
+ result_.append(buffer, size);
}
}
}
diff --git a/tests/cpp/extra/test_json_parser.cc
b/tests/cpp/extra/test_json_parser.cc
index 6c0d90b2..6fb0ad5a 100644
--- a/tests/cpp/extra/test_json_parser.cc
+++ b/tests/cpp/extra/test_json_parser.cc
@@ -22,7 +22,9 @@
#include <tvm/ffi/extra/json.h>
#include <tvm/ffi/extra/structural_equal.h>
+#include <clocale>
#include <cmath>
+#include <string>
namespace {
@@ -120,6 +122,21 @@ TEST(JSONParser, Number) {
EXPECT_EQ(json::Parse("123.456E+10").cast<double>(), 123.456E+10);
}
+TEST(JSONParser, NumericLocale) {
+ struct RestoreLocale {
+ std::string saved = std::setlocale(LC_NUMERIC, nullptr);
+ ~RestoreLocale() { std::setlocale(LC_NUMERIC, saved.c_str()); }
+ } restore_locale;
+ const char* found = nullptr;
+ for (const char* name : {"de_DE.UTF-8", "German_Germany.1252"}) {
+ found = std::setlocale(LC_NUMERIC, name);
+ if (found != nullptr) break;
+ }
+ if (found == nullptr) GTEST_SKIP() << "No locale with a decimal comma is
installed";
+ EXPECT_EQ(json::Parse("1234.5").cast<double>(), 1234.5);
+ EXPECT_EQ(json::Stringify(1234.5), "1234.5");
+}
+
TEST(JSONParser, WrongNumber) {
String error_msg;
EXPECT_EQ(json::Parse("123.456.789", &error_msg), nullptr);
