This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 594eafd19 perf(json): optimize json performance (#3914)
594eafd19 is described below
commit 594eafd1975c08fb1dcb1d2144174bbc76f75df2
Author: Shawn Yang <[email protected]>
AuthorDate: Tue Aug 4 10:06:40 2026 +0800
perf(json): optimize json performance (#3914)
## Why?
## What does this PR do?
## Related issues
## AI Contribution Checklist
- [ ] Substantial AI assistance was used in this PR: `yes` / `no`
- [ ] If `yes`, I included a completed [AI Contribution
Checklist](https://github.com/apache/fory/blob/main/AI_POLICY.md#9-contributor-checklist-for-ai-assisted-prs)
in this PR description and the required `AI Usage Disclosure`.
- [ ] If `yes`, my PR description includes the required `ai_review`
summary and screenshot evidence or equivalent persisted links of the
final clean AI review results from both fresh reviewers described in
`AI_POLICY.md`, the Fory-guided reviewer and the independent general
reviewer, on the current PR diff or current HEAD after the latest code
changes.
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
.agents/languages/java.md | 6 +
.../apache/fory/json/codec/CollectionCodec.java | 60 ++++++----
.../fory/json/codegen/JsonReaderCodegen.java | 126 +++++++++++++++++++--
.../fory/json/codegen/Latin1ReaderCodegen.java | 5 +
.../fory/json/codegen/Utf8ReaderCodegen.java | 10 ++
.../apache/fory/json/reader/Latin1JsonReader.java | 18 +++
.../apache/fory/json/reader/Utf8JsonReader.java | 50 ++++++++
.../apache/fory/json/JsonGeneratedCodecTest.java | 51 +++++++++
.../fory/json/JsonGraphMemoryBudgetTest.java | 32 +++++-
.../java/org/apache/fory/json/JsonStringTest.java | 5 +-
10 files changed, 326 insertions(+), 37 deletions(-)
diff --git a/.agents/languages/java.md b/.agents/languages/java.md
index 2c97a4595..ceb927367 100644
--- a/.agents/languages/java.md
+++ b/.agents/languages/java.md
@@ -141,6 +141,12 @@ Load this file when changing anything under `java/` or
when Java drives a cross-
such a method into a wrapper, add its body back into the caller budget, or
manufacture a boundary
with padding, `@DontInline`, `CompileCommand`, fake receivers, or JVM flags.
Keep escape,
malformed-input, Unicode, arbitrary-length, and other cold fallback work in
separate methods.
+- Generated Latin1 and UTF-8 JSON readers classify arbitrary-order known
fields by a bounded raw
+ prefix and verify the complete compile-time field token in the generated
slow owner. A prefix or
+ token miss must leave the name unread and use the existing hash/table path
for escapes, aliases,
+ unknown names, collisions, and malformed input. Keep Any-property and UTF-16
readers on their
+ existing hash paths. Do not extract the classifier into an independent
scanner owner, add a
+ declaration-order assumption, or replace complete token verification with
prefix equality.
- Generated UTF-8 object writers own their C2 boundaries in their actual
emitted bytecode. A split
writer keeps object framing and the final declaration-order field range in
public `writeUtf8`;
every preceding range is a direct private final helper. Cold source
generation compiles each
diff --git
a/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java
b/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java
index 6b13190ea..90b23da64 100644
---
a/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java
+++
b/java/fory-json/src/main/java/org/apache/fory/json/codec/CollectionCodec.java
@@ -630,11 +630,10 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e3);
return list;
}
- return readLatin1ArrayListTail(reader, e0, e1, e2, e3);
- }
-
- private ArrayList<Object> readLatin1ArrayListTail(
- Latin1JsonReader reader, Object e0, Object e1, Object e2, Object e3) {
+ // Keep this real exact-allocation prefix in the collection owner.
Splitting here makes each
+ // method smaller than C2's hot-inline limit, so a generated caller can
absorb the collection
+ // and element closure solely according to compilation order. The
uncommon longer tail stays
+ // separate below.
Object e4 = readLatin1Element(reader);
if (!reader.consumeNextCommaOrEndArray()) {
reader.exitDepth();
@@ -1389,7 +1388,9 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e7);
return list;
}
- reader.reserveGraphMemory(ARRAY_LIST_OWNER_BYTES + 8 * REFERENCE_BYTES);
+ // Capacity nine is materialized before the ninth child is read, so
charge every backing
+ // slot before allocating the list.
+ reader.reserveGraphMemory(ARRAY_LIST_OWNER_BYTES + 9 * REFERENCE_BYTES);
ArrayList<Object> list = new ArrayList<>(9);
list.add(e0);
list.add(e1);
@@ -1399,14 +1400,15 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e5);
list.add(e6);
list.add(e7);
+ list.add(codec.readLatin1(reader));
int pendingSize = 0;
- do {
+ while (reader.consumeNextCommaOrEndArray()) {
if ((pendingSize & REFERENCE_BATCH_MASK) == REFERENCE_BATCH_MASK) {
reader.reserveGraphMemory(REFERENCE_BATCH_BYTES);
}
list.add(codec.readLatin1(reader));
pendingSize++;
- } while (reader.consumeNextCommaOrEndArray());
+ }
int tailSize = pendingSize & REFERENCE_BATCH_MASK;
if (tailSize != 0) {
reader.reserveGraphMemory(tailSize * REFERENCE_BYTES);
@@ -1538,7 +1540,9 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e7);
return list;
}
- reader.reserveGraphMemory(ARRAY_LIST_OWNER_BYTES + 8 * REFERENCE_BYTES);
+ // Capacity nine is materialized before the ninth child is read, so
charge every backing
+ // slot before allocating the list.
+ reader.reserveGraphMemory(ARRAY_LIST_OWNER_BYTES + 9 * REFERENCE_BYTES);
ArrayList<Object> list = new ArrayList<>(9);
list.add(e0);
list.add(e1);
@@ -1548,14 +1552,15 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e5);
list.add(e6);
list.add(e7);
+ list.add(codec.readUtf16(reader));
int pendingSize = 0;
- do {
+ while (reader.consumeNextCommaOrEndArray()) {
if ((pendingSize & REFERENCE_BATCH_MASK) == REFERENCE_BATCH_MASK) {
reader.reserveGraphMemory(REFERENCE_BATCH_BYTES);
}
list.add(codec.readUtf16(reader));
pendingSize++;
- } while (reader.consumeNextCommaOrEndArray());
+ }
int tailSize = pendingSize & REFERENCE_BATCH_MASK;
if (tailSize != 0) {
reader.reserveGraphMemory(tailSize * REFERENCE_BYTES);
@@ -1611,16 +1616,9 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e3);
return list;
}
- return readUtf8ArrayListTail(reader, codec, e0, e1, e2, e3);
- }
-
- private ArrayList<Object> readUtf8ArrayListTail(
- Utf8JsonReader reader,
- Utf8ReaderCodec<Object> codec,
- Object e0,
- Object e1,
- Object e2,
- Object e3) {
+ // Keep the fifth exact-allocation lane in the collection owner. If this
lane is split after
+ // four elements, both resulting methods fall below C2's hot-inline
limit and let an outer
+ // fallback caller absorb the object-element closure according to
compilation order.
Object e4 = codec.readUtf8(reader);
if (!reader.consumeNextCommaOrEndArray()) {
reader.exitDepth();
@@ -1633,6 +1631,17 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e4);
return list;
}
+ return readUtf8ArrayListTail(reader, codec, e0, e1, e2, e3, e4);
+ }
+
+ private ArrayList<Object> readUtf8ArrayListTail(
+ Utf8JsonReader reader,
+ Utf8ReaderCodec<Object> codec,
+ Object e0,
+ Object e1,
+ Object e2,
+ Object e3,
+ Object e4) {
Object e5 = codec.readUtf8(reader);
if (!reader.consumeNextCommaOrEndArray()) {
reader.exitDepth();
@@ -1687,7 +1696,9 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e7);
return list;
}
- reader.reserveGraphMemory(ARRAY_LIST_OWNER_BYTES + 8 * REFERENCE_BYTES);
+ // Capacity nine is materialized before the ninth child is read, so
charge every backing
+ // slot before allocating the list.
+ reader.reserveGraphMemory(ARRAY_LIST_OWNER_BYTES + 9 * REFERENCE_BYTES);
ArrayList<Object> list = new ArrayList<>(9);
list.add(e0);
list.add(e1);
@@ -1697,14 +1708,15 @@ public abstract class CollectionCodec<T extends
Collection<?>> implements JsonVa
list.add(e5);
list.add(e6);
list.add(e7);
+ list.add(codec.readUtf8(reader));
int pendingSize = 0;
- do {
+ while (reader.consumeNextCommaOrEndArray()) {
if ((pendingSize & REFERENCE_BATCH_MASK) == REFERENCE_BATCH_MASK) {
reader.reserveGraphMemory(REFERENCE_BATCH_BYTES);
}
list.add(codec.readUtf8(reader));
pendingSize++;
- } while (reader.consumeNextCommaOrEndArray());
+ }
int tailSize = pendingSize & REFERENCE_BATCH_MASK;
if (tailSize != 0) {
reader.reserveGraphMemory(tailSize * REFERENCE_BYTES);
diff --git
a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java
b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java
index 1ccd50d7d..a5e6d4ebb 100644
---
a/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java
+++
b/java/fory-json/src/main/java/org/apache/fory/json/codegen/JsonReaderCodegen.java
@@ -125,6 +125,10 @@ abstract class JsonReaderCodegen {
return false;
}
+ boolean rawFieldNameDispatch() {
+ return false;
+ }
+
abstract boolean isDirectName(String name, boolean tokenValueRead);
abstract Expression tryReadNextFieldNameColon(JsonFieldInfo property,
boolean tokenValueRead);
@@ -2884,8 +2888,8 @@ abstract class JsonReaderCodegen {
expressions.add(hashes);
expressions.add(fieldIndex);
Expression anyMapCreated = anyMapCreatedFlag(expressions);
+ expressions.add(expectExpr(':'));
Expression.ListExpression loop = new Expression.ListExpression();
- loop.add(expectExpr(':'));
loop.add(
fieldSwitch(
builder,
@@ -2912,16 +2916,21 @@ abstract class JsonReaderCodegen {
if (fieldStart != null) {
loop.add(fieldStart);
}
- Expression fieldHash = readFieldNameHash("fieldHash");
- loop.add(fieldHash);
- loop.add(assignSlowFieldIndex(fieldIndex, expectedIndex, hashes,
fieldHash, properties));
- if (any != null) {
- loop.add(
- new Expression.Assign(
- new Reference("firstFieldHash", TypeRef.of(long.class)),
fieldHash));
- loop.add(
- new Expression.Assign(
- new Reference("firstFieldStart", TypeRef.of(int.class)),
fieldStart));
+ if (any == null && rawFieldNameDispatch() &&
hasDirectFieldName(properties)) {
+ loop.add(readDirectFieldIndex(fieldIndex, expectedIndex, hashes,
properties));
+ } else {
+ Expression fieldHash = readFieldNameHash("fieldHash");
+ loop.add(fieldHash);
+ loop.add(assignSlowFieldIndex(fieldIndex, expectedIndex, hashes,
fieldHash, properties));
+ loop.add(expectExpr(':'));
+ if (any != null) {
+ loop.add(
+ new Expression.Assign(
+ new Reference("firstFieldHash", TypeRef.of(long.class)),
fieldHash));
+ loop.add(
+ new Expression.Assign(
+ new Reference("firstFieldStart", TypeRef.of(int.class)),
fieldStart));
+ }
}
expressions.add(new Expression.While(Expression.Literal.True, loop));
return expressions;
@@ -2935,6 +2944,9 @@ abstract class JsonReaderCodegen {
Expression hashes,
Expression expectedIndex,
Expression anyMapCreated) {
+ if (any == null && rawFieldNameDispatch() &&
hasDirectFieldName(properties)) {
+ return readNextDirectField(builder, type, properties, object, hashes,
expectedIndex);
+ }
Expression fieldStart =
any == null
? null
@@ -2958,6 +2970,98 @@ abstract class JsonReaderCodegen {
return expressions;
}
+ private Expression readNextDirectField(
+ JsonGeneratedCodecBuilder builder,
+ Class<?> type,
+ JsonFieldInfo[] properties,
+ Expression object,
+ Expression hashes,
+ Expression expectedIndex) {
+ Reference fieldIndex = new Reference("fieldIndex", TypeRef.of(int.class));
+ return new Expression.ListExpression(
+ new Expression.Variable("fieldIndex",
Expression.Literal.ofInt(JsonFieldTable.UNKNOWN)),
+ readDirectFieldIndex(fieldIndex, expectedIndex, hashes, properties),
+ fieldSwitch(builder, type, properties, object, fieldIndex),
+ updateExpectedIndex(expectedIndex, fieldIndex));
+ }
+
+ private Expression readDirectFieldIndex(
+ Expression fieldIndex,
+ Expression expectedIndex,
+ Expression hashes,
+ JsonFieldInfo[] properties) {
+ int unresolved = JsonFieldTable.UNKNOWN;
+ Expression prefix =
+ new Expression.Invoke(
+ readerRef(), "readFieldNamePrefix", "fieldPrefix",
TypeRef.of(int.class), false);
+ Expression fieldHash = readFieldNameHash("fieldHash");
+ Expression fallback =
+ new Expression.ListExpression(
+ fieldHash,
+ assignSlowFieldIndex(fieldIndex, expectedIndex, hashes, fieldHash,
properties),
+ expectExpr(':'));
+ // Keep classification and complete token verification in the generated
slow owner. On a miss
+ // the token matcher leaves the name unread, so one existing hash path
retains every escaped,
+ // aliased, unknown, and malformed-name behavior without adding a second
scanner owner.
+ return new Expression.ListExpression(
+ prefix,
+ new Expression.Assign(fieldIndex,
Expression.Literal.ofInt(unresolved)),
+ directFieldNameSwitch(fieldIndex, prefix, properties),
+ new Expression.If(eq(fieldIndex,
Expression.Literal.ofInt(unresolved)), fallback));
+ }
+
+ private boolean hasDirectFieldName(JsonFieldInfo[] properties) {
+ for (JsonFieldInfo property : properties) {
+ if (!property.name().isEmpty() && isDirectName(property.name(), true)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private Expression directFieldNameSwitch(
+ Expression fieldIndex, Expression prefix, JsonFieldInfo[] properties) {
+ int[] keys = new int[properties.length];
+ int keyCount = 0;
+ for (JsonFieldInfo property : properties) {
+ if (property.name().isEmpty() || !isDirectName(property.name(), true)) {
+ continue;
+ }
+ int key = (int) JsonAsciiToken.prefix(fieldNameToken(property.name()));
+ boolean found = false;
+ for (int i = 0; i < keyCount; i++) {
+ if (keys[i] == key) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ keys[keyCount++] = key;
+ }
+ }
+ Expression.Switch.Case[] cases = new Expression.Switch.Case[keyCount];
+ for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) {
+ int key = keys[keyIndex];
+ Expression resolve = new Expression.Empty();
+ for (int field = properties.length - 1; field >= 0; field--) {
+ JsonFieldInfo property = properties[field];
+ if (!property.name().isEmpty()
+ && isDirectName(property.name(), true)
+ && (int) JsonAsciiToken.prefix(fieldNameToken(property.name())) ==
key) {
+ resolve =
+ new Expression.If(
+ tryReadNextFieldNameColon(property, true),
+ new Expression.Assign(fieldIndex,
Expression.Literal.ofInt(field)),
+ resolve);
+ }
+ }
+ cases[keyIndex] =
+ new Expression.Switch.Case(
+ key, new Expression.ListExpression(resolve, new
Expression.Break()));
+ }
+ return new Expression.Switch(prefix, cases, null);
+ }
+
private Expression fieldSwitch(
JsonGeneratedCodecBuilder builder,
Class<?> type,
diff --git
a/java/fory-json/src/main/java/org/apache/fory/json/codegen/Latin1ReaderCodegen.java
b/java/fory-json/src/main/java/org/apache/fory/json/codegen/Latin1ReaderCodegen.java
index f69d57728..b2b66dfbf 100644
---
a/java/fory-json/src/main/java/org/apache/fory/json/codegen/Latin1ReaderCodegen.java
+++
b/java/fory-json/src/main/java/org/apache/fory/json/codegen/Latin1ReaderCodegen.java
@@ -93,6 +93,11 @@ final class Latin1ReaderCodegen extends JsonReaderCodegen {
return true;
}
+ @Override
+ boolean rawFieldNameDispatch() {
+ return true;
+ }
+
@Override
boolean isDirectName(String name, boolean tokenValueRead) {
return JsonAsciiToken.isLongPackable(fieldNameToken(name));
diff --git
a/java/fory-json/src/main/java/org/apache/fory/json/codegen/Utf8ReaderCodegen.java
b/java/fory-json/src/main/java/org/apache/fory/json/codegen/Utf8ReaderCodegen.java
index 32620108b..953e0ce18 100644
---
a/java/fory-json/src/main/java/org/apache/fory/json/codegen/Utf8ReaderCodegen.java
+++
b/java/fory-json/src/main/java/org/apache/fory/json/codegen/Utf8ReaderCodegen.java
@@ -94,6 +94,16 @@ final class Utf8ReaderCodegen extends JsonReaderCodegen {
return "readUtf8";
}
+ @Override
+ boolean directSlowFieldIndex() {
+ return true;
+ }
+
+ @Override
+ boolean rawFieldNameDispatch() {
+ return true;
+ }
+
@Override
Expression consumeCommaOrEndObjectExpr() {
Expression comma =
diff --git
a/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java
b/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java
index e8579d562..a39fb2b45 100644
---
a/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java
+++
b/java/fory-json/src/main/java/org/apache/fory/json/reader/Latin1JsonReader.java
@@ -25,6 +25,7 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
import java.util.UUID;
+import org.apache.fory.annotation.Internal;
import org.apache.fory.json.JsonConfig;
import org.apache.fory.json.meta.JsonFieldInfo;
import org.apache.fory.json.meta.JsonFieldNameHash;
@@ -2184,6 +2185,23 @@ public final class Latin1JsonReader extends JsonReader {
return readQuotedStringHash();
}
+ /**
+ * Returns the raw four-byte prefix at the next field name after consuming
legal whitespace.
+ *
+ * <p>Generated object readers use this only as a discriminator before a
complete field-token
+ * check. A miss leaves the name unread so the ordinary hash parser retains
escape, Unicode,
+ * alias, unknown-field, and malformed-input handling.
+ */
+ @Internal
+ public int readFieldNamePrefix() {
+ skipWhitespaceFast();
+ int offset = position;
+ if (offset <= input.length - Integer.BYTES) {
+ return LittleEndian.getInt32(input, offset);
+ }
+ return 0;
+ }
+
public boolean tryReadFieldNameColon(long expectedHash, long expectedMask,
int expectedLength) {
int mark = position;
skipWhitespaceFast();
diff --git
a/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java
b/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java
index e7e37a5f7..c587d44a3 100644
---
a/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java
+++
b/java/fory-json/src/main/java/org/apache/fory/json/reader/Utf8JsonReader.java
@@ -2086,6 +2086,11 @@ public final class Utf8JsonReader extends JsonReader {
}
int start = position;
int offset = start;
+ // Keep seven real bounded probes in the token owner. Besides covering
ordinary Strings through
+ // 56 bytes without a helper call, this keeps the complete scanner behind
a natural C2 boundary
+ // so nullable wrappers and generated object readers cannot absorb
duplicate token closures.
+ // A loop or forwarding helper would shrink this owner and restore
compilation-order
+ // sensitivity.
if (offset + Long.BYTES <= inputLength) {
long stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset));
if (stopMask != 0) {
@@ -2104,6 +2109,34 @@ public final class Utf8JsonReader extends JsonReader {
return readStringWordStop(start, offset, stopMask);
}
offset += Long.BYTES;
+ if (offset + Long.BYTES <= inputLength) {
+ stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset));
+ if (stopMask != 0) {
+ return readStringWordStop(start, offset, stopMask);
+ }
+ offset += Long.BYTES;
+ if (offset + Long.BYTES <= inputLength) {
+ stopMask = stringStopMask(LittleEndian.getInt64(bytes, offset));
+ if (stopMask != 0) {
+ return readStringWordStop(start, offset, stopMask);
+ }
+ offset += Long.BYTES;
+ if (offset + Long.BYTES <= inputLength) {
+ stopMask = stringStopMask(LittleEndian.getInt64(bytes,
offset));
+ if (stopMask != 0) {
+ return readStringWordStop(start, offset, stopMask);
+ }
+ offset += Long.BYTES;
+ if (offset + Long.BYTES <= inputLength) {
+ stopMask = stringStopMask(LittleEndian.getInt64(bytes,
offset));
+ if (stopMask != 0) {
+ return readStringWordStop(start, offset, stopMask);
+ }
+ offset += Long.BYTES;
+ }
+ }
+ }
+ }
}
}
}
@@ -2480,6 +2513,23 @@ public final class Utf8JsonReader extends JsonReader {
return readQuotedStringHash();
}
+ /**
+ * Returns the raw four-byte prefix at the next field name after consuming
legal whitespace.
+ *
+ * <p>Generated object readers use this only as a discriminator before a
complete field-token
+ * check. A miss leaves the name unread so the ordinary hash parser retains
escape, UTF-8, alias,
+ * unknown-field, and malformed-input handling.
+ */
+ @Internal
+ public int readFieldNamePrefix() {
+ skipWhitespaceFast();
+ int offset = position;
+ if (offset <= input.length - Integer.BYTES) {
+ return LittleEndian.getInt32(input, offset);
+ }
+ return 0;
+ }
+
public boolean tryReadFieldNameColon(long expectedHash, long expectedMask,
int expectedLength) {
int mark = position;
skipWhitespaceFast();
diff --git
a/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java
b/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java
index eddd0e5a8..010e3cdd0 100644
---
a/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java
+++
b/java/fory-json/src/test/java/org/apache/fory/json/JsonGeneratedCodecTest.java
@@ -255,6 +255,45 @@ public class JsonGeneratedCodecTest extends
ForyJsonTestModels {
assertEquals(mismatch.readNextNullableString(), "pit");
}
+ @Test
+ public void readFieldNamePrefix() {
+ String input = " \n\t\"alpha\":1";
+ int expected = (int) JsonAsciiToken.prefix("\"alpha\":");
+ Latin1JsonReader latin1 = newLatin1Reader(latin1Bytes(input));
+ assertEquals(latin1.readFieldNamePrefix(), expected);
+ assertEquals(latin1.position(), 3);
+ assertTrue(
+ latin1.tryReadNextFieldNameToken0(
+ JsonAsciiToken.prefix("\"alpha\":"), -1L, "\"alpha\":".length()));
+ assertEquals(latin1.readIntTokenValue(), 1);
+
+ Utf8JsonReader utf8 =
newUtf8Reader(input.getBytes(StandardCharsets.UTF_8));
+ assertEquals(utf8.readFieldNamePrefix(), expected);
+ assertEquals(utf8.position(), 3);
+ assertTrue(
+ utf8.tryReadNextFieldNameToken0(
+ JsonAsciiToken.prefix("\"alpha\":"), -1L, "\"alpha\":".length()));
+ assertEquals(utf8.readIntTokenValue(), 1);
+
+ Latin1JsonReader truncatedLatin1 = newLatin1Reader(latin1Bytes(" \"a"));
+ assertEquals(truncatedLatin1.readFieldNamePrefix(), 0);
+ assertEquals(truncatedLatin1.position(), 1);
+ Utf8JsonReader truncatedUtf8 = newUtf8Reader("
\"a".getBytes(StandardCharsets.UTF_8));
+ assertEquals(truncatedUtf8.readFieldNamePrefix(), 0);
+ assertEquals(truncatedUtf8.position(), 1);
+ }
+
+ @Test
+ public void readGeneratedFieldPrefixCollision() {
+ ForyJson json = newJson(true);
+ String input = "{\"unknown\":0, \"alpine\":2, \"\\u0061lpha\":1, \"alpha\"
:4, \"altar\":3}";
+ PrefixFields latin1 = json.fromJson(input, PrefixFields.class);
+ assertPrefixFields(latin1);
+ PrefixFields utf8 = json.fromJson(input.getBytes(StandardCharsets.UTF_8),
PrefixFields.class);
+ assertPrefixFields(utf8);
+ assertGeneratedWhenSupported(json, PrefixFields.class, true);
+ }
+
@Test(dataProvider = "enableCodegen")
public void readGeneratedLongAsciiFields(boolean codegen) {
ForyJson json = newJson(codegen);
@@ -343,6 +382,12 @@ public class JsonGeneratedCodecTest extends
ForyJsonTestModels {
assertEquals(value.shortName, "core");
}
+ private static void assertPrefixFields(PrefixFields value) {
+ assertEquals(value.alpha, 4);
+ assertEquals(value.alpine, 2);
+ assertEquals(value.altar, 3);
+ }
+
private static byte[] latin1Bytes(String value) {
return value.getBytes(StandardCharsets.ISO_8859_1);
}
@@ -354,6 +399,12 @@ public class JsonGeneratedCodecTest extends
ForyJsonTestModels {
public String shortName;
}
+ public static class PrefixFields {
+ public int alpha;
+ public int alpine;
+ public int altar;
+ }
+
public static class WideFields {
public int f0;
public String f1;
diff --git
a/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java
b/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java
index ef218d17f..06d4f6d3d 100644
---
a/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java
+++
b/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java
@@ -285,7 +285,7 @@ public class JsonGraphMemoryBudgetTest extends
ForyJsonTestModels {
@Test
public void stagedCollectionBatches() {
TypeRef<List<CountingChild>> type = new TypeRef<List<CountingChild>>() {};
- int prefixSize = codegenEnabled() ? 9 : 8;
+ int prefixSize = 9;
int completed = prefixSize + 1023;
String input = childArray(completed + 1);
long budget =
@@ -299,6 +299,17 @@ public class JsonGraphMemoryBudgetTest extends
ForyJsonTestModels {
assertEquals(CountingChild.creations, completed);
}
+ @Test
+ public void ninthSlotGuardsArrayListStorage() {
+ TypeRef<List<CountingChild>> type = new TypeRef<List<CountingChild>>() {};
+ long budget = shallow(ArrayList.class) + 8L * REF_BYTES + 9L *
shallow(CountingChild.class);
+
+ assertNinthSlotGuard(childArray(9), type, budget, 8);
+ assertNinthSlotGuard(
+ childArray(9).getBytes(StandardCharsets.UTF_8), type, budget,
codegenEnabled() ? 9 : 8);
+ assertNinthSlotGuard(utf16ChildArray(9), type, budget, 8);
+ }
+
@Test
public void duplicateSlotsAreCharged() {
TypeRef<Set<String>> setType = new TypeRef<Set<String>>() {};
@@ -444,6 +455,25 @@ public class JsonGraphMemoryBudgetTest extends
ForyJsonTestModels {
return input.append(']').toString();
}
+ private static String utf16ChildArray(int size) {
+ String input = childArray(size);
+ return input.replaceFirst("\\{", "{\"ignored\":\"Ā\",");
+ }
+
+ private <T> void assertNinthSlotGuard(
+ String input, TypeRef<T> type, long budget, int expectedCreations) {
+ CountingChild.creations = 0;
+ assertThrows(ForyJsonException.class, () ->
jsonWithBudget(budget).fromJson(input, type));
+ assertEquals(CountingChild.creations, expectedCreations);
+ }
+
+ private <T> void assertNinthSlotGuard(
+ byte[] input, TypeRef<T> type, long budget, int expectedCreations) {
+ CountingChild.creations = 0;
+ assertThrows(ForyJsonException.class, () ->
jsonWithBudget(budget).fromJson(input, type));
+ assertEquals(CountingChild.creations, expectedCreations);
+ }
+
private static String childMap(int size) {
StringBuilder input = new StringBuilder(size * 24);
input.append('{');
diff --git
a/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java
b/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java
index e898c9395..520439e9b 100644
--- a/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java
+++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonStringTest.java
@@ -467,7 +467,10 @@ public class JsonStringTest extends ForyJsonTestModels {
@Test
public void readStringScanBoundaries() {
ForyJson json = newJson();
- for (int length : new int[] {0, 1, 7, 8, 15, 16, 17, 23, 24, 31, 32, 33,
63, 64, 65}) {
+ for (int length :
+ new int[] {
+ 0, 1, 7, 8, 15, 16, 17, 23, 24, 31, 32, 33, 39, 40, 41, 47, 48, 49,
55, 56, 57, 63, 64, 65
+ }) {
String value = repeat('a', length);
String input = "\"" + value + "\"";
assertEquals(json.fromJson(input, String.class), value);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]