This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new 6a0a07bb feat(JavaScript): xlang map code generator (#1571)
6a0a07bb is described below
commit 6a0a07bb973a60d6589604af0c3e54fdcf1ae83d
Author: weipeng <[email protected]>
AuthorDate: Thu Apr 25 22:47:55 2024 +0800
feat(JavaScript): xlang map code generator (#1571)
## What does this PR do?
Implement the code generator for Map. The code generator will produce
JavaScript code based on type descriptions at runtime. The generated
code is entirely inlined to minimize the number of function calls and
reduce the possibility of IC (Inline Cache) misses.
## Input type
```
Type.map(Type.string(), Type.string())
```
## Output Code
```JavaScript
function (fury, external) {
const br = fury.binaryReader;
const bw = fury.binaryWriter;
const cr = fury.classResolver;
const rr = fury.referenceResolver;
const readInner = (fromRef) => {
let count_0 = br.varInt32();
const result_1 = new Map();
if (fromRef) {
rr.reference(result_1)
}
while (count_0 > 0) {
const header = br.uint16();
const keyHeader = header >> 12;
const valueHeader = (header >> 8) & 0b00001111;
const chunkSize = header & 0b11111111;
br.skip(4);
const keyIncludeNone = keyHeader & 2;
const keyTrackingRef = keyHeader & 1;
const valueIncludeNone = valueHeader & 2;
const valueTrackingRef = valueHeader & 1;
for (let index = 0; index < chunkSize; index++) {
let key;
let value;
let flag = 0;
if (keyTrackingRef || keyIncludeNone) {
flag = br.uint8();
}
switch (flag) {
case 0:
key = br.stringOfVarUInt32()
break;
case -2:
key = rr.getReadObject(br.varInt32())
break;
case -3:
key = null;
break;
case -1:
key = br.stringOfVarUInt32()
break;
}
flag = 0;
if (valueTrackingRef || valueIncludeNone) {
flag = br.uint8();
}
switch (flag) {
case 0:
value = br.stringOfVarUInt32()
break;
case -2:
value = rr.getReadObject(br.varInt32())
break;
case -3:
value = null;
break;
case -1:
value = br.stringOfVarUInt32()
break;
}
result_1.set(
key,
value
);
count_0--;
}
}
return result_1
};
const read = () => {
const refFlag_2 = br.int8();
switch (refFlag_2) {
case -1:
case 0:
if (br.int16() === 256) {
cr.readTag(br);
}
return readInner(refFlag_2 === 0)
case -2:
return rr.getReadObject(br.varUInt32())
case -3:
return null
}
};
const writeInner = (v) => {
bw.varInt32(v.size)
let lastKeyIsNull_5 = false;
let lastValueIsNull_6 = false;
let chunkSize_7 = 0;
let chunkSizeOffset_8 = 0;
for (const [k_3, v_4] of v.entries()) {
let keyIsNull = k_3 === null || k_3 === undefined;
let valueIsNull = v_4 === null || v_4 === undefined;
if (lastKeyIsNull_5 !== keyIsNull || lastValueIsNull_6 !==
valueIsNull || chunkSize_7 === 0 || chunkSize_7 === 255) {
if (chunkSize_7 > 0) {
bw.setUint8Position(chunkSizeOffset_8, chunkSize_7);
chunkSize_7 = 0;
}
chunkSizeOffset_8 = bw.getCursor()
bw.uint16(((0 & (keyIsNull ? 2 : 0)) << 4) | (0 & (valueIsNull ? 2
: 0)) << 8)
bw.uint32(3084);
lastKeyIsNull_5 = keyIsNull;
lastValueIsNull_6 = valueIsNull;
}
if (keyIsNull) {
bw.uint8(-3)
}
bw.stringOfVarUInt32(k_3)
if (valueIsNull) {
bw.uint8(-3)
}
bw.stringOfVarUInt32(v_4)
chunkSize_7++;
}
if (chunkSize_7 > 0) {
bw.setUint8Position(chunkSizeOffset_8, chunkSize_7);
}
};
const write = (v) => {
if (v !== null && v !== undefined) {
bw.int24(4351);
writeInner(v);
} else {
bw.int8(-3);
}
};
return {
read,
readInner,
write,
writeInner,
meta: { "fixedSize": 7, "needToWriteRef": false, "type": 15, "typeId":
16 }
};
}
```
## Benchmark
## benchmark/map.js
We can notice that it isn't much faster than any serializer, because
iterating a Map is too expensive. But it is still necessary.
| (index) | Values |
|-----------------|--------|
| any serialize | 215 |
| any deserialize | 144 |
| jit serialize | 291 |
| jit deserialize | 150 |
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
javascript/benchmark/map.js | 43 ++++--
javascript/packages/fury/lib/gen/builder.ts | 12 ++
javascript/packages/fury/lib/gen/map.ts | 207 +++++++++++++++++++++++++---
javascript/test/map.test.ts | 15 +-
4 files changed, 250 insertions(+), 27 deletions(-)
diff --git a/javascript/benchmark/map.js b/javascript/benchmark/map.js
index 8e4df29f..121c02ba 100644
--- a/javascript/benchmark/map.js
+++ b/javascript/benchmark/map.js
@@ -18,33 +18,56 @@
*/
const Fury = require("@furyjs/fury");
+const beautify = require("js-beautify");
const hps = require('@furyjs/hps');
-const fury = new Fury.default({ hps, refTracking: false, useSliceString: true
});
+const fury = new Fury.default({
+ hps, refTracking: false, useSliceString: true, hooks: {
+ afterCodeGenerated: (code) => {
+ return beautify.js(code, { indent_size: 2, space_in_empty_paren: true,
indent_empty_lines: true });
+ }
+ }
+});
const Benchmark = require("benchmark");
const Type = Fury.Type;
-const { serialize, deserialize, serializeVolatile } =
fury.registerSerializer(Type.map(Type.any(), Type.any()));
-const sample = new Map([["foo", "ba1"],["foo1", "ba1"],["foo2",
"ba1"],["foo3", "ba1"],["foo4", "ba1"],["foo5", "ba1"]]);
-const furyAb = serialize(sample);
+const { serialize: serialize1, deserialize: deserialize1, serializeVolatile:
serializeVolatile1 } = fury.registerSerializer(Type.object("any", {
+ f1: Type.map(Type.any(), Type.any()),
+ f2: Type.map(Type.any(), Type.any())
+}));
+
+const { serialize: serialize2, deserialize: deserialize2, serializeVolatile:
serializeVolatile2 } = fury.registerSerializer(Type.object("specific", {
+ f1: Type.map(Type.string(), Type.string()),
+ f2: Type.map(Type.int32(), Type.string())
+}));
+const sample = {
+ f1: new Map([["foo", "ba1"], ["foo1", "ba1"], ["foo2", "ba1"], ["foo3",
"ba1"], ["foo4", "ba1"], ["foo5", "ba1"], ["foo5", "ba1"], ["foo5", "ba1"]]),
+ f2: new Map([[123, "ba1"], [234, "ba1"], [345, "ba1"], [456, "ba1"], [567,
"ba1"], [678, "ba1"], [789, "ba1"], [890, "ba1"]])
+};
+const furyAb1 = serialize1(sample);
+const furyAb2 = serialize2(sample);
async function start() {
let result = {
- serialize: 0,
- deserialize: 0
}
{
var suite = new Benchmark.Suite();
suite
- .add("serialize", function () {
- serializeVolatile(sample).dispose()
+ .add("any serialize", function () {
+ serializeVolatile1(sample).dispose()
+ })
+ .add("any deserialize", function () {
+ deserialize1(furyAb1)
+ })
+ .add("jit serialize", function () {
+ serializeVolatile2(sample).dispose()
})
- .add("deserialize", function () {
- deserialize(furyAb)
+ .add("jit deserialize", function () {
+ deserialize2(furyAb2)
})
.on("complete", function (e) {
e.currentTarget.forEach(({ name, hz }) => {
diff --git a/javascript/packages/fury/lib/gen/builder.ts
b/javascript/packages/fury/lib/gen/builder.ts
index 9859dd88..564eb5de 100644
--- a/javascript/packages/fury/lib/gen/builder.ts
+++ b/javascript/packages/fury/lib/gen/builder.ts
@@ -236,6 +236,18 @@ class BinaryWriterBuilder {
getCursor() {
return `${this.holder}.getCursor()`;
}
+
+ setUint32Position(offset: number | string, v: number | string) {
+ return `${this.holder}.setUint32Position(${offset}, ${v})`;
+ }
+
+ setUint8Position(offset: number | string, v: number | string) {
+ return `${this.holder}.setUint8Position(${offset}, ${v})`;
+ }
+
+ setUint16Position(offset: number | string, v: number | string) {
+ return `${this.holder}.setUint16Position(${offset}, ${v})`;
+ }
}
class ReferenceResolverBuilder {
diff --git a/javascript/packages/fury/lib/gen/map.ts
b/javascript/packages/fury/lib/gen/map.ts
index dc6eda05..fda8b808 100644
--- a/javascript/packages/fury/lib/gen/map.ts
+++ b/javascript/packages/fury/lib/gen/map.ts
@@ -39,7 +39,7 @@ const MapFlags = {
NOT_SAME_TYPE: 0b1000,
};
-class TypeInfo {
+class MapTypeInfo {
private static IS_NULL = 0b10;
private static TRACKING_REF = 0b01;
static elementInfo(typeId: number, isNull: 0 | 1, trackRef: 0 | 1) {
@@ -69,29 +69,30 @@ class MapChunkWriter {
private getHead(keyInfo: number, valueInfo: number) {
let flag = 0;
- if (TypeInfo.isNull(keyInfo)) {
+ if (MapTypeInfo.isNull(keyInfo)) {
flag |= MapFlags.HAS_NULL;
}
- if (TypeInfo.trackingRef(keyInfo)) {
+ if (MapTypeInfo.trackingRef(keyInfo)) {
flag |= MapFlags.TRACKING_REF;
}
flag <<= 4;
- if (TypeInfo.isNull(valueInfo)) {
+ if (MapTypeInfo.isNull(valueInfo)) {
flag |= MapFlags.HAS_NULL;
}
- if (TypeInfo.trackingRef(valueInfo)) {
+ if (MapTypeInfo.trackingRef(valueInfo)) {
flag |= MapFlags.TRACKING_REF;
}
return flag;
}
private writeHead(keyInfo: number, valueInfo: number) {
- // KV header
- const header = this.getHead(keyInfo, valueInfo);
- this.fury.binaryWriter.uint8(header);
// chunkSize, max 255
this.chunkOffset = this.fury.binaryWriter.getCursor();
- this.fury.binaryWriter.uint8(0);
+ // KV header
+ const header = this.getHead(keyInfo, valueInfo);
+ // chunkSize default 0 | KV header
+ this.fury.binaryWriter.uint16(header << 8);
+ // key TypeId | value TypeId
this.fury.binaryWriter.uint32((keyInfo >> 16) | (valueInfo & 0xFFFF0000));
return header;
}
@@ -164,8 +165,8 @@ class MapAnySerializer {
const valueSerializer = this.valueSerializer !== null ?
this.valueSerializer : this.fury.classResolver.getSerializerByData(v);
const header = mapChunkWriter.next(
- TypeInfo.elementInfo(keySerializer!.meta.typeId!, k == null ? 1 : 0,
keySerializer!.meta.needToWriteRef ? 1 : 0),
- TypeInfo.elementInfo(valueSerializer!.meta.typeId!, v == null ? 1 : 0,
valueSerializer!.meta.needToWriteRef ? 1 : 0)
+ MapTypeInfo.elementInfo(keySerializer!.meta.typeId!, k == null ? 1 :
0, keySerializer!.meta.needToWriteRef ? 1 : 0),
+ MapTypeInfo.elementInfo(valueSerializer!.meta.typeId!, v == null ? 1 :
0, valueSerializer!.meta.needToWriteRef ? 1 : 0)
);
this.writeHead(header >> 4, k);
@@ -211,9 +212,9 @@ class MapAnySerializer {
}
while (count > 0) {
const header = this.fury.binaryReader.uint16();
- const chunkSize = header >> 8;
const keyHeader = header >> 12;
- const valueHeader = header & 0b00001111;
+ const valueHeader = (header >> 8) & 0b00001111;
+ const chunkSize = header & 0b11111111;
let keySerializer = null;
let valueSerializer = null;
@@ -249,19 +250,193 @@ export class MapSerializerGenerator extends
BaseSerializerGenerator {
return [this.builder.meta(inner.options.key),
this.builder.meta(inner.options.value)];
}
+ private innerGenerator(description: TypeDescription) {
+ const inner = this.builder.meta(description);
+ const InnerGeneratorClass = CodegenRegistry.get(inner.type);
+ if (!InnerGeneratorClass) {
+ throw new Error(`${inner.type} generator not exists`);
+ }
+ return new InnerGeneratorClass(inner, this.builder, this.scope);
+ }
+
+ private isAny() {
+ const [keyMeta, valueMeta] = this.innerMeta();
+ return keyMeta.type === InternalSerializerType.ANY || valueMeta.type ===
InternalSerializerType.ANY;
+ }
+
+ private writeStmtSpecificType(accessor: string) {
+ const [keyMeta, valueMeta] = this.innerMeta();
+ const keyGenerator = this.innerGenerator(keyMeta);
+ const valueGenerator = this.innerGenerator(valueMeta);
+ const k = this.scope.uniqueName("k");
+ const v = this.scope.uniqueName("v");
+ const keyHeader = (keyMeta.needToWriteRef ? MapFlags.TRACKING_REF : 0);
+ const valueHeader = (keyMeta.needToWriteRef ? MapFlags.TRACKING_REF : 0);
+ const typeId = (keyMeta.typeId! << 8) | valueMeta.typeId!;
+ const lastKeyIsNull = this.scope.uniqueName("lastKeyIsNull");
+ const lastValueIsNull = this.scope.uniqueName("lastValueIsNull");
+ const chunkSize = this.scope.uniqueName("chunkSize");
+ const chunkSizeOffset = this.scope.uniqueName("chunkSizeOffset");
+ const keyRef = this.scope.uniqueName("keyRef");
+ const valueRef = this.scope.uniqueName("valueRef");
+
+ return `
+ ${this.builder.writer.varInt32(`${accessor}.size`)}
+ let ${lastKeyIsNull} = false;
+ let ${lastValueIsNull} = false;
+ let ${chunkSize} = 0;
+ let ${chunkSizeOffset} = 0;
+
+ for (const [${k}, ${v}] of ${accessor}.entries()) {
+ let keyIsNull = ${k} === null || ${k} === undefined;
+ let valueIsNull = ${v} === null || ${v} === undefined;
+
+ if (${lastKeyIsNull} !== keyIsNull || ${lastValueIsNull} !==
valueIsNull || ${chunkSize} === 0 || ${chunkSize} === 255) {
+ if (${chunkSize} > 0) {
+ ${this.builder.writer.setUint8Position(chunkSizeOffset,
chunkSize)};
+ ${chunkSize} = 0;
+ }
+ ${chunkSizeOffset} = ${this.builder.writer.getCursor()}
+ ${
+ this.builder.writer.uint16(
+ `((${keyHeader} & (keyIsNull ? ${MapFlags.HAS_NULL} : 0)) << 4)
| (${valueHeader} & (valueIsNull ? ${MapFlags.HAS_NULL} : 0)) << 8`
+ )
+ }
+ ${this.builder.writer.uint32(typeId)};
+
+ ${lastKeyIsNull} = keyIsNull;
+ ${lastValueIsNull} = valueIsNull;
+ }
+ if (keyIsNull) {
+ ${this.builder.writer.uint8(RefFlags.NullFlag)}
+ }
+ ${keyMeta.needToWriteRef
+? `
+ const ${keyRef} =
${this.builder.referenceResolver.existsWriteObject(v)};
+ if (${keyRef} !== undefined) {
+ ${this.builder.writer.uint8(RefFlags.RefFlag)};
+ ${this.builder.writer.uint16(keyRef)};
+ } else {
+ ${this.builder.writer.uint8(RefFlags.RefValueFlag)};
+ }
+ `
+: ""}
+ ${keyGenerator.toWriteEmbed(k, true)}
+
+ if (valueIsNull) {
+ ${this.builder.writer.uint8(RefFlags.NullFlag)}
+ }
+ ${valueMeta.needToWriteRef
+? `
+ const ${valueRef} =
${this.builder.referenceResolver.existsWriteObject(v)};
+ if (${valueRef} !== undefined) {
+ ${this.builder.writer.uint8(RefFlags.RefFlag)};
+ ${this.builder.writer.uint16(valueRef)};
+ } else {
+ ${this.builder.writer.uint8(RefFlags.RefValueFlag)};
+ }
+ `
+: ""}
+ ${valueGenerator.toWriteEmbed(v, true)}
+
+ ${chunkSize}++;
+ }
+ if (${chunkSize} > 0) {
+ ${this.builder.writer.setUint8Position(chunkSizeOffset, chunkSize)};
+ }
+ `;
+ }
+
writeStmt(accessor: string): string {
const [keyMeta, valueMeta] = this.innerMeta();
const anySerializer = this.builder.getExternal(MapAnySerializer.name);
+ if (!this.isAny()) {
+ return this.writeStmtSpecificType(accessor);
+ }
+ return `new (${anySerializer})(${this.builder.furyName()},
${keyMeta.typeId}, ${valueMeta.typeId}).write(${accessor})`;
+ }
+
+ private readStmtSpecificType(accessor: (expr: string) => string, refState:
RefState) {
+ const count = this.scope.uniqueName("count");
+ const result = this.scope.uniqueName("result");
+ const [keyMeta, valueMeta] = this.innerMeta();
+ const keyGenerator = this.innerGenerator(keyMeta);
+ const valueGenerator = this.innerGenerator(valueMeta);
+
return `
- new (${anySerializer})(${this.builder.furyName()}, ${keyMeta.typeId},
${valueMeta.typeId}).write(${accessor})
+ let ${count} = ${this.builder.reader.varInt32()};
+ const ${result} = new Map();
+ if (${refState.toConditionExpr()}) {
+ ${this.builder.referenceResolver.reference(result)}
+ }
+ while (${count} > 0) {
+ const header = ${this.builder.reader.uint16()};
+ const keyHeader = header >> 12;
+ const valueHeader = (header >> 8) & 0b00001111;
+ const chunkSize = header & 0b11111111;
+ ${this.builder.reader.skip(4)};
+ const keyIncludeNone = keyHeader & ${MapFlags.HAS_NULL};
+ const keyTrackingRef = keyHeader & ${MapFlags.TRACKING_REF};
+ const valueIncludeNone = valueHeader & ${MapFlags.HAS_NULL};
+ const valueTrackingRef = valueHeader & ${MapFlags.TRACKING_REF};
+
+ for (let index = 0; index < chunkSize; index++) {
+ let key;
+ let value;
+ let flag = 0;
+ if (keyTrackingRef || keyIncludeNone) {
+ flag = ${this.builder.reader.uint8()};
+ }
+ switch (flag) {
+ case ${RefFlags.RefValueFlag}:
+ ${keyGenerator.toReadEmbed(x => `key = ${x}`, true,
RefState.fromTrue())}
+ break;
+ case ${RefFlags.RefFlag}:
+ key =
${this.builder.referenceResolver.getReadObject(this.builder.reader.varInt32())}
+ break;
+ case ${RefFlags.NullFlag}:
+ key = null;
+ break;
+ case ${RefFlags.NotNullValueFlag}:
+ ${keyGenerator.toReadEmbed(x => `key = ${x}`, true,
RefState.fromFalse())}
+ break;
+ }
+ flag = 0;
+ if (valueTrackingRef || valueIncludeNone) {
+ flag = ${this.builder.reader.uint8()};
+ }
+ switch (flag) {
+ case ${RefFlags.RefValueFlag}:
+ ${valueGenerator.toReadEmbed(x => `value = ${x}`, true,
RefState.fromTrue())}
+ break;
+ case ${RefFlags.RefFlag}:
+ value =
${this.builder.referenceResolver.getReadObject(this.builder.reader.varInt32())}
+ break;
+ case ${RefFlags.NullFlag}:
+ value = null;
+ break;
+ case ${RefFlags.NotNullValueFlag}:
+ ${valueGenerator.toReadEmbed(x => `value = ${x}`, true,
RefState.fromFalse())}
+ break;
+ }
+ ${result}.set(
+ key,
+ value
+ );
+ ${count}--;
+ }
+ }
+ ${accessor(result)}
`;
}
readStmt(accessor: (expr: string) => string, refState: RefState): string {
const anySerializer = this.builder.getExternal(MapAnySerializer.name);
const [keyMeta, valueMeta] = this.innerMeta();
- return accessor(`new (${anySerializer})(${this.builder.furyName()},
${keyMeta.typeId}, ${valueMeta.typeId}).read(${refState.toConditionExpr()})
- `);
+ if (!this.isAny()) {
+ return this.readStmtSpecificType(accessor, refState);
+ }
+ return accessor(`new (${anySerializer})(${this.builder.furyName()},
${keyMeta.typeId}, ${valueMeta.typeId}).read(${refState.toConditionExpr()})`);
}
}
diff --git a/javascript/test/map.test.ts b/javascript/test/map.test.ts
index 3a6fbb01..b85044c1 100644
--- a/javascript/test/map.test.ts
+++ b/javascript/test/map.test.ts
@@ -17,7 +17,7 @@
* under the License.
*/
-import Fury, { TypeDescription, InternalSerializerType } from
'../packages/fury/index';
+import Fury, { TypeDescription, InternalSerializerType, Type } from
'../packages/fury/index';
import {describe, expect, test} from '@jest/globals';
describe('map', () => {
@@ -30,6 +30,19 @@ describe('map', () => {
);
expect(result).toEqual(new Map([["foo", "bar"],["foo2", "bar2"]]))
});
+
+ test('should map specific type work', () => {
+
+ const fury = new Fury({ refTracking: true });
+ const { serialize, deserialize } =
fury.registerSerializer(Type.object("class.foo", {
+ f1: Type.map(Type.string(), Type.varInt32())
+ }))
+ const bin = serialize({
+ f1: new Map([["hello", 123], ["world", 456]]),
+ })
+ const result = deserialize(bin);
+ expect(result).toEqual({ f1: new Map([["hello", 123],["world", 456]])})
+ });
});
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
