chaokunyang commented on code in PR #1549:
URL: https://github.com/apache/incubator-fury/pull/1549#discussion_r1573600177
##########
javascript/packages/fury/lib/gen/map.ts:
##########
@@ -21,77 +21,237 @@ import { MapTypeDescription, TypeDescription } from
"../description";
import { CodecBuilder } from "./builder";
import { BaseSerializerGenerator, RefState } from "./serializer";
import { CodegenRegistry } from "./router";
-import { InternalSerializerType } from "../type";
+import { InternalSerializerType, RefFlags, Serializer } from "../type";
import { Scope } from "./scope";
+import Fury from "../fury";
-class MapSerializerGenerator extends BaseSerializerGenerator {
- description: MapTypeDescription;
+const MapFlags = {
+ /** Whether track elements ref. */
+ TRACKING_REF: 0b1,
+
+ /** Whether collection has null. */
+ HAS_NULL: 0b10,
+
+ /** Whether collection elements type is not declare type. */
+ NOT_DECL_ELEMENT_TYPE: 0b100,
+
+ /** Whether collection elements type different. */
+ NOT_SAME_TYPE: 0b1000,
+};
+
+class MapChunkWriter {
+ private kFlag = 0;
+ private vFlag = 0;
+
+ private chunkSize = 0;
+ private chunkOffset = 0;
+ private header = 0;
+
+ constructor(private fury: Fury) {
- constructor(description: TypeDescription, builder: CodecBuilder, scope:
Scope) {
- super(description, builder, scope);
- this.description = <MapTypeDescription>description;
}
- private innerMeta() {
- const key = this.description.options.key;
- const value = this.description.options.value;
- return [this.builder.meta(key), this.builder.meta(value)];
+ private getHead(keyFlag: number, valueFlag: number) {
+ let flag = 0;
+ if (keyFlag & 0b10) {
+ flag |= MapFlags.HAS_NULL;
+ }
+ if (keyFlag & 0b01) {
+ flag |= MapFlags.TRACKING_REF;
+ }
+ flag <<= 4;
+ if (valueFlag & 0b10) {
+ flag |= MapFlags.HAS_NULL;
+ }
+ if (valueFlag & 0b01) {
+ flag |= MapFlags.TRACKING_REF;
+ }
+ return flag;
+ }
+
+ private writeHead(kFlag: number, vFlag: number) {
+ // KV header
+ const header = this.getHead(kFlag, vFlag);
+ this.fury.binaryWriter.uint8(header);
+ // chunkSize, max 255
+ this.chunkOffset = this.fury.binaryWriter.getCursor();
+ this.fury.binaryWriter.uint8(0);
+ this.fury.binaryWriter.uint32((kFlag >> 16) | (vFlag & 0xFFFF0000));
+ return header;
}
- private innerGenerator() {
- const key = this.description.options.key;
- const value = this.description.options.value;
+ makeFlag(typeId: number, isNull: 0 | 1, trackRef: 0 | 1) {
+ return typeId << 16 | isNull << 1 | trackRef;
+ }
- const KeyGeneratorClass = CodegenRegistry.get(key.type);
- const ValueGeneratorClass = CodegenRegistry.get(value.type);
- if (!KeyGeneratorClass) {
- throw new Error(`${key.type} generator not exists`);
+ next(kFlag: number, vFlag: number) {
+ // max size of chunk is 255
+ if (this.chunkSize == 255
+ || this.chunkOffset == 0
+ || this.kFlag !== kFlag
+ || this.vFlag !== vFlag
+ ) {
+ // new chunk
+ this.endChunk();
+ this.chunkSize++;
+ this.kFlag = kFlag;
+ this.vFlag = vFlag;
+ return this.header = this.writeHead(kFlag, vFlag);
}
- if (!ValueGeneratorClass) {
- throw new Error(`${value.type} generator not exists`);
+ this.chunkSize++;
+ return this.header;
+ }
+
+ endChunk() {
+ if (this.chunkOffset > 0) {
+ this.fury.binaryWriter.setUint8Position(this.chunkOffset,
this.chunkSize);
+ this.chunkSize = 0;
}
- return [new KeyGeneratorClass(key, this.builder, this.scope), new
ValueGeneratorClass(value, this.builder, this.scope)];
}
+}
- writeStmt(accessor: string): string {
- const [keyMeta, valueMeta] = this.innerMeta();
- const [keyGenerator, valueGenerator] = this.innerGenerator();
- const key = this.scope.uniqueName("key");
- const value = this.scope.uniqueName("value");
+class MapAnySerializer {
+ private keySerializer: Serializer | null = null;
+ private valueSerializer: Serializer | null = null;
- return `
- ${this.builder.writer.varUInt32(`${accessor}.size`)}
- ${this.builder.writer.reserve(`${keyMeta.fixedSize +
valueMeta.fixedSize} * ${accessor}.size`)};
- for (const [${key}, ${value}] of ${accessor}.entries()) {
- ${keyGenerator.toWriteEmbed(key)}
- ${valueGenerator.toWriteEmbed(value)}
- }
- `;
+ constructor(private fury: Fury, keySerializerId: null | number,
valueSerializerId: null | number) {
+ if (keySerializerId !== null) {
+ fury.classResolver.getSerializerById(keySerializerId);
+ }
+ if (valueSerializerId !== null) {
+ fury.classResolver.getSerializerById(valueSerializerId);
+ }
}
- readStmt(accessor: (expr: string) => string, refState: RefState): string {
- const [keyGenerator, valueGenerator] = this.innerGenerator();
- const key = this.scope.uniqueName("key");
- const value = this.scope.uniqueName("value");
+ private writeHead(header: number, v: any) {
+ if (header !== 0) {
+ if (header & MapFlags.HAS_NULL) {
+ if (v === null || v === undefined) {
+ this.fury.binaryWriter.uint8(RefFlags.NullFlag);
+ }
+ }
+ if (header & MapFlags.TRACKING_REF) {
+ const keyRef = this.fury.referenceResolver.existsWriteObject(v);
+ if (keyRef !== undefined) {
+ this.fury.binaryWriter.uint8(RefFlags.RefFlag);
+ this.fury.binaryWriter.uint16(keyRef);
+ } else {
+ this.fury.binaryWriter.uint8(RefFlags.RefValueFlag);
+ }
+ } else {
+ this.fury.binaryWriter.uint8(RefFlags.NotNullValueFlag);
+ }
+ }
+ }
+
+ write(value: Map<any, any>) {
+ const mapChunkWriter = new MapChunkWriter(this.fury);
+ this.fury.binaryWriter.varInt32(value.size);
+ for (const [k, v] of value.entries()) {
+ const keySerializer = this.keySerializer !== null ? this.keySerializer :
this.fury.classResolver.getSerializerByData(k);
+ const valueSerializer = this.valueSerializer !== null ?
this.valueSerializer : this.fury.classResolver.getSerializerByData(v);
- const result = this.scope.uniqueName("result");
- const idx = this.scope.uniqueName("idx");
- const len = this.scope.uniqueName("len");
+ const header = mapChunkWriter.next(
+ mapChunkWriter.makeFlag(keySerializer!.meta.typeId!, k == null ? 1 :
0, keySerializer!.meta.needToWriteRef ? 1 : 0),
+ mapChunkWriter.makeFlag(valueSerializer!.meta.typeId!, v == null ? 1 :
0, valueSerializer!.meta.needToWriteRef ? 1 : 0)
+ );
+ this.writeHead(header >> 4, k);
+ keySerializer!.writeInner(k);
+ this.writeHead(header & 0b00001111, v);
+ valueSerializer!.writeInner(v);
+ }
+ mapChunkWriter.endChunk();
+ }
+
+ private readElement(header: number, serializer: Serializer | null) {
+ if (header === 0) {
+ return serializer!.readInner(false);
+ }
+ const isSame = !(header & MapFlags.NOT_SAME_TYPE);
+ const includeNone = header & MapFlags.HAS_NULL;
+ const trackingRef = header & MapFlags.TRACKING_REF;
+
+ let flag = 0;
+ if (trackingRef || includeNone) {
+ flag = this.fury.binaryReader.uint8();
+ }
+ if (!isSame) {
+ serializer =
this.fury.classResolver.getSerializerByType(this.fury.binaryReader.int16());
+ }
+ switch (flag) {
+ case RefFlags.RefValueFlag:
+ return serializer!.readInner(true);
+ case RefFlags.RefFlag:
+ return
this.fury.referenceResolver.getReadObject(this.fury.binaryReader.varUInt32());
+ case RefFlags.NullFlag:
+ return null;
+ case RefFlags.NotNullValueFlag:
+ return serializer!.readInner(false);
+ }
+ }
+
+ read(fromRef: boolean): any {
+ let count = this.fury.binaryReader.varInt32();
+ const result = new Map();
+ if (fromRef) {
+ this.fury.referenceResolver.reference(result);
+ }
+ while (count > 0) {
+ const header = this.fury.binaryReader.uint16();
Review Comment:
Do we need to rename it as `read/writeXXX` in future?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
