This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-fury.git
The following commit(s) were added to refs/heads/main by this push:
new da848b5b chore: Fury header add language field (#1612)
da848b5b is described below
commit da848b5bd81c8c0a1c1745211d62d65e41e9050a
Author: LiangliangSui <[email protected]>
AuthorDate: Wed May 8 23:50:57 2024 +0800
chore: Fury header add language field (#1612)
## What does this PR do?
Add serialization language field for Fury Header in xlang_spec.
## Related issues
https://github.com/apache/incubator-fury/issues/1607
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [x] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---------
Signed-off-by: LiangliangSui <[email protected]>
---
docs/specification/xlang_serialization_spec.md | 7 ++++---
go/fury/fury.go | 3 +++
java/fury-core/src/main/java/org/apache/fury/config/Language.java | 2 ++
javascript/packages/fury/lib/fury.ts | 2 +-
javascript/packages/fury/lib/type.ts | 2 ++
python/pyfury/_fury.py | 2 ++
rust/fury/src/deserializer.rs | 5 +----
rust/fury/src/serializer.rs | 2 +-
8 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/docs/specification/xlang_serialization_spec.md
b/docs/specification/xlang_serialization_spec.md
index 7b46b556..806ee918 100644
--- a/docs/specification/xlang_serialization_spec.md
+++ b/docs/specification/xlang_serialization_spec.md
@@ -139,9 +139,9 @@ Fury will write the byte order for that object into the
data instead of converti
Fury header consists starts one byte:
```
-| 2 bytes | 4 bits | 1 bit | 1 bit | 1 bit | 1 bit |
optional 4 bytes |
-+--------------+---------------+-------+-------+--------+-------+------------------------------------+
-| magic number | reserved bits | oob | xlang | endian | null | unsigned int
for meta start offset |
+| 2 bytes | 4 bits | 1 bit | 1 bit | 1 bit | 1 bit | 1 byte |
optional 4 bytes |
++--------------+---------------+-------+-------+--------+-------+------------+------------------------------------+
+| magic number | reserved bits | oob | xlang | endian | null | language |
unsigned int for meta start offset |
```
- magic number: used to identify fury serialization protocol, current version
use `0x62d4`.
@@ -149,6 +149,7 @@ Fury header consists starts one byte:
- endian flag: 1 when data is encoded by little endian, 0 for big endian.
- xlang flag: 1 when serialization uses xlang format, 0 when serialization
uses Fury java format.
- oob flag: 1 when passed `BufferCallback` is not null, 0 otherwise.
+- language: the language when serializing objects, such as JAVA, PYTHON, GO,
etc. Fury can use this flag to determine whether spend more time on
serialization to make the deserialization faster for dynamic languages.
If meta share mode is enabled, an uncompressed unsigned int is appended to
indicate the start offset of metadata.
diff --git a/go/fury/fury.go b/go/fury/fury.go
index 5d1e401d..099dcd99 100644
--- a/go/fury/fury.go
+++ b/go/fury/fury.go
@@ -80,7 +80,10 @@ const (
XLANG Language = iota
JAVA
PYTHON
+ CPP
GO
+ JAVASCRIPT
+ RUST
)
const (
diff --git a/java/fury-core/src/main/java/org/apache/fury/config/Language.java
b/java/fury-core/src/main/java/org/apache/fury/config/Language.java
index 61f7bb53..661df39d 100644
--- a/java/fury-core/src/main/java/org/apache/fury/config/Language.java
+++ b/java/fury-core/src/main/java/org/apache/fury/config/Language.java
@@ -26,4 +26,6 @@ public enum Language {
PYTHON,
CPP,
GO,
+ JAVASCRIPT,
+ RUST,
}
diff --git a/javascript/packages/fury/lib/fury.ts
b/javascript/packages/fury/lib/fury.ts
index a7055aaf..4587f7ff 100644
--- a/javascript/packages/fury/lib/fury.ts
+++ b/javascript/packages/fury/lib/fury.ts
@@ -111,7 +111,7 @@ export default class {
bitmap |= ConfigFlags.isCrossLanguageFlag;
this.binaryWriter.int16(MAGIC_NUMBER);
this.binaryWriter.uint8(bitmap);
- this.binaryWriter.uint8(Language.XLANG);
+ this.binaryWriter.uint8(Language.JAVASCRIPT);
const cursor = this.binaryWriter.getCursor();
this.binaryWriter.skip(4); // preserve 4-byte for nativeObjects start
offsets.
this.binaryWriter.uint32(0); // nativeObjects length.
diff --git a/javascript/packages/fury/lib/type.ts
b/javascript/packages/fury/lib/type.ts
index d7746d55..9fac1f59 100644
--- a/javascript/packages/fury/lib/type.ts
+++ b/javascript/packages/fury/lib/type.ts
@@ -115,6 +115,8 @@ export enum Language {
PYTHON = 2,
CPP = 3,
GO = 4,
+ JAVASCRIPT = 5,
+ RUST = 6,
}
export const MAGIC_NUMBER = 0x62D4;
diff --git a/python/pyfury/_fury.py b/python/pyfury/_fury.py
index 8afc1937..6552fe36 100644
--- a/python/pyfury/_fury.py
+++ b/python/pyfury/_fury.py
@@ -586,6 +586,8 @@ class Language(enum.Enum):
PYTHON = 2
CPP = 3
GO = 4
+ JAVA_SCRIPT = 5
+ RUST = 6
@dataclass
diff --git a/rust/fury/src/deserializer.rs b/rust/fury/src/deserializer.rs
index 7bba635c..ac443ea3 100644
--- a/rust/fury/src/deserializer.rs
+++ b/rust/fury/src/deserializer.rs
@@ -241,10 +241,7 @@ impl<'de, 'bf: 'de> DeserializerState<'de, 'bf> {
fn head(&mut self) -> Result<(), Error> {
let _bitmap = self.reader.u8();
- let language: Language = self.reader.u8().try_into()?;
- if Language::XLANG != language {
- return Err(Error::UnsupportLanguage { language });
- }
+ let _language: Language = self.reader.u8().try_into()?;
self.reader.skip(8); // native offset and size
Ok(())
}
diff --git a/rust/fury/src/serializer.rs b/rust/fury/src/serializer.rs
index e1e982e9..fe5c7e75 100644
--- a/rust/fury/src/serializer.rs
+++ b/rust/fury/src/serializer.rs
@@ -347,7 +347,7 @@ impl<'de> SerializerState<'de> {
bitmap |= config_flags::IS_LITTLE_ENDIAN_FLAG;
bitmap |= config_flags::IS_CROSS_LANGUAGE_FLAG;
self.writer.u8(bitmap);
- self.writer.u8(Language::XLANG as u8);
+ self.writer.u8(Language::RUST as u8);
self.writer.skip(4); // native offset
self.writer.skip(4); // native size
self
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
