This is an automated email from the ASF dual-hosted git repository. xiazcy pushed a commit to branch offsetdatetime_ser_backport in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit a54635e2f6d326bc15e8e5082cf5b68de430b948 Author: xiazcy <[email protected]> AuthorDate: Sun Jul 27 21:26:59 2025 -0700 Backport OffsetDateTime serializers in GLVs to enable deserialization from server. Date remains the default serializer for GLV native date types. --- CHANGELOG.asciidoc | 1 + .../Structure/IO/GraphBinary/DataType.cs | 1 + .../IO/GraphBinary/TypeSerializerRegistry.cs | 1 + .../GraphBinary/Types/OffsetDateTimeSerializer.cs | 80 +++++++++++ .../Structure/IO/GraphSON/GraphSONReader.cs | 1 + .../IO/GraphSON/OffsetDateTimeDeserializer.cs | 37 +++++ .../IO/GraphSON/OffsetDateTimeSerializer.cs | 37 +++++ .../Structure/IO/GraphSON/GraphSONReaderTests.cs | 13 ++ gremlin-go/driver/graphBinary.go | 72 ++++++++++ gremlin-go/driver/graphBinary_test.go | 33 +++++ gremlin-go/driver/serializer.go | 8 +- .../lib/structure/io/binary/GraphBinary.js | 1 + .../binary/internals/OffsetDateTimeSerializer.js | 151 +++++++++++++++++++++ .../lib/structure/io/graph-serializer.js | 1 + .../lib/structure/io/type-serializers.js | 8 ++ .../test/unit/graphbinary/AnySerializer-test.js | 9 ++ .../gremlin-javascript/test/unit/graphson-test.js | 6 + .../gremlin_python/structure/io/graphbinaryV1.py | 39 +++++- .../gremlin_python/structure/io/graphsonV2d0.py | 17 +++ .../gremlin_python/structure/io/graphsonV3d0.py | 17 +++ .../tests/structure/io/test_graphbinaryV1.py | 27 +++- .../python/tests/structure/io/test_graphsonV2d0.py | 24 +++- .../python/tests/structure/io/test_graphsonV3d0.py | 24 +++- 23 files changed, 593 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 069d52abff..0d01792b1c 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -55,6 +55,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima * Support hot reloading of SSL certificates. * Increase default `max_content_length`/`max_msg_size` in `gremlin-python` from 4MB to 10MB. * Added the `PopContaining` interface designed to get label and `Pop` combinations held in a `PopInstruction` object. +* Backport `OffsetDateTime` serializers from 4.0.0-beta for deserialization. Note `Date` remains the default serializer for GLV native date types. * Fixed bug preventing a vertex from being dropped and then re-added in the same `TinkerTransaction` [[release-3-7-3]] diff --git a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/DataType.cs b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/DataType.cs index 6ccad02937..161db1b1af 100644 --- a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/DataType.cs +++ b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/DataType.cs @@ -78,6 +78,7 @@ namespace Gremlin.Net.Structure.IO.GraphBinary // TODO: Support metrics and traversal metrics public static readonly DataType Char = new DataType(0x80); public static readonly DataType Duration = new DataType(0x81); + public static readonly DataType OffsetDateTime = new DataType(0x88); #pragma warning restore 1591 /// <summary> diff --git a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/TypeSerializerRegistry.cs b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/TypeSerializerRegistry.cs index e7f1997771..c51a1aceaf 100644 --- a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/TypeSerializerRegistry.cs +++ b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/TypeSerializerRegistry.cs @@ -133,6 +133,7 @@ namespace Gremlin.Net.Structure.IO.GraphBinary {DataType.BulkSet, new BulkSetSerializer<List<object>>()}, {DataType.Char, new CharSerializer()}, {DataType.Duration, new DurationSerializer()}, + {DataType.OffsetDateTime, new OffsetDateTimeSerializer()}, }; private readonly Dictionary<string, CustomTypeSerializer> _serializerByCustomTypeName = diff --git a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/Types/OffsetDateTimeSerializer.cs b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/Types/OffsetDateTimeSerializer.cs new file mode 100644 index 0000000000..a8b1fa6c1f --- /dev/null +++ b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphBinary/Types/OffsetDateTimeSerializer.cs @@ -0,0 +1,80 @@ +#region License + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#endregion + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Gremlin.Net.Structure.IO.GraphBinary.Types +{ + /// <summary> + /// A serializer for the GraphBinary type OffsetDateTime, represented as <see cref="DateTimeOffset"/> + /// in .NET. + /// </summary> + public class OffsetDateTimeSerializer : SimpleTypeSerializer<DateTimeOffset> + { + + /// <summary> + /// Initializes a new instance of the <see cref="OffsetDateTimeSerializer" /> class. + /// </summary> + public OffsetDateTimeSerializer() : base(DataType.OffsetDateTime) + { + } + + /// <inheritdoc /> + protected override async Task WriteValueAsync(DateTimeOffset value, Stream stream, GraphBinaryWriter writer, + CancellationToken cancellationToken = default) + { + await stream.WriteIntAsync(value.Year, cancellationToken).ConfigureAwait(false); + await stream.WriteByteAsync(Convert.ToByte(value.Month), cancellationToken).ConfigureAwait(false); + await stream.WriteByteAsync(Convert.ToByte(value.Day), cancellationToken).ConfigureAwait(false); + // Note that nanosecond precisions were added after .NET 7 + // Get the time of day as TimeSpan + var timeOfDay = value.TimeOfDay; + // Convert ticks to nanoseconds (1 tick = 100 nanoseconds) + var ns = timeOfDay.Ticks * 100; + await stream.WriteLongAsync(Convert.ToInt64(ns), cancellationToken).ConfigureAwait(false); + + var offset = value.Offset; + var os = offset.Hours * 60 * 60 + offset.Minutes * 60 + offset.Seconds; + await stream.WriteIntAsync(os, cancellationToken).ConfigureAwait(false); + } + + /// <inheritdoc /> + protected override async Task<DateTimeOffset> ReadValueAsync(Stream stream, GraphBinaryReader reader, + CancellationToken cancellationToken = default) + { + var year = await stream.ReadIntAsync(cancellationToken).ConfigureAwait(false); + var month = await stream.ReadByteAsync(cancellationToken).ConfigureAwait(false); + var day = await stream.ReadByteAsync(cancellationToken).ConfigureAwait(false); + var ns = await stream.ReadLongAsync(cancellationToken).ConfigureAwait(false); + var timeDelta = TimeSpan.FromMilliseconds(ns / 1e6); + + var os = await stream.ReadIntAsync(cancellationToken).ConfigureAwait(false); + var offset = TimeSpan.FromSeconds(os); + + return new DateTimeOffset(year, Convert.ToInt32(month), Convert.ToInt32(day), 0, 0, 0, offset).Add(timeDelta); + } + } +} \ No newline at end of file diff --git a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/GraphSONReader.cs b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/GraphSONReader.cs index 209c0561f4..779c5594a1 100644 --- a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/GraphSONReader.cs +++ b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/GraphSONReader.cs @@ -58,6 +58,7 @@ namespace Gremlin.Net.Structure.IO.GraphSON { "g:T", new TDeserializer() }, //Extended + { "gx:OffsetDateTime", new OffsetDateTimeDeserializer() }, { "gx:BigDecimal", new DecimalConverter() }, { "gx:Duration", new DurationDeserializer() }, { "gx:BigInteger", new BigIntegerDeserializer() }, diff --git a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/OffsetDateTimeDeserializer.cs b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/OffsetDateTimeDeserializer.cs new file mode 100644 index 0000000000..9840fce0ad --- /dev/null +++ b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/OffsetDateTimeDeserializer.cs @@ -0,0 +1,37 @@ +#region License + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#endregion + +using System; +using System.IO; +using System.Text.Json; + +namespace Gremlin.Net.Structure.IO.GraphSON +{ + internal class OffsetDateTimeDeserializer : IGraphSONDeserializer + { + public dynamic Objectify(JsonElement graphsonObject, GraphSONReader reader) + { + return DateTimeOffset.Parse(graphsonObject.GetString() ?? + throw new IOException("Read null but expected a OffsetDateTime value")); + } + } +} \ No newline at end of file diff --git a/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/OffsetDateTimeSerializer.cs b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/OffsetDateTimeSerializer.cs new file mode 100644 index 0000000000..05de9c4164 --- /dev/null +++ b/gremlin-dotnet/src/Gremlin.Net/Structure/IO/GraphSON/OffsetDateTimeSerializer.cs @@ -0,0 +1,37 @@ +#region License + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#endregion + +using System; +using System.Collections.Generic; + +namespace Gremlin.Net.Structure.IO.GraphSON +{ + internal class OffsetDateTimeSerializer : IGraphSONSerializer + { + public Dictionary<string, dynamic> Dictify(dynamic objectData, GraphSONWriter writer) + { + DateTimeOffset value = objectData; + return GraphSONUtil.ToTypedValue("OffsetDateTime", value.ToString("O"), "gx"); + } + } +} \ No newline at end of file diff --git a/gremlin-dotnet/test/Gremlin.Net.UnitTest/Structure/IO/GraphSON/GraphSONReaderTests.cs b/gremlin-dotnet/test/Gremlin.Net.UnitTest/Structure/IO/GraphSON/GraphSONReaderTests.cs index 591e3b4378..8921e68218 100644 --- a/gremlin-dotnet/test/Gremlin.Net.UnitTest/Structure/IO/GraphSON/GraphSONReaderTests.cs +++ b/gremlin-dotnet/test/Gremlin.Net.UnitTest/Structure/IO/GraphSON/GraphSONReaderTests.cs @@ -108,6 +108,19 @@ namespace Gremlin.Net.UnitTest.Structure.IO.GraphSON Assert.Equal(expectedDateTimeOffset, deserializedValue); } + [Theory, MemberData(nameof(Versions))] + public void ShouldDeserializeOffsetDateTimeToDateTimeOffset(int version) + { + const string graphSon = "{\"@type\":\"gx:OffsetDateTime\",\"@value\":\"2016-10-04T12:17:22.5520000+00:00\"}"; + var reader = CreateStandardGraphSONReader(version); + + var jsonElement = JsonSerializer.Deserialize<JsonElement>(graphSon); + var deserializedValue = reader.ToObject(jsonElement); + + var expectedDateTimeOffset = TestUtils.FromJavaTime(1475583442552); + Assert.Equal(expectedDateTimeOffset, deserializedValue); + } + [Theory, MemberData(nameof(Versions))] public void ShouldDeserializeDictionary(int version) { diff --git a/gremlin-go/driver/graphBinary.go b/gremlin-go/driver/graphBinary.go index 39c6d2f029..0baa9ea773 100644 --- a/gremlin-go/driver/graphBinary.go +++ b/gremlin-go/driver/graphBinary.go @@ -85,6 +85,7 @@ const ( metricsType dataType = 0x2c traversalMetricsType dataType = 0x2d durationType dataType = 0x81 + offsetDateTimeType dataType = 0x88 nullType dataType = 0xFE ) @@ -509,6 +510,40 @@ func timeWriter(value interface{}, buffer *bytes.Buffer, _ *graphBinaryTypeSeria return buffer.Bytes(), nil } +// Datetime remains serialized as Date by default, real use is the ability to deserialize OffsetDateTime +func offsetDateTimeWriter(value interface{}, buffer *bytes.Buffer, _ *graphBinaryTypeSerializer) ([]byte, error) { + t := value.(time.Time) + err := binary.Write(buffer, binary.BigEndian, int32(t.Year())) + if err != nil { + return nil, err + } + + err = binary.Write(buffer, binary.BigEndian, byte(t.Month())) + if err != nil { + return nil, err + } + err = binary.Write(buffer, binary.BigEndian, byte(t.Day())) + if err != nil { + return nil, err + } + // construct time of day in nanoseconds + h := int64(t.Hour()) + m := int64(t.Minute()) + s := int64(t.Second()) + ns := (h * 60 * 60 * 1e9) + (m * 60 * 1e9) + (s * 1e9) + int64(t.Nanosecond()) + err = binary.Write(buffer, binary.BigEndian, ns) + if err != nil { + return nil, err + } + _, os := t.Zone() + err = binary.Write(buffer, binary.BigEndian, int32(os)) + if err != nil { + return nil, err + } + + return buffer.Bytes(), nil +} + func durationWriter(value interface{}, buffer *bytes.Buffer, _ *graphBinaryTypeSerializer) ([]byte, error) { t := value.(time.Duration) sec := int64(t / time.Second) @@ -1044,6 +1079,43 @@ func timeReader(data *[]byte, i *int) (interface{}, error) { return time.UnixMilli(readLongSafe(data, i)), nil } +func offsetDateTimeReader(data *[]byte, i *int) (interface{}, error) { + year := readIntSafe(data, i) + month := readByteSafe(data, i) + day := readByteSafe(data, i) + totalNS := readLongSafe(data, i) + // calculate hour, minute, second, and ns from totalNS (int64) to prevent int overflow in the nanoseconds arg + ns := totalNS % 1e9 + totalS := totalNS / 1e9 + s := totalS % 60 + totalM := totalS / 60 + m := totalM % 60 + h := totalM / 60 + + offset := readIntSafe(data, i) + datetime := time.Date(int(year), time.Month(month), int(day), int(h), int(m), int(s), int(ns), GetTimezoneFromOffset(int(offset))) + return datetime, nil +} + +// GetTimezoneFromOffset is a helper function to convert an offset in seconds to a time.Location +func GetTimezoneFromOffset(offsetSeconds int) *time.Location { + // calculate hours and minutes from seconds + hours := offsetSeconds / 3600 + minutes := (offsetSeconds % 3600) / 60 + + // format the timezone name in the format that go expects + // for example: "UTC+01:00" or "UTC-05:30" + sign := "+" + if hours < 0 { + sign = "-" + hours = -hours + minutes = -minutes + } + tzName := fmt.Sprintf("UTC%s%02d:%02d", sign, hours, minutes) + + return time.FixedZone(tzName, offsetSeconds) +} + func durationReader(data *[]byte, i *int) (interface{}, error) { return time.Duration(readLongSafe(data, i)*int64(time.Second) + int64(readIntSafe(data, i))), nil } diff --git a/gremlin-go/driver/graphBinary_test.go b/gremlin-go/driver/graphBinary_test.go index d8c66d2816..1cf7344270 100644 --- a/gremlin-go/driver/graphBinary_test.go +++ b/gremlin-go/driver/graphBinary_test.go @@ -312,6 +312,39 @@ func TestGraphBinaryV1(t *testing.T) { assert.Nil(t, err) assert.Equal(t, source, res) }) + t.Run("read-write local datetime", func(t *testing.T) { + pos := 0 + var buffer bytes.Buffer + source := time.Date(2022, 5, 10, 9, 51, 0, 0, time.Local) + buf, err := offsetDateTimeWriter(source, &buffer, nil) + assert.Nil(t, err) + res, err := offsetDateTimeReader(&buf, &pos) + assert.Nil(t, err) + // ISO format + assert.Equal(t, source.Format(time.RFC3339Nano), res.(time.Time).Format(time.RFC3339Nano)) + }) + t.Run("read-write UTC datetime", func(t *testing.T) { + pos := 0 + var buffer bytes.Buffer + source := time.Date(2022, 5, 10, 9, 51, 0, 0, time.UTC) + buf, err := offsetDateTimeWriter(source, &buffer, nil) + assert.Nil(t, err) + res, err := offsetDateTimeReader(&buf, &pos) + assert.Nil(t, err) + // ISO format + assert.Equal(t, source.Format(time.RFC3339Nano), res.(time.Time).Format(time.RFC3339Nano)) + }) + t.Run("read-write HST datetime", func(t *testing.T) { + pos := 0 + var buffer bytes.Buffer + source := time.Date(2022, 5, 10, 9, 51, 34, 123456789, GetTimezoneFromOffset(-36000)) + buf, err := offsetDateTimeWriter(source, &buffer, nil) + assert.Nil(t, err) + res, err := offsetDateTimeReader(&buf, &pos) + assert.Nil(t, err) + // ISO format + assert.Equal(t, source.Format(time.RFC3339Nano), res.(time.Time).Format(time.RFC3339Nano)) + }) }) t.Run("error handle tests", func(t *testing.T) { diff --git a/gremlin-go/driver/serializer.go b/gremlin-go/driver/serializer.go index 04a7e468b5..b7aa4a9750 100644 --- a/gremlin-go/driver/serializer.go +++ b/gremlin-go/driver/serializer.go @@ -265,6 +265,7 @@ func initSerializers() { setType: setWriter, dateType: timeWriter, durationType: durationWriter, + offsetDateTimeType: offsetDateTimeWriter, cardinalityType: enumWriter, columnType: enumWriter, directionType: enumWriter, @@ -310,9 +311,10 @@ func initDeserializers() { classType: readClass, // Date Time - dateType: timeReader, - timestampType: timeReader, - durationType: durationReader, + dateType: timeReader, + timestampType: timeReader, + offsetDateTimeType: offsetDateTimeReader, + durationType: durationReader, // Graph traverserType: traverserReader, diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/binary/GraphBinary.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/binary/GraphBinary.js index c5bf3c5720..1742454837 100644 --- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/binary/GraphBinary.js +++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/binary/GraphBinary.js @@ -72,6 +72,7 @@ ioc.longSerializer = new (require('./internals/LongSerializer'))(io ioc.longSerializerNg = new (require('./internals/LongSerializerNg'))(ioc); ioc.stringSerializer = new (require('./internals/StringSerializer'))(ioc, ioc.DataType.STRING); ioc.dateSerializer = new (require('./internals/DateSerializer'))(ioc, ioc.DataType.DATE); +ioc.offsetDateTimeSerializer = new (require('./internals/OffsetDateTimeSerializer'))(ioc, ioc.DataType.OFFSETDATETIME); ioc.timestampSerializer = new (require('./internals/DateSerializer'))(ioc, ioc.DataType.TIMESTAMP); ioc.classSerializer = new (require('./internals/StringSerializer'))(ioc, ioc.DataType.CLASS); ioc.doubleSerializer = new (require('./internals/DoubleSerializer'))(ioc); diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/binary/internals/OffsetDateTimeSerializer.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/binary/internals/OffsetDateTimeSerializer.js new file mode 100644 index 0000000000..30045ffd0f --- /dev/null +++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/binary/internals/OffsetDateTimeSerializer.js @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +'use strict'; + +const { Buffer } = require('buffer'); + +module.exports = class OffsetDateTimeSerializer { + constructor(ioc, ID) { + this.ioc = ioc; + this.ID = ID; + this.ioc.serializers[ID] = this; + } + + canBeUsedFor(value) { + return value instanceof Date; + } + + serialize(item, fullyQualifiedFormat = true) { + if (item === undefined || item === null) { + if (fullyQualifiedFormat) { + return Buffer.from([this.ID, 0x01]); + } + return Buffer.from([0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + } + + const bufs = []; + if (fullyQualifiedFormat) { + bufs.push(Buffer.from([this.ID, 0x00])); + } + + // NOTE: js Date will always display time in UTC, but regular date/hour getters will return in local system time. + // To avoid inconsistency we will always serialize the UTC representation of the Date object with offset of 0. + + // {year} + let v = Buffer.alloc(4); + v.writeInt32BE(item.getUTCFullYear()); + bufs.push(v); + + // {month} + v = Buffer.alloc(1); + v.writeUInt8(item.getUTCMonth() + 1); // Java Core DateTime serializer uses 1 - 12 for months, JS uses indices + bufs.push(v); + + // {day} - in UTC + v = Buffer.alloc(1); + v.writeUInt8(item.getUTCDate()); + bufs.push(v); + + // {nanoseconds} + const h = item.getUTCHours(); // in UTC + const m = item.getUTCMinutes(); + const s = item.getUTCSeconds(); + const ms = item.getUTCMilliseconds(); + const ns = h * 60 * 60 * 1e9 + m * 60 * 1e9 + s * 1e9 + ms * 1e6; + v = Buffer.alloc(8); + v.writeBigInt64BE(BigInt(ns)); + bufs.push(v); + + // {zone offset} - UTC is always used for serialization, as such offset will be 0 + v = Buffer.alloc(4); + v.writeInt32BE(0); + bufs.push(v); + + return Buffer.concat(bufs); + } + + deserialize(buffer, fullyQualifiedFormat = true) { + let len = 0; + let cursor = buffer; + + try { + if (buffer === undefined || buffer === null || !(buffer instanceof Buffer)) { + throw new Error('buffer is missing'); + } + if (buffer.length < 1) { + throw new Error('buffer is empty'); + } + + if (fullyQualifiedFormat) { + const type_code = cursor.readUInt8(); + len++; + if (type_code !== this.ID) { + throw new Error('unexpected {type_code}'); + } + cursor = cursor.slice(1); + + if (cursor.length < 1) { + throw new Error('{value_flag} is missing'); + } + const value_flag = cursor.readUInt8(); + len++; + if (value_flag === 1) { + return { v: null, len }; + } + if (value_flag !== 0) { + throw new Error('unexpected {value_flag}'); + } + cursor = cursor.slice(1); + } + + if (cursor.length < 8) { + throw new Error('unexpected {value} length'); + } + len += 18; + + const year = cursor.readInt32BE(); + cursor = cursor.slice(4); + const month = cursor.readUInt8() - 1; + cursor = cursor.slice(1); + const date = cursor.readUInt8(); + cursor = cursor.slice(1); + const ns = cursor.readBigInt64BE(); + cursor = cursor.slice(8); + const offset = cursor.readInt32BE(); + cursor.slice(4); + + // calculate hour, minute, second, and ms from ns as JS Date doesn't have ns precision + const totalMS = ns / BigInt(1e6); + const ms = Number(totalMS) % 1e3; + const totalS = Math.trunc(Number(totalMS) / 1e3) - offset; // js Date doesn't have a way to set offset properly, account offset here to use UTC + const s = totalS % 60; + const totalM = Math.trunc(totalS / 60); + const m = totalM % 60; + const h = Math.trunc(totalM / 60); + + // use UTC time calculated with offset above + const v = new Date(Date.UTC(year, month, date, h, m, s, ms)); + + return { v, len }; + } catch (err) { + throw this.ioc.utils.des_error({ serializer: this, args: arguments, cursor, err }); + } + } +}; diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/graph-serializer.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/graph-serializer.js index 71b1ba2dcc..5c92789df3 100644 --- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/graph-serializer.js +++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/graph-serializer.js @@ -253,6 +253,7 @@ const graphSON2Deserializers = { 'g:Float': typeSerializers.NumberSerializer, 'g:Double': typeSerializers.NumberSerializer, 'g:Date': typeSerializers.DateSerializer, + 'gx:OffsetDateTime': typeSerializers.OffsetDateTimeSerializer, 'g:Direction': typeSerializers.DirectionSerializer, 'g:Vertex': typeSerializers.VertexSerializer, 'g:Edge': typeSerializers.EdgeSerializer, diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/type-serializers.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/type-serializers.js index 3eefbaad82..0791c6d5cf 100644 --- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/type-serializers.js +++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/structure/io/type-serializers.js @@ -103,6 +103,13 @@ class DateSerializer extends TypeSerializer { } } +class OffsetDateTimeSerializer extends TypeSerializer { + // only deserialize gx:OffsetDateTime objects + deserialize(obj) { + return new Date(obj[valueKey]); + } +} + class LongSerializer extends TypeSerializer { serialize(item) { return { @@ -480,6 +487,7 @@ module.exports = { BulkSetSerializer, BytecodeSerializer, DateSerializer, + OffsetDateTimeSerializer, DirectionSerializer, EdgeSerializer, EnumSerializer, diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphbinary/AnySerializer-test.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphbinary/AnySerializer-test.js index b827ebb9d5..967bccca4f 100644 --- a/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphbinary/AnySerializer-test.js +++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphbinary/AnySerializer-test.js @@ -374,6 +374,15 @@ describe('GraphBinary.AnySerializer', () => { { v:null, b:[0x04,0x01] }, { v:new Date('1969-12-31T23:59:59.999Z'), b:[0x04,0x00, 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF] }, + // OFFSETDATETIME + { v:null, b:[0x88,0x01] }, + { v:new Date('2023-08-02T01:30:00-10:00'), + b:[0x88,0x00, + 0x00,0x00,0x07,0xe7, + 0x08,0x02, + 0x00,0x00,0x04,0xe9,0x49,0x14,0xf0,0x00, + 0xFF,0xFF,0x73,0x60] }, + // TIMESTAMP { v:null, b:[0x05,0x01] }, { v:new Date('1969-12-31T23:59:59.999Z'), b:[0x05,0x00, 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF] }, diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphson-test.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphson-test.js index cc7febc4d6..6b899adb9f 100644 --- a/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphson-test.js +++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/test/unit/graphson-test.js @@ -89,6 +89,12 @@ describe('GraphSONReader', function () { const result = reader.read(obj); assert.ok(result instanceof Date); }); + it('should parse OffsetDateTime', function() { + const obj = { "@type" : "gx:OffsetDateTime", "@value" : "2016-12-14T21:14:36.295Z" }; + const reader = new GraphSONReader(); + const result = reader.read(obj); + assert.ok(result instanceof Date); + }); it('should parse vertices from GraphSON', function () { const obj = { "@type":"g:Vertex", "@value":{"id":{"@type":"g:Int32","@value":1},"label":"person", diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py b/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py index 20228bcd1c..a38f4cc387 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py @@ -106,7 +106,7 @@ class DataType(Enum): localdatetime = 0x85 # todo localtime = 0x86 # todo monthday = 0x87 # todo - offsetdatetime = 0x88 # todo + offsetdatetime = 0x88 offsettime = 0x89 # todo period = 0x8a # todo year = 0x8b # todo @@ -343,6 +343,43 @@ class DateIO(_GraphBinaryTypeIO): nullable) +class OffsetDateTimeDeserializer(_GraphBinaryTypeIO): + # datetime remains serialized as Date by default, real use is the ability to deserialize OffsetDateTime + graphbinary_type = DataType.offsetdatetime + + @classmethod + def dictify(cls, obj, writer, to_extend, as_value=False, nullable=True): + if obj.tzinfo is None: + return DateIO.dictify(obj, writer, to_extend, as_value, nullable) + cls.prefix_bytes(cls.graphbinary_type, as_value, nullable, to_extend) + IntIO.dictify(obj.year, writer, to_extend, True, False) + ByteIO.dictify(obj.month, writer, to_extend, True, False) + ByteIO.dictify(obj.day, writer, to_extend, True, False) + # construct time of day in nanoseconds + h = obj.time().hour + m = obj.time().minute + s = obj.time().second + ms = obj.time().microsecond + ns = round((h*60*60*1e9) + (m*60*1e9) + (s*1e9) + (ms*1e3)) + LongIO.dictify(ns, writer, to_extend, True, False) + os = round(obj.utcoffset().total_seconds()) + IntIO.dictify(os, writer, to_extend, True, False) + return to_extend + + @classmethod + def objectify(cls, buff, reader, nullable=True): + return cls.is_null(buff, reader, cls._read_dt, nullable) + + @classmethod + def _read_dt(cls, b, r): + year = r.to_object(b, DataType.int, False) + month = r.to_object(b, DataType.byte, False) + day = r.to_object(b, DataType.byte, False) + ns = r.to_object(b, DataType.long, False) + offset = r.to_object(b, DataType.int, False) + tz = datetime.timezone(timedelta(seconds=offset)) + return datetime.datetime(year, month, day, tzinfo=tz) + timedelta(microseconds=ns/1000) + # Based on current implementation, this class must always be declared before FloatIO. # Seems pretty fragile for future maintainers. Maybe look into this. class TimestampIO(_GraphBinaryTypeIO): diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py index d4138b6c9d..fa053279a1 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py @@ -360,6 +360,23 @@ class DateIO(_GraphSONTypeIO): return datetime.datetime.utcfromtimestamp(ts / 1000.0) + class OffsetDateTimeIO(_GraphSONTypeIO): + graphson_type = "gx:OffsetDateTime" + graphson_base_type = "OffsetDateTime" + + @classmethod + def dictify(cls, obj, writer): + if obj.tzinfo is None: + return DateIO.dictify(obj, writer) + return GraphSONUtil.typed_value(cls.graphson_base_type, obj.isoformat(), "gx") + + @classmethod + def objectify(cls, dt, reader): + # specially handling as python isoformat does not support zulu until 3.11 + dt_iso = dt[:-1] + '+00:00' if dt.endswith('Z') else dt + return datetime.datetime.fromisoformat(dt_iso) + + # Based on current implementation, this class must always be declared before FloatIO. # Seems pretty fragile for future maintainers. Maybe look into this. class TimestampIO(_GraphSONTypeIO): diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py index 692e4d27b0..87425ba2af 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py @@ -367,6 +367,23 @@ class DateIO(_GraphSONTypeIO): return datetime.datetime.utcfromtimestamp(ts / 1000.0) +class OffsetDateTimeIO(_GraphSONTypeIO): + graphson_type = "gx:OffsetDateTime" + graphson_base_type = "OffsetDateTime" + + @classmethod + def dictify(cls, obj, writer): + if obj.tzinfo is None: + return DateIO.dictify(obj, writer) + return GraphSONUtil.typed_value(cls.graphson_base_type, obj.isoformat(), "gx") + + @classmethod + def objectify(cls, dt, reader): + # specially handling as python isoformat does not support zulu until 3.11 + dt_iso = dt[:-1] + '+00:00' if dt.endswith('Z') else dt + return datetime.datetime.fromisoformat(dt_iso) + + # Based on current implementation, this class must always be declared before FloatIO. # Seems pretty fragile for future maintainers. Maybe look into this. class TimestampIO(_GraphSONTypeIO): diff --git a/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py b/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py index 32201b78a9..e5f9172645 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py @@ -17,11 +17,11 @@ specific language governing permissions and limitations under the License. """ -import datetime import uuid import math -from gremlin_python.statics import timestamp, long, bigint, BigDecimal, SingleByte, SingleChar, ByteBufferType +from datetime import datetime, timedelta, timezone +from gremlin_python.statics import long, bigint, BigDecimal, SingleByte, SingleChar, ByteBufferType, timestamp from gremlin_python.structure.graph import Vertex, Edge, Property, VertexProperty, Path from gremlin_python.structure.io.graphbinaryV1 import GraphBinaryWriter, GraphBinaryReader from gremlin_python.process.traversal import Barrier, Binding, Bytecode, Merge, Direction @@ -31,7 +31,7 @@ class TestGraphBinaryReader(object): graphbinary_reader = GraphBinaryReader() -class TestGraphSONWriter(object): +class TestGraphBinaryWriter(object): graphbinary_writer = GraphBinaryWriter() graphbinary_reader = GraphBinaryReader() @@ -83,7 +83,7 @@ class TestGraphSONWriter(object): assert x.unscaled_value == output.unscaled_value def test_date(self): - x = datetime.datetime(2016, 12, 14, 16, 14, 36, 295000) + x = datetime(2016, 12, 14, 16, 14, 36, 295000) output = self.graphbinary_reader.read_object(self.graphbinary_writer.write_object(x)) assert x == output @@ -92,6 +92,23 @@ class TestGraphSONWriter(object): output = self.graphbinary_reader.read_object(self.graphbinary_writer.write_object(x)) assert x == output + def test_offsetdatetime(self): + tz = timezone(timedelta(seconds=36000)) + ms = 12345678912 + x = datetime(2022, 5, 20, tzinfo=tz) + timedelta(microseconds=ms) + output = self.graphbinary_reader.read_object(bytearray(b'\x88\x00\x00\x00\x07\xe6\x05\x14\x00\x00\x0b:s\xceZ\x00\x00\x00\x8c\xa0')) + assert x == output + + def test_offsetdatetime_format(self): + x = datetime.strptime('2022-05-20T03:25:45.678912Z', '%Y-%m-%dT%H:%M:%S.%f%z') + output = self.graphbinary_reader.read_object(bytearray(b'\x88\x00\x00\x00\x07\xe6\x05\x14\x00\x00\x0b:s\xceZ\x00\x00\x00\x00\x00')) + assert x == output + + def test_offsetdatetime_epoch(self): + x = datetime.fromtimestamp(1690934400).astimezone(timezone.utc) + output = self.graphbinary_reader.read_object(bytearray(b'\x88\x00\x00\x00\x07\xe7\x08\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')) + assert x == output + def test_string(self): x = "serialize this!" output = self.graphbinary_reader.read_object(self.graphbinary_writer.write_object(x)) @@ -233,6 +250,6 @@ class TestGraphSONWriter(object): assert x == output def test_duration(self): - x = datetime.timedelta(seconds=1000, microseconds=1000) + x = timedelta(seconds=1000, microseconds=1000) output = self.graphbinary_reader.read_object(self.graphbinary_writer.write_object(x)) assert x == output diff --git a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py index 4860350013..c453180491 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py @@ -249,10 +249,28 @@ class TestGraphSONReader: expected = datetime.datetime(2016, 12, 14, 16, 14, 36, 295000) pts = calendar.timegm(expected.utctimetuple()) + expected.microsecond / 1e6 ts = int(round(pts * 1000)) - dt = self.graphson_reader.read_object(json.dumps({"@type": "g:Date", "@value": ts})) - assert isinstance(dt, datetime.datetime) + output = self.graphson_reader.read_object(json.dumps({"@type": "g:Date", "@value": ts})) + assert isinstance(output, datetime.datetime) # TINKERPOP-1848 - assert dt == expected + assert expected == output + + def test_offsetdatetime(self): + tz = datetime.timezone(datetime.timedelta(seconds=36000)) + ms = 12345678912 + expected = datetime.datetime(2022, 5, 20, tzinfo=tz) + datetime.timedelta(microseconds=ms) + output = self.graphson_reader.read_object(json.dumps({"@type": "gx:OffsetDateTime", "@value": expected.isoformat()})) + assert isinstance(output, datetime.datetime) + assert expected == output + + def test_offsetdatetime_zulu(self): + tz = datetime.timezone.utc + ms = 12345678912 + expected = datetime.datetime(2022, 5, 20, tzinfo=tz) + datetime.timedelta(microseconds=ms) + # simulate zulu format + expected_zulu = expected.isoformat()[:-6] + 'Z' + output = self.graphson_reader.read_object(json.dumps({"@type": "gx:OffsetDateTime", "@value": expected_zulu})) + assert isinstance(output, datetime.datetime) + assert expected == output def test_timestamp(self): dt = self.graphson_reader.read_object(json.dumps({"@type": "g:Timestamp", "@value": 1481750076295})) diff --git a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py index a6a65a84e4..6025a0c674 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py @@ -295,10 +295,28 @@ class TestGraphSONReader: expected = datetime.datetime(2016, 12, 14, 16, 14, 36, 295000) pts = calendar.timegm(expected.utctimetuple()) + expected.microsecond / 1e6 ts = int(round(pts * 1000)) - dt = self.graphson_reader.read_object(json.dumps({"@type": "g:Date", "@value": ts})) - assert isinstance(dt, datetime.datetime) + output = self.graphson_reader.read_object(json.dumps({"@type": "g:Date", "@value": ts})) + assert isinstance(output, datetime.datetime) # TINKERPOP-1848 - assert dt == expected + assert expected == output + + def test_offsetdatetime(self): + tz = datetime.timezone(datetime.timedelta(seconds=36000)) + ms = 12345678912 + expected = datetime.datetime(2022, 5, 20, tzinfo=tz) + datetime.timedelta(microseconds=ms) + output = self.graphson_reader.read_object(json.dumps({"@type": "gx:OffsetDateTime", "@value": expected.isoformat()})) + assert isinstance(output, datetime.datetime) + assert expected == output + + def test_offsetdatetime_zulu(self): + tz = datetime.timezone.utc + ms = 12345678912 + expected = datetime.datetime(2022, 5, 20, tzinfo=tz) + datetime.timedelta(microseconds=ms) + # simulate zulu format + expected_zulu = expected.isoformat()[:-6] + 'Z' + output = self.graphson_reader.read_object(json.dumps({"@type": "gx:OffsetDateTime", "@value": expected_zulu})) + assert isinstance(output, datetime.datetime) + assert expected == output def test_timestamp(self): dt = self.graphson_reader.read_object(json.dumps({"@type": "g:Timestamp", "@value": 1481750076295}))
