KalleOlaviNiemitalo commented on code in PR #1833: URL: https://github.com/apache/avro/pull/1833#discussion_r951039578
########## lang/csharp/src/apache/main/IO/JsonDecoder.cs: ########## @@ -0,0 +1,787 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Avro.IO.Parsing; +using Newtonsoft.Json; + +namespace Avro.IO +{ + /// <summary> + /// A <seealso cref="Decoder"/> for Avro's JSON data encoding. + /// + /// JsonDecoder is not thread-safe. + /// </summary> + public class JsonDecoder : ParsingDecoder + { + private JsonReader reader; + private readonly Stack<ReorderBuffer> reorderBuffers = new Stack<ReorderBuffer>(); + private ReorderBuffer currentReorderBuffer; + + private class ReorderBuffer + { + public readonly IDictionary<string, IList<JsonElement>> SavedFields = + new Dictionary<string, IList<JsonElement>>(); + + public JsonReader OrigParser; + } + + private JsonDecoder(Symbol root, Stream stream) : base(root) + { + Configure(stream); + } + + private JsonDecoder(Symbol root, string str) : base(root) + { + Configure(str); + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonDecoder"/> class. + /// </summary> + public JsonDecoder(Schema schema, Stream stream) : this(getSymbol(schema), stream) + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonDecoder"/> class. + /// </summary> + public JsonDecoder(Schema schema, string str) : this(getSymbol(schema), str) + { + } + + private static Symbol getSymbol(Schema schema) + { + return (new JsonGrammarGenerator()).Generate(schema); + } + + /// <summary> + /// Reconfigures this JsonDecoder to use the InputStream provided. + /// <p/> + /// Otherwise, this JsonDecoder will reset its state and then reconfigure its + /// input. + /// </summary> Review Comment: Not important: The [para](https://docs.microsoft.com/dotnet/csharp/language-reference/xmldoc/recommended-tags#para) element would be more standard in C# XML documentation comments, and is apparently [supported by Doxygen](https://doxygen.nl/manual/xmlcmds.html). ########## lang/csharp/src/apache/main/IO/Parsing/Parser.cs: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace Avro.IO.Parsing +{ + /// <summary> + /// Parser is the class that maintains the stack for parsing. This class is used + /// by encoders, which are not required to skip. + /// </summary> + public class Parser + { + /// <summary> + /// The parser knows how to handle the terminal and non-terminal symbols. But it + /// needs help from outside to handle implicit and explicit actions. The clients + /// implement this interface to provide this help. + /// </summary> + public interface ActionHandler Review Comment: Please name interfaces starting with "I" according to the .NET convention, at least if they are visible outside the assembly. Likewise in SkipParser.SkipHandler. ########## lang/csharp/src/apache/main/IO/JsonEncoder.cs: ########## @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Avro.IO.Parsing; +using System.Collections; +using System.IO; +using System.Text; +using Newtonsoft.Json; + +namespace Avro.IO +{ + /// <summary> + /// An <seealso cref="Encoder"/> for Avro's JSON data encoding. Review Comment: For inline links, please use [see](https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/xmldoc/recommended-tags#see) rather than seealso. ########## lang/csharp/src/apache/main/IO/JsonEncoder.cs: ########## @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Avro.IO.Parsing; +using System.Collections; +using System.IO; +using System.Text; +using Newtonsoft.Json; + +namespace Avro.IO +{ + /// <summary> + /// An <seealso cref="Encoder"/> for Avro's JSON data encoding. + /// + /// JsonEncoder buffers output, and data may not appear on the output until + /// <seealso cref="Encoder.Flush()"/> is called. + /// + /// JsonEncoder is not thread-safe. + /// </summary> + public class JsonEncoder : ParsingEncoder, Parser.ActionHandler + { + private readonly Parser parser; + private JsonWriter writer; + private bool includeNamespace = true; + + // Has anything been written into the collections? + private readonly BitArray isEmpty = new BitArray(64); + + /// <summary> + /// Initializes a new instance of the <see cref="JsonEncoder"/> class. + /// </summary> + public JsonEncoder(Schema sc, Stream stream) : this(sc, getJsonWriter(stream, false)) + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonEncoder"/> class. + /// </summary> + public JsonEncoder(Schema sc, Stream stream, bool pretty) : this(sc, getJsonWriter(stream, pretty)) + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonEncoder"/> class. + /// </summary> + public JsonEncoder(Schema sc, JsonWriter writer) + { + Configure(writer); + this.parser = new Parser((new JsonGrammarGenerator()).Generate(sc), this); + } + + /// <inheritdoc /> + public override void Flush() + { + parser.ProcessImplicitActions(); + if (writer != null) + { + writer.Flush(); + } + } + + // by default, one object per line. + // with pretty option use default pretty printer with root line separator. + private static JsonWriter getJsonWriter(Stream stream, bool pretty) + { + JsonWriter writer = new JsonTextWriter(new StreamWriter(stream)); + if (pretty) + { + writer.Formatting = Formatting.Indented; + } + + return writer; + } + + /// <summary> + /// Whether to include the namespace. + /// </summary> + public virtual bool IncludeNamespace + { + get { return includeNamespace; } + set { this.includeNamespace = value; } + } + + + /// <summary> + /// Reconfigures this JsonEncoder to use the output stream provided. + /// <p/> + /// Otherwise, this JsonEncoder will flush its current output and then + /// reconfigure its output to use a default UTF8 JsonWriter that writes to the + /// provided OutputStream. Review Comment: OutputStream is a Java thing ########## lang/csharp/src/apache/main/IO/JsonDecoder.cs: ########## @@ -0,0 +1,787 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Avro.IO.Parsing; +using Newtonsoft.Json; + +namespace Avro.IO +{ + /// <summary> + /// A <seealso cref="Decoder"/> for Avro's JSON data encoding. + /// + /// JsonDecoder is not thread-safe. + /// </summary> + public class JsonDecoder : ParsingDecoder + { + private JsonReader reader; + private readonly Stack<ReorderBuffer> reorderBuffers = new Stack<ReorderBuffer>(); + private ReorderBuffer currentReorderBuffer; + + private class ReorderBuffer + { + public readonly IDictionary<string, IList<JsonElement>> SavedFields = + new Dictionary<string, IList<JsonElement>>(); + + public JsonReader OrigParser; + } + + private JsonDecoder(Symbol root, Stream stream) : base(root) + { + Configure(stream); + } + + private JsonDecoder(Symbol root, string str) : base(root) + { + Configure(str); + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonDecoder"/> class. + /// </summary> + public JsonDecoder(Schema schema, Stream stream) : this(getSymbol(schema), stream) + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonDecoder"/> class. + /// </summary> + public JsonDecoder(Schema schema, string str) : this(getSymbol(schema), str) + { + } + + private static Symbol getSymbol(Schema schema) + { + return (new JsonGrammarGenerator()).Generate(schema); + } + + /// <summary> + /// Reconfigures this JsonDecoder to use the InputStream provided. + /// <p/> + /// Otherwise, this JsonDecoder will reset its state and then reconfigure its + /// input. + /// </summary> + /// <param name="stream"> The InputStream to read from. Cannot be null. </param> + /// <returns> this JsonDecoder </returns> + public JsonDecoder Configure(Stream stream) + { + Parser.Reset(); + reorderBuffers.Clear(); + currentReorderBuffer = null; + this.reader = new JsonTextReader(new StreamReader(stream)); + this.reader.Read(); + return this; + } + + /// <summary> + /// Reconfigures this JsonDecoder to use the String provided for input. + /// <p/> + /// Otherwise, this JsonDecoder will reset its state and then reconfigure its + /// input. + /// </summary> + /// <param name="str"> The String to read from. Cannot be null. </param> + /// <returns> this JsonDecoder </returns> + public JsonDecoder Configure(string str) + { + Parser.Reset(); + reorderBuffers.Clear(); + currentReorderBuffer = null; + this.reader = new JsonTextReader(new StringReader(str)); + this.reader.Read(); + return this; + } + + private void advance(Symbol symbol) + { + this.Parser.ProcessTrailingImplicitActions(); + Parser.Advance(symbol); + } + + /// <inheritdoc /> + public override void ReadNull() + { + advance(Symbol.Null); + if (reader.TokenType == JsonToken.Null) + { + reader.Read(); + } + else + { + throw error("null"); + } + } + + /// <inheritdoc /> + public override bool ReadBoolean() + { + advance(Symbol.Boolean); + if (reader.TokenType == JsonToken.Boolean) + { + bool result = Convert.ToBoolean(reader.Value); + reader.Read(); + return result; + } + else + { + throw error("boolean"); + } + } + + /// <inheritdoc /> + public override int ReadInt() + { + advance(Symbol.Int); + if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float) + { + int result = Convert.ToInt32(reader.Value); + reader.Read(); + return result; + } + else + { + throw error("int"); + } + } + + /// <inheritdoc /> + public override long ReadLong() + { + advance(Symbol.Long); + if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float) + { + long result = Convert.ToInt64(reader.Value); + reader.Read(); + return result; + } + else + { + throw error("long"); + } + } + + /// <inheritdoc /> + public override float ReadFloat() + { + advance(Symbol.Float); + if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float) + { + float result = (float)Convert.ToDouble(reader.Value); + reader.Read(); + return result; + } + else + { + throw error("float"); + } + } + + /// <inheritdoc /> + public override double ReadDouble() + { + advance(Symbol.Double); + if (reader.TokenType == JsonToken.Integer || reader.TokenType == JsonToken.Float) + { + double result = Convert.ToDouble(reader.Value); + reader.Read(); + return result; + } + else + { + throw error("double"); + } + } + + /// <inheritdoc /> + public override string ReadString() + { + advance(Symbol.String); + if (Parser.TopSymbol() == Symbol.MapKeyMarker) + { + Parser.Advance(Symbol.MapKeyMarker); + if (reader.TokenType != JsonToken.PropertyName) + { + throw error("map-key"); + } + } + else + { + if (reader.TokenType != JsonToken.String) + { + throw error("string"); + } + } + + string result = Convert.ToString(reader.Value); + reader.Read(); + return result; + } + + /// <inheritdoc /> + public override void SkipString() + { + advance(Symbol.String); + if (Parser.TopSymbol() == Symbol.MapKeyMarker) + { + Parser.Advance(Symbol.MapKeyMarker); + if (reader.TokenType != JsonToken.PropertyName) + { + throw error("map-key"); + } + } + else + { + if (reader.TokenType != JsonToken.String) + { + throw error("string"); + } + } + + reader.Read(); + } + + /// <inheritdoc /> + public override byte[] ReadBytes() + { + advance(Symbol.Bytes); + if (reader.TokenType == JsonToken.String) + { + byte[] result = readByteArray(); + reader.Read(); + return result; + } + else + { + throw error("bytes"); + } + } + + private byte[] readByteArray() + { + Encoding iso = Encoding.GetEncoding("ISO-8859-1"); + byte[] result = iso.GetBytes(Convert.ToString(reader.Value)); + return result; + } + + /// <inheritdoc /> + public override void SkipBytes() + { + advance(Symbol.Bytes); + if (reader.TokenType == JsonToken.String) + { + reader.Read(); + } + else + { + throw error("bytes"); + } + } + + private void checkFixed(int size) + { + advance(Symbol.Fixed); + Symbol.IntCheckAction top = (Symbol.IntCheckAction)Parser.PopSymbol(); + if (size != top.Size) + { + throw new AvroTypeException("Incorrect length for fixed binary: expected " + top.Size + + " but received " + size + " bytes."); + } + } + + /// <inheritdoc /> + public override void ReadFixed(byte[] bytes) + { + ReadFixed(bytes, 0, bytes.Length); + } + + /// <inheritdoc /> + public override void ReadFixed(byte[] bytes, int start, int len) + { + checkFixed(len); + if (reader.TokenType == JsonToken.String) + { + byte[] result = readByteArray(); + reader.Read(); + if (result.Length != len) + { + throw new AvroTypeException("Expected fixed length " + len + ", but got" + result.Length); + } + + Array.Copy(result, 0, bytes, start, len); + } + else + { + throw error("fixed"); + } + } + + /// <inheritdoc /> + public override void SkipFixed(int length) + { + checkFixed(length); + doSkipFixed(length); + } + + private void doSkipFixed(int length) + { + if (reader.TokenType == JsonToken.String) + { + byte[] result = readByteArray(); + reader.Read(); + if (result.Length != length) + { + throw new AvroTypeException("Expected fixed length " + length + ", but got" + result.Length); + } + } + else + { + throw error("fixed"); + } + } + + /// <inheritdoc /> + protected override void SkipFixed() + { + advance(Symbol.Fixed); + Symbol.IntCheckAction top = (Symbol.IntCheckAction)Parser.PopSymbol(); + doSkipFixed(top.Size); + } + + /// <inheritdoc /> + public override int ReadEnum() + { + advance(Symbol.Enum); + Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction)Parser.PopSymbol(); + if (reader.TokenType == JsonToken.String) + { + string label = Convert.ToString(reader.Value); + int n = top.FindLabel(label); + if (n >= 0) + { + reader.Read(); + return n; + } + + throw new AvroTypeException("Unknown symbol in enum " + label); + } + else + { + throw error("fixed"); + } + } + + /// <inheritdoc /> + public override long ReadArrayStart() + { + advance(Symbol.ArrayStart); + if (reader.TokenType == JsonToken.StartArray) + { + reader.Read(); + return doArrayNext(); + } + else + { + throw error("array-start"); + } + } + + /// <inheritdoc /> + public override long ReadArrayNext() + { + advance(Symbol.ItemEnd); + return doArrayNext(); + } + + private long doArrayNext() + { + if (reader.TokenType == JsonToken.EndArray) + { + Parser.Advance(Symbol.ArrayEnd); + reader.Read(); + return 0; + } + else + { + return 1; + } + } + + /// <inheritdoc /> + public override void SkipArray() + { + advance(Symbol.ArrayStart); + if (reader.TokenType == JsonToken.StartArray) + { + reader.Skip(); + reader.Read(); + advance(Symbol.ArrayEnd); + } + else + { + throw error("array-start"); + } + } + + /// <inheritdoc /> + public override long ReadMapStart() + { + advance(Symbol.MapStart); + if (reader.TokenType == JsonToken.StartObject) + { + reader.Read(); + return doMapNext(); + } + else + { + throw error("map-start"); + } + } + + /// <inheritdoc /> + public override long ReadMapNext() + { + advance(Symbol.ItemEnd); + return doMapNext(); + } + + private long doMapNext() + { + if (reader.TokenType == JsonToken.EndObject) + { + reader.Read(); + advance(Symbol.MapEnd); + return 0; + } + else + { + return 1; + } + } + + /// <inheritdoc /> + public override void SkipMap() + { + advance(Symbol.MapStart); + if (reader.TokenType == JsonToken.StartObject) + { + reader.Skip(); + reader.Read(); + advance(Symbol.MapEnd); + } + else + { + throw error("map-start"); + } + } + + /// <inheritdoc /> + public override int ReadUnionIndex() + { + advance(Symbol.Union); + Symbol.Alternative a = (Symbol.Alternative)Parser.PopSymbol(); + + string label; + if (reader.TokenType == JsonToken.Null) + { + label = "null"; + } + else if (reader.TokenType == JsonToken.StartObject) + { + reader.Read(); + if (reader.TokenType == JsonToken.PropertyName) + { + label = Convert.ToString(reader.Value); + reader.Read(); + Parser.PushSymbol(Symbol.UnionEnd); + } + else + { + throw error("start-union"); + } + } + else + { + throw error("start-union"); + } + + int n = a.FindLabel(label); + if (n < 0) + { + throw new AvroTypeException("Unknown union branch " + label); + } + + Parser.PushSymbol(a.GetSymbol(n)); + return n; + } + + /// <inheritdoc /> + public override void SkipNull() + { + ReadNull(); + } + + /// <inheritdoc /> + public override void SkipBoolean() + { + ReadBoolean(); + } + + /// <inheritdoc /> + public override void SkipInt() + { + ReadInt(); + } + + /// <inheritdoc /> + public override void SkipLong() + { + ReadLong(); + } + + /// <inheritdoc /> + public override void SkipFloat() + { + ReadFloat(); + } + + /// <inheritdoc /> + public override void SkipDouble() + { + ReadDouble(); + } + + /// <inheritdoc /> + public override void SkipEnum() + { + ReadEnum(); + } + + /// <inheritdoc /> + public override void SkipUnionIndex() + { + ReadUnionIndex(); + } + + /// <inheritdoc /> + public override Symbol DoAction(Symbol input, Symbol top) + { + if (top is Symbol.FieldAdjustAction) + { + Symbol.FieldAdjustAction fa = (Symbol.FieldAdjustAction)top; + string name = fa.FName; + if (currentReorderBuffer != null) + { + IList<JsonElement> node = currentReorderBuffer.SavedFields[name]; + if (node != null) + { + currentReorderBuffer.SavedFields.Remove(name); + currentReorderBuffer.OrigParser = reader; + reader = makeParser(node); + return null; + } + } + + if (reader.TokenType == JsonToken.PropertyName) + { + do + { + string fn = Convert.ToString(reader.Value); + reader.Read(); + if (name.Equals(fn) || (fa.Aliases != null && fa.Aliases.Contains(fn))) + { + return null; + } + else + { + if (currentReorderBuffer == null) + { + currentReorderBuffer = new ReorderBuffer(); + } + + currentReorderBuffer.SavedFields[fn] = getValueAsTree(reader); + } + } while (reader.TokenType == JsonToken.PropertyName); + + throw new AvroTypeException("Expected field name not found: " + fa.FName); + } + } + else if (top == Symbol.FieldEnd) + { + if (currentReorderBuffer != null && currentReorderBuffer.OrigParser != null) + { + reader = currentReorderBuffer.OrigParser; + currentReorderBuffer.OrigParser = null; + } + } + else if (top == Symbol.RecordStart) + { + if (reader.TokenType == JsonToken.StartObject) + { + reader.Read(); + reorderBuffers.Push(currentReorderBuffer); + currentReorderBuffer = null; + } + else + { + throw error("record-start"); + } + } + else if (top == Symbol.RecordEnd || top == Symbol.UnionEnd) + { + // AVRO-2034 advance to the end of our object + while (reader.TokenType != JsonToken.EndObject) + { + reader.Read(); + } + + if (top == Symbol.RecordEnd) + { + if (currentReorderBuffer != null && currentReorderBuffer.SavedFields.Count > 0) + { + throw error("Unknown fields: " + currentReorderBuffer.SavedFields.Keys); + } + + currentReorderBuffer = reorderBuffers.Pop(); + } + + // AVRO-2034 advance beyond the end object for the next record. + reader.Read(); + } + else + { + throw new AvroTypeException("Unknown action symbol " + top); + } + + return null; + } + + + private class JsonElement + { + public readonly JsonToken Token; + public readonly object Value; + + public JsonElement(JsonToken t, object value) + { + this.Token = t; + this.Value = value; + } + + public JsonElement(JsonToken t) : this(t, null) + { + } + } + + private static IList<JsonElement> getValueAsTree(JsonReader reader) + { + int level = 0; + IList<JsonElement> result = new List<JsonElement>(); + do + { + JsonToken t = reader.TokenType; + switch (t) + { + case JsonToken.StartObject: + case JsonToken.StartArray: + level++; + result.Add(new JsonElement(t)); + break; + case JsonToken.EndObject: + case JsonToken.EndArray: + level--; + result.Add(new JsonElement(t)); + break; + case JsonToken.PropertyName: + case JsonToken.String: + case JsonToken.Integer: + case JsonToken.Float: + case JsonToken.Boolean: + case JsonToken.Null: + result.Add(new JsonElement(t, reader.Value)); + break; + } + + reader.Read(); + } while (level != 0); + + result.Add(new JsonElement(JsonToken.None)); + return result; + } + + private JsonReader makeParser(in IList<JsonElement> elements) + { + return new JsonElementReader(elements); + } + + private class JsonElementReader : JsonReader + { + private readonly IList<JsonElement> elements; + + public JsonElementReader(IList<JsonElement> elements) + { + this.elements = elements; + pos = 0; + } + + private int pos; + + public override object Value + { + get { return elements[pos].Value; } + } + + public override JsonToken TokenType + { + get { return elements[pos].Token; } + } + + public override bool Read() + { + pos++; + return true; + } + + public new void Skip() Review Comment: Is JsonElementReader.Skip() ever called? JsonDecoder.SkipArray() and JsonDecoder.SkipMap() can call reader.Skip(), but that refers to the field `private JsonReader reader;`, so the call goes to [JsonReader.Skip()](https://www.newtonsoft.com/json/help/html/M_Newtonsoft_Json_JsonReader_Skip.htm), which is not virtual. ########## lang/csharp/src/apache/main/IO/ParsingDecoder.cs: ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Avro.IO.Parsing; + +namespace Avro.IO +{ + /// <summary> + /// Base class for <a href="parsing/package-summary.html">parser</a>-based Review Comment: Not sure what "parsing/package-summary.html" refers to. Likewise in ParsingEncoder. ########## lang/csharp/src/apache/main/IO/JsonEncoder.cs: ########## @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Avro.IO.Parsing; +using System.Collections; +using System.IO; +using System.Text; +using Newtonsoft.Json; + +namespace Avro.IO +{ + /// <summary> + /// An <seealso cref="Encoder"/> for Avro's JSON data encoding. + /// + /// JsonEncoder buffers output, and data may not appear on the output until + /// <seealso cref="Encoder.Flush()"/> is called. + /// + /// JsonEncoder is not thread-safe. + /// </summary> + public class JsonEncoder : ParsingEncoder, Parser.ActionHandler + { + private readonly Parser parser; + private JsonWriter writer; + private bool includeNamespace = true; + + // Has anything been written into the collections? + private readonly BitArray isEmpty = new BitArray(64); + + /// <summary> + /// Initializes a new instance of the <see cref="JsonEncoder"/> class. + /// </summary> + public JsonEncoder(Schema sc, Stream stream) : this(sc, getJsonWriter(stream, false)) + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonEncoder"/> class. + /// </summary> + public JsonEncoder(Schema sc, Stream stream, bool pretty) : this(sc, getJsonWriter(stream, pretty)) + { + } + + /// <summary> + /// Initializes a new instance of the <see cref="JsonEncoder"/> class. + /// </summary> + public JsonEncoder(Schema sc, JsonWriter writer) + { + Configure(writer); + this.parser = new Parser((new JsonGrammarGenerator()).Generate(sc), this); + } + + /// <inheritdoc /> + public override void Flush() + { + parser.ProcessImplicitActions(); + if (writer != null) + { + writer.Flush(); + } + } + + // by default, one object per line. + // with pretty option use default pretty printer with root line separator. + private static JsonWriter getJsonWriter(Stream stream, bool pretty) + { + JsonWriter writer = new JsonTextWriter(new StreamWriter(stream)); + if (pretty) + { + writer.Formatting = Formatting.Indented; + } + + return writer; + } + + /// <summary> + /// Whether to include the namespace. + /// </summary> + public virtual bool IncludeNamespace + { + get { return includeNamespace; } + set { this.includeNamespace = value; } + } + + + /// <summary> + /// Reconfigures this JsonEncoder to use the output stream provided. + /// <p/> + /// Otherwise, this JsonEncoder will flush its current output and then + /// reconfigure its output to use a default UTF8 JsonWriter that writes to the + /// provided OutputStream. + /// </summary> + /// <param name="stream"> The OutputStream to direct output to. Cannot be null. </param> + /// <returns> this JsonEncoder </returns> + public JsonEncoder Configure(Stream stream) + { + this.Configure(getJsonWriter(stream, false)); + return this; + } + + /// <summary> + /// Reconfigures this JsonEncoder to output to the JsonWriter provided. + /// <p/> + /// Otherwise, this JsonEncoder will flush its current output and then + /// reconfigure its output to use the provided JsonWriter. + /// </summary> + /// <param name="jsonWriter"> The JsonWriter to direct output to. Cannot be null. </param> + /// <returns> this JsonEncoder </returns> + public JsonEncoder Configure(JsonWriter jsonWriter) + { + if (null != parser) + { + Flush(); + } + + this.writer = jsonWriter; + return this; + } + + /// <inheritdoc /> + public override void WriteNull() + { + parser.Advance(Symbol.Null); + writer.WriteNull(); + } + + /// <inheritdoc /> + public override void WriteBoolean(bool b) + { + parser.Advance(Symbol.Boolean); + writer.WriteValue(b); + } + + /// <inheritdoc /> + public override void WriteInt(int n) + { + parser.Advance(Symbol.Int); + writer.WriteValue(n); + } + + /// <inheritdoc /> + public override void WriteLong(long n) + { + parser.Advance(Symbol.Long); + writer.WriteValue(n); + } + + /// <inheritdoc /> + public override void WriteFloat(float f) + { + parser.Advance(Symbol.Float); + writer.WriteValue(f); + } + + /// <inheritdoc /> + public override void WriteDouble(double d) + { + parser.Advance(Symbol.Double); + writer.WriteValue(d); + } + + /// <inheritdoc /> + public override void WriteString(string str) + { + parser.Advance(Symbol.String); + if (parser.TopSymbol() == Symbol.MapKeyMarker) + { + parser.Advance(Symbol.MapKeyMarker); + writer.WritePropertyName(str); + } + else + { + writer.WriteValue(str); + } + } + + /// <inheritdoc /> + public override void WriteBytes(byte[] bytes) + { + WriteBytes(bytes, 0, bytes.Length); + } + + /// <inheritdoc /> + public override void WriteBytes(byte[] bytes, int start, int len) + { + parser.Advance(Symbol.Bytes); + writeByteArray(bytes, start, len); + } + + private void writeByteArray(byte[] bytes, int start, int len) + { + Encoding iso = Encoding.GetEncoding("ISO-8859-1"); + writer.WriteValue(iso.GetString(bytes, start, len)); + } + + /// <inheritdoc /> + public override void WriteFixed(byte[] bytes) + { + WriteFixed(bytes, 0, bytes.Length); + } + + /// <inheritdoc /> + public override void WriteFixed(byte[] bytes, int start, int len) + { + parser.Advance(Symbol.Fixed); + Symbol.IntCheckAction top = (Symbol.IntCheckAction)parser.PopSymbol(); + if (len != top.Size) + { + throw new AvroTypeException("Incorrect length for fixed binary: expected " + top.Size + + " but received " + len + " bytes."); + } + + writeByteArray(bytes, start, len); + } + + /// <inheritdoc /> + public override void WriteEnum(int e) + { + parser.Advance(Symbol.Enum); + Symbol.EnumLabelsAction top = (Symbol.EnumLabelsAction)parser.PopSymbol(); + if (e < 0 || e >= top.Size) + { + throw new AvroTypeException("Enumeration out of range: max is " + top.Size + " but received " + e); + } + + writer.WriteValue(top.GetLabel(e)); + } + + /// <inheritdoc /> + public override void WriteArrayStart() + { + parser.Advance(Symbol.ArrayStart); + writer.WriteStartArray(); + Push(); + if (Depth() >= isEmpty.Length) + { + isEmpty.Length += isEmpty.Length; + } + + isEmpty.Set(Depth(), true); + } + + /// <inheritdoc /> + public override void WriteArrayEnd() + { + if (!isEmpty.Get(Pos)) + { + parser.Advance(Symbol.ItemEnd); + } + + Pop(); + parser.Advance(Symbol.ArrayEnd); + writer.WriteEndArray(); + } + + /// <inheritdoc /> + public override void WriteMapStart() + { + Push(); + if (Depth() >= isEmpty.Length) + { + isEmpty.Length += isEmpty.Length; + } + + isEmpty.Set(Depth(), true); + + parser.Advance(Symbol.MapStart); + writer.WriteStartObject(); + } + + /// <inheritdoc /> + public override void WriteMapEnd() + { + if (!isEmpty.Get(Pos)) + { + parser.Advance(Symbol.ItemEnd); + } + + Pop(); + + parser.Advance(Symbol.MapEnd); + writer.WriteEndObject(); + } + + /// <summary> + /// Start an array item. + /// </summary> + public new void StartItem() Review Comment: Is JsonEncoder.StartItem() ever called? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
