This is an automated email from the ASF dual-hosted git repository.
clesaec pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 51851d49a Avro 3876 jackson util (#2529)
51851d49a is described below
commit 51851d49af3283cd31caecf6bdc7b3a4e973a5a6
Author: Christophe Le Saec <[email protected]>
AuthorDate: Thu Oct 5 14:14:01 2023 +0200
Avro 3876 jackson util (#2529)
* AVRO-3876: Jackson util symmetric
---
.../main/java/org/apache/avro/io/JsonEncoder.java | 2 +-
.../apache/avro/util/internal/JacksonUtils.java | 26 +++++++---
.../java/org/apache/avro/TestSchemaBuilder.java | 57 +++++++++++-----------
.../java/org/apache/avro/io/TestBinaryDecoder.java | 25 ++++++++++
.../java/org/apache/avro/io/TestJsonDecoder.java | 26 ++++++++++
.../avro/util/internal/TestJacksonUtils.java | 36 ++++++++++++--
6 files changed, 134 insertions(+), 38 deletions(-)
diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
b/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
index 7e3a67eb6..46fb6b5cb 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
@@ -208,7 +208,7 @@ public class JsonEncoder extends ParsingEncoder implements
Parser.ActionHandler
@Override
public void writeFloat(float f) throws IOException {
parser.advance(Symbol.FLOAT);
- out.writeNumber(f);
+ out.writeNumber(f + 0d);
}
@Override
diff --git
a/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java
b/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java
index 1a822899f..02a3872d4 100644
---
a/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java
+++
b/lang/java/avro/src/main/java/org/apache/avro/util/internal/JacksonUtils.java
@@ -18,6 +18,7 @@
package org.apache.avro.util.internal;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
@@ -75,7 +76,7 @@ public class JacksonUtils {
}
generator.writeEndArray();
} else if (datum instanceof byte[]) { // bytes, fixed
- generator.writeString(new String((byte[]) datum,
StandardCharsets.ISO_8859_1));
+ generator.writeBinary((byte[]) datum);// writeString(new String((byte[])
datum, StandardCharsets.ISO_8859_1));
} else if (datum instanceof CharSequence || datum instanceof Enum<?>) { //
string, enum
generator.writeString(datum.toString());
} else if (datum instanceof Double) { // double
@@ -136,10 +137,23 @@ public class JacksonUtils {
return jsonNode.asDouble();
}
} else if (jsonNode.isDouble() || jsonNode.isFloat()) {
- if (schema == null || schema.getType().equals(Schema.Type.DOUBLE)) {
- return jsonNode.asDouble();
- } else if (schema.getType().equals(Schema.Type.FLOAT)) {
- return (float) jsonNode.asDouble();
+ if (schema != null) {
+ if (schema.getType().equals(Schema.Type.DOUBLE)) {
+ return jsonNode.doubleValue();
+ } else if (schema.getType().equals(Schema.Type.FLOAT)) {
+ return jsonNode.floatValue();
+ }
+ } else if (jsonNode.isDouble()) {
+ return jsonNode.doubleValue();
+ } else {
+ return jsonNode.floatValue();
+ }
+ } else if (jsonNode.isBinary()) {
+ try {
+ return jsonNode.binaryValue();
+ } catch (IOException ex) {
+ // only for TextNode, so, can't happen with binaryNode.
+ throw new UncheckedIOException(ex);
}
} else if (jsonNode.isTextual()) {
if (schema == null || schema.getType().equals(Schema.Type.STRING) ||
schema.getType().equals(Schema.Type.ENUM)) {
@@ -175,7 +189,7 @@ public class JacksonUtils {
/**
* Convert an object into a map
- *
+ *
* @param datum The object
* @return Its Map representation
*/
diff --git
a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java
b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java
index fcbaae655..293554e8e 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/TestSchemaBuilder.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.node.NullNode;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -111,14 +112,14 @@ public class TestSchemaBuilder {
assertTrue(s.getObjectProp("intProp") instanceof Integer);
assertTrue(s.getObjectProp("longProp") instanceof Long);
assertEquals(Long.MAX_VALUE, s.getObjectProp("longProp"));
- assertTrue(s.getObjectProp("floatProp") instanceof Double);
+ assertTrue(s.getObjectProp("floatProp") instanceof Float);
// float converts to double
- assertEquals(1.0d, s.getObjectProp("floatProp"));
+ assertEquals(1.0f, s.getObjectProp("floatProp"));
assertTrue(s.getObjectProp("doubleProp") instanceof Double);
assertEquals(Double.MAX_VALUE, s.getObjectProp("doubleProp"));
// byte[] converts to string
- assertTrue(s.getObjectProp("byteProp") instanceof String);
- assertEquals("ABC", s.getObjectProp("byteProp"));
+ assertTrue(s.getObjectProp("byteProp") instanceof byte[]);
+ assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[])
s.getObjectProp("byteProp"));
assertTrue(s.getObjectProp("stringProp") instanceof String);
assertEquals("abc", s.getObjectProp("stringProp"));
}
@@ -141,14 +142,14 @@ public class TestSchemaBuilder {
assertTrue(f.getObjectProp("intProp") instanceof Integer);
assertTrue(f.getObjectProp("longProp") instanceof Long);
assertEquals(Long.MAX_VALUE, f.getObjectProp("longProp"));
- assertTrue(f.getObjectProp("floatProp") instanceof Double);
+ assertTrue(f.getObjectProp("floatProp") instanceof Float);
// float converts to double
- assertEquals(1.0d, f.getObjectProp("floatProp"));
+ assertEquals(1.0f, f.getObjectProp("floatProp"));
assertTrue(f.getObjectProp("doubleProp") instanceof Double);
assertEquals(Double.MAX_VALUE, f.getObjectProp("doubleProp"));
// byte[] converts to string
- assertTrue(f.getObjectProp("byteProp") instanceof String);
- assertEquals("ABC", f.getObjectProp("byteProp"));
+ assertTrue(f.getObjectProp("byteProp") instanceof byte[]);
+ assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[])
f.getObjectProp("byteProp"));
assertTrue(f.getObjectProp("stringProp") instanceof String);
assertEquals("abc", f.getObjectProp("stringProp"));
@@ -180,11 +181,11 @@ public class TestSchemaBuilder {
assertEquals(true, iter.next());
assertEquals(Integer.MAX_VALUE, iter.next());
assertEquals(Long.MAX_VALUE, iter.next());
- // float converts to double
- assertEquals(1.0d, iter.next());
+
+ assertEquals(1.0f, iter.next());
assertEquals(Double.MAX_VALUE, iter.next());
- // byte[] converts to string
- assertEquals("ABC", iter.next());
+
+ assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[]) iter.next());
assertEquals("abc", iter.next());
}
@@ -215,11 +216,11 @@ public class TestSchemaBuilder {
assertEquals(true, iter.next());
assertEquals(Integer.MAX_VALUE, iter.next());
assertEquals(Long.MAX_VALUE, iter.next());
- // float converts to double
- assertEquals(1.0d, iter.next());
+
+ assertEquals(1.0f, iter.next());
assertEquals(Double.MAX_VALUE, iter.next());
- // byte[] converts to string
- assertEquals("ABC", iter.next());
+
+ assertArrayEquals(new byte[] { 0x41, 0x42, 0x43 }, (byte[]) iter.next());
assertEquals("abc", iter.next());
}
@@ -248,14 +249,14 @@ public class TestSchemaBuilder {
assertEquals(Integer.MAX_VALUE, valueMap.get("intKey"));
assertTrue(valueMap.get("longKey") instanceof Long);
assertEquals(Long.MAX_VALUE, valueMap.get("longKey"));
- // float converts to double
- assertTrue(valueMap.get("floatKey") instanceof Double);
- assertEquals(1.0d, valueMap.get("floatKey"));
+
+ assertTrue(valueMap.get("floatKey") instanceof Float);
+ assertEquals(1.0f, valueMap.get("floatKey"));
assertTrue(valueMap.get("doubleKey") instanceof Double);
assertEquals(Double.MAX_VALUE, valueMap.get("doubleKey"));
- // byte[] converts to string
- assertTrue(valueMap.get("byteKey") instanceof String);
- assertEquals("ABC", valueMap.get("byteKey"));
+
+ assertTrue(valueMap.get("byteKey") instanceof byte[]);
+ assertArrayEquals("ABC".getBytes(StandardCharsets.UTF_8), (byte[])
valueMap.get("byteKey"));
assertTrue(valueMap.get("stringKey") instanceof String);
assertEquals("abc", valueMap.get("stringKey"));
}
@@ -288,14 +289,14 @@ public class TestSchemaBuilder {
assertEquals(Integer.MAX_VALUE, valueMap.get("intKey"));
assertTrue(valueMap.get("longKey") instanceof Long);
assertEquals(Long.MAX_VALUE, valueMap.get("longKey"));
- // float converts to double
- assertTrue(valueMap.get("floatKey") instanceof Double);
- assertEquals(1.0d, valueMap.get("floatKey"));
+
+ assertTrue(valueMap.get("floatKey") instanceof Float);
+ assertEquals(1.0f, valueMap.get("floatKey"));
assertTrue(valueMap.get("doubleKey") instanceof Double);
assertEquals(Double.MAX_VALUE, valueMap.get("doubleKey"));
- // byte[] converts to string
- assertTrue(valueMap.get("byteKey") instanceof String);
- assertEquals("ABC", valueMap.get("byteKey"));
+
+ assertTrue(valueMap.get("byteKey") instanceof byte[]);
+ assertEquals("ABC", new String((byte[]) valueMap.get("byteKey")));
assertTrue(valueMap.get("stringKey") instanceof String);
assertEquals("abc", valueMap.get("stringKey"));
}
diff --git
a/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java
b/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java
index 6010fc9c6..805335857 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestBinaryDecoder.java
@@ -20,8 +20,10 @@ package org.apache.avro.io;
import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.SystemLimitException;
+import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.ByteBufferInputStream;
import org.apache.avro.util.ByteBufferOutputStream;
import org.apache.avro.util.RandomData;
@@ -43,6 +45,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import static org.apache.avro.TestSystemLimitException.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestBinaryDecoder {
// prime number buffer size so that looping tests hit the buffer edge
@@ -682,4 +685,26 @@ public class TestBinaryDecoder {
Assertions.assertThrows(EOFException.class, () -> d.readInt());
}
+ @Test
+ void testFloatPrecision() throws Exception {
+ String def = "{\"type\":\"record\",\"name\":\"X\",\"fields\":" +
"[{\"type\":\"float\",\"name\":\"n\"}]}";
+ Schema schema = new Schema.Parser().parse(def);
+ DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
+
+ float value = 33.33000183105469f;
+
+ GenericData.Record record = new GenericData.Record(schema);
+ record.put(0, value);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
+
+ DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
+ writer.write(record, encoder);
+ encoder.flush();
+
+ Decoder decoder = DecoderFactory.get().directBinaryDecoder(new
ByteArrayInputStream(out.toByteArray()), null);
+ GenericRecord r = reader.read(null, decoder);
+ assertEquals(value + 0d, ((float) r.get("n")) + 0d);
+ }
+
}
diff --git
a/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java
b/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java
index 050571396..1f44344e3 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestJsonDecoder.java
@@ -22,12 +22,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import org.apache.avro.AvroTypeException;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
+import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
public class TestJsonDecoder {
@@ -66,6 +69,29 @@ public class TestJsonDecoder {
}
}
+ @Test
+ void testFloatPrecision() throws Exception {
+ String def = "{\"type\":\"record\",\"name\":\"X\",\"fields\":" +
"[{\"type\":\"float\",\"name\":\"n\"}]}";
+ Schema schema = new Schema.Parser().parse(def);
+ DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
+
+ float value = 33.33000183105469f;
+ GenericData.Record record = new GenericData.Record(schema);
+ record.put(0, value);
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
+
+ DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
+ writer.write(record, encoder);
+ encoder.flush();
+ // check the whole float precision is kept.
+ assertEquals("{\"n\":33.33000183105469}", out.toString());
+
+ Decoder decoder = DecoderFactory.get().jsonDecoder(schema, out.toString());
+ GenericRecord r = reader.read(null, decoder);
+ assertEquals(value + 0d, ((float) r.get("n")) + 0d);
+ }
+
// Ensure that even if the order of fields in JSON is different from the
order
// in schema,
// it works.
diff --git
a/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java
b/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java
index cd6fe3f74..4a272ae5b 100644
---
a/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java
+++
b/lang/java/avro/src/test/java/org/apache/avro/util/internal/TestJacksonUtils.java
@@ -21,8 +21,10 @@ import static
org.apache.avro.util.internal.JacksonUtils.toJsonNode;
import static org.apache.avro.util.internal.JacksonUtils.toObject;
import static org.junit.jupiter.api.Assertions.*;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.BigIntegerNode;
+import com.fasterxml.jackson.databind.node.BinaryNode;
import com.fasterxml.jackson.databind.node.BooleanNode;
import com.fasterxml.jackson.databind.node.DecimalNode;
import com.fasterxml.jackson.databind.node.DoubleNode;
@@ -31,15 +33,24 @@ import com.fasterxml.jackson.databind.node.IntNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.LongNode;
import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.databind.node.NumericNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
+import java.util.stream.Stream;
+
import org.apache.avro.JsonProperties;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
+
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
public class TestJacksonUtils {
@@ -55,8 +66,9 @@ public class TestJacksonUtils {
assertEquals(IntNode.valueOf(1), toJsonNode(1));
assertEquals(LongNode.valueOf(2), toJsonNode(2L));
assertEquals(FloatNode.valueOf(1.0f), toJsonNode(1.0f));
- assertEquals(DoubleNode.valueOf(2.0), toJsonNode(2.0));
- assertEquals(TextNode.valueOf("\u0001\u0002"), toJsonNode(new byte[] { 1,
2 }));
+ assertEquals(FloatNode.valueOf(33.33000183105469f),
toJsonNode(33.33000183105469f));
+ assertEquals(DoubleNode.valueOf(2.0), toJsonNode(2.0d));
+ assertEquals(BinaryNode.valueOf(new byte[] { 1, 2 }), toJsonNode(new
byte[] { 1, 2 }));
assertEquals(TextNode.valueOf("a"), toJsonNode("a"));
assertEquals(TextNode.valueOf("UP"), toJsonNode(Direction.UP));
assertEquals(BigIntegerNode.valueOf(BigInteger.ONE),
toJsonNode(BigInteger.ONE));
@@ -80,7 +92,7 @@ public class TestJacksonUtils {
assertEquals(2L, toObject(IntNode.valueOf(2),
Schema.create(Schema.Type.LONG)));
assertEquals(1.0f, toObject(DoubleNode.valueOf(1.0),
Schema.create(Schema.Type.FLOAT)));
assertEquals(2.0, toObject(DoubleNode.valueOf(2.0)));
- assertEquals(TextNode.valueOf("\u0001\u0002"), toJsonNode(new byte[] { 1,
2 }));
+ assertEquals(BinaryNode.valueOf(new byte[] { 1, 2 }), toJsonNode(new
byte[] { 1, 2 }));
assertArrayEquals(new byte[] { 1, 2 },
(byte[]) toObject(TextNode.valueOf("\u0001\u0002"),
Schema.create(Schema.Type.BYTES)));
assertEquals("a", toObject(TextNode.valueOf("a")));
@@ -102,4 +114,22 @@ public class TestJacksonUtils {
assertEquals("a", toObject(TextNode.valueOf("a"),
SchemaBuilder.unionOf().stringType().and().intType().endUnion()));
}
+ @ParameterizedTest
+ @MethodSource("nodes")
+ void cycle(JsonNode input) {
+ Object object = JacksonUtils.toObject(input);
+ JsonNode node = JacksonUtils.toJsonNode(object);
+ Assertions.assertEquals(input, node);
+ }
+
+ public static Stream<Arguments> nodes() {
+ ObjectNode o1 = JsonNodeFactory.instance.objectNode();
+ o1.put("intField", 123);
+ o1.put("floatField", 33.33000183105469f);
+ o1.put("doubleField", 33.33000183105469245d);
+ return Stream.of(JsonNodeFactory.instance.numberNode(33.33000183105469f),
+
JsonNodeFactory.instance.binaryNode("Hello".getBytes(StandardCharsets.ISO_8859_1)),
+
JsonNodeFactory.instance.arrayNode().add(1).add("Hello").add(o1)).map(Arguments::of);
+ }
+
}
