Author: tomwhite
Date: Fri Apr 4 10:33:02 2014
New Revision: 1584605
URL: http://svn.apache.org/r1584605
Log:
AVRO-1402. Support for DECIMAL type (as a record mapping).
Added:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java
Modified:
avro/trunk/CHANGES.txt
avro/trunk/doc/src/content/xdocs/spec.xml
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
Modified: avro/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Apr 4 10:33:02 2014
@@ -11,6 +11,8 @@ Trunk (not yet released)
AVRO-1471. Java: Permit writing generated code in different
character encodings. (Eugene Mustaphin via cutting)
+ AVRO-1402. Support for DECIMAL type (as a record mapping). (tomwhite)
+
OPTIMIZATIONS
AVRO-1455. Deep copy does not need to create new instances for primitives.
Modified: avro/trunk/doc/src/content/xdocs/spec.xml
URL:
http://svn.apache.org/viewvc/avro/trunk/doc/src/content/xdocs/spec.xml?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/doc/src/content/xdocs/spec.xml (original)
+++ avro/trunk/doc/src/content/xdocs/spec.xml Fri Apr 4 10:33:02 2014
@@ -1336,6 +1336,34 @@ void initFPTable() {
</section>
</section>
+ <section>
+ <title>Record Mappings</title>
+
+ <p>Implementations may optionally map the following record schemas to an
appropriate
+ native type.</p>
+
+ <section>
+ <title>Decimal</title>
+ <p>The <code>Decimal</code> type represents arbitrary-precision signed
decimal
+ numbers. A <code>Decimal</code> is encoded as an <code>int</code>
<em>scale</em>
+ followed by a <code>bytes</code> <em>value</em> field containing the
+ two's-complement representation of the unscaled integer value in
big-endian
+ byte order.</p>
+ <p>The value of the number represented by this <code>Decimal</code>
type is
+ <em>value × 10<sup>-scale</sup></em>.</p>
+ <source>
+{
+ "type": "record",
+ "name": "org.apache.avro.Decimal",
+ "fields": [
+ {"name": "scale", "type": "int"},
+ {"name": "value", "type": "bytes"}
+ ]
+}
+ </source>
+ </section>
+ </section>
+
<p><em>Apache Avro, Avro, Apache, and the Avro and Apache logos are
trademarks of The Apache Software Foundation.</em></p>
Added:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java?rev=1584605&view=auto
==============================================================================
---
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
(added)
+++
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
Fri Apr 4 10:33:02 2014
@@ -0,0 +1,65 @@
+package org.apache.avro.generic;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.SchemaBuilder.RecordBuilder;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.Encoder;
+
+/**
+ * <p>This {@link RecordMapping} writes a {@code BigDecimal}
+ * object as an Avro record with the following schema:</p>
+ * <pre><code>
+ * {
+ * "type": "record",
+ * "name": "org.apache.avro.Decimal",
+ * "fields": [
+ * {"name": "scale", "type": "int"},
+ * {"name": "value", "type": "bytes"}
+ * ]
+ * }
+ * </code></pre>
+ */
+public class DecimalRecordMapping extends RecordMapping<BigDecimal> {
+ public DecimalRecordMapping() {
+ this(null, null);
+ }
+
+ public DecimalRecordMapping(Integer maxPrecision, Integer maxScale) {
+ super(schema(maxPrecision, maxScale), BigDecimal.class);
+ }
+
+ private static Schema schema(Integer maxPrecision, Integer maxScale) {
+ RecordBuilder<Schema> builder =
SchemaBuilder.record("org.apache.avro.Decimal");
+ if (maxPrecision != null && maxScale != null) {
+ builder.prop("maxPrecision", Integer.toString(maxPrecision))
+ .prop("maxScale", Integer.toString(maxScale));
+ }
+ return builder.fields()
+ .requiredInt("scale")
+ .requiredBytes("value")
+ .endRecord();
+ }
+
+ @Override
+ public void write(Object datum, Encoder out) throws IOException {
+ BigDecimal decimal = (BigDecimal) datum;
+ out.writeInt(decimal.scale());
+ out.writeBytes(decimal.unscaledValue().toByteArray());
+ }
+
+ @Override
+ public BigDecimal read(Object reuse, Decoder in) throws IOException {
+ // BigDecimal instances are immutable so can't reuse
+ int scale = in.readInt();
+ ByteBuffer byteBuffer = in.readBytes(null);
+ byte[] value = new byte[byteBuffer.remaining()];
+ byteBuffer.get(value);
+ return new BigDecimal(new BigInteger(value), scale);
+ }
+
+}
Modified:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
---
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
(original)
+++
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
Fri Apr 4 10:33:02 2014
@@ -64,6 +64,12 @@ public class GenericData {
private final ClassLoader classLoader;
+ private final Map<String, RecordMapping<?>> recordMappings =
+ new HashMap<String, RecordMapping<?>>();
+
+ private final Map<Class<?>, RecordMapping<?>> recordMappingClasses =
+ new HashMap<Class<?>, RecordMapping<?>>();
+
/** Set the Java type to be used when reading this schema. Meaningful only
* only string schemas and map schemas (for the keys). */
public static void setStringType(Schema s, StringType stringType) {
@@ -91,6 +97,22 @@ public class GenericData {
/** Return the class loader that's used (by subclasses). */
public ClassLoader getClassLoader() { return classLoader; }
+ /** Register a {@link org.apache.avro.generic.RecordMapping} to be used when
reading or
+ * writing records using {@link org.apache.avro.generic.GenericDatumReader}
or
+ * {@link org.apache.avro.generic.GenericDatumWriter}.
+ */
+ public <T> void addRecordMapping(RecordMapping<T> recordMapping) {
+ recordMappings.put(recordMapping.getSchema().getFullName(), recordMapping);
+ recordMappingClasses.put(recordMapping.getRecordClass(), recordMapping);
+ }
+
+ /** Return the {@link org.apache.avro.generic.RecordMapping} for the given
record
+ * schema, or <code>null</code> if there is no mapping registered.
+ */
+ public RecordMapping<?> getRecordMapping(Schema schema) {
+ return recordMappings.get(schema.getFullName());
+ }
+
/** Default implementation of {@link GenericRecord}. Note that this
implementation
* does not fill in default values for fields if they are not specified; use
{@link
* GenericRecordBuilder} in that case.
@@ -677,14 +699,25 @@ public class GenericData {
/** Called by the default implementation of {@link #instanceOf}.*/
protected boolean isRecord(Object datum) {
- return datum instanceof IndexedRecord;
+ return datum instanceof IndexedRecord || isRecordMapping(datum);
+ }
+
+ /** Return true if the given <code>datum</code> has a registered record
mapping class.
+ * Called by the default implementation of {@link #isRecord(Object)}.
+ */
+ protected boolean isRecordMapping(Object datum) {
+ return datum != null && recordMappingClasses.containsKey(datum.getClass());
}
/** Called to obtain the schema of a record. By default calls
- * {GenericContainer#getSchema(). May be overridden for alternate record
- * representations. */
+ * {@link GenericContainer#getSchema()}, or uses a registered record mapper
if not a
+ * {@link GenericContainer}. May be overridden for alternate
+ * record representations. */
protected Schema getRecordSchema(Object record) {
- return ((GenericContainer)record).getSchema();
+ if (record instanceof GenericContainer) {
+ return ((GenericContainer) record).getSchema();
+ }
+ return recordMappingClasses.get(record.getClass()).getSchema();
}
/** Called by the default implementation of {@link #instanceOf}.*/
Modified:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
---
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
(original)
+++
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
Fri Apr 4 10:33:02 2014
@@ -170,6 +170,10 @@ public class GenericDatumReader<D> imple
* representations.*/
protected Object readRecord(Object old, Schema expected,
ResolvingDecoder in) throws IOException {
+ RecordMapping<?> recordMapping = data.getRecordMapping(expected);
+ if (recordMapping != null) {
+ return recordMapping.read(old, in);
+ }
Object r = data.newRecord(old, expected);
Object state = data.getRecordState(r, expected);
Modified:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
---
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
(original)
+++
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
Fri Apr 4 10:33:02 2014
@@ -99,6 +99,11 @@ public class GenericDatumWriter<D> imple
* representations.*/
protected void writeRecord(Schema schema, Object datum, Encoder out)
throws IOException {
+ RecordMapping<?> recordMapping = data.getRecordMapping(schema);
+ if (recordMapping != null) {
+ recordMapping.write(datum, out);
+ return;
+ }
Object state = data.getRecordState(datum, schema);
for (Field f : schema.getFields()) {
writeField(datum, f, out, state);
Added:
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java?rev=1584605&view=auto
==============================================================================
---
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java
(added)
+++
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java
Fri Apr 4 10:33:02 2014
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.generic;
+
+import java.io.IOException;
+import org.apache.avro.Schema;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.Encoder;
+
+/**
+ * Expert: a custom mapping that writes an object directly as an Avro record.
+ * No validation is performed to check that the encoding conforms to the
schema.
+ * Invalid implementations may result in an unreadable file.
+ * The use of {@link org.apache.avro.io.ValidatingEncoder} is recommended.
+ *
+ * @param <T> The class of objects that can be serialized as Avro records
using this
+ * record mapping.
+ * @see org.apache.avro.generic.GenericData#addRecordMapping(RecordMapping)
+ */
+public abstract class RecordMapping<T> {
+
+ private final Schema schema;
+ private final Class<T> recordClass;
+
+ /**
+ * Create a record mapping for the given record {@link
org.apache.avro.Schema}
+ * for serializing objects of the given {@link java.lang.Class}.
+ */
+ public RecordMapping(Schema schema, Class<T> recordClass) {
+ this.schema = schema;
+ this.recordClass = recordClass;
+ }
+
+ /**
+ * @return the schema describing the records that are serialized using this
record
+ * mapping.
+ */
+ public Schema getSchema() {
+ return schema;
+ }
+
+ /**
+ * @return the class of objects that can be serialized as Avro records using
this
+ * record mapping.
+ */
+ public Class<T> getRecordClass() {
+ return recordClass;
+ }
+
+ public abstract void write(Object datum, Encoder out) throws IOException;
+
+ public abstract T read(Object reuse, Decoder in) throws IOException;
+
+}
Modified: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
URL:
http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
(original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java Fri
Apr 4 10:33:02 2014
@@ -26,6 +26,7 @@ import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -42,6 +43,7 @@ import org.apache.avro.data.Json;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.DecimalRecordMapping;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.Decoder;
@@ -540,6 +542,40 @@ public class TestSchema {
}
}
+ @Test
+ public void testDecimalRecordMapping() throws Exception {
+ String recordJson = "{\"type\":\"record\"," +
+ "\"name\":\"org.apache.avro.Decimal\"," +
+ "\"fields\":[\n" +
+ " {\"name\":\"scale\",\"type\":\"int\"},\n" +
+ " {\"name\":\"value\",\"type\":\"bytes\"}\n" +
+ "]}";
+ Schema schema = Schema.parse(recordJson);
+ BigDecimal decimal = new BigDecimal("12.45");
+ GenericData data = new GenericData();
+ data.addRecordMapping(new DecimalRecordMapping());
+ checkBinary(schema, decimal,
+ new GenericDatumWriter<Object>(schema, data),
+ new GenericDatumReader<Object>(schema, schema, data));
+ }
+
+ @Test
+ public void testDecimalRecordMappingUnion() throws Exception {
+ String recordJson = "{\"type\":\"record\"," +
+ "\"name\":\"org.apache.avro.Decimal\"," +
+ "\"fields\":[\n" +
+ " {\"name\":\"scale\",\"type\":\"int\"},\n" +
+ " {\"name\":\"value\",\"type\":\"bytes\"}\n" +
+ "]}";
+ Schema schema = Schema.parse("[\"null\",\"string\"," + recordJson + "]");
+ BigDecimal decimal = new BigDecimal("12.45");
+ GenericData data = new GenericData();
+ data.addRecordMapping(new DecimalRecordMapping());
+ checkBinary(schema, decimal,
+ new GenericDatumWriter<Object>(schema, data),
+ new GenericDatumReader<Object>(schema, schema, data));
+ }
+
private static void checkParseError(String json) {
try {
Schema.parse(json);
