This is an automated email from the ASF dual-hosted git repository. emkornfield pushed a commit to branch decimal256 in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/decimal256 by this push: new 9965614 Decimal256 java implementation with working integration tests. (#8281) 9965614 is described below commit 99656142ad139dd6f16903cec50af15877b10de6 Author: emkornfield <emkornfi...@gmail.com> AuthorDate: Fri Sep 25 20:50:45 2020 -0700 Decimal256 java implementation with working integration tests. (#8281) This PR completes round trip between C++ and Java integration tests. --- cpp/src/arrow/testing/json_internal.cc | 14 ++- dev/archery/archery/integration/datagen.py | 30 ++++-- .../java/org/apache/arrow/AvroToArrowUtils.java | 2 +- .../arrow/adapter/jdbc/JdbcToArrowUtils.java | 2 +- java/vector/src/main/codegen/data/ArrowTypes.tdd | 2 +- .../src/main/codegen/data/ValueVectorTypes.tdd | 17 +++ .../codegen/templates/AbstractFieldWriter.java | 8 +- .../templates/AbstractPromotableFieldWriter.java | 26 ++++- .../src/main/codegen/templates/ArrowType.java | 5 +- .../src/main/codegen/templates/ComplexCopier.java | 9 +- .../src/main/codegen/templates/ComplexWriters.java | 27 ++--- .../main/codegen/templates/DenseUnionReader.java | 4 +- .../main/codegen/templates/DenseUnionVector.java | 12 +-- .../main/codegen/templates/DenseUnionWriter.java | 12 +-- .../main/codegen/templates/HolderReaderImpl.java | 5 + .../src/main/codegen/templates/StructWriters.java | 4 +- .../templates/UnionFixedSizeListWriter.java | 54 ++++++++++ .../main/codegen/templates/UnionListWriter.java | 46 ++++++++ .../src/main/codegen/templates/UnionMapWriter.java | 15 +++ .../src/main/codegen/templates/UnionReader.java | 8 +- .../src/main/codegen/templates/UnionVector.java | 28 ++--- .../src/main/codegen/templates/UnionWriter.java | 50 ++++----- .../{DecimalVector.java => BigDecimalVector.java} | 84 +++++++-------- .../java/org/apache/arrow/vector/BufferLayout.java | 5 +- .../org/apache/arrow/vector/DecimalVector.java | 6 +- .../java/org/apache/arrow/vector/TypeLayout.java | 2 +- .../vector/complex/impl/PromotableWriter.java | 38 ++++++- .../apache/arrow/vector/ipc/JsonFileReader.java | 24 ++++- .../apache/arrow/vector/ipc/JsonFileWriter.java | 13 ++- .../java/org/apache/arrow/vector/types/Types.java | 19 ++++ .../apache/arrow/vector/util/DecimalUtility.java | 65 ++++++----- .../vector/validate/ValidateVectorTypeVisitor.java | 3 +- ...ecimalVector.java => TestBigDecimalVector.java} | 76 ++++++------- .../org/apache/arrow/vector/TestDecimalVector.java | 20 ++-- .../org/apache/arrow/vector/TestTypeLayout.java | 6 +- .../org/apache/arrow/vector/TestVectorAlloc.java | 2 +- .../vector/complex/impl/TestComplexCopier.java | 18 ++-- .../vector/complex/writer/TestComplexWriter.java | 6 +- .../apache/arrow/vector/types/pojo/TestSchema.java | 2 +- .../arrow/vector/util/DecimalUtilityTest.java | 119 +++++++++++---------- 40 files changed, 592 insertions(+), 296 deletions(-) diff --git a/cpp/src/arrow/testing/json_internal.cc b/cpp/src/arrow/testing/json_internal.cc index 4a191a6..fae0e35 100644 --- a/cpp/src/arrow/testing/json_internal.cc +++ b/cpp/src/arrow/testing/json_internal.cc @@ -839,8 +839,20 @@ Status GetDecimal(const RjObject& json_type, std::shared_ptr<DataType>* type) { ARROW_ASSIGN_OR_RAISE(const int32_t precision, GetMemberInt<int32_t>(json_type, "precision")); ARROW_ASSIGN_OR_RAISE(const int32_t scale, GetMemberInt<int32_t>(json_type, "scale")); + int32_t bit_width = 128; + Result<int32_t> maybe_bit_width = GetMemberInt<int32_t>(json_type, "bitWidth"); + if (maybe_bit_width.ok()) { + bit_width = maybe_bit_width.ValueOrDie(); + } - *type = decimal(precision, scale); + if (bit_width == 128) { + *type = decimal128(precision, scale); + } else if (bit_width == 256) { + *type = decimal256(precision, scale); + } else { + return Status::Invalid("Only 128 bit and 256 Decimals are supported. Received", + bit_width); + } return Status::OK(); } diff --git a/dev/archery/archery/integration/datagen.py b/dev/archery/archery/integration/datagen.py index b740198..5d62835 100644 --- a/dev/archery/archery/integration/datagen.py +++ b/dev/archery/archery/integration/datagen.py @@ -401,7 +401,7 @@ class FloatingPointField(PrimitiveField): DECIMAL_PRECISION_TO_VALUE = { key: (1 << (8 * i - 1)) - 1 for i, key in enumerate( [1, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24, 27, 29, 32, 34, 36, - 38, 40, 42, 44, 50, 60, 70], + 40, 42, 44, 50, 60, 70], start=1, ) } @@ -1274,20 +1274,29 @@ def generate_null_trivial_case(batch_sizes): return _generate_file('null_trivial', fields, batch_sizes) -def generate_decimal_case(): +def generate_decimal128_case(): fields = [ DecimalField(name='f{}'.format(i), precision=precision, scale=2, - bit_width=128) + bit_width=128) for i, precision in enumerate(range(3, 39)) - ] + [ + ] + + possible_batch_sizes = 7, 10 + batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))] + return _generate_file('decimal128', fields, batch_sizes) + + +def generate_decimal256_case(): + fields = [ DecimalField(name='f{}'.format(i), precision=precision, scale=5, - bit_width=256) + bit_width=256) for i, precision in enumerate(range(37, 70)) ] possible_batch_sizes = 7, 10 batch_sizes = [possible_batch_sizes[i % 2] for i in range(len(fields))] - return _generate_file('decimal', fields, batch_sizes) + return _generate_file('decimal256', fields, batch_sizes) + def generate_datetime_case(): @@ -1520,10 +1529,15 @@ def get_generated_json_files(tempdir=None): .skip_category('JS') # TODO(ARROW-7900) .skip_category('Go'), # TODO(ARROW-7901) - generate_decimal_case() + generate_decimal128_case() + .skip_category('Go') # TODO(ARROW-7948): Decimal + Go + .skip_category('Rust'), + + generate_decimal256_case() .skip_category('Go') # TODO(ARROW-7948): Decimal + Go + .skip_category('JS') .skip_category('Rust'), - .skip_category('Java'), + generate_datetime_case() diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java b/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java index 29e44da..80293c8 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/AvroToArrowUtils.java @@ -298,7 +298,7 @@ public class AvroToArrowUtils { Preconditions.checkArgument(scale <= precision, "Invalid decimal scale: %s (greater than precision: %s)", scale, precision); - return new ArrowType.Decimal(precision, scale); + return new ArrowType.Decimal(precision, scale, 128); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index e534d20..f64f178 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -251,7 +251,7 @@ public class JdbcToArrowUtils { case Types.DECIMAL: int precision = fieldInfo.getPrecision(); int scale = fieldInfo.getScale(); - return new ArrowType.Decimal(precision, scale); + return new ArrowType.Decimal(precision, scale, 128); case Types.REAL: case Types.FLOAT: return new ArrowType.FloatingPoint(SINGLE); diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd index 4d2a540..3cf9a96 100644 --- a/java/vector/src/main/codegen/data/ArrowTypes.tdd +++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd @@ -92,7 +92,7 @@ }, { name: "Decimal", - fields: [{name: "precision", type: int}, {name: "scale", type: int}], + fields: [{name: "precision", type: int}, {name: "scale", type: int}, {name: "bitWidth", type: int}], complex: false }, { diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd index b9e0529..cff1b73 100644 --- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd +++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd @@ -115,6 +115,22 @@ }, { major: "Fixed", + width: 32, + javaType: "ArrowBuf", + boxedType: "ArrowBuf", + + minor: [ + { + class: "BigDecimal", + maxPrecisionDigits: 76, nDecimalDigits: 4, friendlyType: "BigDecimal", + typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}], + arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal", + fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}] + } + ] + }, + { + major: "Fixed", width: 16, javaType: "ArrowBuf", boxedType: "ArrowBuf", @@ -129,6 +145,7 @@ } ] }, + { major: "Fixed", width: -1, diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java index 4f6d5ea..5252345 100644 --- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java @@ -76,20 +76,20 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr fail("${name}"); } - <#if minor.class == "Decimal"> + <#if minor.class?ends_with("Decimal")> public void write${minor.class}(${friendlyType} value) { fail("${name}"); } - public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, ArrowType arrowType</#if>) { + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, ArrowType arrowType) { fail("${name}"); } - public void writeBigEndianBytesToDecimal(byte[] value) { + public void writeBigEndianBytesTo${minor.class}(byte[] value) { fail("${name}"); } - public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) { + public void writeBigEndianBytesTo${minor.class}(byte[] value, ArrowType arrowType) { fail("${name}"); } </#if> diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java index 5566c80..b31277f 100644 --- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java +++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java @@ -75,7 +75,7 @@ abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> - <#if minor.class != "Decimal"> + <#if minor.class != "Decimal" && minor.class != "BigDecimal"> @Override public void write(${name}Holder holder) { getWriter(MinorType.${name?upper_case}).write(holder); @@ -85,7 +85,7 @@ abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter { getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>); } - <#else> + <#elseif minor.class == "Decimal"> @Override public void write(DecimalHolder holder) { getWriter(MinorType.DECIMAL).write(holder); @@ -106,6 +106,28 @@ abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter { public void writeBigEndianBytesToDecimal(byte[] value) { getWriter(MinorType.DECIMAL).writeBigEndianBytesToDecimal(value); } + <#elseif minor.class == "BigDecimal"> + @Override + public void write(BigDecimalHolder holder) { + getWriter(MinorType.BIGDECIMAL).write(holder); + } + + public void writeBigDecimal(int start, ArrowBuf buffer, ArrowType arrowType) { + getWriter(MinorType.BIGDECIMAL).writeBigDecimal(start, buffer, arrowType); + } + + public void writeBigDecimal(int start, ArrowBuf buffer) { + getWriter(MinorType.BIGDECIMAL).writeBigDecimal(start, buffer); + } + public void writeBigEndianBytesToBigDecimal(byte[] value, ArrowType arrowType) { + getWriter(MinorType.BIGDECIMAL).writeBigEndianBytesToBigDecimal(value, arrowType); + } + + public void writeBigEndianBytesToBigDecimal(byte[] value) { + getWriter(MinorType.BIGDECIMAL).writeBigEndianBytesToBigDecimal(value); + } + + </#if> </#list></#list> diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java index 77894af..082a923 100644 --- a/java/vector/src/main/codegen/templates/ArrowType.java +++ b/java/vector/src/main/codegen/templates/ArrowType.java @@ -327,9 +327,8 @@ public abstract class ArrowType { </#if> </#list> <#if type.name == "Decimal"> - int bitWidth = ${nameLower}Type.bitWidth(); - if (bitWidth != defaultDecimalBitWidth) { - throw new IllegalArgumentException("Library only supports 128-bit decimal values"); + if (bitWidth != defaultDecimalBitWidth && bitWidth != 256) { + throw new IllegalArgumentException("Library only supports 128-bit and 256-bit decimal values"); } </#if> return new ${name}(<#list type.fields as field><#if field.valueType??>${field.valueType}.fromFlatbufID(${field.name})<#else>${field.name}</#if><#if field_has_next>, </#if></#list>); diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java index 1189e8e..fa0a9cb 100644 --- a/java/vector/src/main/codegen/templates/ComplexCopier.java +++ b/java/vector/src/main/codegen/templates/ComplexCopier.java @@ -117,14 +117,14 @@ public class ComplexCopier { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class?starts_with("Decimal") > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > case ${name?upper_case}: if (reader.isSet()) { Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder(); reader.read(${uncappedName}Holder); if (${uncappedName}Holder.isSet == 1) { - writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, new ArrowType.Decimal(decimalHolder.precision, decimalHolder.scale)</#if>); + writer.write${name}(<#list fields as field>${uncappedName}Holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class?ends_with("Decimal")>, new ArrowType.Decimal(${uncappedName}Holder.precision, ${uncappedName}Holder.scale, ${name}Holder.WIDTH * 8)</#if>); } } else { writer.writeNull(); @@ -145,7 +145,7 @@ public class ComplexCopier { case ${name?upper_case}: return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(name); </#if> - <#if minor.class == "Decimal"> + <#if minor.class?ends_with("Decimal")> case ${name?upper_case}: if (reader.getField().getType() instanceof ArrowType.Decimal) { ArrowType.Decimal type = (ArrowType.Decimal) reader.getField().getType(); @@ -154,6 +154,7 @@ public class ComplexCopier { return (FieldWriter) writer.${uncappedName}(name); } </#if> + </#list></#list> case STRUCT: return (FieldWriter) writer.struct(name); @@ -171,7 +172,7 @@ public class ComplexCopier { <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class?starts_with("Decimal") > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > case ${name?upper_case}: return (FieldWriter) writer.<#if name == "Int">integer<#else>${uncappedName}</#if>(); </#if> diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java index ab99ac3..e1cbf13 100644 --- a/java/vector/src/main/codegen/templates/ComplexWriters.java +++ b/java/vector/src/main/codegen/templates/ComplexWriters.java @@ -99,7 +99,7 @@ public class ${eName}WriterImpl extends AbstractFieldWriter { <#else> - <#if minor.class != "Decimal"> + <#if !minor.class?ends_with("Decimal")> public void write(${minor.class}Holder h) { vector.setSafe(idx(), h); vector.setValueCount(idx()+1); @@ -123,15 +123,15 @@ public class ${eName}WriterImpl extends AbstractFieldWriter { } </#if> - <#if minor.class == "Decimal"> + <#if minor.class?ends_with("Decimal")> - public void write(DecimalHolder h){ + public void write(${minor.class}Holder h){ DecimalUtility.checkPrecisionAndScale(h.precision, h.scale, vector.getPrecision(), vector.getScale()); vector.setSafe(idx(), h); vector.setValueCount(idx() + 1); } - public void write(NullableDecimalHolder h){ + public void write(Nullable${minor.class}Holder h){ if (h.isSet == 1) { DecimalUtility.checkPrecisionAndScale(h.precision, h.scale, vector.getPrecision(), vector.getScale()); } @@ -139,37 +139,38 @@ public class ${eName}WriterImpl extends AbstractFieldWriter { vector.setValueCount(idx() + 1); } - public void writeDecimal(int start, ArrowBuf buffer){ + public void write${minor.class}(int start, ArrowBuf buffer){ vector.setSafe(idx(), 1, start, buffer); vector.setValueCount(idx() + 1); } - public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType){ + public void write${minor.class}(int start, ArrowBuf buffer, ArrowType arrowType){ DecimalUtility.checkPrecisionAndScale(((ArrowType.Decimal) arrowType).getPrecision(), ((ArrowType.Decimal) arrowType).getScale(), vector.getPrecision(), vector.getScale()); vector.setSafe(idx(), 1, start, buffer); vector.setValueCount(idx() + 1); } - public void writeDecimal(BigDecimal value){ + public void write${minor.class}(BigDecimal value){ // vector.setSafe already does precision and scale checking vector.setSafe(idx(), value); vector.setValueCount(idx() + 1); } - public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType){ + public void writeBigEndianBytesTo${minor.class}(byte[] value, ArrowType arrowType){ DecimalUtility.checkPrecisionAndScale(((ArrowType.Decimal) arrowType).getPrecision(), ((ArrowType.Decimal) arrowType).getScale(), vector.getPrecision(), vector.getScale()); vector.setBigEndianSafe(idx(), value); vector.setValueCount(idx() + 1); } - public void writeBigEndianBytesToDecimal(byte[] value){ + public void writeBigEndianBytesTo${minor.class}(byte[] value){ vector.setBigEndianSafe(idx(), value); vector.setValueCount(idx() + 1); } </#if> + public void writeNull() { vector.setNull(idx()); vector.setValueCount(idx()+1); @@ -190,18 +191,18 @@ package org.apache.arrow.vector.complex.writer; public interface ${eName}Writer extends BaseWriter { public void write(${minor.class}Holder h); - <#if minor.class == "Decimal">@Deprecated</#if> + <#if minor.class?ends_with("Decimal")>@Deprecated</#if> public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>); -<#if minor.class == "Decimal"> +<#if minor.class?ends_with("Decimal")> public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, ArrowType arrowType); public void write${minor.class}(${friendlyType} value); - public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType); + public void writeBigEndianBytesTo${minor.class}(byte[] value, ArrowType arrowType); @Deprecated - public void writeBigEndianBytesToDecimal(byte[] value); + public void writeBigEndianBytesTo${minor.class}(byte[] value); </#if> } diff --git a/java/vector/src/main/codegen/templates/DenseUnionReader.java b/java/vector/src/main/codegen/templates/DenseUnionReader.java index 51bd7d1..0b3a66c 100644 --- a/java/vector/src/main/codegen/templates/DenseUnionReader.java +++ b/java/vector/src/main/codegen/templates/DenseUnionReader.java @@ -92,7 +92,7 @@ public class DenseUnionReader extends AbstractFieldReader { <#list type.minor as minor> <#assign name = minor.class?cap_first /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> case ${name?upper_case}: reader = (FieldReader) get${name}(typeId); break; @@ -165,7 +165,7 @@ public class DenseUnionReader extends AbstractFieldReader { <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> private ${name}ReaderImpl get${name}(byte typeId) { ${name}ReaderImpl reader = (${name}ReaderImpl) readers[typeId]; diff --git a/java/vector/src/main/codegen/templates/DenseUnionVector.java b/java/vector/src/main/codegen/templates/DenseUnionVector.java index 8902f29..2ef4358 100644 --- a/java/vector/src/main/codegen/templates/DenseUnionVector.java +++ b/java/vector/src/main/codegen/templates/DenseUnionVector.java @@ -306,13 +306,13 @@ public class DenseUnionVector implements FieldVector { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> <#assign lowerCaseName = name?lower_case/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> - public ${name}Vector get${name}Vector(byte typeId<#if minor.class == "Decimal">, ArrowType arrowType</#if>) { + public ${name}Vector get${name}Vector(byte typeId<#if minor.class?ends_with("Decimal")>, ArrowType arrowType</#if>) { ValueVector vector = typeId < 0 ? null : childVectors[typeId]; if (vector == null) { int vectorCount = internalStruct.size(); - vector = addOrGet(typeId, MinorType.${name?upper_case}<#if minor.class == "Decimal">, arrowType</#if>, ${name}Vector.class); + vector = addOrGet(typeId, MinorType.${name?upper_case}<#if minor.class?ends_with("Decimal")>, arrowType</#if>, ${name}Vector.class); childVectors[typeId] = vector; if (internalStruct.size() > vectorCount) { vector.allocateNew(); @@ -810,7 +810,7 @@ public class DenseUnionVector implements FieldVector { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> case ${name?upper_case}: Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder(); reader.read(${uncappedName}Holder); @@ -834,13 +834,13 @@ public class DenseUnionVector implements FieldVector { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> public void setSafe(int index, Nullable${name}Holder holder) { while (index >= getOffsetBufferValueCapacity()) { reallocOffsetBuffer(); } byte typeId = getTypeId(index); - ${name}Vector vector = get${name}Vector(typeId<#if minor.class == "Decimal">, new ArrowType.Decimal(holder.precision, holder.scale)</#if>); + ${name}Vector vector = get${name}Vector(typeId<#if minor.class?ends_with("Decimal")>, new ArrowType.Decimal(holder.precision, holder.scale, holder.WIDTH * 8)</#if>); int offset = vector.getValueCount(); vector.setValueCount(offset + 1); vector.setSafe(offset, holder); diff --git a/java/vector/src/main/codegen/templates/DenseUnionWriter.java b/java/vector/src/main/codegen/templates/DenseUnionWriter.java index ee6f614..992cc9f 100644 --- a/java/vector/src/main/codegen/templates/DenseUnionWriter.java +++ b/java/vector/src/main/codegen/templates/DenseUnionWriter.java @@ -123,7 +123,7 @@ public class DenseUnionWriter extends AbstractFieldWriter implements FieldWriter <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> case ${name?upper_case}: return get${name}Writer(typeId); </#if> @@ -138,7 +138,7 @@ public class DenseUnionWriter extends AbstractFieldWriter implements FieldWriter <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> private ${name}Writer get${name}Writer(byte typeId) { ${name}Writer writer = (${name}Writer) writers[typeId]; @@ -159,10 +159,10 @@ public class DenseUnionWriter extends AbstractFieldWriter implements FieldWriter throw new UnsupportedOperationException(); } - public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, byte typeId<#if minor.class == "Decimal">, ArrowType arrowType</#if>) { + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>, byte typeId<#if minor.class?ends_with("Decimal")>, ArrowType arrowType</#if>) { data.setTypeId(idx(), typeId); get${name}Writer(typeId).setPosition(data.getOffset(idx())); - get${name}Writer(typeId).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, arrowType</#if>); + get${name}Writer(typeId).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class?ends_with("Decimal")>, arrowType</#if>); } </#if> </#list> @@ -208,7 +208,7 @@ public class DenseUnionWriter extends AbstractFieldWriter implements FieldWriter <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if !minor.typeParams?? || minor.class == "Decimal" > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > @Override public ${capName}Writer ${lowerName}(String name) { byte typeId = data.getTypeId(idx()); @@ -225,7 +225,7 @@ public class DenseUnionWriter extends AbstractFieldWriter implements FieldWriter return getListWriter(typeId).${lowerName}(); } </#if> - <#if minor.class == "Decimal"> + <#if minor.class?ends_with("Decimal")> public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { byte typeId = data.getTypeId(idx()); data.setTypeId(idx(), typeId); diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java index fa7e83a..8c33959 100644 --- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java +++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java @@ -129,6 +129,11 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader { holder.buffer.getBytes(holder.start, bytes, 0, ${type.width}); ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale); return value; + <#elseif minor.class == "BigDecimal"> + byte[] bytes = new byte[${type.width}]; + holder.buffer.getBytes(holder.start, bytes, 0, ${type.width}); + ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale); + return value; <#elseif minor.class == "FixedSizeBinary"> byte[] value = new byte [holder.byteWidth]; holder.buffer.getBytes(0, value, 0, holder.byteWidth); diff --git a/java/vector/src/main/codegen/templates/StructWriters.java b/java/vector/src/main/codegen/templates/StructWriters.java index 7df2217..46d8309 100644 --- a/java/vector/src/main/codegen/templates/StructWriters.java +++ b/java/vector/src/main/codegen/templates/StructWriters.java @@ -255,7 +255,7 @@ public class ${mode}StructWriter extends AbstractFieldWriter { <#assign constructorParams = constructorParams + [ typeParam.name ] /> </#list> </#if> - new ${minor.arrowType}(${constructorParams?join(", ")}) + new ${minor.arrowType}(${constructorParams?join(", ")}<#if minor.class?ends_with("Decimal")>, ${vectName}Vector.TYPE_WIDTH * 8</#if>) <#else> MinorType.${upperName}.getType() </#if> @@ -274,7 +274,7 @@ public class ${mode}StructWriter extends AbstractFieldWriter { } else { if (writer instanceof PromotableWriter) { // ensure writers are initialized - ((PromotableWriter)writer).getWriter(MinorType.${upperName}<#if minor.class == "Decimal">, new ${minor.arrowType}(precision, scale)</#if>); + ((PromotableWriter)writer).getWriter(MinorType.${upperName}<#if minor.class?ends_with("Decimal")>, new ${minor.arrowType}(precision, scale, ${vectName}Vector.TYPE_WIDTH * 8)</#if>); } } return writer; diff --git a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java index 0574dcf..ed59ea9 100644 --- a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java @@ -16,9 +16,12 @@ */ import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.complex.writer.BigDecimalWriter; import org.apache.arrow.vector.complex.writer.DecimalWriter; +import org.apache.arrow.vector.holders.BigDecimalHolder; import org.apache.arrow.vector.holders.DecimalHolder; + import java.lang.UnsupportedOperationException; import java.math.BigDecimal; @@ -127,6 +130,22 @@ public class UnionFixedSizeListWriter extends AbstractFieldWriter { return writer.decimal(name); } + + @Override + public BigDecimalWriter bigDecimal() { + return this; + } + + @Override + public BigDecimalWriter bigDecimal(String name, int scale, int precision) { + return writer.bigDecimal(name, scale, precision); + } + + @Override + public BigDecimalWriter bigDecimal(String name) { + return writer.bigDecimal(name); + } + @Override public StructWriter struct() { inStruct = true; @@ -180,6 +199,16 @@ public class UnionFixedSizeListWriter extends AbstractFieldWriter { writer.write(holder); writer.setPosition(writer.idx() + 1); } + + @Override + public void write(BigDecimalHolder holder) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.write(holder); + writer.setPosition(writer.idx() + 1); + } + @Override public void writeNull() { @@ -213,6 +242,31 @@ public class UnionFixedSizeListWriter extends AbstractFieldWriter { writer.setPosition(writer.idx() + 1); } + public void writeBigDecimal(int start, ArrowBuf buffer, ArrowType arrowType) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.writeBigDecimal(start, buffer, arrowType); + writer.setPosition(writer.idx() + 1); + } + + public void writeBigDecimal(BigDecimal value) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.writeBigDecimal(value); + writer.setPosition(writer.idx() + 1); + } + + public void writeBigEndianBytesToBigDecimal(byte[] value, ArrowType arrowType) { + if (writer.idx() >= (idx() + 1) * listSize) { + throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize)); + } + writer.writeBigEndianBytesToBigDecimal(value, arrowType); + writer.setPosition(writer.idx() + 1); + } + + <#list vv.types as type> <#list type.minor as minor> <#assign name = minor.class?cap_first /> diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java index a266443..432ef92 100644 --- a/java/vector/src/main/codegen/templates/UnionListWriter.java +++ b/java/vector/src/main/codegen/templates/UnionListWriter.java @@ -16,9 +16,12 @@ */ import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.complex.writer.BigDecimalWriter; import org.apache.arrow.vector.complex.writer.DecimalWriter; +import org.apache.arrow.vector.holders.BigDecimalHolder; import org.apache.arrow.vector.holders.DecimalHolder; + import java.lang.UnsupportedOperationException; import java.math.BigDecimal; @@ -134,6 +137,22 @@ public class Union${listName}Writer extends AbstractFieldWriter { } @Override + public BigDecimalWriter bigDecimal() { + return this; + } + + @Override + public BigDecimalWriter bigDecimal(String name, int scale, int precision) { + return writer.bigDecimal(name, scale, precision); + } + + @Override + public BigDecimalWriter bigDecimal(String name) { + return writer.bigDecimal(name); + } + + + @Override public StructWriter struct() { inStruct = true; return this; @@ -200,6 +219,12 @@ public class Union${listName}Writer extends AbstractFieldWriter { } @Override + public void write(BigDecimalHolder holder) { + writer.write(holder); + writer.setPosition(writer.idx()+1); + } + + @Override public void writeNull() { writer.writeNull(); } @@ -224,6 +249,27 @@ public class Union${listName}Writer extends AbstractFieldWriter { writer.setPosition(writer.idx() + 1); } + public void writeBigDecimal(int start, ArrowBuf buffer, ArrowType arrowType) { + writer.writeBigDecimal(start, buffer, arrowType); + writer.setPosition(writer.idx()+1); + } + + public void writeBigDecimal(int start, ArrowBuf buffer) { + writer.writeBigDecimal(start, buffer); + writer.setPosition(writer.idx()+1); + } + + public void writeBigDecimal(BigDecimal value) { + writer.writeBigDecimal(value); + writer.setPosition(writer.idx()+1); + } + + public void writeBigEndianBytesToBigDecimal(byte[] value, ArrowType arrowType){ + writer.writeBigEndianBytesToBigDecimal(value, arrowType); + writer.setPosition(writer.idx() + 1); + } + + <#list vv.types as type> <#list type.minor as minor> <#assign name = minor.class?cap_first /> diff --git a/java/vector/src/main/codegen/templates/UnionMapWriter.java b/java/vector/src/main/codegen/templates/UnionMapWriter.java index 01b3713..8edc2ee 100644 --- a/java/vector/src/main/codegen/templates/UnionMapWriter.java +++ b/java/vector/src/main/codegen/templates/UnionMapWriter.java @@ -16,7 +16,9 @@ */ import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.complex.writer.BigDecimalWriter; import org.apache.arrow.vector.complex.writer.DecimalWriter; +import org.apache.arrow.vector.holders.BigDecimalHolder; import org.apache.arrow.vector.holders.DecimalHolder; import java.lang.UnsupportedOperationException; @@ -170,6 +172,19 @@ public class UnionMapWriter extends UnionListWriter { } @Override + public BigDecimalWriter bigDecimal() { + switch (mode) { + case KEY: + return entryWriter.bigDecimal(MapVector.KEY_NAME); + case VALUE: + return entryWriter.bigDecimal(MapVector.VALUE_NAME); + default: + return this; + } + } + + + @Override public StructWriter struct() { switch (mode) { case KEY: diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java index 20fdb41..467b5ba 100644 --- a/java/vector/src/main/codegen/templates/UnionReader.java +++ b/java/vector/src/main/codegen/templates/UnionReader.java @@ -34,7 +34,7 @@ package org.apache.arrow.vector.complex.impl; @SuppressWarnings("unused") public class UnionReader extends AbstractFieldReader { - private BaseReader[] readers = new BaseReader[43]; + private BaseReader[] readers = new BaseReader[44]; public UnionVector data; public UnionReader(UnionVector data) { @@ -45,7 +45,7 @@ public class UnionReader extends AbstractFieldReader { return TYPES[data.getTypeValue(idx())]; } - private static MinorType[] TYPES = new MinorType[43]; + private static MinorType[] TYPES = new MinorType[44]; static { for (MinorType minorType : MinorType.values()) { @@ -88,7 +88,7 @@ public class UnionReader extends AbstractFieldReader { <#list type.minor as minor> <#assign name = minor.class?cap_first /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> case ${name?upper_case}: return (FieldReader) get${name}(); </#if> @@ -157,7 +157,7 @@ public class UnionReader extends AbstractFieldReader { <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> <#assign safeType=friendlyType /> <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if> - <#if !minor.typeParams?? || minor.class == "Decimal" > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > private ${name}ReaderImpl ${uncappedName}Reader; diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index b63aa87..906db7c 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -273,18 +273,18 @@ public class UnionVector implements FieldVector { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> <#assign lowerCaseName = name?lower_case/> - <#if !minor.typeParams?? || minor.class == "Decimal" > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > private ${name}Vector ${uncappedName}Vector; - public ${name}Vector get${name}Vector(<#if minor.class == "Decimal"> ArrowType arrowType</#if>) { - return get${name}Vector(null<#if minor.class == "Decimal">, arrowType</#if>); + public ${name}Vector get${name}Vector(<#if minor.class?ends_with("Decimal")> ArrowType arrowType</#if>) { + return get${name}Vector(null<#if minor.class?ends_with("Decimal")>, arrowType</#if>); } - public ${name}Vector get${name}Vector(String name<#if minor.class == "Decimal">, ArrowType arrowType</#if>) { + public ${name}Vector get${name}Vector(String name<#if minor.class?ends_with("Decimal")>, ArrowType arrowType</#if>) { if (${uncappedName}Vector == null) { int vectorCount = internalStruct.size(); - ${uncappedName}Vector = addOrGet(name, MinorType.${name?upper_case},<#if minor.class == "Decimal"> arrowType,</#if> ${name}Vector.class); + ${uncappedName}Vector = addOrGet(name, MinorType.${name?upper_case},<#if minor.class?ends_with("Decimal")> arrowType,</#if> ${name}Vector.class); if (internalStruct.size() > vectorCount) { ${uncappedName}Vector.allocateNew(); if (callBack != null) { @@ -294,10 +294,10 @@ public class UnionVector implements FieldVector { } return ${uncappedName}Vector; } - <#if minor.class == "Decimal"> + <#if minor.class?ends_with("Decimal")> public ${name}Vector get${name}Vector() { if (${uncappedName}Vector == null) { - throw new IllegalArgumentException("No Decimal Vector present. Provide ArrowType argument to create a new vector"); + throw new IllegalArgumentException("No Decimal ${uncappedName} present. Provide ArrowType argument to create a new vector"); } return ${uncappedName}Vector; } @@ -638,9 +638,9 @@ public class UnionVector implements FieldVector { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal" > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > case ${name?upper_case}: - return get${name}Vector(name<#if minor.class == "Decimal">, arrowType</#if>); + return get${name}Vector(name<#if minor.class?ends_with("Decimal")>, arrowType</#if>); </#if> </#list> </#list> @@ -723,11 +723,11 @@ public class UnionVector implements FieldVector { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal" > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > case ${name?upper_case}: Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder(); reader.read(${uncappedName}Holder); - setSafe(index, ${uncappedName}Holder<#if minor.class == "Decimal">, arrowType</#if>); + setSafe(index, ${uncappedName}Holder<#if minor.class?ends_with("Decimal")>, arrowType</#if>); break; </#if> </#list> @@ -749,10 +749,10 @@ public class UnionVector implements FieldVector { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal" > - public void setSafe(int index, Nullable${name}Holder holder<#if minor.class == "Decimal">, ArrowType arrowType</#if>) { + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > + public void setSafe(int index, Nullable${name}Holder holder<#if minor.class?ends_with("Decimal")>, ArrowType arrowType</#if>) { setType(index, MinorType.${name?upper_case}); - get${name}Vector(null<#if minor.class == "Decimal">, arrowType</#if>).setSafe(index, holder); + get${name}Vector(null<#if minor.class?ends_with("Decimal")>, arrowType</#if>).setSafe(index, holder); } </#if> diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java index 6f2b2e1..754c8c3 100644 --- a/java/vector/src/main/codegen/templates/UnionWriter.java +++ b/java/vector/src/main/codegen/templates/UnionWriter.java @@ -125,9 +125,9 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { <#assign name = minor.class?cap_first /> <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> - <#if !minor.typeParams?? || minor.class == "Decimal"> + <#if !minor.typeParams?? || minor.class?ends_with("Decimal")> case ${name?upper_case}: - return get${name}Writer(<#if minor.class == "Decimal" >arrowType</#if>); + return get${name}Writer(<#if minor.class?ends_with("Decimal") >arrowType</#if>); </#if> </#list> </#list> @@ -141,49 +141,49 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { <#assign fields = minor.fields!type.fields /> <#assign uncappedName = name?uncap_first/> <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) /> - <#if !minor.typeParams?? || minor.class == "Decimal" > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > private ${name}Writer ${name?uncap_first}Writer; - private ${name}Writer get${name}Writer(<#if minor.class == "Decimal">ArrowType arrowType</#if>) { + private ${name}Writer get${name}Writer(<#if minor.class?ends_with("Decimal")>ArrowType arrowType</#if>) { if (${uncappedName}Writer == null) { - ${uncappedName}Writer = new ${name}WriterImpl(data.get${name}Vector(<#if minor.class == "Decimal">arrowType</#if>)); + ${uncappedName}Writer = new ${name}WriterImpl(data.get${name}Vector(<#if minor.class?ends_with("Decimal")>arrowType</#if>)); ${uncappedName}Writer.setPosition(idx()); writers.add(${uncappedName}Writer); } return ${uncappedName}Writer; } - public ${name}Writer as${name}(<#if minor.class == "Decimal">ArrowType arrowType</#if>) { + public ${name}Writer as${name}(<#if minor.class?ends_with("Decimal")>ArrowType arrowType</#if>) { data.setType(idx(), MinorType.${name?upper_case}); - return get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>); + return get${name}Writer(<#if minor.class?ends_with("Decimal")>arrowType</#if>); } @Override public void write(${name}Holder holder) { data.setType(idx(), MinorType.${name?upper_case}); - <#if minor.class == "Decimal">ArrowType arrowType = new ArrowType.Decimal(holder.precision, holder.scale);</#if> - get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).setPosition(idx()); - get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, arrowType</#if>); + <#if minor.class?ends_with("Decimal")>ArrowType arrowType = new ArrowType.Decimal(holder.precision, holder.scale, ${name}Holder.WIDTH * 8);</#if> + get${name}Writer(<#if minor.class?ends_with("Decimal")>arrowType</#if>).setPosition(idx()); + get${name}Writer(<#if minor.class?ends_with("Decimal")>arrowType</#if>).write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list><#if minor.class?ends_with("Decimal")>, arrowType</#if>); } - public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, ArrowType arrowType</#if>) { + public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list><#if minor.class?ends_with("Decimal")>, ArrowType arrowType</#if>) { data.setType(idx(), MinorType.${name?upper_case}); - get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).setPosition(idx()); - get${name}Writer(<#if minor.class == "Decimal">arrowType</#if>).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class == "Decimal">, arrowType</#if>); + get${name}Writer(<#if minor.class?ends_with("Decimal")>arrowType</#if>).setPosition(idx()); + get${name}Writer(<#if minor.class?ends_with("Decimal")>arrowType</#if>).write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list><#if minor.class?ends_with("Decimal")>, arrowType</#if>); } - <#if minor.class == "Decimal"> - public void write${minor.class}(${friendlyType} value) { - data.setType(idx(), MinorType.DECIMAL); - ArrowType arrowType = new ArrowType.Decimal(value.precision(), value.scale()); - getDecimalWriter(arrowType).setPosition(idx()); - getDecimalWriter(arrowType).writeDecimal(value); + <#if minor.class?ends_with("Decimal")> + public void write${name}(${friendlyType} value) { + data.setType(idx(), MinorType.${name?upper_case}); + ArrowType arrowType = new ArrowType.Decimal(value.precision(), value.scale(), ${name}Vector.TYPE_WIDTH * 8); + get${name}Writer(arrowType).setPosition(idx()); + get${name}Writer(arrowType).write${name}(value); } - public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) { - data.setType(idx(), MinorType.DECIMAL); - getDecimalWriter(arrowType).setPosition(idx()); - getDecimalWriter(arrowType).writeBigEndianBytesToDecimal(value, arrowType); + public void writeBigEndianBytesTo${name}(byte[] value, ArrowType arrowType) { + data.setType(idx(), MinorType.${name?upper_case}); + get${name}Writer(arrowType).setPosition(idx()); + get${name}Writer(arrowType).writeBigEndianBytesTo${name}(value, arrowType); } </#if> </#if> @@ -226,7 +226,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { <#if lowerName == "int" ><#assign lowerName = "integer" /></#if> <#assign upperName = minor.class?upper_case /> <#assign capName = minor.class?cap_first /> - <#if !minor.typeParams?? || minor.class == "Decimal" > + <#if !minor.typeParams?? || minor.class?ends_with("Decimal") > @Override public ${capName}Writer ${lowerName}(String name) { data.setType(idx(), MinorType.STRUCT); @@ -241,7 +241,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter { return getListWriter().${lowerName}(); } </#if> - <#if minor.class == "Decimal"> + <#if minor.class?ends_with("Decimal")> @Override public ${capName}Writer ${lowerName}(String name<#list minor.typeParams as typeParam>, ${typeParam.type} ${typeParam.name}</#list>) { data.setType(idx(), MinorType.STRUCT); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BigDecimalVector.java similarity index 86% copy from java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java copy to java/vector/src/main/java/org/apache/arrow/vector/BigDecimalVector.java index 554e174..c3a36a2 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BigDecimalVector.java @@ -23,10 +23,10 @@ import java.math.BigDecimal; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.complex.impl.DecimalReaderImpl; +import org.apache.arrow.vector.complex.impl.BigDecimalReaderImpl; import org.apache.arrow.vector.complex.reader.FieldReader; -import org.apache.arrow.vector.holders.DecimalHolder; -import org.apache.arrow.vector.holders.NullableDecimalHolder; +import org.apache.arrow.vector.holders.BigDecimalHolder; +import org.apache.arrow.vector.holders.NullableBigDecimalHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -37,52 +37,52 @@ import org.apache.arrow.vector.util.TransferPair; import io.netty.util.internal.PlatformDependent; /** - * DecimalVector implements a fixed width vector (16 bytes) of + * BigDecimalVector implements a fixed width vector (32 bytes) of * decimal values which could be null. A validity buffer (bit vector) is * maintained to track which elements in the vector are null. */ -public final class DecimalVector extends BaseFixedWidthVector { - public static final byte TYPE_WIDTH = 16; +public final class BigDecimalVector extends BaseFixedWidthVector { + public static final byte TYPE_WIDTH = 32; private final FieldReader reader; private final int precision; private final int scale; /** - * Instantiate a DecimalVector. This doesn't allocate any memory for + * Instantiate a BigDecimalVector. This doesn't allocate any memory for * the data in vector. * * @param name name of the vector * @param allocator allocator for memory management. */ - public DecimalVector(String name, BufferAllocator allocator, + public BigDecimalVector(String name, BufferAllocator allocator, int precision, int scale) { - this(name, FieldType.nullable(new ArrowType.Decimal(precision, scale)), allocator); + this(name, FieldType.nullable(new ArrowType.Decimal(precision, scale, /*bitWidth=*/TYPE_WIDTH * 8)), allocator); } /** - * Instantiate a DecimalVector. This doesn't allocate any memory for + * Instantiate a BigDecimalVector. This doesn't allocate any memory for * the data in vector. * * @param name name of the vector * @param fieldType type of Field materialized by this vector * @param allocator allocator for memory management. */ - public DecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { + public BigDecimalVector(String name, FieldType fieldType, BufferAllocator allocator) { this(new Field(name, fieldType, null), allocator); } /** - * Instantiate a DecimalVector. This doesn't allocate any memory for + * Instantiate a BigDecimalVector. This doesn't allocate any memory for * the data in vector. * * @param field field materialized by this vector * @param allocator allocator for memory management. */ - public DecimalVector(Field field, BufferAllocator allocator) { + public BigDecimalVector(Field field, BufferAllocator allocator) { super(field, allocator, TYPE_WIDTH); ArrowType.Decimal arrowType = (ArrowType.Decimal) field.getFieldType().getType(); - reader = new DecimalReaderImpl(DecimalVector.this); + reader = new BigDecimalReaderImpl(BigDecimalVector.this); this.precision = arrowType.getPrecision(); this.scale = arrowType.getScale(); } @@ -105,7 +105,7 @@ public final class DecimalVector extends BaseFixedWidthVector { */ @Override public MinorType getMinorType() { - return MinorType.DECIMAL; + return MinorType.BIGDECIMAL; } @@ -136,7 +136,7 @@ public final class DecimalVector extends BaseFixedWidthVector { * * @param index position of element */ - public void get(int index, NullableDecimalHolder holder) { + public void get(int index, NullableBigDecimalHolder holder) { if (isSet(index) == 0) { holder.isSet = 0; return; @@ -158,7 +158,7 @@ public final class DecimalVector extends BaseFixedWidthVector { if (isSet(index) == 0) { return null; } else { - return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); + return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale, TYPE_WIDTH); } } @@ -197,7 +197,7 @@ public final class DecimalVector extends BaseFixedWidthVector { /** * Set the decimal element at given index to the provided array of bytes. - * Decimal is now implemented as Little Endian. This API allows the user + * BigDecimal is now implemented as Little Endian. This API allows the user * to pass a decimal value in the form of byte array in BE byte order. * * <p>Consumers of Arrow code can use this API instead of first swapping @@ -205,7 +205,7 @@ public final class DecimalVector extends BaseFixedWidthVector { * ArrowBuf of decimal vector. * * <p>This method takes care of adding the necessary padding if the length - * of byte array is less then 16 (length of decimal type). + * of byte array is less then 32 (length of decimal type). * * @param index position of element * @param value array of bytes containing decimal in big endian byte order. @@ -228,14 +228,14 @@ public final class DecimalVector extends BaseFixedWidthVector { } if (length == 0) { - PlatformDependent.setMemory(outAddress, DecimalVector.TYPE_WIDTH, (byte) 0); + PlatformDependent.setMemory(outAddress, BigDecimalVector.TYPE_WIDTH, (byte) 0); } else if (length < TYPE_WIDTH) { // sign extend final byte pad = (byte) (value[0] < 0 ? 0xFF : 0x00); - PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); + PlatformDependent.setMemory(outAddress + length, BigDecimalVector.TYPE_WIDTH - length, pad); } else { throw new IllegalArgumentException( - "Invalid decimal value length. Valid length in [1 - 16], got " + length); + "Invalid decimal value length. Valid length in [1 - 32], got " + length); } } @@ -252,7 +252,7 @@ public final class DecimalVector extends BaseFixedWidthVector { } /** - * Sets the element at given index using the buffer whose size maybe <= 16 bytes. + * Sets the element at given index using the buffer whose size maybe <= 32 bytes. * @param index index to write the decimal to * @param start start of value in the buffer * @param buffer contains the decimal in little endian bytes @@ -270,16 +270,16 @@ public final class DecimalVector extends BaseFixedWidthVector { long outAddress = valueBuffer.memoryAddress() + (long) index * TYPE_WIDTH; PlatformDependent.copyMemory(inAddress, outAddress, length); // sign extend - if (length < 16) { + if (length < 32) { byte msb = PlatformDependent.getByte(inAddress + length - 1); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); + PlatformDependent.setMemory(outAddress + length, BigDecimalVector.TYPE_WIDTH - length, pad); } } /** - * Sets the element at given index using the buffer whose size maybe <= 16 bytes. + * Sets the element at given index using the buffer whose size maybe <= 32 bytes. * @param index index to write the decimal to * @param start start of value in the buffer * @param buffer contains the decimal in big endian bytes @@ -302,10 +302,10 @@ public final class DecimalVector extends BaseFixedWidthVector { PlatformDependent.putByte(outAddress + byteIdx, val); } // sign extend - if (length < 16) { + if (length < 32) { byte msb = PlatformDependent.getByte(inAddress); final byte pad = (byte) (msb < 0 ? 0xFF : 0x00); - PlatformDependent.setMemory(outAddress + length, DecimalVector.TYPE_WIDTH - length, pad); + PlatformDependent.setMemory(outAddress + length, BigDecimalVector.TYPE_WIDTH - length, pad); } } @@ -318,7 +318,7 @@ public final class DecimalVector extends BaseFixedWidthVector { public void set(int index, BigDecimal value) { BitVectorHelper.setBit(validityBuffer, index); DecimalUtility.checkPrecisionAndScale(value, precision, scale); - DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); + DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index, TYPE_WIDTH); } /** @@ -329,7 +329,7 @@ public final class DecimalVector extends BaseFixedWidthVector { */ public void set(int index, long value) { BitVectorHelper.setBit(validityBuffer, index); - DecimalUtility.writeLongToArrowBuf(value, valueBuffer, index); + DecimalUtility.writeLongToArrowBufBigDecimal(value, valueBuffer, index); } /** @@ -340,7 +340,7 @@ public final class DecimalVector extends BaseFixedWidthVector { * @param index position of element * @param holder nullable data holder for value of element */ - public void set(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + public void set(int index, NullableBigDecimalHolder holder) throws IllegalArgumentException { if (holder.isSet < 0) { throw new IllegalArgumentException(); } else if (holder.isSet > 0) { @@ -357,7 +357,7 @@ public final class DecimalVector extends BaseFixedWidthVector { * @param index position of element * @param holder data holder for value of element */ - public void set(int index, DecimalHolder holder) { + public void set(int index, BigDecimalHolder holder) { BitVectorHelper.setBit(validityBuffer, index); valueBuffer.setBytes((long) index * TYPE_WIDTH, holder.buffer, holder.start, TYPE_WIDTH); } @@ -433,20 +433,20 @@ public final class DecimalVector extends BaseFixedWidthVector { * @param index position of element * @param holder nullable data holder for value of element */ - public void setSafe(int index, NullableDecimalHolder holder) throws IllegalArgumentException { + public void setSafe(int index, NullableBigDecimalHolder holder) throws IllegalArgumentException { handleSafe(index); set(index, holder); } /** - * Same as {@link #set(int, DecimalHolder)} except that it handles the + * Same as {@link #set(int, BigDecimalHolder)} except that it handles the * case when index is greater than or equal to existing * value capacity {@link #getValueCapacity()}. * * @param index position of element * @param holder data holder for value of element */ - public void setSafe(int index, DecimalHolder holder) { + public void setSafe(int index, BigDecimalHolder holder) { handleSafe(index); set(index, holder); } @@ -511,23 +511,23 @@ public final class DecimalVector extends BaseFixedWidthVector { */ @Override public TransferPair makeTransferPair(ValueVector to) { - return new TransferImpl((DecimalVector) to); + return new TransferImpl((BigDecimalVector) to); } private class TransferImpl implements TransferPair { - DecimalVector to; + BigDecimalVector to; public TransferImpl(String ref, BufferAllocator allocator) { - to = new DecimalVector(ref, allocator, DecimalVector.this.precision, - DecimalVector.this.scale); + to = new BigDecimalVector(ref, allocator, BigDecimalVector.this.precision, + BigDecimalVector.this.scale); } - public TransferImpl(DecimalVector to) { + public TransferImpl(BigDecimalVector to) { this.to = to; } @Override - public DecimalVector getTo() { + public BigDecimalVector getTo() { return to; } @@ -543,7 +543,7 @@ public final class DecimalVector extends BaseFixedWidthVector { @Override public void copyValueSafe(int fromIndex, int toIndex) { - to.copyFromSafe(fromIndex, toIndex, DecimalVector.this); + to.copyFromSafe(fromIndex, toIndex, BigDecimalVector.this); } } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java index 0bd64c0..09c874e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java @@ -51,6 +51,7 @@ public class BufferLayout { private static final BufferLayout LARGE_OFFSET_BUFFER = new BufferLayout(BufferType.OFFSET, 64); private static final BufferLayout TYPE_BUFFER = new BufferLayout(BufferType.TYPE, 32); private static final BufferLayout BIT_BUFFER = new BufferLayout(BufferType.DATA, 1); + private static final BufferLayout VALUES_256 = new BufferLayout(BufferType.DATA, 256); private static final BufferLayout VALUES_128 = new BufferLayout(BufferType.DATA, 128); private static final BufferLayout VALUES_64 = new BufferLayout(BufferType.DATA, 64); private static final BufferLayout VALUES_32 = new BufferLayout(BufferType.DATA, 32); @@ -85,8 +86,10 @@ public class BufferLayout { return VALUES_64; case 128: return VALUES_128; + case 256: + return VALUES_256; default: - throw new IllegalArgumentException("only 8, 16, 32, 64, or 128 bits supported"); + throw new IllegalArgumentException("only 8, 16, 32, 64, 128, or 256 bits supported"); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index 554e174..53458ad 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -57,7 +57,7 @@ public final class DecimalVector extends BaseFixedWidthVector { */ public DecimalVector(String name, BufferAllocator allocator, int precision, int scale) { - this(name, FieldType.nullable(new ArrowType.Decimal(precision, scale)), allocator); + this(name, FieldType.nullable(new ArrowType.Decimal(precision, scale, TYPE_WIDTH * 8)), allocator); } /** @@ -158,7 +158,7 @@ public final class DecimalVector extends BaseFixedWidthVector { if (isSet(index) == 0) { return null; } else { - return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale); + return DecimalUtility.getBigDecimalFromArrowBuf(valueBuffer, index, scale, TYPE_WIDTH); } } @@ -318,7 +318,7 @@ public final class DecimalVector extends BaseFixedWidthVector { public void set(int index, BigDecimal value) { BitVectorHelper.setBit(validityBuffer, index); DecimalUtility.checkPrecisionAndScale(value, precision, scale); - DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index); + DecimalUtility.writeBigDecimalToArrowBuf(value, valueBuffer, index, TYPE_WIDTH); } /** diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java index 501ca98..1004ce1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java @@ -155,7 +155,7 @@ public class TypeLayout { @Override public TypeLayout visit(Decimal type) { - return newFixedWidthTypeLayout(BufferLayout.dataBuffer(128)); + return newFixedWidthTypeLayout(BufferLayout.dataBuffer(type.getBitWidth())); } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java index 6f40836..be9056e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java @@ -30,6 +30,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.complex.writer.FieldWriter; +import org.apache.arrow.vector.holders.BigDecimalHolder; import org.apache.arrow.vector.holders.DecimalHolder; import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -54,6 +55,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { private final NullableStructWriterFactory nullableStructWriterFactory; private int position; private static final int MAX_DECIMAL_PRECISION = 38; + private static final int MAX_BIG_DECIMAL_PRECISION = 76; private enum State { UNTYPED, SINGLE, UNION @@ -316,27 +318,55 @@ public class PromotableWriter extends AbstractPromotableFieldWriter { @Override public void write(DecimalHolder holder) { - getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION, holder.scale)).write(holder); + getWriter(MinorType.DECIMAL, + new ArrowType.Decimal(MAX_DECIMAL_PRECISION, holder.scale, /*bitWidth=*/128)).write(holder); } @Override public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) { getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION, - ((ArrowType.Decimal) arrowType).getScale())).writeDecimal(start, buffer, arrowType); + ((ArrowType.Decimal) arrowType).getScale(), /*bitWidth=*/128)).writeDecimal(start, buffer, arrowType); } @Override public void writeDecimal(BigDecimal value) { - getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION, value.scale())).writeDecimal(value); + getWriter(MinorType.DECIMAL, + new ArrowType.Decimal(MAX_DECIMAL_PRECISION, value.scale(), /*bitWidth=*/128)).writeDecimal(value); } @Override public void writeBigEndianBytesToDecimal(byte[] value, ArrowType arrowType) { getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION, - ((ArrowType.Decimal) arrowType).getScale())).writeBigEndianBytesToDecimal(value, arrowType); + ((ArrowType.Decimal) arrowType).getScale(), /*bitWidth=*/128)).writeBigEndianBytesToDecimal(value, arrowType); } @Override + public void write(BigDecimalHolder holder) { + getWriter(MinorType.BIGDECIMAL, + new ArrowType.Decimal(MAX_BIG_DECIMAL_PRECISION, holder.scale, /*bitWidth=*/256)).write(holder); + } + + @Override + public void writeBigDecimal(int start, ArrowBuf buffer, ArrowType arrowType) { + getWriter(MinorType.BIGDECIMAL, new ArrowType.Decimal(MAX_BIG_DECIMAL_PRECISION, + ((ArrowType.Decimal) arrowType).getScale(), /*bitWidth=*/256)).writeBigDecimal(start, buffer, arrowType); + } + + @Override + public void writeBigDecimal(BigDecimal value) { + getWriter(MinorType.BIGDECIMAL, + new ArrowType.Decimal(MAX_BIG_DECIMAL_PRECISION, value.scale(), /*bitWidth=*/256)).writeBigDecimal(value); + } + + @Override + public void writeBigEndianBytesToBigDecimal(byte[] value, ArrowType arrowType) { + getWriter(MinorType.BIGDECIMAL, new ArrowType.Decimal(MAX_BIG_DECIMAL_PRECISION, + ((ArrowType.Decimal) arrowType).getScale(), + /*bitWidth=*/256)).writeBigEndianBytesToBigDecimal(value, arrowType); + } + + + @Override public void allocate() { getWriter().allocate(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java index 13935ef..48acc2b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java @@ -42,6 +42,7 @@ import java.util.Objects; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.BigDecimalVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.BufferLayout.BufferType; @@ -438,7 +439,7 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider { for (int i = 0; i < count; i++) { parser.nextToken(); BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); - DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i); + DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i, DecimalVector.TYPE_WIDTH); } buf.writerIndex(size); @@ -446,6 +447,24 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider { } }; + BufferReader BIG_DECIMAL = new BufferReader() { + @Override + protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { + final int size = count * BigDecimalVector.TYPE_WIDTH; + ArrowBuf buf = allocator.buffer(size); + + for (int i = 0; i < count; i++) { + parser.nextToken(); + BigDecimal decimalValue = new BigDecimal(parser.readValueAs(String.class)); + DecimalUtility.writeBigDecimalToArrowBuf(decimalValue, buf, i, BigDecimalVector.TYPE_WIDTH); + } + + buf.writerIndex(size); + return buf; + } + }; + + BufferReader FIXEDSIZEBINARY = new BufferReader() { @Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { @@ -615,6 +634,9 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider { case DECIMAL: reader = helper.DECIMAL; break; + case BIGDECIMAL: + reader = helper.BIG_DECIMAL; + break; case FIXEDSIZEBINARY: reader = helper.FIXEDSIZEBINARY; break; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java index e210b00..043c7ef 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BaseVariableWidthVector; +import org.apache.arrow.vector.BigDecimalVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.BufferLayout.BufferType; @@ -377,11 +378,21 @@ public class JsonFileWriter implements AutoCloseable { } case DECIMAL: { int scale = ((DecimalVector) vector).getScale(); - BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale); + BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale, + DecimalVector.TYPE_WIDTH); // We write the unscaled value, because the scale is stored in the type metadata. generator.writeString(decimalValue.unscaledValue().toString()); break; } + case BIGDECIMAL: { + int scale = ((BigDecimalVector) vector).getScale(); + BigDecimal decimalValue = DecimalUtility.getBigDecimalFromArrowBuf(buffer, index, scale, + BigDecimalVector.TYPE_WIDTH); + // We write the unscaled value, because the scale is stored in the type metadata. + generator.writeString(decimalValue.unscaledValue().toString()); + break; + } + default: throw new UnsupportedOperationException("minor type: " + vector.getMinorType()); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index 886478c..d43342e 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -23,6 +23,7 @@ import static org.apache.arrow.vector.types.UnionMode.Dense; import static org.apache.arrow.vector.types.UnionMode.Sparse; import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BigDecimalVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; @@ -68,6 +69,7 @@ import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; +import org.apache.arrow.vector.complex.impl.BigDecimalWriterImpl; import org.apache.arrow.vector.complex.impl.BigIntWriterImpl; import org.apache.arrow.vector.complex.impl.BitWriterImpl; import org.apache.arrow.vector.complex.impl.DateDayWriterImpl; @@ -528,6 +530,20 @@ public class Types { return new DecimalWriterImpl((DecimalVector) vector); } }, + BIGDECIMAL(null) { + @Override + public FieldVector getNewVector( + Field field, + BufferAllocator allocator, + CallBack schemaChangeCallback) { + return new BigDecimalVector(field, allocator); + } + + @Override + public FieldWriter getNewFieldWriter(ValueVector vector) { + return new BigDecimalWriterImpl((BigDecimalVector) vector); + } + }, FIXEDSIZEBINARY(null) { @Override public FieldVector getNewVector( @@ -899,6 +915,9 @@ public class Types { @Override public MinorType visit(Decimal type) { + if (type.getBitWidth() == 256) { + return MinorType.BIGDECIMAL; + } return MinorType.DECIMAL; } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index 711fa3b..8f891cc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -32,24 +32,26 @@ public class DecimalUtility { private DecimalUtility() {} public static final int DECIMAL_BYTE_LENGTH = 16; - public static final byte [] zeroes = new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - public static final byte [] minus_one = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; + public static final byte [] zeroes = new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + public static final byte [] minus_one = new byte[] {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; /** * Read an ArrowType.Decimal at the given value index in the ArrowBuf and convert to a BigDecimal * with the given scale. */ - public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) { - byte[] value = new byte[DECIMAL_BYTE_LENGTH]; + public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale, int byteWidth) { + byte[] value = new byte[byteWidth]; byte temp; - final int startIndex = index * DECIMAL_BYTE_LENGTH; + final int startIndex = index * byteWidth; // Decimal stored as little endian, need to swap bytes to make BigDecimal - bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH); - int stop = DECIMAL_BYTE_LENGTH / 2; + bytebuf.getBytes(startIndex, value, 0, byteWidth); + int stop = byteWidth / 2; for (int i = 0, j; i < stop; i++) { temp = value[i]; - j = (DECIMAL_BYTE_LENGTH - 1) - i; + j = (byteWidth - 1) - i; value[i] = value[j]; value[j] = temp; } @@ -61,8 +63,8 @@ public class DecimalUtility { * Read an ArrowType.Decimal from the ByteBuffer and convert to a BigDecimal with the given * scale. */ - public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int scale) { - byte[] value = new byte[DECIMAL_BYTE_LENGTH]; + public static BigDecimal getBigDecimalFromByteBuffer(ByteBuffer bytebuf, int scale, int byteWidth) { + byte[] value = new byte[byteWidth]; bytebuf.get(value); BigInteger unscaledValue = new BigInteger(value); return new BigDecimal(unscaledValue, scale); @@ -72,10 +74,10 @@ public class DecimalUtility { * Read an ArrowType.Decimal from the ArrowBuf at the given value index and return it as a byte * array. */ - public static byte[] getByteArrayFromArrowBuf(ArrowBuf bytebuf, int index) { - final byte[] value = new byte[DECIMAL_BYTE_LENGTH]; - final long startIndex = (long) index * DECIMAL_BYTE_LENGTH; - bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH); + public static byte[] getByteArrayFromArrowBuf(ArrowBuf bytebuf, int index, int byteWidth) { + final byte[] value = new byte[byteWidth]; + final long startIndex = (long) index * byteWidth; + bytebuf.getBytes(startIndex, value, 0, byteWidth); return value; } @@ -119,34 +121,47 @@ public class DecimalUtility { * UnsupportedOperationException if the decimal size is greater than the Decimal vector byte * width. */ - public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, int index) { + public static void writeBigDecimalToArrowBuf(BigDecimal value, ArrowBuf bytebuf, int index, int byteWidth) { final byte[] bytes = value.unscaledValue().toByteArray(); - writeByteArrayToArrowBufHelper(bytes, bytebuf, index); + writeByteArrayToArrowBufHelper(bytes, bytebuf, index, byteWidth); } /** * Write the given long to the ArrowBuf at the given value index. */ public static void writeLongToArrowBuf(long value, ArrowBuf bytebuf, int index) { - final long addressOfValue = bytebuf.memoryAddress() + (long) index * DECIMAL_BYTE_LENGTH; + final long addressOfValue = bytebuf.memoryAddress() + (long) index * 16; PlatformDependent.putLong(addressOfValue, value); final long padValue = Long.signum(value) == -1 ? -1L : 0L; PlatformDependent.putLong(addressOfValue + Long.BYTES, padValue); } /** + * Write value to the buffer extending it to 32 bytes at the given index. + */ + public static void writeLongToArrowBufBigDecimal(long value, ArrowBuf bytebuf, int index) { + final long addressOfValue = bytebuf.memoryAddress() + (long) index * 32; + PlatformDependent.putLong(addressOfValue, value); + final long padValue = Long.signum(value) == -1 ? -1L : 0L; + PlatformDependent.putLong(addressOfValue + Long.BYTES, padValue); + PlatformDependent.putLong(addressOfValue + 2 * Long.BYTES, padValue); + PlatformDependent.putLong(addressOfValue + 3 * Long.BYTES, padValue); + } + + + /** * Write the given byte array to the ArrowBuf at the given value index. Will throw an * UnsupportedOperationException if the decimal size is greater than the Decimal vector byte * width. */ - public static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int index) { - writeByteArrayToArrowBufHelper(bytes, bytebuf, index); + public static void writeByteArrayToArrowBuf(byte[] bytes, ArrowBuf bytebuf, int index, int byteWidth) { + writeByteArrayToArrowBufHelper(bytes, bytebuf, index, byteWidth); } - private static void writeByteArrayToArrowBufHelper(byte[] bytes, ArrowBuf bytebuf, int index) { - final long startIndex = (long) index * DECIMAL_BYTE_LENGTH; - if (bytes.length > DECIMAL_BYTE_LENGTH) { - throw new UnsupportedOperationException("Decimal size greater than 16 bytes"); + private static void writeByteArrayToArrowBufHelper(byte[] bytes, ArrowBuf bytebuf, int index, int byteWidth) { + final long startIndex = (long) index * byteWidth; + if (bytes.length > byteWidth) { + throw new UnsupportedOperationException("Decimal size greater than " + byteWidth + " bytes: " + bytes.length); } // Decimal stored as little endian, need to swap data bytes before writing to ArrowBuf @@ -156,8 +171,8 @@ public class DecimalUtility { } // Write LE data - byte [] padByes = bytes[0] < 0 ? minus_one : zeroes; + byte [] padBytes = bytes[0] < 0 ? minus_one : zeroes; bytebuf.setBytes(startIndex, bytesLE, 0, bytes.length); - bytebuf.setBytes(startIndex + bytes.length, padByes, 0, DECIMAL_BYTE_LENGTH - bytes.length); + bytebuf.setBytes(startIndex + bytes.length, padBytes, 0, byteWidth - bytes.length); } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java index d9b0e7b..57c5c6b 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java @@ -22,6 +22,7 @@ import static org.apache.arrow.vector.validate.ValidateUtil.validateOrThrow; import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; +import org.apache.arrow.vector.BigDecimalVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; @@ -172,7 +173,7 @@ public class ValidateVectorTypeVisitor implements VectorVisitor<Void, Void> { validateIntVector(vector, 64, false); } else if (vector instanceof BitVector) { validateVectorCommon(vector, ArrowType.Bool.class); - } else if (vector instanceof DecimalVector) { + } else if (vector instanceof DecimalVector || vector instanceof BigDecimalVector) { validateVectorCommon(vector, ArrowType.Decimal.class); ArrowType.Decimal arrowType = (ArrowType.Decimal) vector.getField().getType(); validateOrThrow(arrowType.getScale() > 0, "The scale of decimal %s is not positive.", arrowType.getScale()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBigDecimalVector.java similarity index 81% copy from java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java copy to java/vector/src/test/java/org/apache/arrow/vector/TestBigDecimalVector.java index 28d799b..7542a14 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBigDecimalVector.java @@ -31,7 +31,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -public class TestDecimalVector { +public class TestBigDecimalVector { private static long[] intValues; @@ -59,10 +59,10 @@ public class TestDecimalVector { @Test public void testValuesWriteRead() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(10, scale), allocator);) { + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(10, scale, 256), allocator);) { - try (DecimalVector oldConstructor = new DecimalVector("decimal", allocator, 10, scale);) { + try (BigDecimalVector oldConstructor = new BigDecimalVector("decimal", allocator, 10, scale);) { assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType()); } @@ -85,8 +85,8 @@ public class TestDecimalVector { @Test public void testBigDecimalDifferentScaleAndPrecision() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(4, 2), allocator);) { + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(4, 2, 256), allocator);) { decimalVector.allocateNew(); // test BigDecimal with different scale @@ -115,17 +115,17 @@ public class TestDecimalVector { @Test public void testWriteBigEndian() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(38, 9), allocator);) { + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(38, 18, 256), allocator);) { decimalVector.allocateNew(); - BigDecimal decimal1 = new BigDecimal("123456789.000000000"); - BigDecimal decimal2 = new BigDecimal("11.123456789"); - BigDecimal decimal3 = new BigDecimal("1.000000000"); - BigDecimal decimal4 = new BigDecimal("0.111111111"); - BigDecimal decimal5 = new BigDecimal("987654321.123456789"); - BigDecimal decimal6 = new BigDecimal("222222222222.222222222"); - BigDecimal decimal7 = new BigDecimal("7777777777777.666666667"); - BigDecimal decimal8 = new BigDecimal("1212121212.343434343"); + BigDecimal decimal1 = new BigDecimal("123456789.000000000000000000"); + BigDecimal decimal2 = new BigDecimal("11.123456789123456789"); + BigDecimal decimal3 = new BigDecimal("1.000000000000000000"); + BigDecimal decimal4 = new BigDecimal("0.111111111000000000"); + BigDecimal decimal5 = new BigDecimal("987654321.123456789000000000"); + BigDecimal decimal6 = new BigDecimal("222222222222.222222222000000000"); + BigDecimal decimal7 = new BigDecimal("7777777777777.666666667000000000"); + BigDecimal decimal8 = new BigDecimal("1212121212.343434343000000000"); byte[] decimalValue1 = decimal1.unscaledValue().toByteArray(); byte[] decimalValue2 = decimal2.unscaledValue().toByteArray(); @@ -160,8 +160,8 @@ public class TestDecimalVector { @Test public void testLongReadWrite() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(38, 0), allocator)) { + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(38, 0, 256), allocator)) { decimalVector.allocateNew(); long[] longValues = {0L, -2L, Long.MAX_VALUE, Long.MIN_VALUE, 187L}; @@ -181,8 +181,8 @@ public class TestDecimalVector { @Test public void testBigDecimalReadWrite() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(38, 9), allocator);) { + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(38, 9, 256), allocator);) { decimalVector.allocateNew(); BigDecimal decimal1 = new BigDecimal("123456789.000000000"); BigDecimal decimal2 = new BigDecimal("11.123456789"); @@ -216,13 +216,13 @@ public class TestDecimalVector { } /** - * Test {@link DecimalVector#setBigEndian(int, byte[])} which takes BE layout input and stores in LE layout. + * Test {@link BigDecimalVector#setBigEndian(int, byte[])} which takes BE layout input and stores in LE layout. * Cases to cover: input byte array in different lengths in range [1-16] and negative values. */ @Test public void decimalBE2LE() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(21, 2), allocator)) { + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(23, 2, 256), allocator)) { decimalVector.allocateNew(); BigInteger[] testBigInts = new BigInteger[] { @@ -233,8 +233,8 @@ public class TestDecimalVector { new BigInteger("-234234234"), new BigInteger("234234234234"), new BigInteger("-56345345345345"), - new BigInteger("29823462983462893462934679234653456345"), // converts to 16 byte array - new BigInteger("-3894572983475982374598324598234346536"), // converts to 16 byte array + new BigInteger("2982346298346289346293467923465345634500"), // converts to 16+ byte array + new BigInteger("-389457298347598237459832459823434653600"), // converts to 16+ byte array new BigInteger("-345345"), new BigInteger("754533") }; @@ -248,12 +248,12 @@ public class TestDecimalVector { // insert a zero length buffer decimalVector.setBigEndian(insertionIdx++, new byte[0]); - // Try inserting a buffer larger than 16bytes and expect a failure + // Try inserting a buffer larger than 33 bytes and expect a failure try { - decimalVector.setBigEndian(insertionIdx, new byte[17]); + decimalVector.setBigEndian(insertionIdx, new byte[33]); fail("above statement should have failed"); } catch (IllegalArgumentException ex) { - assertTrue(ex.getMessage().equals("Invalid decimal value length. Valid length in [1 - 16], got 17")); + assertTrue(ex.getMessage().equals("Invalid decimal value length. Valid length in [1 - 32], got 33")); } decimalVector.setValueCount(insertionIdx); @@ -271,8 +271,8 @@ public class TestDecimalVector { @Test public void setUsingArrowBufOfLEInts() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(5, 2), allocator); + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(5, 2, 256), allocator); ArrowBuf buf = allocator.buffer(8);) { decimalVector.allocateNew(); @@ -300,8 +300,8 @@ public class TestDecimalVector { @Test public void setUsingArrowLongLEBytes() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(18, 0), allocator); + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(18, 0, 256), allocator); ArrowBuf buf = allocator.buffer(16);) { decimalVector.allocateNew(); @@ -326,25 +326,25 @@ public class TestDecimalVector { @Test public void setUsingArrowBufOfBEBytes() { - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(5, 2), allocator); + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(5, 2, 256), allocator); ArrowBuf buf = allocator.buffer(9);) { BigDecimal [] expectedValues = new BigDecimal[] {BigDecimal.valueOf(705.32), BigDecimal .valueOf(-705.32), BigDecimal.valueOf(705.32)}; verifyWritingArrowBufWithBigEndianBytes(decimalVector, buf, expectedValues, 3); } - try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(36, 2), allocator); + try (BigDecimalVector decimalVector = TestUtils.newVector(BigDecimalVector.class, "decimal", + new ArrowType.Decimal(43, 2, 256), allocator); ArrowBuf buf = allocator.buffer(45);) { - BigDecimal[] expectedValues = new BigDecimal[] {new BigDecimal("2982346298346289346293467923465345.63"), + BigDecimal[] expectedValues = new BigDecimal[] {new BigDecimal("29823462983462893462934679234653450000000.63"), new BigDecimal("-2982346298346289346293467923465345.63"), new BigDecimal("2982346298346289346293467923465345.63")}; verifyWritingArrowBufWithBigEndianBytes(decimalVector, buf, expectedValues, 15); } } - private void verifyWritingArrowBufWithBigEndianBytes(DecimalVector decimalVector, + private void verifyWritingArrowBufWithBigEndianBytes(BigDecimalVector decimalVector, ArrowBuf buf, BigDecimal[] expectedValues, int length) { decimalVector.allocateNew(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index 28d799b..25f4801 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -60,7 +60,7 @@ public class TestDecimalVector { @Test public void testValuesWriteRead() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(10, scale), allocator);) { + new ArrowType.Decimal(10, scale, 128), allocator);) { try (DecimalVector oldConstructor = new DecimalVector("decimal", allocator, 10, scale);) { assertEquals(decimalVector.getField().getType(), oldConstructor.getField().getType()); @@ -86,7 +86,7 @@ public class TestDecimalVector { @Test public void testBigDecimalDifferentScaleAndPrecision() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(4, 2), allocator);) { + new ArrowType.Decimal(4, 2, 128), allocator);) { decimalVector.allocateNew(); // test BigDecimal with different scale @@ -116,7 +116,7 @@ public class TestDecimalVector { @Test public void testWriteBigEndian() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(38, 9), allocator);) { + new ArrowType.Decimal(38, 9, 128), allocator);) { decimalVector.allocateNew(); BigDecimal decimal1 = new BigDecimal("123456789.000000000"); BigDecimal decimal2 = new BigDecimal("11.123456789"); @@ -161,7 +161,7 @@ public class TestDecimalVector { @Test public void testLongReadWrite() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(38, 0), allocator)) { + new ArrowType.Decimal(38, 0, 128), allocator)) { decimalVector.allocateNew(); long[] longValues = {0L, -2L, Long.MAX_VALUE, Long.MIN_VALUE, 187L}; @@ -182,7 +182,7 @@ public class TestDecimalVector { @Test public void testBigDecimalReadWrite() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(38, 9), allocator);) { + new ArrowType.Decimal(38, 9, 128), allocator);) { decimalVector.allocateNew(); BigDecimal decimal1 = new BigDecimal("123456789.000000000"); BigDecimal decimal2 = new BigDecimal("11.123456789"); @@ -222,7 +222,7 @@ public class TestDecimalVector { @Test public void decimalBE2LE() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(21, 2), allocator)) { + new ArrowType.Decimal(21, 2, 128), allocator)) { decimalVector.allocateNew(); BigInteger[] testBigInts = new BigInteger[] { @@ -272,7 +272,7 @@ public class TestDecimalVector { @Test public void setUsingArrowBufOfLEInts() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(5, 2), allocator); + new ArrowType.Decimal(5, 2, 128), allocator); ArrowBuf buf = allocator.buffer(8);) { decimalVector.allocateNew(); @@ -301,7 +301,7 @@ public class TestDecimalVector { @Test public void setUsingArrowLongLEBytes() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(18, 0), allocator); + new ArrowType.Decimal(18, 0, 128), allocator); ArrowBuf buf = allocator.buffer(16);) { decimalVector.allocateNew(); @@ -327,7 +327,7 @@ public class TestDecimalVector { @Test public void setUsingArrowBufOfBEBytes() { try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(5, 2), allocator); + new ArrowType.Decimal(5, 2, 128), allocator); ArrowBuf buf = allocator.buffer(9);) { BigDecimal [] expectedValues = new BigDecimal[] {BigDecimal.valueOf(705.32), BigDecimal .valueOf(-705.32), BigDecimal.valueOf(705.32)}; @@ -335,7 +335,7 @@ public class TestDecimalVector { } try (DecimalVector decimalVector = TestUtils.newVector(DecimalVector.class, "decimal", - new ArrowType.Decimal(36, 2), allocator); + new ArrowType.Decimal(36, 2, 128), allocator); ArrowBuf buf = allocator.buffer(45);) { BigDecimal[] expectedValues = new BigDecimal[] {new BigDecimal("2982346298346289346293467923465345.63"), new BigDecimal("-2982346298346289346293467923465345.63"), diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java b/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java index 1817527..97930f4 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestTypeLayout.java @@ -61,9 +61,13 @@ public class TestTypeLayout { type = new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size()); - type = new ArrowType.Decimal(10, 10); + type = new ArrowType.Decimal(10, 10, 128); assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size()); + type = new ArrowType.Decimal(10, 10, 256); + assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size()); + + type = new ArrowType.FixedSizeBinary(5); assertEquals(TypeLayout.getTypeBufferCount(type), TypeLayout.getTypeLayout(type).getBufferLayouts().size()); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java index 089f1f8..b9e7c86 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java @@ -72,7 +72,7 @@ public class TestVectorAlloc { field("UTF8", MinorType.VARCHAR.getType()), field("VARBINARY", MinorType.VARBINARY.getType()), field("BIT", MinorType.BIT.getType()), - field("DECIMAL", new Decimal(38, 5)), + field("DECIMAL", new Decimal(38, 5, 128)), field("FIXEDSIZEBINARY", new FixedSizeBinary(50)), field("DATEDAY", MinorType.DATEDAY.getType()), field("DATEMILLI", MinorType.DATEMILLI.getType()), diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index a0f3505..0992ffc 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -522,12 +522,12 @@ public class TestComplexCopier { public void testCopyFixedSizedListOfDecimalsVector() { try (FixedSizeListVector from = FixedSizeListVector.empty("v", 4, allocator); FixedSizeListVector to = FixedSizeListVector.empty("v", 4, allocator)) { - from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0))); - to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0))); + from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0, 128))); + to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 0, 128))); DecimalHolder holder = new DecimalHolder(); holder.buffer = allocator.buffer(DecimalUtility.DECIMAL_BYTE_LENGTH); - ArrowType arrowType = new ArrowType.Decimal(3, 0); + ArrowType arrowType = new ArrowType.Decimal(3, 0, 128); // populate from vector UnionFixedSizeListWriter writer = from.getWriter(); @@ -535,13 +535,13 @@ public class TestComplexCopier { writer.startList(); writer.decimal().writeDecimal(BigDecimal.valueOf(i)); - DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 2), holder.buffer, 0); + DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 2), holder.buffer, 0, /*byteWidth=*/16); holder.start = 0; holder.scale = 0; holder.precision = 3; writer.decimal().write(holder); - DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 3), holder.buffer, 0); + DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(i * 3), holder.buffer, 0, /*byteWidth=*/16); writer.decimal().writeDecimal(0, holder.buffer, arrowType); writer.decimal().writeBigEndianBytesToDecimal(BigDecimal.valueOf(i * 4).unscaledValue().toByteArray(), @@ -582,7 +582,7 @@ public class TestComplexCopier { listWriter.decimal().writeDecimal(BigDecimal.valueOf(i * 2)); listWriter.integer().writeInt(i); listWriter.decimal().writeBigEndianBytesToDecimal(BigDecimal.valueOf(i * 3).unscaledValue().toByteArray(), - new ArrowType.Decimal(3, 0)); + new ArrowType.Decimal(3, 0, 128)); listWriter.endList(); } @@ -623,7 +623,7 @@ public class TestComplexCopier { innerStructWriter.integer("innerint").writeInt(i * 3); innerStructWriter.decimal("innerdec", 0, 38).writeDecimal(BigDecimal.valueOf(i * 4)); innerStructWriter.decimal("innerdec", 0, 38).writeBigEndianBytesToDecimal(BigDecimal.valueOf(i * 4) - .unscaledValue().toByteArray(), new ArrowType.Decimal(3, 0)); + .unscaledValue().toByteArray(), new ArrowType.Decimal(3, 0, 128)); innerStructWriter.end(); structWriter.end(); } @@ -649,8 +649,8 @@ public class TestComplexCopier { public void testCopyDecimalVectorWrongScale() { try (FixedSizeListVector from = FixedSizeListVector.empty("v", 3, allocator); FixedSizeListVector to = FixedSizeListVector.empty("v", 3, allocator)) { - from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 2))); - to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 1))); + from.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 2, 128))); + to.addOrGetVector(FieldType.nullable(new ArrowType.Decimal(3, 1, 128))); // populate from vector UnionFixedSizeListWriter writer = from.getWriter(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 769a94f..1e6fe49 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -311,20 +311,20 @@ public class TestComplexWriter { UnionListWriter listWriter = new UnionListWriter(listVector); DecimalHolder holder = new DecimalHolder(); holder.buffer = allocator.buffer(DecimalUtility.DECIMAL_BYTE_LENGTH); - ArrowType arrowType = new ArrowType.Decimal(10, 0); + ArrowType arrowType = new ArrowType.Decimal(10, 0, 128); for (int i = 0; i < COUNT; i++) { listWriter.startList(); for (int j = 0; j < i % 7; j++) { if (j % 4 == 0) { listWriter.writeDecimal(new BigDecimal(j)); } else if (j % 4 == 1) { - DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0); + DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0, 16); holder.start = 0; holder.scale = 0; holder.precision = 10; listWriter.write(holder); } else if (j % 4 == 2) { - DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0); + DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0, 16); listWriter.writeDecimal(0, holder.buffer, arrowType); } else { byte[] value = BigDecimal.valueOf(j).unscaledValue().toByteArray(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java index 3ca8d0a..3d93407 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java @@ -90,7 +90,7 @@ public class TestSchema { field("g", new Utf8()), field("h", new Binary()), field("i", new Bool()), - field("j", new Decimal(5, 5)), + field("j", new Decimal(5, 5, 128)), field("k", new Date(DateUnit.DAY)), field("l", new Date(DateUnit.MILLISECOND)), field("m", new Time(TimeUnit.SECOND, 32)), diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java b/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java index 667e962..2e255e6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java @@ -27,76 +27,81 @@ import org.junit.Assert; import org.junit.Test; public class DecimalUtilityTest { - private static final BigInteger MAX_BIG_INT = java.math.BigInteger.valueOf(10).pow(38) - .subtract(java.math.BigInteger.ONE); - private static final BigDecimal MAX_DECIMAL = new java.math.BigDecimal(MAX_BIG_INT, 0); - private static final BigInteger MIN_BIG_INT = MAX_BIG_INT.multiply(BigInteger.valueOf(-1)); - private static final BigDecimal MIN_DECIMAL = new java.math.BigDecimal(MIN_BIG_INT, 0); + private static final BigInteger[] MAX_BIG_INT = new BigInteger[]{BigInteger.valueOf(10).pow(38) + .subtract(java.math.BigInteger.ONE), java.math.BigInteger.valueOf(10).pow(76)}; + private static final BigInteger[] MIN_BIG_INT = new BigInteger[]{MAX_BIG_INT[0].multiply(BigInteger.valueOf(-1)), + MAX_BIG_INT[1].multiply(BigInteger.valueOf(-1))}; @Test public void testSetByteArrayInDecimalArrowBuf() { - try (BufferAllocator allocator = new RootAllocator(128); - ArrowBuf buf = allocator.buffer(16); - ) { - int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0}; - for (int val : intValues) { - buf.clear(); - DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0); - BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0); - BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); - } + int[] byteLengths = new int[]{16, 32}; + for (int x = 0; x < 2; x++) { + try (BufferAllocator allocator = new RootAllocator(128); + ArrowBuf buf = allocator.buffer(byteLengths[x]); + ) { + int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0}; + for (int val : intValues) { + buf.clear(); + DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0, byteLengths[x]); + BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); + BigDecimal expected = BigDecimal.valueOf(val); + Assert.assertEquals(expected, actual); + } - long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE}; - for (long val : longValues) { - buf.clear(); - DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0); - BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0); - BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); - } + long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE}; + for (long val : longValues) { + buf.clear(); + DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0, byteLengths[x]); + BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); + BigDecimal expected = BigDecimal.valueOf(val); + Assert.assertEquals(expected, actual); + } - BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT, new BigInteger("0"), MIN_BIG_INT}; - for (BigInteger val : decimals) { - buf.clear(); - DecimalUtility.writeByteArrayToArrowBuf(val.toByteArray(), buf, 0); - BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0); - BigDecimal expected = new BigDecimal(val); - Assert.assertEquals(expected, actual); + BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT[x], new BigInteger("0"), MIN_BIG_INT[x]}; + for (BigInteger val : decimals) { + buf.clear(); + DecimalUtility.writeByteArrayToArrowBuf(val.toByteArray(), buf, 0, byteLengths[x]); + BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); + BigDecimal expected = new BigDecimal(val); + Assert.assertEquals(expected, actual); + } } } } @Test public void testSetBigDecimalInDecimalArrowBuf() { - try (BufferAllocator allocator = new RootAllocator(128); - ArrowBuf buf = allocator.buffer(16); - ) { - int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0}; - for (int val : intValues) { - buf.clear(); - DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0); - BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0); - BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); - } + int[] byteLengths = new int[]{16, 32}; + for (int x = 0; x < 2; x++) { + try (BufferAllocator allocator = new RootAllocator(128); + ArrowBuf buf = allocator.buffer(byteLengths[x]); + ) { + int [] intValues = new int [] {Integer.MAX_VALUE, Integer.MIN_VALUE, 0}; + for (int val : intValues) { + buf.clear(); + DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0, byteLengths[x]); + BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); + BigDecimal expected = BigDecimal.valueOf(val); + Assert.assertEquals(expected, actual); + } - long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE}; - for (long val : longValues) { - buf.clear(); - DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0); - BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0); - BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); - } + long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE}; + for (long val : longValues) { + buf.clear(); + DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0, byteLengths[x]); + BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); + BigDecimal expected = BigDecimal.valueOf(val); + Assert.assertEquals(expected, actual); + } - BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT, new BigInteger("0"), MIN_BIG_INT}; - for (BigInteger val : decimals) { - buf.clear(); - DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(val), buf, 0); - BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0); - BigDecimal expected = new BigDecimal(val); - Assert.assertEquals(expected, actual); + BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT[x], new BigInteger("0"), MIN_BIG_INT[x]}; + for (BigInteger val : decimals) { + buf.clear(); + DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(val), buf, 0, byteLengths[x]); + BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); + BigDecimal expected = new BigDecimal(val); + Assert.assertEquals(expected, actual); + } } } }