[
https://issues.apache.org/jira/browse/ARROW-1946?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16307063#comment-16307063
]
ASF GitHub Bot commented on ARROW-1946:
---------------------------------------
siddharthteotia closed pull request #1443: ARROW-1946: [JAVA] Add APIs to
decimal vector for writing big endian data
URL: https://github.com/apache/arrow/pull/1443
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 853f67fd0..fce687602 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -67,6 +67,10 @@ public void write(${name}Holder holder) {
public void write${minor.class}(${friendlyType} value) {
fail("${name}");
}
+
+ public void writeBigEndianBytesToDecimal(byte[] value) {
+ fail("${name}");
+ }
</#if>
</#list></#list>
diff --git
a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 228c2c531..7f4a13d4f 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.vector.types.Types;
import org.apache.drill.common.types.TypeProtos.MinorType;
<@pp.dropOutputFile />
@@ -82,6 +84,12 @@ public void write(${name}Holder holder) {
getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields
as field>${field.name}<#if field_has_next>, </#if></#list>);
}
+ <#if minor.class == "Decimal">
+ public void writeBigEndianBytesToDecimal(byte[] value) {
+ getWriter(Types.MinorType.DECIMAL).writeBigEndianBytesToDecimal(value);
+ }
+ </#if>
+
</#list></#list>
public void writeNull() {
}
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java
b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 4a1545a17..24994d72a 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -120,6 +120,11 @@ public void write(Nullable${minor.class}Holder h) {
vector.setSafe(idx(), value);
vector.setValueCount(idx()+1);
}
+
+ public void writeBigEndianBytesToDecimal(byte[] value) {
+ vector.setBigEndianSafe(idx(), value);
+ vector.setValueCount(idx()+1);
+ }
</#if>
public void writeNull() {
@@ -146,6 +151,8 @@ public void writeNull() {
<#if minor.class == "Decimal">
public void write${minor.class}(${friendlyType} value);
+
+ public void writeBigEndianBytesToDecimal(byte[] value);
</#if>
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index 71a74cb9f..a04357508 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -195,6 +195,43 @@ public void set(int index, ArrowBuf buffer) {
valueBuffer.setBytes(index * TYPE_WIDTH, buffer, 0, TYPE_WIDTH);
}
+ /**
+ * Set the decimal element at given index to the provided array of bytes.
+ * Decimal is now implemented as Little Endian. This API allows the user
+ * to pass a decimal value in the form of byte array in BE byte order.
+ *
+ * Consumers of Arrow code can use this API instead of first swapping
+ * the source bytes (doing a write and read) and then finally writing to
+ * ArrowBuf of decimal vector.
+ *
+ * This method takes care of adding the necessary padding if the length
+ * of byte array is less then 16 (length of decimal type).
+ *
+ * @param index position of element
+ * @param value array of bytes containing decimal in big endian byte order.
+ */
+ public void setBigEndian(int index, byte[] value) {
+ assert value.length <= TYPE_WIDTH;
+ BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+ final int length = value.length;
+ int startIndex = index * TYPE_WIDTH;
+ if (length == TYPE_WIDTH) {
+ for (int i = TYPE_WIDTH - 1; i >= 3; i-=4) {
+ valueBuffer.setByte(startIndex, value[i]);
+ valueBuffer.setByte(startIndex + 1, value[i-1]);
+ valueBuffer.setByte(startIndex + 2, value[i-2]);
+ valueBuffer.setByte(startIndex + 3, value[i-3]);
+ startIndex += 4;
+ }
+ } else {
+ for (int i = length - 1; i >= 0; i--) {
+ valueBuffer.setByte(startIndex, value[i]);
+ startIndex++;
+ }
+ valueBuffer.setZero(startIndex, TYPE_WIDTH - length);
+ }
+ }
+
/**
* Set the element at the given index to the given value.
*
@@ -262,6 +299,16 @@ public void setSafe(int index, ArrowBuf buffer) {
set(index, buffer);
}
+ /**
+ * Same as {@link #setBigEndian(int, byte[])} except that it handles the
+ * case when index is greater than or equal to existing
+ * value capacity {@link #getValueCapacity()}.
+ */
+ public void setBigEndianSafe(int index, byte[] value) {
+ handleSafe(index);
+ setBigEndian(index, value);
+ }
+
/**
* Same as {@link #set(int, int, ArrowBuf)} except that it handles the
* case when index is greater than or equal to existing
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index a48c03571..8c86452fc 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -111,4 +111,83 @@ public void testBigDecimalDifferentScaleAndPrecision() {
}
}
}
+
+ @Test
+ public void testWriteBigEndian() {
+ try (DecimalVector decimalVector =
TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38,
9), allocator);) {
+ decimalVector.allocateNew();
+ BigDecimal decimal1 = new BigDecimal("123456789.000000000");
+ BigDecimal decimal2 = new BigDecimal("11.123456789");
+ BigDecimal decimal3 = new BigDecimal("1.000000000");
+ BigDecimal decimal4 = new BigDecimal("0.111111111");
+ BigDecimal decimal5 = new BigDecimal("987654321.123456789");
+ BigDecimal decimal6 = new BigDecimal("222222222222.222222222");
+ BigDecimal decimal7 = new BigDecimal("7777777777777.666666667");
+ BigDecimal decimal8 = new BigDecimal("1212121212.343434343");
+
+ byte[] decimalValue1 = decimal1.unscaledValue().toByteArray();
+ byte[] decimalValue2 = decimal2.unscaledValue().toByteArray();
+ byte[] decimalValue3 = decimal3.unscaledValue().toByteArray();
+ byte[] decimalValue4 = decimal4.unscaledValue().toByteArray();
+ byte[] decimalValue5 = decimal5.unscaledValue().toByteArray();
+ byte[] decimalValue6 = decimal6.unscaledValue().toByteArray();
+ byte[] decimalValue7 = decimal7.unscaledValue().toByteArray();
+ byte[] decimalValue8 = decimal8.unscaledValue().toByteArray();
+
+ decimalVector.setBigEndian(0, decimalValue1);
+ decimalVector.setBigEndian(1, decimalValue2);
+ decimalVector.setBigEndian(2, decimalValue3);
+ decimalVector.setBigEndian(3, decimalValue4);
+ decimalVector.setBigEndian(4, decimalValue5);
+ decimalVector.setBigEndian(5, decimalValue6);
+ decimalVector.setBigEndian(6, decimalValue7);
+ decimalVector.setBigEndian(7, decimalValue8);
+
+ decimalVector.setValueCount(8);
+ assertEquals(8, decimalVector.getValueCount());
+ assertEquals(decimal1, decimalVector.getObject(0));
+ assertEquals(decimal2, decimalVector.getObject(1));
+ assertEquals(decimal3, decimalVector.getObject(2));
+ assertEquals(decimal4, decimalVector.getObject(3));
+ assertEquals(decimal5, decimalVector.getObject(4));
+ assertEquals(decimal6, decimalVector.getObject(5));
+ assertEquals(decimal7, decimalVector.getObject(6));
+ assertEquals(decimal8, decimalVector.getObject(7));
+ }
+ }
+
+ @Test
+ public void testBigDecimalReadWrite() {
+ try (DecimalVector decimalVector =
TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38,
9), allocator);) {
+ decimalVector.allocateNew();
+ BigDecimal decimal1 = new BigDecimal("123456789.000000000");
+ BigDecimal decimal2 = new BigDecimal("11.123456789");
+ BigDecimal decimal3 = new BigDecimal("1.000000000");
+ BigDecimal decimal4 = new BigDecimal("-0.111111111");
+ BigDecimal decimal5 = new BigDecimal("-987654321.123456789");
+ BigDecimal decimal6 = new BigDecimal("-222222222222.222222222");
+ BigDecimal decimal7 = new BigDecimal("7777777777777.666666667");
+ BigDecimal decimal8 = new BigDecimal("1212121212.343434343");
+
+ decimalVector.set(0, decimal1);
+ decimalVector.set(1, decimal2);
+ decimalVector.set(2, decimal3);
+ decimalVector.set(3, decimal4);
+ decimalVector.set(4, decimal5);
+ decimalVector.set(5, decimal6);
+ decimalVector.set(6, decimal7);
+ decimalVector.set(7, decimal8);
+
+ decimalVector.setValueCount(8);
+ assertEquals(8, decimalVector.getValueCount());
+ assertEquals(decimal1, decimalVector.getObject(0));
+ assertEquals(decimal2, decimalVector.getObject(1));
+ assertEquals(decimal3, decimalVector.getObject(2));
+ assertEquals(decimal4, decimalVector.getObject(3));
+ assertEquals(decimal5, decimalVector.getObject(4));
+ assertEquals(decimal6, decimalVector.getObject(5));
+ assertEquals(decimal7, decimalVector.getObject(6));
+ assertEquals(decimal8, decimalVector.getObject(7));
+ }
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> Add APIs to decimal vector for writing big endian data
> ------------------------------------------------------
>
> Key: ARROW-1946
> URL: https://issues.apache.org/jira/browse/ARROW-1946
> Project: Apache Arrow
> Issue Type: Bug
> Components: Java - Vectors
> Reporter: Siddharth Teotia
> Assignee: Siddharth Teotia
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> We recently moved Dremio to LE Decimal format (similar to Arrow). As part of
> that we introduces some APIs in decimal vector which take a big endian data
> and swap the bytes while writing into the ArrowBuf of decimal vector.
> The advantage of these APIs is that caller would not have to allocate an
> additional memory and write( and read) source big endian twice for swapping
> into new memory and using that to write into the vector.
> We can directly swap bytes while writing into the vector -- just read once
> and swap while writing.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
