This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 37c3265 ARROW-1946: [JAVA] Add APIs to decimal vector for writing big
endian data
37c3265 is described below
commit 37c3265b8a7a9e3ccb3063faeafa537d5b334830
Author: siddharth <[email protected]>
AuthorDate: Sat Dec 30 20:05:08 2017 -0800
ARROW-1946: [JAVA] Add APIs to decimal vector for writing big endian data
We recently moved Dremio to LE Decimal format (similar to Arrow). As part
of that we introduced some APIs in decimal vector which take a big endian data
and swap the bytes while writing into the ArrowBuf of decimal vector.
The advantage of these APIs is that caller would not have to allocate an
additional memory and write( and read) source big endian twice for swapping
into new memory and using that to write into the vector.
We can directly swap bytes while writing into the vector – just read once
and swap while writing.
cc @jacques-n , @BryanCutler , @icexelloss
Author: siddharth <[email protected]>
Closes #1443 from siddharthteotia/ARROW-1946 and squashes the following
commits:
7805b62b [siddharth] unit tests
c89efbf8 [siddharth] ARROW-1946: Add APIs to decimal vector for writing big
endian data
---
.../codegen/templates/AbstractFieldWriter.java | 4 ++
.../templates/AbstractPromotableFieldWriter.java | 8 +++
.../src/main/codegen/templates/ComplexWriters.java | 7 ++
.../org/apache/arrow/vector/DecimalVector.java | 47 +++++++++++++
.../org/apache/arrow/vector/TestDecimalVector.java | 79 ++++++++++++++++++++++
5 files changed, 145 insertions(+)
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 853f67f..fce6876 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -67,6 +67,10 @@ abstract class AbstractFieldWriter extends
AbstractBaseWriter implements FieldWr
public void write${minor.class}(${friendlyType} value) {
fail("${name}");
}
+
+ public void writeBigEndianBytesToDecimal(byte[] value) {
+ fail("${name}");
+ }
</#if>
</#list></#list>
diff --git
a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 228c2c5..7f4a13d 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.vector.types.Types;
import org.apache.drill.common.types.TypeProtos.MinorType;
<@pp.dropOutputFile />
@@ -82,6 +84,12 @@ abstract class AbstractPromotableFieldWriter extends
AbstractFieldWriter {
getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields
as field>${field.name}<#if field_has_next>, </#if></#list>);
}
+ <#if minor.class == "Decimal">
+ public void writeBigEndianBytesToDecimal(byte[] value) {
+ getWriter(Types.MinorType.DECIMAL).writeBigEndianBytesToDecimal(value);
+ }
+ </#if>
+
</#list></#list>
public void writeNull() {
}
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java
b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 4a1545a..24994d7 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -120,6 +120,11 @@ public class ${eName}WriterImpl extends
AbstractFieldWriter {
vector.setSafe(idx(), value);
vector.setValueCount(idx()+1);
}
+
+ public void writeBigEndianBytesToDecimal(byte[] value) {
+ vector.setBigEndianSafe(idx(), value);
+ vector.setValueCount(idx()+1);
+ }
</#if>
public void writeNull() {
@@ -146,6 +151,8 @@ public interface ${eName}Writer extends BaseWriter {
<#if minor.class == "Decimal">
public void write${minor.class}(${friendlyType} value);
+
+ public void writeBigEndianBytesToDecimal(byte[] value);
</#if>
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index 71a74cb..a043575 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -196,6 +196,43 @@ public class DecimalVector extends BaseFixedWidthVector {
}
/**
+ * Set the decimal element at given index to the provided array of bytes.
+ * Decimal is now implemented as Little Endian. This API allows the user
+ * to pass a decimal value in the form of byte array in BE byte order.
+ *
+ * Consumers of Arrow code can use this API instead of first swapping
+ * the source bytes (doing a write and read) and then finally writing to
+ * ArrowBuf of decimal vector.
+ *
+ * This method takes care of adding the necessary padding if the length
+ * of byte array is less then 16 (length of decimal type).
+ *
+ * @param index position of element
+ * @param value array of bytes containing decimal in big endian byte order.
+ */
+ public void setBigEndian(int index, byte[] value) {
+ assert value.length <= TYPE_WIDTH;
+ BitVectorHelper.setValidityBitToOne(validityBuffer, index);
+ final int length = value.length;
+ int startIndex = index * TYPE_WIDTH;
+ if (length == TYPE_WIDTH) {
+ for (int i = TYPE_WIDTH - 1; i >= 3; i-=4) {
+ valueBuffer.setByte(startIndex, value[i]);
+ valueBuffer.setByte(startIndex + 1, value[i-1]);
+ valueBuffer.setByte(startIndex + 2, value[i-2]);
+ valueBuffer.setByte(startIndex + 3, value[i-3]);
+ startIndex += 4;
+ }
+ } else {
+ for (int i = length - 1; i >= 0; i--) {
+ valueBuffer.setByte(startIndex, value[i]);
+ startIndex++;
+ }
+ valueBuffer.setZero(startIndex, TYPE_WIDTH - length);
+ }
+ }
+
+ /**
* Set the element at the given index to the given value.
*
* @param index position of element
@@ -263,6 +300,16 @@ public class DecimalVector extends BaseFixedWidthVector {
}
/**
+ * Same as {@link #setBigEndian(int, byte[])} except that it handles the
+ * case when index is greater than or equal to existing
+ * value capacity {@link #getValueCapacity()}.
+ */
+ public void setBigEndianSafe(int index, byte[] value) {
+ handleSafe(index);
+ setBigEndian(index, value);
+ }
+
+ /**
* Same as {@link #set(int, int, ArrowBuf)} except that it handles the
* case when index is greater than or equal to existing
* value capacity {@link #getValueCapacity()}.
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
index a48c035..8c86452 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java
@@ -111,4 +111,83 @@ public class TestDecimalVector {
}
}
}
+
+ @Test
+ public void testWriteBigEndian() {
+ try (DecimalVector decimalVector =
TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38,
9), allocator);) {
+ decimalVector.allocateNew();
+ BigDecimal decimal1 = new BigDecimal("123456789.000000000");
+ BigDecimal decimal2 = new BigDecimal("11.123456789");
+ BigDecimal decimal3 = new BigDecimal("1.000000000");
+ BigDecimal decimal4 = new BigDecimal("0.111111111");
+ BigDecimal decimal5 = new BigDecimal("987654321.123456789");
+ BigDecimal decimal6 = new BigDecimal("222222222222.222222222");
+ BigDecimal decimal7 = new BigDecimal("7777777777777.666666667");
+ BigDecimal decimal8 = new BigDecimal("1212121212.343434343");
+
+ byte[] decimalValue1 = decimal1.unscaledValue().toByteArray();
+ byte[] decimalValue2 = decimal2.unscaledValue().toByteArray();
+ byte[] decimalValue3 = decimal3.unscaledValue().toByteArray();
+ byte[] decimalValue4 = decimal4.unscaledValue().toByteArray();
+ byte[] decimalValue5 = decimal5.unscaledValue().toByteArray();
+ byte[] decimalValue6 = decimal6.unscaledValue().toByteArray();
+ byte[] decimalValue7 = decimal7.unscaledValue().toByteArray();
+ byte[] decimalValue8 = decimal8.unscaledValue().toByteArray();
+
+ decimalVector.setBigEndian(0, decimalValue1);
+ decimalVector.setBigEndian(1, decimalValue2);
+ decimalVector.setBigEndian(2, decimalValue3);
+ decimalVector.setBigEndian(3, decimalValue4);
+ decimalVector.setBigEndian(4, decimalValue5);
+ decimalVector.setBigEndian(5, decimalValue6);
+ decimalVector.setBigEndian(6, decimalValue7);
+ decimalVector.setBigEndian(7, decimalValue8);
+
+ decimalVector.setValueCount(8);
+ assertEquals(8, decimalVector.getValueCount());
+ assertEquals(decimal1, decimalVector.getObject(0));
+ assertEquals(decimal2, decimalVector.getObject(1));
+ assertEquals(decimal3, decimalVector.getObject(2));
+ assertEquals(decimal4, decimalVector.getObject(3));
+ assertEquals(decimal5, decimalVector.getObject(4));
+ assertEquals(decimal6, decimalVector.getObject(5));
+ assertEquals(decimal7, decimalVector.getObject(6));
+ assertEquals(decimal8, decimalVector.getObject(7));
+ }
+ }
+
+ @Test
+ public void testBigDecimalReadWrite() {
+ try (DecimalVector decimalVector =
TestUtils.newVector(DecimalVector.class, "decimal", new ArrowType.Decimal(38,
9), allocator);) {
+ decimalVector.allocateNew();
+ BigDecimal decimal1 = new BigDecimal("123456789.000000000");
+ BigDecimal decimal2 = new BigDecimal("11.123456789");
+ BigDecimal decimal3 = new BigDecimal("1.000000000");
+ BigDecimal decimal4 = new BigDecimal("-0.111111111");
+ BigDecimal decimal5 = new BigDecimal("-987654321.123456789");
+ BigDecimal decimal6 = new BigDecimal("-222222222222.222222222");
+ BigDecimal decimal7 = new BigDecimal("7777777777777.666666667");
+ BigDecimal decimal8 = new BigDecimal("1212121212.343434343");
+
+ decimalVector.set(0, decimal1);
+ decimalVector.set(1, decimal2);
+ decimalVector.set(2, decimal3);
+ decimalVector.set(3, decimal4);
+ decimalVector.set(4, decimal5);
+ decimalVector.set(5, decimal6);
+ decimalVector.set(6, decimal7);
+ decimalVector.set(7, decimal8);
+
+ decimalVector.setValueCount(8);
+ assertEquals(8, decimalVector.getValueCount());
+ assertEquals(decimal1, decimalVector.getObject(0));
+ assertEquals(decimal2, decimalVector.getObject(1));
+ assertEquals(decimal3, decimalVector.getObject(2));
+ assertEquals(decimal4, decimalVector.getObject(3));
+ assertEquals(decimal5, decimalVector.getObject(4));
+ assertEquals(decimal6, decimalVector.getObject(5));
+ assertEquals(decimal7, decimalVector.getObject(6));
+ assertEquals(decimal8, decimalVector.getObject(7));
+ }
+ }
}
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
