vibhatha commented on code in PR #41861:
URL: https://github.com/apache/arrow/pull/41861#discussion_r1621567382
##########
java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java:
##########
@@ -826,7 +839,158 @@ public void transferTo(BaseVariableWidthViewVector
target) {
*/
public void splitAndTransferTo(int startIndex, int length,
BaseVariableWidthViewVector target) {
- throw new UnsupportedOperationException("splitAndTransferTo function not
supported!");
+ Preconditions.checkArgument(startIndex >= 0 && length >= 0 && startIndex +
length <= valueCount,
+ "Invalid parameters startIndex: %s, length: %s for valueCount: %s",
startIndex, length, valueCount);
+ compareTypes(target, "splitAndTransferTo");
+ target.clear();
+ if (length > 0) {
+ splitAndTransferValidityBuffer(startIndex, length, target);
+ splitAndTransferViewBufferAndDataBuffer(startIndex, length, target);
+ target.setLastSet(length - 1);
+ target.setValueCount(length);
+ }
+ }
+
+ /* allocate validity buffer */
+ private void allocateValidityBuffer(final long size) {
+ final int curSize = (int) size;
+ validityBuffer = allocator.buffer(curSize);
+ validityBuffer.readerIndex(0);
+ initValidityBuffer();
+ }
+
+ /*
+ * Transfer the validity.
+ */
+ private void splitAndTransferValidityBuffer(int startIndex, int length,
+ BaseVariableWidthViewVector target) {
+ if (length <= 0) {
+ return;
+ }
+
+ final int firstByteSource = BitVectorHelper.byteIndex(startIndex);
+ final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1);
+ final int byteSizeTarget = getValidityBufferSizeFromCount(length);
+ final int offset = startIndex % 8;
+
+ if (offset == 0) {
+ // slice
+ if (target.validityBuffer != null) {
+ target.validityBuffer.getReferenceManager().release();
+ }
+ final ArrowBuf slicedValidityBuffer =
validityBuffer.slice(firstByteSource, byteSizeTarget);
+ target.validityBuffer = transferBuffer(slicedValidityBuffer,
target.allocator);
+ return;
+ }
+
+ /* Copy data
+ * When the first bit starts from the middle of a byte (offset != 0),
+ * copy data from src BitVector.
+ * Each byte in the target is composed by a part in i-th byte,
+ * another part in (i+1)-th byte.
+ */
+ target.allocateValidityBuffer(byteSizeTarget);
+
+ for (int i = 0; i < byteSizeTarget - 1; i++) {
+ byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
firstByteSource + i, offset);
+ byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
firstByteSource + i + 1, offset);
+
+ target.validityBuffer.setByte(i, (b1 + b2));
+ }
+ /* Copying the last piece is done in the following manner:
+ * if the source vector has 1 or more bytes remaining, we copy
+ * the last piece as a byte formed by shifting data
+ * from the current byte and the next byte.
+ *
+ * if the source vector has no more bytes remaining
+ * (we are at the last byte), we copy the last piece as a byte
+ * by shifting data from the current byte.
+ */
+ if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
+ byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+ firstByteSource + byteSizeTarget - 1, offset);
+ byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer,
+ firstByteSource + byteSizeTarget, offset);
+
+ target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
+ } else {
+ byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer,
+ firstByteSource + byteSizeTarget - 1, offset);
+ target.validityBuffer.setByte(byteSizeTarget - 1, b1);
+ }
+ }
+
+ /**
+ * In split and transfer, the view buffer and the data buffer will be
allocated.
+ * Then the values will be copied from the source vector to the target
vector.
+ * Allocation and setting are preferred over transfer
+ * since the buf index and buf offset needs to be overwritten
+ * when large strings are added.
+ * @param startIndex starting index
+ * @param length number of elements to be copied
+ * @param target target vector
+ */
+ private void splitAndTransferViewBufferAndDataBuffer(int startIndex, int
length,
+ BaseVariableWidthViewVector target) {
+ if (length == 0) {
+ return;
+ }
+
+ if (target.viewBuffer != null) {
+ target.viewBuffer.getReferenceManager().release();
+ }
+
+ // allocate target view buffer
+ target.viewBuffer = target.allocator.buffer(length * ELEMENT_SIZE);
+
+ for (int i = startIndex; i < startIndex + length; i++) {
+ final int stringLength = getValueLength(i);
+
+ // keeping track of writing index in the target view buffer
+ int writePosition = (i - startIndex) * ELEMENT_SIZE;
+ // keeping track of reading index in the source view buffer
+ int readPosition = i * ELEMENT_SIZE;
+
+ // to clear the memory segment of view being written to
+ // this is helpful in case of overwriting the value
+ target.viewBuffer.setZero(writePosition, ELEMENT_SIZE);
Review Comment:
removed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
