This is an automated email from the ASF dual-hosted git repository.
liyafan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new e396d4f ARROW-13872: [Java] ExtensionTypeVector does not work with
RangeEqualsVisitor
e396d4f is described below
commit e396d4fa64429d8b7aa1a72118b941ad8e9b06be
Author: Bryan Cutler <[email protected]>
AuthorDate: Wed Sep 8 14:28:13 2021 +0800
ARROW-13872: [Java] ExtensionTypeVector does not work with
RangeEqualsVisitor
This adds the ExtensionTypeVector to the VectorVisitor interface and
implements that method for the existing visitors. Now when visitor like
RangeEqualsVisitor that uses another ExtensionTypeVector, they can both use the
visitor on the underlying storage vector.
Closes #11073 from BryanCutler/java-extension-compare
Authored-by: Bryan Cutler <[email protected]>
Signed-off-by: liyafan82 <[email protected]>
---
.../apache/arrow/vector/ExtensionTypeVector.java | 2 +-
.../arrow/vector/compare/RangeEqualsVisitor.java | 13 ++++++
.../arrow/vector/compare/TypeEqualsVisitor.java | 6 +++
.../apache/arrow/vector/compare/VectorVisitor.java | 3 ++
.../apache/arrow/vector/util/VectorAppender.java | 9 ++++
.../validate/ValidateVectorBufferVisitor.java | 7 ++++
.../vector/validate/ValidateVectorDataVisitor.java | 7 ++++
.../vector/validate/ValidateVectorTypeVisitor.java | 18 ++++++++
.../vector/validate/ValidateVectorVisitor.java | 9 +++-
.../arrow/vector/types/pojo/TestExtensionType.java | 48 ++++++++++++++++++++++
10 files changed, 120 insertions(+), 2 deletions(-)
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
index 516077d..3722250 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ExtensionTypeVector.java
@@ -261,6 +261,6 @@ public abstract class ExtensionTypeVector<T extends
ValueVector & FieldVector> e
@Override
public <OUT, IN> OUT accept(VectorVisitor<OUT, IN> visitor, IN value) {
- return getUnderlyingVector().accept(visitor, value);
+ return visitor.visit(this, value);
}
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java
b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java
index 6805d7c..35b4936 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java
@@ -27,6 +27,7 @@ import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseLargeVariableWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.BaseRepeatedValueVector;
@@ -214,6 +215,18 @@ public class RangeEqualsVisitor implements
VectorVisitor<Boolean, Range> {
return true;
}
+ @Override
+ public Boolean visit(ExtensionTypeVector<?> left, Range range) {
+ if (!(right instanceof ExtensionTypeVector<?>) || !validate(left)) {
+ return false;
+ }
+ ValueVector rightUnderlying = ((ExtensionTypeVector<?>)
right).getUnderlyingVector();
+ TypeEqualsVisitor typeVisitor = new TypeEqualsVisitor(rightUnderlying);
+ RangeEqualsVisitor underlyingVisitor =
+ createInnerVisitor(left.getUnderlyingVector(), rightUnderlying,
(l, r) -> typeVisitor.equals(l));
+ return underlyingVisitor.rangeEquals(range);
+ }
+
protected RangeEqualsVisitor createInnerVisitor(
ValueVector leftInner, ValueVector rightInner,
BiFunction<ValueVector, ValueVector, Boolean> typeComparator) {
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java
b/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java
index 95db792..443ee1f 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/compare/TypeEqualsVisitor.java
@@ -23,6 +23,7 @@ import java.util.Objects;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseLargeVariableWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.DenseUnionVector;
@@ -119,6 +120,11 @@ public class TypeEqualsVisitor implements
VectorVisitor<Boolean, Void> {
return compareField(left.getField(), right.getField());
}
+ @Override
+ public Boolean visit(ExtensionTypeVector<?> left, Void value) {
+ return compareField(left.getField(), right.getField());
+ }
+
private boolean compareField(Field leftField, Field rightField) {
if (leftField == rightField) {
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java
b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java
index 14f3434..aee0907 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/compare/VectorVisitor.java
@@ -20,6 +20,7 @@ package org.apache.arrow.vector.compare;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseLargeVariableWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.complex.DenseUnionVector;
import org.apache.arrow.vector.complex.FixedSizeListVector;
@@ -54,5 +55,7 @@ public interface VectorVisitor<OUT, IN> {
OUT visit(DenseUnionVector left, IN value);
OUT visit(NullVector left, IN value);
+
+ OUT visit(ExtensionTypeVector<?> left, IN value);
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java
b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java
index 7703fed..e5809e9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java
@@ -26,6 +26,7 @@ import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseLargeVariableWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
import org.apache.arrow.vector.BitVectorHelper;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.compare.TypeEqualsVisitor;
@@ -530,4 +531,12 @@ class VectorAppender implements VectorVisitor<ValueVector,
Void> {
"The targetVector to append must have the same type as the
targetVector being appended");
return targetVector;
}
+
+ @Override
+ public ValueVector visit(ExtensionTypeVector<?> deltaVector, Void value) {
+ ValueVector targetUnderlying = ((ExtensionTypeVector<?>)
targetVector).getUnderlyingVector();
+ VectorAppender underlyingAppender = new VectorAppender(targetUnderlying);
+ deltaVector.getUnderlyingVector().accept(underlyingAppender, null);
+ return targetVector;
+ }
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java
index 46064c3..d4abaa1 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java
@@ -24,6 +24,7 @@ import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseLargeVariableWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.TypeLayout;
@@ -236,4 +237,10 @@ public class ValidateVectorBufferVisitor implements
VectorVisitor<Void, Void> {
public Void visit(NullVector vector, Void value) {
return null;
}
+
+ @Override
+ public Void visit(ExtensionTypeVector<?> vector, Void value) {
+ vector.getUnderlyingVector().accept(this, value);
+ return null;
+ }
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java
index 23a0bee..cdeb4f1 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java
@@ -23,6 +23,7 @@ import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseLargeVariableWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.compare.VectorVisitor;
@@ -170,4 +171,10 @@ public class ValidateVectorDataVisitor implements
VectorVisitor<Void, Void> {
public Void visit(NullVector vector, Void value) {
return null;
}
+
+ @Override
+ public Void visit(ExtensionTypeVector<?> vector, Void value) {
+ vector.getUnderlyingVector().accept(this, value);
+ return null;
+ }
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
index b7aa44c..65795b4 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
@@ -29,6 +29,7 @@ import org.apache.arrow.vector.DateMilliVector;
import org.apache.arrow.vector.Decimal256Vector;
import org.apache.arrow.vector.DecimalVector;
import org.apache.arrow.vector.DurationVector;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.FixedSizeBinaryVector;
import org.apache.arrow.vector.Float4Vector;
import org.apache.arrow.vector.Float8Vector;
@@ -146,6 +147,17 @@ public class ValidateVectorTypeVisitor implements
VectorVisitor<Void, Void> {
}
}
+ private void validateExtensionTypeVector(ExtensionTypeVector<?> vector) {
+ validateOrThrow(vector.getField().getFieldType().getType() instanceof
ArrowType.ExtensionType,
+ "Vector %s is not an extension type vector.", vector.getClass());
+
validateOrThrow(vector.getField().getMetadata().containsKey(ArrowType.ExtensionType.EXTENSION_METADATA_KEY_NAME),
+ "Field %s does not have proper extension type metadata: %s",
+ vector.getField().getName(),
+ vector.getField().getMetadata());
+ // Validate the storage vector type
+ vector.getUnderlyingVector().accept(this, null);
+ }
+
@Override
public Void visit(BaseFixedWidthVector vector, Void value) {
if (vector instanceof TinyIntVector) {
@@ -357,4 +369,10 @@ public class ValidateVectorTypeVisitor implements
VectorVisitor<Void, Void> {
validateVectorCommon(vector, ArrowType.Null.class);
return null;
}
+
+ @Override
+ public Void visit(ExtensionTypeVector<?> vector, Void value) {
+ validateExtensionTypeVector(vector);
+ return null;
+ }
}
diff --git
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java
index 3e44c26..7e99b1f 100644
---
a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java
+++
b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorVisitor.java
@@ -23,6 +23,7 @@ import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseLargeVariableWidthVector;
import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.ExtensionTypeVector;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.ValueVector;
@@ -39,7 +40,7 @@ import org.apache.arrow.vector.util.ValueVectorUtility;
/**
* Visitor to validate vector (without validating data).
* This visitor could be used for {@link ValueVector#accept(VectorVisitor,
Object)} API,
- * and also users could simply use {@link
ValueVectorUtility#validate(FieldVector)}.
+ * and also users could simply use {@link
ValueVectorUtility#validate(ValueVector)}.
*/
public class ValidateVectorVisitor implements VectorVisitor<Void, Void> {
@@ -263,4 +264,10 @@ public class ValidateVectorVisitor implements
VectorVisitor<Void, Void> {
public Void visit(NullVector vector, Void value) {
return null;
}
+
+ @Override
+ public Void visit(ExtensionTypeVector<?> vector, Void value) {
+ vector.getUnderlyingVector().accept(this, value);
+ return null;
+ }
}
diff --git
a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java
b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java
index 53f009c..8b27432 100644
---
a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java
+++
b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java
@@ -17,6 +17,8 @@
package org.apache.arrow.vector.types.pojo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -40,11 +42,15 @@ import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.FixedSizeBinaryVector;
import org.apache.arrow.vector.Float4Vector;
import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.compare.Range;
+import org.apache.arrow.vector.compare.RangeEqualsVisitor;
import org.apache.arrow.vector.complex.StructVector;
import org.apache.arrow.vector.ipc.ArrowFileReader;
import org.apache.arrow.vector.ipc.ArrowFileWriter;
import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType;
+import org.apache.arrow.vector.util.VectorBatchAppender;
+import org.apache.arrow.vector.validate.ValidateVectorVisitor;
import org.junit.Assert;
import org.junit.Test;
@@ -230,6 +236,48 @@ public class TestExtensionType {
}
}
+ @Test
+ public void testVectorCompare() {
+ UuidType uuidType = new UuidType();
+ ExtensionTypeRegistry.register(uuidType);
+ try (final BufferAllocator allocator = new
RootAllocator(Integer.MAX_VALUE);
+ UuidVector a1 = (UuidVector) uuidType.getNewVector("a",
FieldType.nullable(uuidType), allocator);
+ UuidVector a2 = (UuidVector) uuidType.getNewVector("a",
FieldType.nullable(uuidType), allocator);
+ UuidVector bb = (UuidVector) uuidType.getNewVector("a",
FieldType.nullable(uuidType), allocator)
+ ) {
+ UUID u1 = UUID.randomUUID();
+ UUID u2 = UUID.randomUUID();
+
+ // Test out type and vector validation visitors for an
ExtensionTypeVector
+ ValidateVectorVisitor validateVisitor = new ValidateVectorVisitor();
+ validateVisitor.visit(a1, null);
+
+ a1.setValueCount(2);
+ a1.set(0, u1);
+ a1.set(1, u2);
+
+ a2.setValueCount(2);
+ a2.set(0, u1);
+ a2.set(1, u2);
+
+ bb.setValueCount(2);
+ bb.set(0, u2);
+ bb.set(1, u1);
+
+ Range range = new Range(0, 0, a1.getValueCount());
+ RangeEqualsVisitor visitor = new RangeEqualsVisitor(a1, a2);
+ assertTrue(visitor.rangeEquals(range));
+
+ visitor = new RangeEqualsVisitor(a1, bb);
+ assertFalse(visitor.rangeEquals(range));
+
+ // Test out vector appender
+ VectorBatchAppender.batchAppend(a1, a2, bb);
+ assertEquals(a1.getValueCount(), 6);
+ validateVisitor.visit(a1, null);
+ }
+ }
+
static class UuidType extends ExtensionType {
@Override
