Fokko commented on code in PR #3876:
URL: https://github.com/apache/parquet-java/pull/3876#discussion_r4455578798


##########
parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java:
##########
@@ -34,6 +34,7 @@ public final class MessageType extends GroupType {
    */
   public MessageType(String name, Type... fields) {
     super(Repetition.REPEATED, name, fields);
+    TypeUtil.checkValidFiniteVectorSchema(this);

Review Comment:
   I still think this isn't the right place, let's do something like:
   
   ```
   diff --git 
a/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java 
b/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java
   index 790174557..1aba0cc6a 100644
   --- a/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java
   +++ b/parquet-column/src/main/java/org/apache/parquet/schema/MessageType.java
   @@ -34,7 +34,6 @@ public final class MessageType extends GroupType {
       */
      public MessageType(String name, Type... fields) {
        super(Repetition.REPEATED, name, fields);
   -    TypeUtil.checkValidFiniteVectorSchema(this);
      }
    
      /**
   @@ -43,7 +42,6 @@ public final class MessageType extends GroupType {
       */
      public MessageType(String name, List<Type> fields) {
        super(Repetition.REPEATED, name, fields);
   -    TypeUtil.checkValidFiniteVectorSchema(this);
      }
    
      /**
   diff --git 
a/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java 
b/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java
   index c18a5cef7..b32efce39 100644
   --- a/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java
   +++ b/parquet-column/src/main/java/org/apache/parquet/schema/TypeUtil.java
   @@ -18,29 +18,9 @@
     */
    package org.apache.parquet.schema;
    
   -import 
org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
    import 
org.apache.parquet.schema.LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation;
   -import 
org.apache.parquet.schema.LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
   -import 
org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation;
   -import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
   -import org.apache.parquet.schema.Type.Repetition;
    
    public class TypeUtil {
   -  public static boolean checkValidFiniteVectorSchema(Type type) {
   -    boolean found = type.getLogicalTypeAnnotation() instanceof 
FiniteVectorLogicalTypeAnnotation;
   -    if (found) {
   -      if (type.isPrimitive()) {
   -        throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
type);
   -      }
   -      checkVector(type.asGroupType());
   -    } else if (!type.isPrimitive()) {
   -      for (Type child : type.asGroupType().getFields()) {
   -        found |= checkValidFiniteVectorSchema(child);
   -      }
   -    }
   -    return found;
   -  }
   -
      public static boolean hasFiniteVector(Type type) {
        if (type.getLogicalTypeAnnotation() instanceof 
FiniteVectorLogicalTypeAnnotation) return true;
        if (!type.isPrimitive()) {
   @@ -51,57 +31,6 @@ public class TypeUtil {
        return false;
      }
    
   -  private static void checkVector(GroupType vector) {
   -    if (vector.getRepetition() == Repetition.REPEATED || 
vector.getFieldCount() != 1) {
   -      throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   -    }
   -    Type listType = vector.getType(0);
   -    if (listType.isPrimitive()
   -        || !listType.getName().equals("list")
   -        || listType.getRepetition() != Repetition.REPEATED
   -        || listType.getLogicalTypeAnnotation() != null) {
   -      throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   -    }
   -    GroupType list = listType.asGroupType();
   -    if (list.getFieldCount() != 1) {
   -      throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   -    }
   -    Type elementType = list.getType(0);
   -    if (!elementType.isPrimitive()
   -        || !elementType.getName().equals("element")
   -        || elementType.getRepetition() != Repetition.REQUIRED) {
   -      throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   -    }
   -    PrimitiveType element = elementType.asPrimitiveType();
   -    PrimitiveTypeName physical = element.getPrimitiveTypeName();
   -    LogicalTypeAnnotation logical = element.getLogicalTypeAnnotation();
   -    boolean allowed = switch (physical) {
   -      case BOOLEAN, FLOAT, DOUBLE -> logical == null;
   -      case INT32, INT64 ->
   -        logical == null
   -            || logical instanceof IntLogicalTypeAnnotation
   -            || logical instanceof DecimalLogicalTypeAnnotation;
   -      case FIXED_LEN_BYTE_ARRAY ->
   -        logical instanceof Float16LogicalTypeAnnotation || logical 
instanceof DecimalLogicalTypeAnnotation;
   -      default -> false;
   -    };
   -    if (!allowed) {
   -      throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   -    }
   -    try {
   -      Types.PrimitiveBuilder<PrimitiveType> builder = 
Types.primitive(physical, element.getRepetition());
   -      if (physical == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
   -        builder.length(element.getTypeLength());
   -      }
   -      if (logical != null) {
   -        builder.as(logical);
   -      }
   -      builder.named(element.getName());
   -    } catch (IllegalArgumentException | IllegalStateException e) {
   -      throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector, e);
   -    }
   -  }
   -
      public static FiniteVectorLogicalTypeAnnotation 
getFiniteVectorType(GroupType schema, String[] path) {
        if (path.length < 3) {
          return null;
   diff --git 
a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java 
b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
   index 2d6f0cbf8..a4b1381db 100644
   --- a/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
   +++ b/parquet-column/src/main/java/org/apache/parquet/schema/Types.java
   @@ -29,6 +29,10 @@ import java.util.Optional;
    import java.util.stream.Collectors;
    import org.apache.parquet.Preconditions;
    import org.apache.parquet.schema.ColumnOrder.ColumnOrderName;
   +import 
org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
   +import 
org.apache.parquet.schema.LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation;
   +import 
org.apache.parquet.schema.LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
   +import 
org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation;
    import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
    import org.apache.parquet.schema.Type.ID;
    import org.slf4j.Logger;
   @@ -846,10 +850,50 @@ public class Types {
    
        @Override
        protected GroupType build(String name) {
   -      if (newLogicalTypeSet) {
   -        return new GroupType(repetition, name, logicalTypeAnnotation, 
fields, id);
   -      } else {
   -        return new GroupType(repetition, name, getOriginalType(), fields, 
id);
   +      GroupType group = newLogicalTypeSet
   +          ? new GroupType(repetition, name, logicalTypeAnnotation, fields, 
id)
   +          : new GroupType(repetition, name, getOriginalType(), fields, id);
   +      if (logicalTypeAnnotation instanceof 
FiniteVectorLogicalTypeAnnotation) {
   +        checkFiniteVector(group);
   +      }
   +      return group;
   +    }
   +
   +    /**
   +     * FINITE_VECTOR must annotate the canonical 3-level LIST structure 
with a
   +     * required numeric or boolean element.
   +     */
   +    private static void checkFiniteVector(GroupType vector) {
   +      if (vector.isRepetition(Type.Repetition.REPEATED) || 
vector.getFieldCount() != 1) {
   +        throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   +      }
   +      Type list = vector.getType(0);
   +      if (list.isPrimitive()
   +          || !list.getName().equals("list")
   +          || !list.isRepetition(Type.Repetition.REPEATED)
   +          || list.getLogicalTypeAnnotation() != null
   +          || list.asGroupType().getFieldCount() != 1) {
   +        throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   +      }
   +      Type element = list.asGroupType().getType(0);
   +      if (!element.isPrimitive()
   +          || !element.getName().equals("element")
   +          || !element.isRepetition(Type.Repetition.REQUIRED)) {
   +        throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
   +      }
   +      LogicalTypeAnnotation logical = element.getLogicalTypeAnnotation();
   +      boolean allowed = switch 
(element.asPrimitiveType().getPrimitiveTypeName()) {
   +        case BOOLEAN, FLOAT, DOUBLE -> logical == null;
   +        case INT32, INT64 ->
   +          logical == null
   +              || logical instanceof IntLogicalTypeAnnotation
   +              || logical instanceof DecimalLogicalTypeAnnotation;
   +        case FIXED_LEN_BYTE_ARRAY ->
   +          logical instanceof Float16LogicalTypeAnnotation || logical 
instanceof DecimalLogicalTypeAnnotation;
   +        default -> false;
   +      };
   +      if (!allowed) {
   +        throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + 
vector);
          }
        }
    
   diff --git 
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
 
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
   index 61507cc2c..a6ebf5c9a 100644
   --- 
a/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
   +++ 
b/parquet-column/src/test/java/org/apache/parquet/schema/TestTypeBuildersWithLogicalTypes.java
   @@ -45,7 +45,6 @@ import static 
org.apache.parquet.schema.Type.Repetition.REQUIRED;
    import static org.assertj.core.api.Assertions.assertThat;
    import static org.assertj.core.api.Assertions.assertThatThrownBy;
    
   -import java.util.Collections;
    import java.util.Optional;
    import java.util.stream.Stream;
    import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
   @@ -144,33 +143,48 @@ public class TestTypeBuildersWithLogicalTypes {
                
vector(Types.required(FIXED_LEN_BYTE_ARRAY).length(2).named("element"))),
            Arguments.of(vector(Types.required(INT96).named("element"))),
            
Arguments.of(vector(Types.required(BINARY).as(stringType()).named("element"))),
   -        
Arguments.of(vector(Types.required(INT32).as(dateType()).named("element"))),
   -        Arguments.of(
   -            
vector(Types.required(INT32).named("element").withLogicalTypeAnnotation(intType(64,
 true)))));
   +        
Arguments.of(vector(Types.required(INT32).as(dateType()).named("element"))));
      }
    
      @ParameterizedTest
      @MethodSource("invalidVectors")
      public void testInvalidVectorSchema(GroupType vector) {
   -    assertThatThrownBy(() -> new MessageType("m", vector))
   +    assertThatThrownBy(() -> Types.buildGroup(vector.getRepetition())
   +            .as(vector.getLogicalTypeAnnotation())
   +            .addFields(vector.getFields().toArray(new Type[0]))
   +            .named(vector.getName()))
            .isInstanceOf(InvalidSchemaException.class)
            .hasMessage("Invalid FINITE_VECTOR schema: " + vector);
      }
    
      private static Stream<Executable> vectorConstructionBoundaries() {
   -    Type element = Types.optional(FLOAT).named("element");
   -    GroupType invalid = vector(element);
   -    GroupType bad = new GroupType(
   -        REQUIRED, "bad", finiteVectorType(5), new 
GroupType(Repetition.REPEATED, "list", element));
   +    GroupType list =
   +        new GroupType(Repetition.REPEATED, "list", 
Types.optional(FLOAT).named("element"));
        return Stream.of(
   -        () -> new MessageType("m", Collections.singletonList(invalid)),
   -        () -> Types.buildMessage().addField(invalid).named("m"),
   +        () -> Types.requiredGroup()
   +            .as(finiteVectorType(5))
   +            .addField(list)
   +            .named("v"),
   +        () -> Types.buildMessage()
   +            .requiredGroup()
   +            .as(finiteVectorType(5))
   +            .repeatedGroup()
   +            .optional(FLOAT)
   +            .named("element")
   +            .named("list")
   +            .named("v")
   +            .named("m"),
   +        () -> Types.buildMessage()
   +            .requiredGroup()
   +            .addField(vector(Types.required(FLOAT).named("element")))
   +            .requiredGroup()
   +            .as(finiteVectorType(5))
   +            .addField(list)
   +            .named("bad")
   +            .named("parent")
   +            .named("m"),
            () -> MessageTypeParser.parseMessageType("message m { required 
group v (FINITE_VECTOR(5)) { "
   -            + "repeated group list { optional float element; } } }"),
   -        () -> new MessageType(
   -            "m",
   -            new GroupType(
   -                REQUIRED, "parent", 
vector(Types.required(FLOAT).named("element")), bad)));
   +            + "repeated group list { optional float element; } } }"));
      }
    
      @ParameterizedTest
   diff --git 
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
 
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
   index 61cd86707..4e1aa1749 100644
   --- 
a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
   +++ 
b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
   @@ -2036,8 +2036,8 @@ public class ParquetMetadataConverter {
          builder.id(root.field_id);
        }
        Iterator<ColumnOrder> columnOrderIterator = columnOrders == null ? null 
: columnOrders.iterator();
   -    buildChildren(builder, iterator, root.getNum_children(), 
columnOrderIterator);
        try {
   +      buildChildren(builder, iterator, root.getNum_children(), 
columnOrderIterator);
          return builder.named(root.name);
        } catch (InvalidSchemaException e) {
          throw new ParquetDecodingException("Invalid FINITE_VECTOR schema in 
file metadata", e);
   
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to