This is an automated email from the ASF dual-hosted git repository. omalley pushed a commit to branch branch-1.5 in repository https://gitbox.apache.org/repos/asf/orc.git
commit bad383e9074b1597af50e85e77e7b6ce79147e39 Author: Arvin Zheng <[email protected]> AuthorDate: Mon Jul 27 17:32:04 2020 -0700 ORC-644: Support positional mapping for nested types. Resolves #522 Signed-off-by: Owen O'Malley <[email protected]> --- java/core/src/java/org/apache/orc/Reader.java | 4 +- .../java/org/apache/orc/impl/SchemaEvolution.java | 2 +- .../org/apache/orc/impl/TestSchemaEvolution.java | 43 ++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/java/core/src/java/org/apache/orc/Reader.java b/java/core/src/java/org/apache/orc/Reader.java index 2117b45..8403e99 100644 --- a/java/core/src/java/org/apache/orc/Reader.java +++ b/java/core/src/java/org/apache/orc/Reader.java @@ -271,9 +271,9 @@ public interface Reader extends Closeable { } /** - * Set no. of levels to force schema evolution to be positional instead of + * Set number of levels to force schema evolution to be positional instead of * based on the column names. - * @param value force positional evolution + * @param value number of levels of positional schema evolution * @return this */ public Options positionalEvolutionLevel(int value) { diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java index b70c0e4..3184d5e 100644 --- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java +++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java @@ -485,7 +485,7 @@ public class SchemaEvolution { if (fileChildren.size() == readerChildren.size()) { for(int i=0; i < fileChildren.size(); ++i) { buildConversion(fileChildren.get(i), - readerChildren.get(i), 0); + readerChildren.get(i), positionalLevels - 1); } } else { isOk = false; diff --git a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java index 789bf55..f707ce4 100644 --- a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java +++ b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java @@ -1716,6 +1716,49 @@ public class TestSchemaEvolution { assertEquals(4, evo.getFileType(4).getId()); } + @Test + public void testPositionalEvolutionForStructInArray() throws IOException { + options.forcePositionalEvolution(true); + options.positionalEvolutionLevel(Integer.MAX_VALUE); + TypeDescription file = TypeDescription.fromString("array<struct<x:int,y:int,z:int>>"); + TypeDescription read = TypeDescription.fromString("array<struct<z:int,x:int,a:int,b:int>>"); + SchemaEvolution evo = new SchemaEvolution(file, read, options); + assertEquals(1, evo.getFileType(1).getId()); + assertEquals(2, evo.getFileType(2).getId()); + assertEquals(3, evo.getFileType(3).getId()); + assertEquals(4, evo.getFileType(4).getId()); + assertEquals(null, evo.getFileType(5)); + } + + @Test + public void testPositionalEvolutionForTwoLayerNestedStruct() throws IOException { + options.forcePositionalEvolution(true); + options.positionalEvolutionLevel(Integer.MAX_VALUE); + TypeDescription file = TypeDescription.fromString("struct<s:struct<x:int,y:int,z:int>>"); + TypeDescription read = TypeDescription.fromString("struct<s:struct<z:int,x:int,a:int,b:int>>"); + SchemaEvolution evo = new SchemaEvolution(file, read, options); + assertEquals(1, evo.getFileType(1).getId()); + assertEquals(2, evo.getFileType(2).getId()); + assertEquals(3, evo.getFileType(3).getId()); + assertEquals(4, evo.getFileType(4).getId()); + assertNull(evo.getFileType(5)); + } + + @Test + public void testPositionalEvolutionForThreeLayerNestedStruct() throws IOException { + options.forcePositionalEvolution(true); + options.positionalEvolutionLevel(Integer.MAX_VALUE); + TypeDescription file = TypeDescription.fromString("struct<s1:struct<s2:struct<x:int,y:int,z:int>>>"); + TypeDescription read = TypeDescription.fromString("struct<s1:struct<s:struct<z:int,x:int,a:int,b:int>>>"); + SchemaEvolution evo = new SchemaEvolution(file, read, options); + assertEquals(1, evo.getFileType(1).getId()); + assertEquals(2, evo.getFileType(2).getId()); + assertEquals(3, evo.getFileType(3).getId()); + assertEquals(4, evo.getFileType(4).getId()); + assertEquals(5, evo.getFileType(5).getId()); + assertNull(evo.getFileType(6)); + } + // These are helper methods that pull some of the common code into one // place.
