Repository: orc Updated Branches: refs/heads/master 639a7667d -> d97654911
ORC-264. Provide option to make schema evolution case insensitive. Fixes #190 Signed-off-by: Owen O'Malley <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/orc/repo Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/d9765491 Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/d9765491 Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/d9765491 Branch: refs/heads/master Commit: d976549118a9480d2c52bbcf1af3488948219158 Parents: 639a766 Author: piyush.mukati <[email protected]> Authored: Wed Nov 15 11:00:43 2017 +0530 Committer: Owen O'Malley <[email protected]> Committed: Wed Dec 20 10:47:02 2017 -0800 ---------------------------------------------------------------------- java/core/src/java/org/apache/orc/OrcConf.java | 4 ++- java/core/src/java/org/apache/orc/Reader.java | 18 +++++++++++ .../org/apache/orc/impl/SchemaEvolution.java | 33 ++++++++++++++++++-- .../apache/orc/impl/TestSchemaEvolution.java | 30 ++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/orc/blob/d9765491/java/core/src/java/org/apache/orc/OrcConf.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/OrcConf.java b/java/core/src/java/org/apache/orc/OrcConf.java index 319366e..443da87 100644 --- a/java/core/src/java/org/apache/orc/OrcConf.java +++ b/java/core/src/java/org/apache/orc/OrcConf.java @@ -150,7 +150,9 @@ public enum OrcConf { "added to all of the writers. Valid range is [1,10000] and is primarily meant for" + "n\testing. Setting this too low may negatively affect performance."), OVERWRITE_OUTPUT_FILE("orc.overwrite.output.file", "orc.overwrite.output.file", false, - "A boolean flag to enable overwriting of the output file if it already exists.\n") + "A boolean flag to enable overwriting of the output file if it already exists.\n"), + IS_SCHEMA_EVOLUTION_CASE_SENSITIVE("orc.schema.evolution.case.sensitive", "orc.schema.evolution.case.sensitive", true, + "A boolean flag to determine if the comparision of field names in schema evolution is case sensitive .\n") ; private final String attribute; http://git-wip-us.apache.org/repos/asf/orc/blob/d9765491/java/core/src/java/org/apache/orc/Reader.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/Reader.java b/java/core/src/java/org/apache/orc/Reader.java index 2ef64d7..d4e6b56 100644 --- a/java/core/src/java/org/apache/orc/Reader.java +++ b/java/core/src/java/org/apache/orc/Reader.java @@ -160,6 +160,7 @@ public interface Reader { private DataReader dataReader = null; private Boolean tolerateMissingSchema = null; private boolean forcePositionalEvolution; + private boolean isSchemaEvolutionCaseAware; public Options() { // PASS @@ -170,6 +171,8 @@ public interface Reader { skipCorruptRecords = OrcConf.SKIP_CORRUPT_DATA.getBoolean(conf); tolerateMissingSchema = OrcConf.TOLERATE_MISSING_SCHEMA.getBoolean(conf); forcePositionalEvolution = OrcConf.FORCE_POSITIONAL_EVOLUTION.getBoolean(conf); + isSchemaEvolutionCaseAware = + OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.getBoolean(conf); } /** @@ -262,6 +265,17 @@ public interface Reader { return this; } + /** + * Set boolean flag to determine if the comparision of field names in schema + * evolution is case sensitive + * @param value the flag for schema evolution is case sensitive or not. + * @return + */ + public Options isSchemaEvolutionCaseAware(boolean value) { + this.isSchemaEvolutionCaseAware = value; + return this; + } + public boolean[] getInclude() { return include; } @@ -310,6 +324,10 @@ public interface Reader { return forcePositionalEvolution; } + public boolean getIsSchemaEvolutionCaseAware() { + return isSchemaEvolutionCaseAware; + } + public Options clone() { try { Options result = (Options) super.clone(); http://git-wip-us.apache.org/repos/asf/orc/blob/d9765491/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java ---------------------------------------------------------------------- diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java index 886d585..78acd41 100644 --- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java +++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java @@ -48,6 +48,7 @@ public class SchemaEvolution { private final TypeDescription readerSchema; private boolean hasConversion = false; private final boolean isAcid; + private final boolean isSchemaEvolutionCaseAware; // indexed by reader column id private final boolean[] ppdSafeConversion; @@ -57,6 +58,7 @@ public class SchemaEvolution { private static final Pattern missingMetadataPattern = Pattern.compile("_col\\d+"); + public static class IllegalEvolutionException extends RuntimeException { public IllegalEvolutionException(String msg) { super(msg); @@ -68,6 +70,7 @@ public class SchemaEvolution { Reader.Options options) { boolean allowMissingMetadata = options.getTolerateMissingSchema(); boolean[] includedCols = options.getInclude(); + this.isSchemaEvolutionCaseAware=options.getIsSchemaEvolutionCaseAware(); this.readerIncluded = includedCols == null ? null : Arrays.copyOf(includedCols, includedCols.length); this.fileIncluded = new boolean[fileSchema.getMaximumId() + 1]; @@ -398,13 +401,20 @@ public class SchemaEvolution { if (positionalLevels == 0) { List<String> readerFieldNames = readerType.getFieldNames(); List<String> fileFieldNames = fileType.getFieldNames(); - Map<String, TypeDescription> fileTypesIdx = new HashMap<>(); + + final Map<String, TypeDescription> fileTypesIdx; + if (isSchemaEvolutionCaseAware) { + fileTypesIdx = new HashMap<>(); + } else { + fileTypesIdx = new CaseInsensitiveMap<TypeDescription>(); + } for (int i = 0; i < fileFieldNames.size(); i++) { - fileTypesIdx.put(fileFieldNames.get(i), fileChildren.get(i)); + final String fileFieldName = fileFieldNames.get(i); + fileTypesIdx.put(fileFieldName, fileChildren.get(i)); } for (int i = 0; i < readerFieldNames.size(); i++) { - String readerFieldName = readerFieldNames.get(i); + final String readerFieldName = readerFieldNames.get(i); TypeDescription readerField = readerChildren.get(i); TypeDescription fileField = fileTypesIdx.get(readerFieldName); @@ -511,4 +521,21 @@ public class SchemaEvolution { acidEventFieldNames.add("currentTransaction"); acidEventFieldNames.add("row"); } + + private static class CaseInsensitiveMap<V> extends HashMap<String,V> { + @Override + public V put(String key, V value) { + return super.put(key.toLowerCase(), value); + } + + @Override + public V get(Object key) { + return this.get((String) key); + } + + // not @Override as key to be of type Object + public V get(String key) { + return super.get(key.toLowerCase()); + } + } } http://git-wip-us.apache.org/repos/asf/orc/blob/d9765491/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java ---------------------------------------------------------------------- diff --git a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java index 82f823a..510e00c 100644 --- a/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java +++ b/java/core/src/test/org/apache/orc/impl/TestSchemaEvolution.java @@ -46,6 +46,7 @@ import org.apache.orc.RecordReader; import org.apache.orc.TypeDescription; import org.apache.orc.Writer; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -718,6 +719,35 @@ public class TestSchemaEvolution { original = fileType.getChildren().get(1); assertSame(original, mapped); } + @Test + public void testCaseMismatchInReaderAndWriterSchema() { + TypeDescription fileType = + TypeDescription.fromString("struct<a:struct<b:int>,c:string>"); + TypeDescription readerType = + TypeDescription.fromString("struct<A:struct<b:int>,c:string>"); + boolean[] included = includeAll(readerType); + options.tolerateMissingSchema(false); + SchemaEvolution transition = + new SchemaEvolution(fileType, readerType, options.include(included).isSchemaEvolutionCaseAware(false)); + + // a -> A + TypeDescription reader = readerType.getChildren().get(0); + TypeDescription mapped = transition.getFileType(reader); + TypeDescription original = fileType.getChildren().get(0); + assertSame(original, mapped); + + // a.b -> a.b + TypeDescription readerChild = reader.getChildren().get(0); + mapped = transition.getFileType(readerChild); + TypeDescription originalChild = original.getChildren().get(0); + assertSame(originalChild, mapped); + + // c -> c + reader = readerType.getChildren().get(1); + mapped = transition.getFileType(reader); + original = fileType.getChildren().get(1); + assertSame(original, mapped); + } /** * Two structs can be equal but in different locations. They can converge to this.
