IMPALA-4033: Treat string-partition key values as case sensitive. This commit makes ADD PARTITION operations treat string partition-key values as case sensitive in consistent with other related partition DDL operations.
Change-Id: I6fbe67d99df8a50a16a18456fde85d03d622c7a1 Reviewed-on: http://gerrit.cloudera.org:8080/5535 Reviewed-by: Alex Behm <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b3636c97 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b3636c97 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b3636c97 Branch: refs/heads/master Commit: b3636c97d4b872e1640955974409c57459d655e0 Parents: 226a2e6 Author: Amos Bird <[email protected]> Authored: Fri Dec 16 14:23:22 2016 +0800 Committer: Internal Jenkins <[email protected]> Committed: Thu Dec 22 10:45:39 2016 +0000 ---------------------------------------------------------------------- .../apache/impala/analysis/PartitionSet.java | 8 ++------ .../org/apache/impala/catalog/HdfsTable.java | 4 ++-- .../apache/impala/analysis/AnalyzeDDLTest.java | 2 +- .../partition-ddl-predicates-all-fs.test | 20 ++++++++++++++++++++ 4 files changed, 25 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java index 3ba2ad2..d5f0e70 100644 --- a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java +++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java @@ -137,12 +137,11 @@ public class PartitionSet extends PartitionSpecBase { } } - // Transform <COL> = NULL into IsNull expr; <String COL> = '' into IsNull expr and - // <String COL> = 'String Value' into lower case. + // Transform <COL> = NULL into IsNull expr; <String COL> = '' into IsNull expr. // The reason is that COL = NULL is allowed for selecting the NULL // partition, but a COL = NULL predicate can never be true, so we // need to transform such predicates before feeding them into the - // partition pruner. Same logic goes to String transformation. + // partition pruner. private List<Expr> transformPartitionConjuncts(Analyzer analyzer, List<Expr> conjuncts) throws AnalysisException { List<Expr> transformedConjuncts = Lists.newArrayList(); @@ -162,9 +161,6 @@ public class PartitionSet extends PartitionSpecBase { } else if (leftChild != null && stringChild != null) { if (stringChild.getStringValue().isEmpty()) { result = new IsNullPredicate(leftChild, false); - } else { - stringChild = new StringLiteral(stringChild.getStringValue().toLowerCase()); - result.setChild(1, stringChild); } } } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java index c2c569f..904c90f 100644 --- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java @@ -570,7 +570,7 @@ public class HdfsTable extends Table { for (FieldSchema fs: getMetaStoreTable().getPartitionKeys()) { for (TPartitionKeyValue kv: partitionSpec) { if (fs.getName().toLowerCase().equals(kv.getName().toLowerCase())) { - targetValues.add(kv.getValue().toLowerCase()); + targetValues.add(kv.getValue()); // Same key was specified twice if (!keys.add(kv.getName().toLowerCase())) { return null; @@ -604,7 +604,7 @@ public class HdfsTable extends Table { // backwards compatibility with Hive, and is clearly broken. if (value.isEmpty()) value = getNullPartitionKeyValue(); } - if (!targetValues.get(i).equals(value.toLowerCase())) { + if (!targetValues.get(i).equals(value)) { matchFound = false; break; } http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java index 935edc5..5a08f98 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java @@ -424,7 +424,7 @@ public class AnalyzeDDLTest extends FrontendTestBase { AnalyzesOk("alter table functional.stringpartitionkey PARTITION " + "(string_col='partition1') set fileformat parquet"); AnalyzesOk("alter table functional.stringpartitionkey PARTITION " + - "(string_col='PaRtiTion1') set location '/a/b/c'"); + "(string_col='partition1') set location '/a/b/c'"); AnalyzesOk("alter table functional.alltypes PARTITION (year=2010, month=11) " + "set tblproperties('a'='1')"); AnalyzesOk("alter table functional.alltypes PARTITION (year<=2010, month=11) " + http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3636c97/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test index 36104c3..1aa5c51 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates-all-fs.test @@ -133,3 +133,23 @@ show partitions p1 ---- TYPES STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING ==== +---- QUERY +# Tests case-sensitivity of string-typed partition columns. +alter table p1 add partition (j=2,k="D"); +alter table p1 add partition (j=2,k="E"); +alter table p1 add partition (j=2,k="F"); +==== +---- QUERY +show partitions p1 +---- RESULTS +'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g +'2','D',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=D +'2','E',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=E +'2','F',-1,0,regex:.+,regex:.+,regex:.+,'TEXT',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=F +'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d +'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e +'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f +'Total','',0,0,regex:.+,regex:.+,'','','','' +---- TYPES +STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING +====
