Repository: incubator-impala Updated Branches: refs/heads/master b55ec3f64 -> dced73167
IMPALA-5757: Make tbl property toSql deterministic On Ubuntu 16.04, it seems that table properties may be returned in a different order from that expected by TestShowCreateTable in test_kudu.py. The fix is to change ToSqlUtils to print properties in a deterministic way. Change-Id: I2391e04eb40e398098704b77588ad352d514df7d Reviewed-on: http://gerrit.cloudera.org:8080/7580 Reviewed-by: Matthew Jacobs <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/dced7316 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/dced7316 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/dced7316 Branch: refs/heads/master Commit: dced731671d28249f35d707854605e7ada82a6e0 Parents: b55ec3f Author: Matthew Jacobs <[email protected]> Authored: Thu Aug 3 15:31:36 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Aug 4 19:51:46 2017 +0000 ---------------------------------------------------------------------- .../java/org/apache/impala/analysis/ToSqlUtils.java | 14 ++++++++++++-- .../java/org/apache/impala/analysis/ToSqlTest.java | 8 ++++---- 2 files changed, 16 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/dced7316/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java index 01517e1..471947c 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java +++ b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java @@ -18,10 +18,12 @@ package org.apache.impala.analysis; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -32,6 +34,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.Token; +import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.metastore.TableType; @@ -425,8 +428,15 @@ public class ToSqlUtils { } private static String propertyMapToSql(Map<String, String> propertyMap) { + // Sort entries on the key to ensure output is deterministic for tests (IMPALA-5757). + List<Entry<String, String>> mapEntries = Lists.newArrayList(propertyMap.entrySet()); + Collections.sort(mapEntries, new Comparator<Entry<String, String>>() { + public int compare(Entry<String, String> o1, Entry<String, String> o2) { + return ObjectUtils.compare(o1.getKey(), o2.getKey()); + } }); + List<String> properties = Lists.newArrayList(); - for (Map.Entry<String, String> entry: propertyMap.entrySet()) { + for (Map.Entry<String, String> entry: mapEntries) { properties.add(String.format("'%s'='%s'", entry.getKey(), // Properties may contain characters that need to be escaped. // e.g. If the row format escape delimiter is '\', the map of serde properties http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/dced7316/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java index 6ef4b04..4cd8934 100644 --- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java @@ -314,8 +314,8 @@ public class ToSqlTest extends FrontendTestBase { "CREATE TABLE default.p ( a BIGINT PRIMARY KEY, b TIMESTAMP " + "DEFAULT '1987-05-19' ) PARTITION BY HASH (a) PARTITIONS 3 " + "STORED AS KUDU TBLPROPERTIES ('kudu.master_addresses'='foo', " + - "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler', " + - "'kudu.table_name'='impala::default.p')", true); + "'kudu.table_name'='impala::default.p', " + + "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler')", true); } @Test @@ -347,8 +347,8 @@ public class ToSqlTest extends FrontendTestBase { "CREATE TABLE default.p PRIMARY KEY (a, b) PARTITION BY HASH (a) PARTITIONS 3, " + "RANGE (b) (PARTITION VALUE = 1) STORED AS KUDU TBLPROPERTIES " + "('kudu.master_addresses'='foo', " + - "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler', " + - "'kudu.table_name'='impala::default.p') AS " + + "'kudu.table_name'='impala::default.p', " + + "'storage_handler'='com.cloudera.kudu.hive.KuduStorageHandler') AS " + "SELECT int_col a, bigint_col b FROM functional.alltypes", true); }
