Repository: incubator-impala Updated Branches: refs/heads/master 45ff0f9e6 -> 0ad935b63
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0ad935b6/common/thrift/ImpalaInternalService.thrift ---------------------------------------------------------------------- diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift index cfe0080..c7401b3 100644 --- a/common/thrift/ImpalaInternalService.thrift +++ b/common/thrift/ImpalaInternalService.thrift @@ -133,11 +133,14 @@ struct TQueryOptions { // produce different results than the scan based approach in some edge cases. 32: optional bool optimize_partition_key_scans = 0 - // Configure whether scheduling of scans over multiple non-cached replicas will break - // ties between multiple, otherwise equivalent locations at random or deterministically. - // The former will pick a random replica, the latter will use the replica order from the - // metastore. This setting will not affect tie-breaking for cached replicas. Instead, - // they will always break ties randomly. + // Specify the prefered locality level of replicas during scan scheduling. + // Replicas with an equal or better locality will be preferred. + 33: optional PlanNodes.TReplicaPreference replica_preference = + PlanNodes.TReplicaPreference.CACHE_LOCAL + + // Configure whether scan ranges with local replicas will be assigned by starting from + // the same replica for every query or by starting with a new, pseudo-random replica for + // subsequent queries. The default is to start with the same replica for every query. 34: optional bool schedule_random_replica = 0 // For scan nodes with any conjuncts, use codegen to evaluate the conjuncts if http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0ad935b6/common/thrift/ImpalaService.thrift ---------------------------------------------------------------------- diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift index 218a339..f96a119 100644 --- a/common/thrift/ImpalaService.thrift +++ b/common/thrift/ImpalaService.thrift @@ -172,7 +172,12 @@ enum TImpalaQueryOptions { // produce different results than the scan based approach in some edge cases. OPTIMIZE_PARTITION_KEY_SCANS, - // Determines tie breaking policy when picking locations. + // Prefered memory distance of replicas. This parameter determines the pool of replicas + // among which scans will be scheduled in terms of the distance of the replica storage + // from the impalad. + REPLICA_PREFERENCE, + + // Enables random backend selection during scheduling. SCHEDULE_RANDOM_REPLICA, // For scan nodes with any conjuncts, use codegen to evaluate the conjuncts if http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0ad935b6/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java b/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java index cfb8c6e..b2ae8a0 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/TableRef.java @@ -388,7 +388,16 @@ public class TableRef implements ParseNode { analyzer.addWarning("Table hints only supported for Hdfs tables"); } for (String hint: tableHints_) { - if (hint.equalsIgnoreCase("SCHEDULE_RANDOM_REPLICA")) { + if (hint.equalsIgnoreCase("SCHEDULE_CACHE_LOCAL")) { + analyzer.setHasPlanHints(); + replicaPreference_ = TReplicaPreference.CACHE_LOCAL; + } else if (hint.equalsIgnoreCase("SCHEDULE_DISK_LOCAL")) { + analyzer.setHasPlanHints(); + replicaPreference_ = TReplicaPreference.DISK_LOCAL; + } else if (hint.equalsIgnoreCase("SCHEDULE_REMOTE")) { + analyzer.setHasPlanHints(); + replicaPreference_ = TReplicaPreference.REMOTE; + } else if (hint.equalsIgnoreCase("SCHEDULE_RANDOM_REPLICA")) { analyzer.setHasPlanHints(); randomReplica_ = true; } else { http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0ad935b6/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java index 2c6b9bf..6f13b60 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java @@ -1618,8 +1618,20 @@ public class AnalyzeStmtsTest extends AnalyzerTest { String suffix = hintStyle[1]; for (String alias : new String[] { "", "a" }) { AnalyzesOk( - String.format("select * from functional.alltypes %s " + - "%sschedule_random_replica%s", alias, prefix, suffix)); + String.format("select * from functional.alltypes %s %sschedule_cache_local%s", + alias, prefix, suffix)); + AnalyzesOk( + String.format("select * from functional.alltypes %s %sschedule_disk_local%s", + alias, prefix, suffix)); + AnalyzesOk( + String.format("select * from functional.alltypes %s %sschedule_remote%s", + alias, prefix, suffix)); + AnalyzesOk( + String.format("select * from functional.alltypes %s %sschedule_remote," + + "schedule_random_replica%s", alias, prefix, suffix)); + AnalyzesOk( + String.format("select * from functional.alltypes %s %s" + + "schedule_random_replica,schedule_remote%s", alias, prefix, suffix)); String name = alias.isEmpty() ? "functional.alltypes" : alias; AnalyzesOk(String.format("select * from functional.alltypes %s %sFOO%s", alias, http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0ad935b6/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java index f39f902..aa58056 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java @@ -392,14 +392,29 @@ public class ParserTest { // Test TableRef hints. TestTableHints(String.format( - "select * from functional.alltypes %sschedule_random_replica%s", prefix, - suffix), "schedule_random_replica"); + "select * from functional.alltypes %sschedule_disk_local%s", prefix, suffix), + "schedule_disk_local"); + TestTableHints(String.format( + "select * from functional.alltypes %sschedule_cache_local," + + "schedule_random_replica%s", prefix, suffix), "schedule_cache_local", + "schedule_random_replica"); + TestTableHints(String.format( + "select * from functional.alltypes a %sschedule_cache_local," + + "schedule_random_replica%s", prefix, suffix), "schedule_cache_local", + "schedule_random_replica"); + TestTableHints(String.format( + "select * from functional.alltypes a %sschedule_cache_local," + + "schedule_random_replica%s" + ", functional.alltypes b %sschedule_remote%s", + prefix, suffix, prefix, suffix), "schedule_cache_local", + "schedule_random_replica", "schedule_remote"); // Test both TableRef and join hints. TestTableAndJoinHints(String.format( - "select * from functional.alltypes a %sschedule_random_replica%s join " + - "%sbroadcast%s functional.alltypes b using(id)", prefix, suffix, prefix, suffix, - prefix, suffix), "schedule_random_replica", "broadcast"); + "select * from functional.alltypes a %sschedule_cache_local," + + "schedule_random_replica%s join %sbroadcast%s functional.alltypes b " + + "%sschedule_remote%s using(id)", prefix, suffix, prefix, suffix, prefix, + suffix), "schedule_cache_local", "schedule_random_replica", "broadcast", + "schedule_remote"); // Test select-list hints (e.g., straight_join). The legacy-style hint has no // prefix and suffix. http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0ad935b6/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java index 6005d35..151254c 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java @@ -469,11 +469,17 @@ public class ToSqlTest extends AnalyzerTest { // Table hint testToSql(String.format( "select * from functional.alltypes at %sschedule_random_replica%s", prefix, - suffix), "SELECT * FROM functional.alltypes at \n-- +schedule_random_replica\n" - ); + suffix), + "SELECT * FROM functional.alltypes at \n-- +schedule_random_replica\n"); testToSql(String.format( "select * from functional.alltypes %sschedule_random_replica%s", prefix, - suffix), "SELECT * FROM functional.alltypes \n-- +schedule_random_replica\n"); + suffix), + "SELECT * FROM functional.alltypes \n-- +schedule_random_replica\n"); + testToSql(String.format( + "select * from functional.alltypes %sschedule_random_replica," + + "schedule_disk_local%s", prefix, suffix), + "SELECT * FROM functional.alltypes \n-- +schedule_random_replica," + + "schedule_disk_local\n"); testToSql(String.format( "select c1 from (select at.tinyint_col as c1 from functional.alltypes at " + "%sschedule_random_replica%s) s1", prefix, suffix),
