This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 1dd2e8595 IMPALA-14488: Calcite planner: Support fallback to Original
planner
1dd2e8595 is described below
commit 1dd2e8595c2ddb4e41e9bd6af3a2bb68142752ee
Author: Steve Carlin <[email protected]>
AuthorDate: Tue Sep 23 09:22:37 2025 -0700
IMPALA-14488: Calcite planner: Support fallback to Original planner
Introduced the query options PLANNER and FALLBACK_PLANNER which are
of type TPlannerType
The 3 options for TPlannerType are:
ORIGINAL (default) : Use the Original planner for queries.
CALCITE : Use the Calcite planner for queries.
NONE : Convenient option for setting the FALLBACK_PLANNER.
This should never be used for the PLANNER option.
The Calcite planner currently does not support DDL and other various
queries (e.g. complex types). If a known unsupported query is detected,
the original planner will be used regardless of what is chosen for
FALLBACK_PLANNER.
The fallback planner will allow for easier transition for users who
want to use the new Calcite planner without having to worry about
backward compatibility issues that the compiler doesn't support.
One example that is currently not supported (in runtime_planners.test)
is the "ignore nulls" phrase which needs to be inside the parens for
Impala, but outside the parens for the Calcite parser.
At the time of this commit, there are also various compiler exceptions
thrown for Calcite which are not yet supported. The PLANNER=CALCITE
and FALLBACK_PLANNER=ORIGINAL options are set if the
--use_calcite_planner=true server option is specified. This will allow
easier transitioning to the Calcite planner because the user will be
assured that even if there is a compilation failure in Calcite,
their existing query will still run with the original planner.
Change-Id: Id1fdc5ef92fff84e89af0e19c4246cc15e2ea823
Reviewed-on: http://gerrit.cloudera.org:8080/23900
Tested-by: Impala Public Jenkins <[email protected]>
Reviewed-by: Michael Smith <[email protected]>
---
be/src/service/impala-server.cc | 4 +-
be/src/service/query-options.cc | 17 +++
be/src/service/query-options.h | 7 +-
common/thrift/ImpalaService.thrift | 17 ++-
common/thrift/PlanNodes.thrift | 6 +
common/thrift/Query.thrift | 6 +
.../java/org/apache/impala/service/Frontend.java | 122 ++++++++++++-------
.../impala/calcite/planner/CalcitePlannerTest.java | 7 +-
.../calcite/planner/TpcdsCpuCostPlannerTest.java | 5 +-
.../queries/QueryTest/fallback_planner.test | 135 +++++++++++++++++++++
tests/authorization/test_ranger.py | 25 ++--
tests/custom_cluster/test_calcite_planner.py | 11 +-
.../test_calcite_planner.py | 39 ++++--
13 files changed, 316 insertions(+), 85 deletions(-)
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 4bd5f0dde..f7a58469f 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -2031,7 +2031,9 @@ void ImpalaServer::InitializeConfigVariables() {
// Set idle_session_timeout here to let the SET command return the value of
// the command line option FLAGS_idle_session_timeout
default_query_options_.__set_idle_session_timeout(FLAGS_idle_session_timeout);
- default_query_options_.__set_use_calcite_planner(FLAGS_use_calcite_planner);
+ if (FLAGS_use_calcite_planner) {
+ default_query_options_.__set_planner(TPlannerType::CALCITE);
+ }
// The next query options used to be set with flags. Setting them in
// default_query_options_ here in order to make default_query_options
// take precedence over the legacy flags.
diff --git a/be/src/service/query-options.cc b/be/src/service/query-options.cc
index 09e4b3132..92dc76f78 100644
--- a/be/src/service/query-options.cc
+++ b/be/src/service/query-options.cc
@@ -1432,6 +1432,23 @@ Status impala::SetQueryOption(TImpalaQueryOptions::type
option, const string& va
query_options->__set_show_create_table_partition_limit(int32_t_val);
break;
}
+ case TImpalaQueryOptions::PLANNER: {
+ TPlannerType::type planner_type;
+ RETURN_IF_ERROR(GetThriftEnum(value, "Planner",
+ _TPlannerType_VALUES_TO_NAMES, &planner_type));
+ if (planner_type == TPlannerType::NONE) {
+ return Status("Planner must be set to ORIGINAL or CALCITE");
+ }
+ query_options->__set_planner(planner_type);
+ break;
+ }
+ case TImpalaQueryOptions::FALLBACK_PLANNER: {
+ TPlannerType::type planner_type;
+ RETURN_IF_ERROR(GetThriftEnum(value, "Fallback Planner",
+ _TPlannerType_VALUES_TO_NAMES, &planner_type));
+ query_options->__set_fallback_planner(planner_type);
+ break;
+ }
default:
string key = to_string(option);
if (IsRemovedQueryOption(key)) {
diff --git a/be/src/service/query-options.h b/be/src/service/query-options.h
index c89fecbe7..e0b3f725a 100644
--- a/be/src/service/query-options.h
+++ b/be/src/service/query-options.h
@@ -51,7 +51,7 @@ typedef std::unordered_map<string,
beeswax::TQueryOptionLevel::type>
// plus one. Thus, the second argument to the DCHECK has to be updated every
// time we add or remove a query option to/from the enum TImpalaQueryOptions.
constexpr unsigned NUM_QUERY_OPTIONS =
- TImpalaQueryOptions::SHOW_CREATE_TABLE_PARTITION_LIMIT + 1;
+ TImpalaQueryOptions::FALLBACK_PLANNER + 1;
#define QUERY_OPTS_TABLE
\
DCHECK_EQ(_TImpalaQueryOptions_VALUES_TO_NAMES.size(), NUM_QUERY_OPTIONS);
\
REMOVED_QUERY_OPT_FN(abort_on_default_limit_exceeded,
ABORT_ON_DEFAULT_LIMIT_EXCEEDED) \
@@ -374,8 +374,7 @@ constexpr unsigned NUM_QUERY_OPTIONS =
SKIP_UNNEEDED_UPDATES_COL_LIMIT, TQueryOptionLevel::ADVANCED)
\
QUERY_OPT_FN(mem_estimate_scale_for_spilling_operator,
\
MEM_ESTIMATE_SCALE_FOR_SPILLING_OPERATOR,
TQueryOptionLevel::DEVELOPMENT) \
- QUERY_OPT_FN(use_calcite_planner, USE_CALCITE_PLANNER,
\
- TQueryOptionLevel::ADVANCED)
\
+ REMOVED_QUERY_OPT_FN(use_calcite_planner, USE_CALCITE_PLANNER)
\
QUERY_OPT_FN(json_binary_format, JSON_BINARY_FORMAT,
TQueryOptionLevel::REGULAR) \
QUERY_OPT_FN(hide_analyzed_query, HIDE_ANALYZED_QUERY,
TQueryOptionLevel::ADVANCED) \
QUERY_OPT_FN(broadcast_cost_scale_factor, BROADCAST_COST_SCALE_FACTOR,
\
@@ -387,6 +386,8 @@ constexpr unsigned NUM_QUERY_OPTIONS =
TUPLE_CACHE_EXEMPT_QUERY_OPT_FN(tuple_cache_budget_bytes_per_executor,
\
TUPLE_CACHE_BUDGET_BYTES_PER_EXECUTOR, TQueryOptionLevel::ADVANCED)
\
QUERY_OPT_FN(show_create_table_partition_limit,
SHOW_CREATE_TABLE_PARTITION_LIMIT, TQueryOptionLevel::REGULAR) \
+ QUERY_OPT_FN(planner, PLANNER, TQueryOptionLevel::ADVANCED)
\
+ QUERY_OPT_FN(fallback_planner, FALLBACK_PLANNER,
TQueryOptionLevel::ADVANCED) \
;
/// Enforce practical limits on some query options to avoid undesired query
state.
diff --git a/common/thrift/ImpalaService.thrift
b/common/thrift/ImpalaService.thrift
index b5753733a..0c733822c 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -1025,7 +1025,7 @@ enum TImpalaQueryOptions {
MEM_ESTIMATE_SCALE_FOR_SPILLING_OPERATOR = 190
// If True, use the Calcite planner for compilation
- USE_CALCITE_PLANNER = 191
+ USE_CALCITE_PLANNER = 191 // Removed
// The default format for reading JSON binary columns, can be overridden by
table
// property 'json.binary.format' (if set). The valid values are:
@@ -1073,6 +1073,21 @@ enum TImpalaQueryOptions {
// Maximum number of partitions to show in SHOW CREATE TABLE WITH STATS.
// 0 means no limit. Default is 1000.
SHOW_CREATE_TABLE_PARTITION_LIMIT = 198
+
+ // Planner to use for compiling query. Choices are:
+ // ORIGINAL : use Original planner
+ // CALCITE : use Calcite planner
+ // default is ORIGINAL
+ PLANNER = 199
+
+ // Planner to use for fallback if chosen planner fails at compilation time.
+ // If the planner and fallback_planner are the same, there will be no
fallback.
+ // Choices are:
+ // ORIGINAL : use Original planner
+ // CALCITE : use Calcite planner
+ // NONE : No fallback planner to be used.
+ // default is ORIGINAL
+ FALLBACK_PLANNER = 200
}
// The summary of a DML statement.
diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift
index cba7d2131..ade976ad2 100644
--- a/common/thrift/PlanNodes.thrift
+++ b/common/thrift/PlanNodes.thrift
@@ -139,6 +139,12 @@ enum TRuntimeFilterType {
IN_LIST = 2
}
+enum TPlannerType {
+ ORIGINAL = 0
+ CALCITE = 1
+ NONE = 2
+}
+
// The level of filtering of enabled min/max filters to be applied to Parquet
scan nodes.
enum TMinmaxFilteringLevel {
ROW_GROUP = 1
diff --git a/common/thrift/Query.thrift b/common/thrift/Query.thrift
index ff46f940d..fcd04e290 100644
--- a/common/thrift/Query.thrift
+++ b/common/thrift/Query.thrift
@@ -811,6 +811,12 @@ struct TQueryOptions {
// See comment in ImpalaService.thrift
199: optional i32 show_create_table_partition_limit = 1000
+
+ // See comment in ImpalaService.thrift
+ 200: optional PlanNodes.TPlannerType planner = TPlannerType.ORIGINAL
+
+ // See comment in ImpalaService.thrift
+ 201: optional PlanNodes.TPlannerType fallback_planner = TPlannerType.ORIGINAL
}
// Impala currently has three types of sessions: Beeswax, HiveServer2 and
external
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java
b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 6f9d95221..a5d3ba216 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -25,6 +25,7 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -221,6 +222,7 @@ import org.apache.impala.thrift.TLoadDataResp;
import org.apache.impala.thrift.TMetadataOpRequest;
import org.apache.impala.thrift.TPlanExecInfo;
import org.apache.impala.thrift.TPlanFragment;
+import org.apache.impala.thrift.TPlannerType;
import org.apache.impala.thrift.TPoolConfig;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TQueryExecRequest;
@@ -301,7 +303,8 @@ public class Frontend {
private static final String CPU_ASK_BOUNDED = "CpuAskBounded";
private static final String AVG_ADMISSION_SLOTS_PER_EXECUTOR =
"AvgAdmissionSlotsPerExecutor";
- private static final String CALCITE_FAILURE_REASON = "CalciteFailureReason";
+ private static final String FIRST_ATTEMPT_PLANNER = "FirstAttemptPlanner";
+ private static final String FIRST_ATTEMPT_FAILURE_REASON =
"FirstAttemptFailureReason";
// info about the planner used. In this code, we will always use the
Original planner,
// but other planners may set their own planner values
@@ -310,7 +313,9 @@ public class Frontend {
// The Calcite compiler is loaded on demand, not linked in. The factory is a
// singleton.
- private static CompilerFactory calciteCompilerFactory =
loadCalciteCompilerFactory();
+ private static CompilerFactory CALCITE_COMPILER_FACTORY =
loadCalciteCompilerFactory();
+
+ private static CompilerFactory ORIGINAL_COMPILER_FACTORY = new
CompilerFactoryImpl();
/**
* Plan-time context that allows capturing various artifacts created
@@ -2411,46 +2416,63 @@ public class Frontend {
private TExecRequest getTExecRequestWithFallback(
PlanCtx planCtx, EventSequence timeline) throws ImpalaException {
- TExecRequest request = null;
- CompilerFactory compilerFactory = getCalciteCompilerFactory(planCtx);
- String exceptionClass = null;
- if (compilerFactory != null) {
+ TQueryOptions queryOptions =
+ planCtx.getQueryContext().client_request.getQuery_options();
+ TPlannerType planner = queryOptions.getPlanner();
+ TPlannerType fallbackPlanner = queryOptions.getFallback_planner();
+ List<TPlannerType> plannerTypes =
+ (planner == fallbackPlanner || fallbackPlanner == TPlannerType.NONE)
+ ? ImmutableList.of(planner)
+ : ImmutableList.of(planner, fallbackPlanner);
+
+ LOG.info("Searching for planner to use...");
+ boolean onlyCalcite = (plannerTypes.size() == 1 && planner ==
TPlannerType.CALCITE);
+ Throwable error = null;
+ CompilerFactory compilerFactory;
+ String attemptedPlanner = null;
+ for (TPlannerType plannerType : plannerTypes) {
+ compilerFactory = getCompilerFactory(plannerType);
try {
- request = getTExecRequest(compilerFactory, planCtx, timeline);
- } catch (Exception e) {
- if (!shouldFallbackToRegularPlanner(planCtx, e)) {
- throw e;
- }
- LOG.info("Calcite planner failed: {}", e.getClass());
- exceptionClass = e.getClass().toString();
- timeline.markEvent("Failing over from Calcite planner");
+ TExecRequest request = getTExecRequest(compilerFactory, planCtx,
timeline);
+ addPlannerToProfile(compilerFactory.getPlannerString(),
attemptedPlanner, error);
+ return request;
+ } catch (Throwable e) {
+ // If there is any kind of exception from the first planner, we hit
this code.
+ error = e;
+ attemptedPlanner = compilerFactory.getPlannerString();
+ LOG.info(attemptedPlanner + " planner failed: {}", e.getClass());
}
}
- if (request == null) {
- LOG.info("Using Original Planner.");
- // use the original planner if Calcite planner is not set or fallback
- // for Calcite planner is enabled.
- compilerFactory = new CompilerFactoryImpl();
- request = getTExecRequest(compilerFactory, planCtx, timeline);
+
+ // None of the planners worked. Exceptional case: If only Calcite,
fallback for
+ // unsupported SQLs
+ if (error != null && onlyCalcite && isUnsupportedCalciteSQL(planCtx,
error)) {
+ compilerFactory = getCompilerFactory(TPlannerType.ORIGINAL);
+ TExecRequest request = getTExecRequest(compilerFactory, planCtx,
timeline);
+ addPlannerToProfile(compilerFactory.getPlannerString(),
attemptedPlanner, error);
+ return request;
+ }
+
+ if (error instanceof ImpalaException) {
+ throw (ImpalaException) error;
}
- addPlannerToProfile(compilerFactory.getPlannerString(), exceptionClass);
- return request;
+ throw new RuntimeException(error);
}
- private boolean shouldFallbackToRegularPlanner(PlanCtx planCtx, Exception e)
{
- // TODO: Need a fallback flag for various modes. In production, we will
most
- // likely want to fallback to the original planner, but in testing, we
might want
- // the query to fail.
- // There are some cases where we will always want to fallback, e.g. if the
statement
- // fails at parse time because it is not a select statement.
- if (e instanceof UnsupportedFeatureException) {
- return true;
- }
+ private boolean isUnsupportedCalciteSQL(PlanCtx planCtx, Throwable e) {
TQueryCtx queryCtx = planCtx.getQueryContext();
try {
- return !(Parser.parse(queryCtx.client_request.stmt,
- queryCtx.client_request.query_options) instanceof QueryStmt);
+ // return true either if
+ // a) The UnsupportedFeatureException was explicitly thrown by the
Calcite planner
+ // b) The SQL is parseable by the original planner and it's something
other than
+ // a query statememnt (e.g. DDL) which should always be run on the
orignal
+ // planner. The "Parser.parse()" method is the original planner
parser.
+ return (e instanceof UnsupportedFeatureException) ||
+ !(Parser.parse(queryCtx.client_request.stmt,
+ queryCtx.client_request.query_options) instanceof QueryStmt);
} catch (Exception f) {
+ // If an exception was thrown, it failed to parse in the original
planner, so there
+ // is no reason to compile it there.
return false;
}
}
@@ -2860,13 +2882,17 @@ public class Frontend {
}
}
- public static void addPlannerToProfile(String planner, String
exceptionClass) {
+ public static void addPlannerToProfile(String planner, String
firstAttemptPlanner,
+ Throwable firstAttemptException) {
TRuntimeProfileNode profile = createTRuntimeProfileNode(PLANNER_PROFILE);
addInfoString(profile, PLANNER_TYPE, planner);
- if (exceptionClass != null) {
- addInfoString(profile, CALCITE_FAILURE_REASON, exceptionClass);
+ if (firstAttemptPlanner != null) {
+ addInfoString(profile, FIRST_ATTEMPT_PLANNER, firstAttemptPlanner);
+ addInfoString(profile, FIRST_ATTEMPT_FAILURE_REASON,
+ firstAttemptException.getClass().toString());
}
FrontendProfile.getCurrent().addChildrenProfile(profile);
+
}
/**
@@ -3788,17 +3814,21 @@ public class Frontend {
}
}
- @Nullable
- private CompilerFactory getCalciteCompilerFactory(PlanCtx ctx)
- throws ImpalaException {
- TQueryOptions queryOptions =
ctx.getQueryContext().client_request.getQuery_options();
- if (queryOptions.isUse_calcite_planner()) {
- if (calciteCompilerFactory == null) {
- throw new InternalException("Could not find Calcite planner");
- }
- return calciteCompilerFactory;
+ private CompilerFactory getCompilerFactory(TPlannerType plannerType) {
+ switch (plannerType) {
+ case ORIGINAL:
+ LOG.info("Using Original Planner.");
+ return ORIGINAL_COMPILER_FACTORY;
+ case CALCITE:
+ if (CALCITE_COMPILER_FACTORY == null) {
+ LOG.info("Could not find Calcite planner.");
+ throw new RuntimeException("Could not find Calcite Planner");
+ }
+ LOG.info("Using Calcite Planner.");
+ return CALCITE_COMPILER_FACTORY;
+ default:
+ throw new RuntimeException("Unknown planner type: [" + plannerType +
"]");
}
- return null;
}
private static CompilerFactory loadCalciteCompilerFactory() {
diff --git
a/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/CalcitePlannerTest.java
b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/CalcitePlannerTest.java
index 3604d1e9e..54422eb10 100644
---
a/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/CalcitePlannerTest.java
+++
b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/CalcitePlannerTest.java
@@ -20,6 +20,7 @@ package org.apache.impala.calcite.planner;
import java.nio.file.Paths;
import org.apache.impala.planner.PlannerTestBase;
+import org.apache.impala.thrift.TPlannerType;
import org.apache.impala.thrift.TQueryOptions;
import org.junit.Test;
@@ -37,14 +38,16 @@ public class CalcitePlannerTest extends PlannerTestBase {
// basic limit pushdown tests with it disabled.
TQueryOptions options = defaultQueryOptions();
options.setAnalytic_rank_pushdown_threshold(0);
- options.setUse_calcite_planner(true);
+ options.setPlanner(TPlannerType.CALCITE);
+ options.setFallback_planner(TPlannerType.CALCITE);
runPlannerTestFile("limit-pushdown-analytic-calcite", options);
}
@Test
public void testAnalyticRankPushdown() {
TQueryOptions options = defaultQueryOptions();
- options.setUse_calcite_planner(true);
+ options.setPlanner(TPlannerType.CALCITE);
+ options.setFallback_planner(TPlannerType.CALCITE);
runPlannerTestFile("analytic-rank-pushdown-calcite", options);
}
diff --git
a/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TpcdsCpuCostPlannerTest.java
b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TpcdsCpuCostPlannerTest.java
index f64db8a4b..196c157d9 100644
---
a/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TpcdsCpuCostPlannerTest.java
+++
b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TpcdsCpuCostPlannerTest.java
@@ -17,6 +17,7 @@
package org.apache.impala.calcite.planner;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
@@ -25,6 +26,7 @@ import org.apache.impala.common.ByteUnits;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.planner.PlannerTestBase;
import org.apache.impala.thrift.TExecutorGroupSet;
+import org.apache.impala.thrift.TPlannerType;
import org.apache.impala.thrift.TQueryOptions;
import org.apache.impala.thrift.TReplicaPreference;
import org.apache.impala.thrift.TSlotCountStrategy;
@@ -77,7 +79,8 @@ public class TpcdsCpuCostPlannerTest extends PlannerTestBase {
// Required so that output doesn't vary by whether scanned tables
have stats &
// numRows property or not.
.setDisable_hdfs_num_rows_estimate(true)
- .setUse_calcite_planner(true);
+ .setPlanner(TPlannerType.CALCITE)
+ .setFallback_planner(TPlannerType.CALCITE);
// Database name to run this test.
private static String testDb = "tpcds_partitioned_parquet_snap";
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/fallback_planner.test
b/testdata/workloads/functional-query/queries/QueryTest/fallback_planner.test
new file mode 100644
index 000000000..ff94149a4
--- /dev/null
+++
b/testdata/workloads/functional-query/queries/QueryTest/fallback_planner.test
@@ -0,0 +1,135 @@
+====
+---- QUERY
+set planner=CALCITE;
+set fallback_planner=CALCITE;
+select id from functional.alltypestiny where id = 1 limit 1;
+---- RESULTS
+1
+---- RUNTIME_PROFILE
+row_regex: PlannerType: CalcitePlanner
+====
+---- QUERY
+set planner=CALCITE;
+set fallback_planner=NONE;
+select id from functional.alltypestiny where id = 1 limit 1;
+---- RESULTS
+1
+---- RUNTIME_PROFILE
+row_regex: PlannerType: CalcitePlanner
+====
+---- QUERY
+set planner=ORIGINAL;
+set fallback_planner=ORIGINAL;
+create table calcite_alltypes as select * from functional.alltypes order by id
limit 5;
+---- RUNTIME_PROFILE
+row_regex: PlannerType: OriginalPlanner
+====
+---- QUERY
+set planner=ORIGINAL;
+set fallback_planner=NONE;
+create table calcite_alltypes2 as select * from functional.alltypes order by
id limit 5;
+---- RUNTIME_PROFILE
+row_regex: PlannerType: OriginalPlanner
+====
+---- QUERY
+# fallback should happen on all CTAS queries
+set planner=CALCITE;
+set fallback_planner=CALCITE;
+create table calcite_alltypes3 as select * from functional.alltypes order by
id limit 5;
+---- RUNTIME_PROFILE
+row_regex: PlannerType: OriginalPlanner
+====
+---- QUERY
+# fallback should happen on all CTAS queries
+set planner=CALCITE;
+set fallback_planner=NONE;
+create table calcite_alltypes4 as select * from functional.alltypes order by
id limit 5;
+---- RUNTIME_PROFILE
+row_regex: PlannerType: OriginalPlanner
+====
+---- QUERY
+# fallback should happen on complex types which are not supported, even though
no
+# fallback planner is specified.
+set planner=CALCITE;
+set fallback_planner=CALCITE;
+select int_array_col from functional.allcomplextypes
+---- RUNTIME_PROFILE
+row_regex: PlannerType: OriginalPlanner
+====
+---- QUERY
+# fallback should happen on complex types which are not supported, even though
no
+# fallback planner is specified.
+set planner=CALCITE;
+set fallback_planner=NONE;
+select int_array_col from functional.allcomplextypes
+---- RUNTIME_PROFILE
+row_regex: PlannerType: OriginalPlanner
+====
+---- QUERY
+set planner=CALCITE;
+set fallback_planner=CALCITE;
+# query does not compile in Calcite, Calcite currently expects ignore nulls to
be
+# outside the parens.
+select id, tinyint_col,
+ last_value(tinyint_col ignore nulls) over
+ (order by id rows between unbounded preceding and 1 preceding),
+from functional.alltypesagg where id < 21
+---- CATCH
+ParseException
+====
+---- QUERY
+set planner=CALCITE;
+set fallback_planner=NONE;
+# query does not compile in Calcite, Calcite currently expects ignore nulls to
be
+# outside the parens.
+select id, tinyint_col,
+ last_value(tinyint_col ignore nulls) over
+ (order by id rows between unbounded preceding and 1 preceding),
+from functional.alltypesagg where id < 21
+---- CATCH
+ParseException
+====
+---- QUERY
+set planner=CALCITE;
+set fallback_planner=ORIGINAL;
+# query does not compile in Calcite, Fallback planner kicks in
+select id, tinyint_col,
+ last_value(tinyint_col ignore nulls) over
+ (order by id rows between unbounded preceding and 1 preceding)
+from functional.alltypesagg where id < 21
+---- RUNTIME_PROFILE
+row_regex: PlannerType: OriginalPlanner
+====
+---- QUERY
+set planner=ORIGINAL;
+set fallback_planner=ORIGINAL;
+# query does not compile in Original planner
+select id, tinyint_col,
+ last_value(tinyint_col) ignore nulls over
+ (order by id rows between unbounded preceding and 1 preceding)
+from functional.alltypesagg where id < 21
+---- CATCH
+ParseException
+====
+---- QUERY
+set planner=ORIGINAL;
+set fallback_planner=NONE;
+# query does not compile in Original planner
+select id, tinyint_col,
+ last_value(tinyint_col) ignore nulls over
+ (order by id rows between unbounded preceding and 1 preceding)
+from functional.alltypesagg where id < 21
+---- CATCH
+ParseException
+====
+---- QUERY
+set planner=ORIGINAL;
+set fallback_planner=CALCITE;
+# query does not compile in Original planner, falls back to Calcite planner
+select id, tinyint_col,
+ last_value(tinyint_col) ignore nulls over
+ (order by id rows between unbounded preceding and 1 preceding)
+from functional.alltypesagg where id < 21
+---- RUNTIME_PROFILE
+row_regex: PlannerType: CalcitePlanner
+====
diff --git a/tests/authorization/test_ranger.py
b/tests/authorization/test_ranger.py
index 458984a7c..c5b4eb2e4 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -1724,8 +1724,10 @@ class TestRanger(CustomClusterTestSuite):
grantee_user = "non_owner"
with self.create_impala_client(user=ADMIN) as admin_client, \
self.create_impala_client(user=grantee_user) as non_owner_client:
- # Set the query option of 'use_calcite_planner' to 1 to use the Calcite
planner.
- non_owner_client.set_configuration({"use_calcite_planner": 1})
+ # Set the query option planner to CALCITE and ensure there is no
fallback to
+ # the original planner
+ non_owner_client.set_configuration({"planner": "CALCITE"})
+ non_owner_client.set_configuration({"fallback_planner": "CALCITE"})
unique_database = unique_name + "_db"
unique_table = unique_name + "_tbl"
test_select_query = "select * from {0}.{1}".format(unique_database,
unique_table)
@@ -1784,8 +1786,10 @@ class TestRanger(CustomClusterTestSuite):
with self.create_impala_client(user=ADMIN) as admin_client, \
self.create_impala_client(user=grantee_user) as non_owner_client:
if use_calcite_planner is True:
- # Set the query option of 'use_calcite_planner' to 1 to use the
Calcite planner.
- non_owner_client.set_configuration({"use_calcite_planner": 1})
+ # Set the query option planner to CALCITE and ensure there is no
fallback to
+ # the original planner
+ non_owner_client.set_configuration({"planner": "CALCITE"})
+ non_owner_client.set_configuration({"fallback_planner": "CALCITE"})
test_select_query = "select * from {0}".format(v2_name)
select_error_prefix = "User '{0}' does not have privileges to execute " \
"'SELECT' on:".format(grantee_user)
@@ -1923,7 +1927,7 @@ class TestRanger(CustomClusterTestSuite):
grantee_user = "non_owner"
with self.create_impala_client(user=ADMIN) as admin_client, \
self.create_impala_client(user=grantee_user) as non_owner_client:
- non_owner_client.set_configuration({"use_calcite_planner": 1})
+ non_owner_client.set_configuration({"planner": "CALCITE"})
database = "functional"
table_1 = "alltypes"
table_2 = "alltypestiny"
@@ -4032,8 +4036,10 @@ class TestRangerWithCalcite(TestRanger):
"""
with self.create_impala_client(user=ADMIN) as admin_client, \
self.create_impala_client(user=NON_OWNER) as non_owner_client:
- # Set the query option of 'use_calcite_planner' to 1 to use the Calcite
planner.
- non_owner_client.set_configuration({"use_calcite_planner": 1})
+ # Set the query option planner to CALCITE and ensure there is no
fallback to
+ # the original planner
+ non_owner_client.set_configuration({"planner": "CALCITE"})
+ non_owner_client.set_configuration({"fallback_planner": "CALCITE"})
database = "functional"
table_1 = database + "." + "alltypes"
table_2 = database + "." "alltypestiny"
@@ -4069,8 +4075,9 @@ class TestRangerWithCalcite(TestRanger):
non_owner_client.execute(test_query_3)
non_owner_client.execute(test_query_4)
- # Set the query option of 'use_calcite_planner' to 0 to use the
classic frontend.
- non_owner_client.set_configuration({"use_calcite_planner": 0})
+ # Set the query option planner back to ORIGINAL
+ non_owner_client.set_configuration({"planner": "ORIGINAL"})
+ non_owner_client.set_configuration({"fallback_planner": "ORIGINAL"})
# Impala's classic frontend supports 'test_query_3'.
non_owner_client.execute(test_query_3)
# Impala's classic frontend could not support 'test_query_4'.
diff --git a/tests/custom_cluster/test_calcite_planner.py
b/tests/custom_cluster/test_calcite_planner.py
index 93e1b8ab6..8cae1d294 100644
--- a/tests/custom_cluster/test_calcite_planner.py
+++ b/tests/custom_cluster/test_calcite_planner.py
@@ -34,17 +34,10 @@ class TestCalcitePlanner(CustomClusterTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestCalcitePlanner, cls).add_test_dimensions()
- add_mandatory_exec_option(cls, 'use_calcite_planner', 'true')
+ add_mandatory_exec_option(cls, 'planner', 'CALCITE')
+ add_mandatory_exec_option(cls, 'fallback_planner', 'CALCITE')
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(start_args="--env_vars=USE_CALCITE_PLANNER=true")
def test_calcite_frontend(self, vector, unique_database):
- """Calcite planner does not work in local catalog mode yet."""
self.run_test_case('QueryTest/calcite', vector, use_db=unique_database)
-
- @pytest.mark.execute_serially
-
@CustomClusterTestSuite.with_args(start_args="--env_vars=USE_CALCITE_PLANNER=true")
- def test_semicolon(self):
- with self.create_impala_client() as client:
- client.execute("set use_calcite_planner=true;")
- client.execute("select 4;")
diff --git a/tests/custom_cluster/test_calcite_planner.py
b/tests/query_test/test_calcite_planner.py
similarity index 55%
copy from tests/custom_cluster/test_calcite_planner.py
copy to tests/query_test/test_calcite_planner.py
index 93e1b8ab6..19fbb0ac9 100644
--- a/tests/custom_cluster/test_calcite_planner.py
+++ b/tests/query_test/test_calcite_planner.py
@@ -17,15 +17,14 @@
from __future__ import absolute_import, division, print_function
import logging
-import pytest
-from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (add_mandatory_exec_option)
LOG = logging.getLogger(__name__)
-class TestCalcitePlanner(CustomClusterTestSuite):
+class TestCalcitePlanner(ImpalaTestSuite):
@classmethod
def setup_class(cls):
@@ -34,17 +33,31 @@ class TestCalcitePlanner(CustomClusterTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestCalcitePlanner, cls).add_test_dimensions()
- add_mandatory_exec_option(cls, 'use_calcite_planner', 'true')
+ add_mandatory_exec_option(cls, 'planner', 'CALCITE')
+ add_mandatory_exec_option(cls, 'fallback_planner', 'CALCITE')
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ v.get_value('table_format').file_format == 'parquet'
+ and v.get_value('table_format').compression_codec == 'none')
- @pytest.mark.execute_serially
-
@CustomClusterTestSuite.with_args(start_args="--env_vars=USE_CALCITE_PLANNER=true")
def test_calcite_frontend(self, vector, unique_database):
- """Calcite planner does not work in local catalog mode yet."""
self.run_test_case('QueryTest/calcite', vector, use_db=unique_database)
- @pytest.mark.execute_serially
-
@CustomClusterTestSuite.with_args(start_args="--env_vars=USE_CALCITE_PLANNER=true")
- def test_semicolon(self):
- with self.create_impala_client() as client:
- client.execute("set use_calcite_planner=true;")
- client.execute("select 4;")
+ def test_semicolon(self, cursor):
+ cursor.execute("select 4;")
+
+
+class TestFallbackPlanner(ImpalaTestSuite):
+
+ @classmethod
+ def setup_class(cls):
+ super(TestFallbackPlanner, cls).setup_class()
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestFallbackPlanner, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ v.get_value('table_format').file_format == 'parquet'
+ and v.get_value('table_format').compression_codec == 'none')
+
+ def test_fallback_planner(self, vector, unique_database):
+ self.run_test_case('QueryTest/fallback_planner', vector,
use_db=unique_database)
