This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 825d423079726072c07021aff14b6e8ed77625e8 Author: Steve Carlin <[email protected]> AuthorDate: Mon Feb 2 10:28:08 2026 -0800 IMPALA-14541: Remove deprecated CalciteJniFrontend class The CalciteJniFrontend class is no longer used. The main interface for the Calcite code is now CalciteCompilerFactory which is instantiated by the Frontend class. Various other method and classes only used by the CalciteJniFrontend method have also been deleted. The test framework has been modified to use the CalciteCompilerFactory interface. One minor change also in this commit: Some tests were moved to the calcite/planner directory to be more consistent. Change-Id: I3797acf0f23da4f94098e2f8f5f7aa378c9ae91a Reviewed-on: http://gerrit.cloudera.org:8080/23929 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Steve Carlin <[email protected]> --- .../org/apache/impala/common/FrontendTestBase.java | 11 +- .../impala/calcite/service/CalciteJniFrontend.java | 301 -------------- .../calcite/service/CalciteMetadataHandler.java | 116 +----- .../impala/calcite/service/CalciteOptimizer.java | 9 - .../calcite/service/CalcitePhysPlanCreator.java | 8 - .../impala/calcite/service/CalciteQueryParser.java | 4 - .../calcite/service/CalciteRelNodeConverter.java | 14 - .../impala/calcite/service/CalciteValidator.java | 101 ----- .../impala/calcite/service/ExecRequestCreator.java | 434 --------------------- .../{ => calcite}/planner/CalcitePlannerTest.java | 3 +- .../{ => calcite}/planner/TestCalciteStats.java | 37 +- .../planner/TestReduceExprShuttle.java | 98 +++-- 12 files changed, 71 insertions(+), 1065 deletions(-) diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java index 6a80c7068..0fc3a7c75 100644 --- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java +++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java @@ -345,11 +345,20 @@ public class FrontendTestBase extends AbstractFrontendTest { return parseAndAnalyze(stmt, ctx, frontend_); } + protected AnalysisResult parseAndAnalyze(String stmt, AnalysisContext ctx, + CompilerFactory compilerFactory) throws ImpalaException { + return parseAndAnalyze(stmt, ctx, frontend_, compilerFactory); + } + protected AnalysisResult parseAndAnalyze(String stmt, AnalysisContext ctx, Frontend fe) throws ImpalaException { + return parseAndAnalyze(stmt, ctx, fe, new CompilerFactoryImpl()); + } + + protected AnalysisResult parseAndAnalyze(String stmt, AnalysisContext ctx, Frontend fe, + CompilerFactory compilerFactory) throws ImpalaException { try (FrontendProfile.Scope scope = FrontendProfile.createNewWithScope()) { ctx.getQueryCtx().getClient_request().setStmt(stmt); - CompilerFactory compilerFactory = new CompilerFactoryImpl(); ParsedStatement parsedStmt = compilerFactory.createParsedStatement(ctx.getQueryCtx()); User user = new User(TSessionStateUtil.getEffectiveUser(ctx.getQueryCtx().session)); diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteJniFrontend.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteJniFrontend.java deleted file mode 100644 index 65bfe0b95..000000000 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteJniFrontend.java +++ /dev/null @@ -1,301 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.impala.calcite.service; - -import org.apache.impala.util.EventSequence; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; -import org.apache.calcite.rel.metadata.JaninoRelMetadataProvider; -import org.apache.calcite.rel.metadata.RelMetadataQuery; -import org.apache.calcite.sql.SqlExplain; -import org.apache.calcite.sql.SqlNode; -import org.apache.impala.analysis.Parser; -import org.apache.impala.analysis.SelectStmt; -import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache; -import org.apache.impala.calcite.functions.FunctionResolver; -import org.apache.impala.calcite.operators.ImpalaOperatorTable; -import org.apache.impala.calcite.rel.node.NodeWithExprs; -import org.apache.impala.calcite.rel.node.ImpalaPlanRel; -import org.apache.impala.catalog.BuiltinsDb; -import org.apache.impala.common.ImpalaException; -import org.apache.impala.common.JniUtil; -import org.apache.impala.common.ParseException; -import org.apache.impala.common.UnsupportedFeatureException; -import org.apache.impala.service.Frontend; -import org.apache.impala.service.FrontendProfile; -import org.apache.impala.service.JniFrontend; -import org.apache.impala.thrift.TClientRequest; -import org.apache.impala.thrift.TExecRequest; -import org.apache.impala.thrift.TQueryCtx; -import org.apache.impala.thrift.TQueryOptions; -import org.apache.thrift.TException; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; - -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * CalciteJniFrontend. This is a frontend that uses the Calcite code - * to walk through all the steps of compiling the query (e.g. parsing, validating, - * etc... to generate a TExecRequest that can be used by the execution engine. - */ -public class CalciteJniFrontend extends JniFrontend { - - protected static final Logger LOG = - LoggerFactory.getLogger(CalciteJniFrontend.class.getName()); - - private static Pattern LEFT_SEMI = Pattern.compile(".*\\bleft\\ssemi\\b.*", - Pattern.CASE_INSENSITIVE); - - private static Pattern RIGHT_SEMI = Pattern.compile(".*\\bright\\ssemi\\b.*", - Pattern.CASE_INSENSITIVE); - - private static Pattern LEFT_ANTI = Pattern.compile(".*\\bleft\\santi\\b.*", - Pattern.CASE_INSENSITIVE); - - private static Pattern RIGHT_ANTI = Pattern.compile(".*\\bright\\santi\\b.*", - Pattern.CASE_INSENSITIVE); - - private static Pattern INPUT_FILE_NAME = Pattern.compile(".*\\binput__file__name\\b.*", - Pattern.CASE_INSENSITIVE); - - private static Pattern FILE_POSITION = Pattern.compile(".*\\bfile__position\\b.*", - Pattern.CASE_INSENSITIVE); - - private static Pattern TABLE_NOT_FOUND = - Pattern.compile(".*\\bTable '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE); - - private static Pattern COLUMN_NOT_FOUND = - Pattern.compile(".*\\bColumn '(.*)' not found\\b.*", Pattern.CASE_INSENSITIVE); - - public CalciteJniFrontend(byte[] thriftBackendConfig, boolean isBackendTest) - throws ImpalaException, TException { - super(thriftBackendConfig, isBackendTest); - loadCalciteImpalaFunctions(); - } - - /** - * Jni wrapper for Frontend.createExecRequest(). Accepts a serialized - * TQueryContext; returns a serialized TQueryExecRequest. - */ - @Override - public byte[] createExecRequest(byte[] thriftQueryContext) - throws ImpalaException { - // Needed for Calcite's JaninoRelMetadataProvider - Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader()); - - QueryContext queryCtx = new QueryContext(thriftQueryContext, getFrontend()); - - CalciteMetadataHandler mdHandler = null; - - try { - CalciteCompilerFactory.checkOptionSupportedInCalcite(queryCtx.getTQueryCtx()); - } catch (UnsupportedFeatureException e) { - return runThroughOriginalPlanner(thriftQueryContext, queryCtx); - } - - try (FrontendProfile.Scope scope = FrontendProfile.createNewWithScope()) { - LOG.info("Using Calcite Planner for the following query: " + queryCtx.getStmt()); - - // Parse the query - RelMetadataQuery.THREAD_PROVIDERS.set( - JaninoRelMetadataProvider.of(DefaultRelMetadataProvider.INSTANCE)); - CalciteQueryParser queryParser = new CalciteQueryParser(queryCtx); - SqlNode parsedSqlNode = queryParser.parse(); - markEvent(queryParser, parsedSqlNode, queryCtx, "Parsed query"); - - // Make sure the metadata cache has all the info for the query. - mdHandler = new CalciteMetadataHandler(parsedSqlNode, queryCtx); - markEvent(mdHandler, null, queryCtx, "Loaded tables"); - - boolean isExplain = false; - if (parsedSqlNode instanceof SqlExplain) { - isExplain = true; - parsedSqlNode = ((SqlExplain) parsedSqlNode).getExplicandum(); - } - // Validate the parsed query - CalciteValidator validator = new CalciteValidator(mdHandler, queryCtx); - SqlNode validatedNode = validator.validate(parsedSqlNode); - markEvent(mdHandler, validatedNode, queryCtx, "Validated query"); - - // Convert the query to RelNodes which can be optimized - CalciteRelNodeConverter relNodeConverter = new CalciteRelNodeConverter(validator); - RelNode logicalPlan = relNodeConverter.convert(validatedNode); - markEvent(mdHandler, logicalPlan, queryCtx, "Created initial logical plan"); - - // Optimize the query - CalciteOptimizer optimizer = new CalciteOptimizer(validator, - mdHandler.getAnalyzer(), queryCtx.getTimeline(), queryCtx.getTQueryCtx()); - ImpalaPlanRel optimizedPlan = optimizer.optimize(logicalPlan); - markEvent(mdHandler, optimizedPlan, queryCtx, "Optimized logical plan"); - - // Create Physical Impala PlanNodes - CalcitePhysPlanCreator physPlanCreator = - new CalcitePhysPlanCreator(mdHandler, queryCtx); - NodeWithExprs rootNode = physPlanCreator.create(optimizedPlan); - markEvent(mdHandler, rootNode, queryCtx, "Created physical plan"); - - // Create exec request for the server - ExecRequestCreator execRequestCreator = - new ExecRequestCreator(physPlanCreator, queryCtx, mdHandler, isExplain); - TExecRequest execRequest = execRequestCreator.create(rootNode); - markEvent(mdHandler, execRequest, queryCtx, "Created exec request"); - - byte[] serializedRequest = JniUtil.serializeToThrift(execRequest); - queryCtx.getTimeline().markEvent("Serialized request"); - - return serializedRequest; - } catch (ParseException e) { - throwUnsupportedIfKnownException(e); - // do a quick parse just to make sure it's not a select stmt. If it is - // a select statement, we fail the query since all select statements - // should be run through the Calcite Planner. - if (Parser.parse(queryCtx.getStmt()) instanceof SelectStmt) { - throw e; - } - LOG.info("Calcite planner failed to parse query: " + queryCtx.getStmt()); - LOG.info("Going to use original Impala planner."); - return runThroughOriginalPlanner(thriftQueryContext, queryCtx); - } catch (ImpalaException e) { - if (mdHandler != null) { - throwUnsupportedIfKnownException(e, mdHandler.getStmtTableCache()); - } - throw e; - } catch (Exception e) { - throw e; - } - } - - /** - * Fallback planner method - */ - public byte[] runThroughOriginalPlanner(byte[] thriftQueryContext, - QueryContext queryCtx) throws ImpalaException { - LOG.info("Using Impala Planner for the following query: " + queryCtx.getStmt()); - return super.createExecRequest(thriftQueryContext); - } - - private void markEvent(CompilerStep compilerStep, Object stepResult, - QueryContext queryCtx, String stepMessage) { - LOG.info(stepMessage); - queryCtx.getTimeline().markEvent(stepMessage); - compilerStep.logDebug(stepResult); - } - - private static void loadCalciteImpalaFunctions() { - ImpalaOperatorTable.create(BuiltinsDb.getInstance()); - } - - private static void throwUnsupportedIfKnownException(Exception e) - throws ImpalaException { - String s = e.toString().replace("\n"," ");; - if (LEFT_ANTI.matcher(s).matches() || RIGHT_ANTI.matcher(s).matches()) { - throw new UnsupportedFeatureException("Anti joins not supported."); - } - if (LEFT_SEMI.matcher(s).matches() || RIGHT_SEMI.matcher(s).matches()) { - throw new UnsupportedFeatureException("Semi joins not supported."); - } - if (INPUT_FILE_NAME.matcher(s).matches() || FILE_POSITION.matcher(s).matches()) { - throw new UnsupportedFeatureException("Virtual columns not supported."); - } - } - - public static void throwUnsupportedIfKnownException(ImpalaException e, - StmtTableCache stmtTableCache) throws ImpalaException { - throwUnsupportedIfKnownException(e); - String s = e.toString().replace("\n"," ");; - Matcher m = TABLE_NOT_FOUND.matcher(s); - if (m.matches()) { - if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) { - throw new UnsupportedFeatureException( - "Complex column " + m.group(1) + " not supported."); - } - } - - m = COLUMN_NOT_FOUND.matcher(s); - if (m.matches()) { - if (CalciteMetadataHandler.anyTableContainsColumn(stmtTableCache, m.group(1))) { - throw new UnsupportedFeatureException( - "Complex column " + m.group(1) + " not supported."); - } - } - } - - public static class QueryContext { - private final TQueryCtx queryCtx_; - private final String stmt_; - private final String currentDb_; - private final Frontend frontend_; - private final EventSequence timeline_; - - public QueryContext(byte[] thriftQueryContext, - Frontend frontend) throws ImpalaException { - this.queryCtx_ = new TQueryCtx(); - JniUtil.deserializeThrift(queryCtx_, thriftQueryContext); - - // hack to match the code in Frontend.java: - // If unset, set MT_DOP to 0 to simplify the rest of the code. - if (queryCtx_.getClient_request() != null && - queryCtx_.getClient_request().getQuery_options() != null) { - if (!queryCtx_.getClient_request().getQuery_options().isSetMt_dop()) { - queryCtx_.getClient_request().getQuery_options().setMt_dop(0); - } - } - - this.frontend_ = frontend; - this.stmt_ = queryCtx_.getClient_request().getStmt(); - this.currentDb_ = queryCtx_.getSession().getDatabase(); - this.timeline_ = new EventSequence("Frontend Timeline (Calcite Planner)"); - } - - public QueryContext(TQueryOptions options, Frontend frontend, - String stmt) throws ImpalaException { - this.queryCtx_ = new TQueryCtx(); - this.queryCtx_.setClient_request(new TClientRequest("FeTests", options)); - this.frontend_ = frontend; - this.stmt_ = stmt; - this.currentDb_ = "default"; - this.timeline_ = new EventSequence("Frontend Timeline (Calcite Planner)"); - } - - public TQueryCtx getTQueryCtx() { - return queryCtx_; - } - - public Frontend getFrontend() { - return frontend_; - } - - public String getStmt() { - return stmt_; - } - - public String getCurrentDb() { - return currentDb_; - } - - public EventSequence getTimeline() { - return timeline_; - } - } -} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteMetadataHandler.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteMetadataHandler.java index 67e4e5674..63f6b0c93 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteMetadataHandler.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteMetadataHandler.java @@ -71,47 +71,11 @@ import org.slf4j.LoggerFactory; * from catalogd into the coordinator and populating the Calcite schema with * these tables. */ -public class CalciteMetadataHandler implements CompilerStep { +public class CalciteMetadataHandler { protected static final Logger LOG = LoggerFactory.getLogger(CalciteMetadataHandler.class.getName()); - // StmtTableCache needed by Impala's Analyzer class at planning time. - private final StmtMetadataLoader.StmtTableCache stmtTableCache_; - - // CalciteCatalogReader is a context class that holds global information that - // may be needed by the CalciteTable object - private final CalciteCatalogReader reader_; - - private final Analyzer analyzer_; - - public CalciteMetadataHandler(SqlNode parsedNode, - CalciteJniFrontend.QueryContext queryCtx) throws ImpalaException { - - StmtMetadataLoader stmtMetadataLoader = new StmtMetadataLoader( - queryCtx.getFrontend(), queryCtx.getCurrentDb(), queryCtx.getTimeline()); - - // retrieve all the tablenames in the query, will be in tableVisitor.tableNames - TableVisitor tableVisitor = new TableVisitor(queryCtx.getCurrentDb()); - parsedNode.accept(tableVisitor); - - // load the relevant tables in the query from catalogd - this.stmtTableCache_ = stmtMetadataLoader.loadTables(tableVisitor.tableNames_); - - this.reader_ = createCalciteCatalogReader(stmtTableCache_, - queryCtx.getTQueryCtx(), queryCtx.getCurrentDb()); - - this.analyzer_ = createAnalyzer(stmtTableCache_, queryCtx); - - // populate calcite schema. This step needs to be done after the loader because the - // schema needs to contain the columns in the table for validation, which cannot - // be done when it's an IncompleteTable - List<String> errorTables = populateCalciteSchema(reader_, - queryCtx.getFrontend().getCatalog(), stmtTableCache_, analyzer_); - - tableVisitor.checkForComplexTable(stmtTableCache_, errorTables, queryCtx); - } - /** * Creates CalciteCatalogReader object which will contain information about the schema. * Since the individual Table objects have reference to the Schema, this also serves @@ -170,26 +134,6 @@ public class CalciteMetadataHandler implements CompilerStep { return notFoundTables; } - public StmtMetadataLoader.StmtTableCache getStmtTableCache() { - return stmtTableCache_; - } - - public CalciteCatalogReader getCalciteCatalogReader() { - return reader_; - } - - public Analyzer getAnalyzer() { - return analyzer_; - } - - private Analyzer createAnalyzer(StmtMetadataLoader.StmtTableCache stmtTableCache, - CalciteJniFrontend.QueryContext queryCtx) throws ImpalaException { - // XXX: using NoopAuthorizationFactory, but this part of the code will - // eventually either be deprecated or only used in the test environment. - return new SimplifiedAnalyzer(stmtTableCache, queryCtx.getTQueryCtx(), - new NoopAuthorizationFactory(), null); - } - /** * TableVisitor walks through the AST and places all the tables into * tableNames @@ -290,55 +234,6 @@ public class CalciteMetadataHandler implements CompilerStep { } return false; } - - /** - * Check if the error table is actually a table with a complex column. There is Impala - * syntax where a complex column uses the same syntax in the FROM clause as a table. - * This method is passed in all the tables that are not found and checks to see if - * the table turned out to be a complex column rather than an actual table. If so, - * this throws an unsupported feature exception (for the time being). If it's not - * a table with a complex column, a table not found error will eventually be thrown - * in a different place. - */ - public void checkForComplexTable(StmtMetadataLoader.StmtTableCache stmtTableCache, - List<String> errorTables, CalciteJniFrontend.QueryContext queryCtx) - throws ImpalaException { - List<String> allErrorTables = new ArrayList<>(); - allErrorTables.addAll(errorTables_); - allErrorTables.addAll(errorTables); - for (String errorTable : allErrorTables) { - List<String> parts = Splitter.on('.').splitToList(errorTable); - // if there are 3 parts, then it has to be db.table.column and must be a - // complex column. - if (parts.size() > 2) { - throw new UnsupportedFeatureException("Complex column " + - errorTable + " not supported."); - } - // if there are 2 parts, then it is either a missing db.table or a - // table.column. We look to see if the column can be found in any - // of the tables, in which case, it is a complex column being referenced. - if (parts.size() == 2) { - // first check the already existing cache for the error table. - if (anyTableContainsColumn(stmtTableCache, parts.get(1))) { - throw new UnsupportedFeatureException("Complex column " + - errorTable + " not supported."); - } - // it's possible that the table wasn't loaded yet because this method is - // only called when there is an error finding a table. Try loading the table - // from catalogd just in case, and check to see if it's a complex column. - TableName potentialComplexTable = new TableName( - currentDb_.toLowerCase(), parts.get(0).toLowerCase()); - StmtMetadataLoader errorLoader = new StmtMetadataLoader( - queryCtx.getFrontend(), queryCtx.getCurrentDb(), queryCtx.getTimeline()); - StmtMetadataLoader.StmtTableCache errorCache = - errorLoader.loadTables(Sets.newHashSet(potentialComplexTable)); - if (anyTableContainsColumn(errorCache, parts.get(1))) { - throw new UnsupportedFeatureException("Complex column " + - errorTable + " not supported."); - } - } - } - } } public static boolean anyTableContainsColumn( @@ -351,13 +246,4 @@ public class CalciteMetadataHandler implements CompilerStep { } return false; } - - @Override - public void logDebug(Object resultObject) { - if (!LOG.isDebugEnabled()) return; - String allTables = stmtTableCache_.tables.values().stream() - .map(feTable -> feTable.getName().toString()) - .collect(Collectors.joining( ", " )); - LOG.debug("Loaded tables: {}", allTables); - } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java index fd6305254..a1306850b 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java @@ -106,15 +106,6 @@ public class CalciteOptimizer implements CompilerStep { this.queryCtx_ = analyzer_.getQueryCtx(); } - public CalciteOptimizer(CalciteValidator validator, Analyzer analyzer, - EventSequence timeline, TQueryCtx queryCtx) { - this.reader_ = validator.getCatalogReader(); - this.validator_ = validator.getSqlValidator(); - this.timeline_ = timeline; - this.queryCtx_ = queryCtx; - this.analyzer_ = analyzer; - } - public ImpalaPlanRel optimize(RelNode logPlan) throws ImpalaException { RelBuilder relBuilder = ImpalaCoreRules.LOGICAL_BUILDER_NO_SIMPLIFY.create( logPlan.getCluster(), reader_); diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java index a1950345b..4d2d7ad17 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java @@ -48,14 +48,6 @@ public class CalcitePhysPlanCreator implements CompilerStep { plannerContext_ = ctx; } - public CalcitePhysPlanCreator(CalciteMetadataHandler mdHandler, - CalciteJniFrontend.QueryContext queryCtx) throws ImpalaException { - this.analyzer_ = mdHandler.getAnalyzer(); - this.plannerContext_ = - new PlannerContext(analyzer_, queryCtx.getTQueryCtx(), queryCtx.getTimeline()); - - } - /** * returns the root plan node along with its output expressions. */ diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java index a72b7c9c3..ce2911646 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteQueryParser.java @@ -42,10 +42,6 @@ public class CalciteQueryParser implements CompilerStep { private final String sqlStmt_; - public CalciteQueryParser(CalciteJniFrontend.QueryContext queryCtx) { - this.sqlStmt_ = queryCtx.getStmt(); - } - public CalciteQueryParser(String stmt) { this.sqlStmt_ = stmt; } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java index e82e3531f..71c4233d9 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteRelNodeConverter.java @@ -97,20 +97,6 @@ public class CalciteRelNodeConverter implements CompilerStep { cluster_.setMetadataProvider(ImpalaRelMetadataProvider.DEFAULT); } - public CalciteRelNodeConverter(CalciteValidator validator) { - this.typeFactory_ = validator.getTypeFactory(); - this.reader_ = validator.getCatalogReader(); - this.sqlValidator_ = validator.getSqlValidator(); - this.planner_ = new VolcanoPlanner(); - planner_.addRelTraitDef(ConventionTraitDef.INSTANCE); - planner_.setExecutor(new RemoveUnraggedCharCastRexExecutor()); - cluster_ = - RelOptCluster.create(planner_, new RexBuilder(typeFactory_)); - viewExpander_ = createViewExpander(validator.getCatalogReader() - .getRootSchema().plus()); - cluster_.setMetadataProvider(ImpalaRelMetadataProvider.DEFAULT); - } - private static RelOptTable.ViewExpander createViewExpander(SchemaPlus schemaPlus) { SqlParser.Config parserConfig = SqlParser.configBuilder().setCaseSensitive(false).build() diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteValidator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteValidator.java deleted file mode 100644 index 11d2d6d6a..000000000 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteValidator.java +++ /dev/null @@ -1,101 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.impala.calcite.service; - -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.prepare.CalciteCatalogReader; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.runtime.CalciteContextException; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.validate.SqlValidator; -import org.apache.calcite.sql.validate.SqlValidatorUtil; -import org.apache.impala.calcite.operators.ImpalaOperatorTable; -import org.apache.impala.calcite.type.ImpalaTypeCoercionFactory; -import org.apache.impala.calcite.type.ImpalaTypeSystemImpl; -import org.apache.impala.calcite.validate.ImpalaConformance; -import org.apache.impala.common.AnalysisException; - - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * SqlValidator. Responsible for validating the parsed SQL AST. - */ -public class CalciteValidator implements CompilerStep { - protected static final Logger LOG = - LoggerFactory.getLogger(CalciteValidator.class.getName()); - - private final CalciteMetadataHandler mdHandler; - private final CalciteJniFrontend.QueryContext queryCtx; - private final RelDataTypeFactory typeFactory; - private final CalciteCatalogReader catalogReader; - private final SqlValidator sqlValidator; - - public CalciteValidator(CalciteMetadataHandler mdHandler, - CalciteJniFrontend.QueryContext queryCtx) { - this.mdHandler = mdHandler; - this.queryCtx = queryCtx; - this.typeFactory = new JavaTypeFactoryImpl(new ImpalaTypeSystemImpl()); - this.catalogReader = mdHandler.getCalciteCatalogReader(); - - this.sqlValidator = SqlValidatorUtil.newValidator( - ImpalaOperatorTable.getInstance(), - catalogReader, typeFactory, - SqlValidator.Config.DEFAULT - // Impala requires identifier expansion (tpcds test queries fail - // without this) - .withIdentifierExpansion(true) - .withConformance(ImpalaConformance.INSTANCE) - .withTypeCoercionEnabled(true) - .withTypeCoercionFactory(new ImpalaTypeCoercionFactory()) - ); - } - - public SqlNode validate(SqlNode parsedNode) throws AnalysisException { - try { - // Validate the initial AST - SqlNode node = sqlValidator.validate(parsedNode); - return node; - } catch (CalciteContextException e) { - throw new AnalysisException(e.getMessage(), e.getCause()); - } - } - - public RelDataTypeFactory getTypeFactory() { - return typeFactory; - } - - public SqlValidator getSqlValidator() { - return sqlValidator; - } - - public CalciteCatalogReader getCatalogReader() { - return catalogReader; - } - - @Override - public void logDebug(Object resultObject) { - if (!(resultObject instanceof SqlNode)) { - LOG.debug("Finished validator step, but unknown result: {}", resultObject); - return; - } - LOG.debug("Validated node: {}", resultObject); - } -} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/ExecRequestCreator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/ExecRequestCreator.java deleted file mode 100644 index 66efff738..000000000 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/ExecRequestCreator.java +++ /dev/null @@ -1,434 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.impala.calcite.service; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -import org.apache.impala.analysis.Analyzer; -import org.apache.impala.analysis.Expr; -import org.apache.impala.analysis.JoinOperator; -import org.apache.impala.calcite.rel.node.NodeWithExprs; -import org.apache.impala.catalog.FeFsTable; -import org.apache.impala.catalog.FeTable; -import org.apache.impala.catalog.HdfsTable; -import org.apache.impala.common.ImpalaException; -import org.apache.impala.planner.DataPartition; -import org.apache.impala.planner.DistributedPlanner; -import org.apache.impala.planner.JoinNode; -import org.apache.impala.planner.NestedLoopJoinNode; -import org.apache.impala.planner.ParallelPlanner; -import org.apache.impala.planner.PlanFragment; -import org.apache.impala.planner.PlanNode; -import org.apache.impala.planner.PlanRootSink; -import org.apache.impala.planner.Planner; -import org.apache.impala.planner.PlannerContext; -import org.apache.impala.planner.RuntimeFilterGenerator; -import org.apache.impala.planner.SingleNodePlanner; -import org.apache.impala.planner.SingularRowSrcNode; -import org.apache.impala.planner.SubplanNode; -import org.apache.impala.service.Frontend; -import org.apache.impala.service.FrontendProfile; -import org.apache.impala.thrift.TColumn; -import org.apache.impala.thrift.TExecRequest; -import org.apache.impala.thrift.TExplainLevel; -import org.apache.impala.thrift.TNetworkAddress; -import org.apache.impala.thrift.TPlanExecInfo; -import org.apache.impala.thrift.TPlanFragment; -import org.apache.impala.thrift.TQueryCtx; -import org.apache.impala.thrift.TQueryExecRequest; -import org.apache.impala.thrift.TResultSetMetadata; -import org.apache.impala.thrift.TRuntimeFilterMode; -import org.apache.impala.thrift.TRuntimeProfileNode; -import org.apache.impala.thrift.TStmtType; -import org.apache.impala.util.EventSequence; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * ExecRequestCreator. Responsible for taking a PlanNode and the output Expr list - * from the top level PlanNode and convert it into a TExecRequest thrift object - * which is needed by the backend executor code. The input PlanNode tree is - * an optimized logical tree based on the Calcite rules. This class is also - * responsible for creating physical node optimizations which are located in - * the SingleNodePlanner and DistributedPlanner. - * - * TODO: This class is very similar to the Frontend.createExecRequest method and - * contains duplicate code. Accordingly, This class and that method should be - * refactored to prevent this duplication. - **/ -public class ExecRequestCreator implements CompilerStep { - protected static final Logger LOG = - LoggerFactory.getLogger(ExecRequestCreator.class.getName()); - - private final CalcitePhysPlanCreator physPlanCreator; - private final CalciteJniFrontend.QueryContext queryCtx; - private final CalciteMetadataHandler mdHandler; - private final boolean isExplain; - - public ExecRequestCreator(CalcitePhysPlanCreator physPlanCreator, - CalciteJniFrontend.QueryContext queryCtx, CalciteMetadataHandler mdHandler, - boolean isExplain) { - this.physPlanCreator = physPlanCreator; - this.queryCtx = queryCtx; - this.mdHandler = mdHandler; - this.isExplain = isExplain; - } - - /** - * create() is the main public method responsible for taking the NodeWithExprs - * object containing the PlanNode and output Expr list and returning the TExecRequest. - */ - public TExecRequest create(NodeWithExprs nodeWithExprs) throws ImpalaException { - TExecRequest request = createExecRequest(nodeWithExprs.planNode_, - queryCtx.getTQueryCtx(), physPlanCreator.getPlannerContext(), - physPlanCreator.getAnalyzer(), nodeWithExprs.outputExprs_, - nodeWithExprs.fieldNames_, - mdHandler.getStmtTableCache().tables.values()); - - return request; - } - - /** - * Create an exec request for Impala to execute based on the supplied plan. This - * method is similar to the Frontend.createExecRequest method and needs to be - * refactored. - */ - private TExecRequest createExecRequest(PlanNode planNodeRoot, TQueryCtx queryCtx, - PlannerContext plannerContext, Analyzer analyzer, List<Expr> outputExprs, - List<String> fieldNames, - Collection<FeTable> tables) throws ImpalaException { - List<PlanFragment> fragments = - createPlans(planNodeRoot, analyzer, plannerContext, outputExprs); - PlanFragment planFragmentRoot = fragments.get(0); - - TQueryExecRequest queryExecRequest = new TQueryExecRequest(); - TExecRequest result = createExecRequest(queryCtx, planFragmentRoot, - queryExecRequest, fieldNames); - queryExecRequest.setHost_list(getHostLocations(tables)); - queryExecRequest.setCores_required(-1); - - // compute resource requirements of the final plan - Planner.computeResourceReqs(fragments, queryCtx, queryExecRequest, - plannerContext, true /*isQuery*/); - - // create the plan's exec-info and assign fragment idx - int idx = 0; - for (PlanFragment planRoot : fragments) { - TPlanExecInfo tPlanExecInfo = Frontend.createPlanExecInfo(planRoot, queryCtx); - queryExecRequest.addToPlan_exec_info(tPlanExecInfo); - for (TPlanFragment fragment : tPlanExecInfo.fragments) { - fragment.setIdx(idx++); - } - } - - // create EXPLAIN output after setting everything else - queryExecRequest.setQuery_ctx(queryCtx); // needed by getExplainString() - - List<PlanFragment> allFragments = planFragmentRoot.getNodesPreOrder(); - // to mimic the original planner behavior, use EXTENDED mode explain except for - // EXPLAIN statements. - TExplainLevel explainLevel = - isExplain ? plannerContext.getQueryOptions().getExplain_level() : - TExplainLevel.EXTENDED; - if (isExplain) { - result.setStmt_type(TStmtType.EXPLAIN); - } - String explainString = getExplainString(allFragments, explainLevel, plannerContext); - queryExecRequest.setQuery_plan(explainString); - - queryCtx.setDesc_tbl_serialized( - plannerContext.getRootAnalyzer().getDescTbl().toSerializedThrift()); - - plannerContext.getTimeline().markEvent("Execution request created"); - EventSequence eventSequence = plannerContext.getTimeline(); - result.setTimeline(eventSequence.toThrift()); - - TRuntimeProfileNode calciteProfile = - this.queryCtx.getFrontend().createTRuntimeProfileNode(Frontend.PLANNER_PROFILE); - this.queryCtx.getFrontend().addPlannerToProfile("CalcitePlanner", null); - result.setProfile(FrontendProfile.getCurrent().emitAsThrift()); - result.setProfile_children(FrontendProfile.getCurrent().emitChildrenAsThrift()); - if (isExplain) { - this.queryCtx.getFrontend().createExplainRequest(explainString, result); - } - return result; - } - - List<PlanFragment> createPlans(PlanNode planNodeRoot, Analyzer analyzer, - PlannerContext ctx, List<Expr> outputExprs) throws ImpalaException { - // Create the values transfer graph in the Analyzer. Note that Calcite plans - // don't register equijoin predicates in the Analyzer's GlobalState since - // Calcite should have already done the predicate inferencing analysis. - // Hence, the GlobalState's registeredValueTransfers will be empty. It is - // still necessary to instantiate the graph because otherwise - // RuntimeFilterGenerator tries to de-reference it and encounters NPE. - analyzer.computeValueTransferGraph(); - Planner.checkForSmallQueryOptimization(planNodeRoot, ctx); - - // Although the Calcite plan creates the relative order among different - // joins, currently it does not swap left and right inputs if the right - // input has higher estimated cardinality. Do this through Impala's method - // since we are using Impala's cardinality estimates in the physical planning. - invertJoins(planNodeRoot, ctx.isSingleNodeExec(), ctx.getRootAnalyzer()); - SingleNodePlanner.validatePlan(ctx, planNodeRoot); - - List<PlanFragment> fragments = - createPlanFragments(planNodeRoot, ctx, analyzer, outputExprs); - PlanFragment planFragmentRoot = fragments.get(0); - List<PlanFragment> rootFragments; - if (Planner.useParallelPlan(ctx)) { - ParallelPlanner parallelPlanner = new ParallelPlanner(ctx); - // The rootFragmentList contains the 'root' fragments of each of - // the parallel plans - rootFragments = parallelPlanner.createPlans(planFragmentRoot); - ctx.getTimeline().markEvent("Parallel plans created"); - } else { - rootFragments = new ArrayList(Arrays.asList(planFragmentRoot)); - } - return rootFragments; - } - - /** - * Create one or more plan fragments corresponding to the supplied single node physical - * plan. This function calls Impala's DistributedPlanner to create the plan fragments - * and does some post-processing. It is loosely based on Impala's Planner.createPlan() - * function. - */ - private List<PlanFragment> createPlanFragments(PlanNode planNodeRoot, - PlannerContext ctx, Analyzer analyzer, - List<Expr> outputExprs) throws ImpalaException { - - DistributedPlanner distributedPlanner = new DistributedPlanner(ctx); - List<PlanFragment> fragments; - - if (ctx.isSingleNodeExec()) { - // create one fragment containing the entire single-node plan tree - fragments = Lists.newArrayList(new PlanFragment( - ctx.getNextFragmentId(), planNodeRoot, DataPartition.UNPARTITIONED)); - } else { - fragments = new ArrayList<>(); - // Create distributed plan. For insert/CTAS without limit, - // isPartitioned should be true. - // TODO: only query statements are currently supported - // final boolean isPartitioned = stmtType_ == - // TStmtType.DML && !planNodeRoot.hasLimit(); - boolean isPartitioned = false; - distributedPlanner.createPlanFragments(planNodeRoot, isPartitioned, fragments); - } - - PlanFragment rootFragment = fragments.get(fragments.size() - 1); - // Create runtime filters. - if (ctx.getQueryOptions().getRuntime_filter_mode() != TRuntimeFilterMode.OFF) { - RuntimeFilterGenerator.generateRuntimeFilters(ctx, rootFragment.getPlanRoot()); - ctx.getTimeline().markEvent("Runtime filters computed"); - } - - rootFragment.verifyTree(); - - List<Expr> resultExprs = outputExprs; - rootFragment.setSink(PlanRootSink.create(ctx, resultExprs, true)); - - Planner.checkForDisableCodegen(rootFragment.getPlanRoot(), ctx); - // finalize exchanges: this ensures that for hash partitioned joins, the partitioning - // keys on both sides of the join have compatible data types - for (PlanFragment fragment: fragments) { - fragment.finalizeExchanges(analyzer); - } - - Collections.reverse(fragments); - ctx.getTimeline().markEvent("Distributed plan created"); - return fragments; - } - - private TExecRequest createExecRequest(TQueryCtx queryCtx, - PlanFragment planFragmentRoot, TQueryExecRequest queryExecRequest, - List<String> fieldNames) { - TExecRequest result = new TExecRequest(); - // NOTE: the below 4 are mandatory fields - result.setQuery_options(queryCtx.getClient_request().getQuery_options()); - - // TODO: Need to populate these 3 fields - result.setAccess_events(new ArrayList<>()); - result.setAnalysis_warnings(new ArrayList<>()); - result.setUser_has_profile_access(true); - - result.setQuery_exec_request(queryExecRequest); - - // TODO: only query currently supported - // result.setStmt_type(stmtType_); - // result.getQuery_exec_request().setStmt_type(stmtType_); - result.setStmt_type(TStmtType.QUERY); - result.getQuery_exec_request().setStmt_type(TStmtType.QUERY); - - // fill in the metadata using the root fragment's PlanRootSink - Preconditions.checkState(planFragmentRoot.hasSink()); - List<Expr> outputExprs = new ArrayList<>(); - - planFragmentRoot.getSink().collectExprs(outputExprs); - result.setResult_set_metadata(createQueryResultSetMetadata(outputExprs, fieldNames)); - - return result; - } - - // TODO: Refactor and share Impala's getExplainString() - private String getExplainString(List<PlanFragment> fragments, - TExplainLevel explainLevel, PlannerContext ctx) { - if (explainLevel.ordinal() < TExplainLevel.VERBOSE.ordinal()) { - // Print the non-fragmented parallel plan. - return fragments.get(0).getExplainString(ctx.getQueryOptions(), explainLevel); - } - - StringBuffer sb = new StringBuffer(); - // Print the fragmented parallel plan. - for (int i = 0; i < fragments.size(); ++i) { - PlanFragment fragment = fragments.get(i); - sb.append(fragment.getExplainString(ctx.getQueryOptions(), explainLevel)); - if (i < fragments.size() - 1) { - sb.append("\n"); - } - } - return sb.toString(); - } - - private TResultSetMetadata createQueryResultSetMetadata(List<Expr> outputExprs, - List<String> fieldNames) { - TResultSetMetadata metadata = new TResultSetMetadata(); - int colCnt = outputExprs.size(); - for (int i = 0; i < colCnt; ++i) { - TColumn colDesc = new TColumn(fieldNames.get(i).toLowerCase(), - outputExprs.get(i).getType().toThrift()); - metadata.addToColumns(colDesc); - } - return metadata; - } - - private List<TNetworkAddress> getHostLocations(Collection<FeTable> tables) { - Set<TNetworkAddress> hostLocations = new HashSet<>(); - for (FeTable table : tables) { - if (table instanceof FeFsTable) { - hostLocations.addAll(((FeFsTable) table).getHostIndex().getList()); - } - } - return new ArrayList<>(hostLocations); - } - - /** - * Traverses the plan tree rooted at 'root' and inverts joins in the following - * situations: - * 1. If the left-hand side is a SingularRowSrcNode then we invert the join because - * then the build side is guaranteed to have only a single row. - * 2. There is no backend support for distributed non-equi right outer/semi joins, - * so we invert them (any distributed left semi/outer join is ok). - * 3. If we estimate that the inverted join is cheaper (see isInvertedJoinCheaper()). - * Do not invert if relevant stats are missing. - * The first two inversion rules are independent of the presence/absence of stats. - * Left Null Aware Anti Joins are never inverted due to lack of backend support. - * Joins that originate from query blocks with a straight join hint are not inverted. - * The 'isLocalPlan' parameter indicates whether the plan tree rooted at 'root' - * will be executed locally within one machine, i.e., without any data exchanges. - * Return true if any join in the plan rooted at 'root' was inverted. - * - * TODO: This should be replaced once we conclude the changes contained in this method - * are safe to be pushed to Planner.invertJoins, i.e., they do not cause any - * performance regressions with Impala FE. A couple of differences in this version - * of invertJoins include: - * - The computeStats is done here. The Calcite planner will eventually have - * the computeStats built in during optimization time, but the join stats here - * are specific for Impala. So we need an explicit computeStats call only for - * Impala here. - * - We only need to do the computeStats when the plan changes via inversion, so - * the method returns a boolean whenever the inversion happens. - * - A Jira has been filed for this: IMPALA-12958. Probably the best fix for this - * is to put the inversion inside a Calcite rule. - * - **/ - private static boolean invertJoins(PlanNode root, boolean isLocalPlan, - Analyzer analyzer) { - boolean inverted = false; - if (root instanceof SubplanNode) { - inverted |= invertJoins(root.getChild(0), isLocalPlan, analyzer); - inverted |= invertJoins(root.getChild(1), true, analyzer); - } else { - for (PlanNode child: root.getChildren()) { - inverted |= invertJoins(child, isLocalPlan, analyzer); - } - } - - if (root instanceof JoinNode) { - JoinNode joinNode = (JoinNode) root; - JoinOperator joinOp = joinNode.getJoinOp(); - - if (!joinNode.isInvertible(isLocalPlan)) { - if (inverted) { - // Re-compute tuple ids since their order must correspond to the order - // of children. - root.computeTupleIds(); - // Re-compute stats since PK-FK inference and cardinality may have changed after - // inversion. - root.computeStats(analyzer); - } - return inverted; - } - - if (joinNode.getChild(0) instanceof SingularRowSrcNode) { - // Always place a singular row src on the build side because it - // only produces a single row. - joinNode.invertJoin(); - inverted = true; - } else if (!isLocalPlan && joinNode instanceof NestedLoopJoinNode && - (joinOp.isRightSemiJoin() || joinOp.isRightOuterJoin())) { - // The current join is a distributed non-equi right outer or semi join - // which has no backend support. Invert the join to make it executable. - joinNode.invertJoin(); - inverted = true; - } else if (Planner.isInvertedJoinCheaper(joinNode, isLocalPlan)) { - joinNode.invertJoin(); - inverted = true; - } - // Re-compute the numNodes and numInstances based on the new input order - joinNode.recomputeNodes(); - } - - if (inverted) { - // Re-compute tuple ids because the backend assumes that their order corresponds to - // the order of children. - root.computeTupleIds(); - // Re-compute stats since PK-FK inference and cardinality may have changed after - // inversion. - root.computeStats(analyzer); - } - return inverted; - } - - @Override - public void logDebug(Object resultObject) { - if (!(resultObject instanceof TExecRequest)) { - LOG.debug("Finished exec request step, but unknown result: {}", resultObject); - } - LOG.debug("Exec request: {}", resultObject); - } -} diff --git a/java/calcite-planner/src/test/java/org/apache/impala/planner/CalcitePlannerTest.java b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/CalcitePlannerTest.java similarity index 95% rename from java/calcite-planner/src/test/java/org/apache/impala/planner/CalcitePlannerTest.java rename to java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/CalcitePlannerTest.java index 34d0fefdd..3604d1e9e 100644 --- a/java/calcite-planner/src/test/java/org/apache/impala/planner/CalcitePlannerTest.java +++ b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/CalcitePlannerTest.java @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.impala.planner; +package org.apache.impala.calcite.planner; import java.nio.file.Paths; +import org.apache.impala.planner.PlannerTestBase; import org.apache.impala.thrift.TQueryOptions; import org.junit.Test; diff --git a/java/calcite-planner/src/test/java/org/apache/impala/planner/TestCalciteStats.java b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TestCalciteStats.java similarity index 95% rename from java/calcite-planner/src/test/java/org/apache/impala/planner/TestCalciteStats.java rename to java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TestCalciteStats.java index c113b85d6..ff90cea4b 100644 --- a/java/calcite-planner/src/test/java/org/apache/impala/planner/TestCalciteStats.java +++ b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TestCalciteStats.java @@ -15,13 +15,8 @@ // specific language governing permissions and limitations // under the License. -package org.apache.impala.planner; +package org.apache.impala.calcite.planner; - -import org.apache.impala.common.ImpalaException; -import org.apache.impala.common.RuntimeEnv; -import org.apache.impala.thrift.TQueryOptions; -import org.apache.impala.thrift.TSessionState; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -54,14 +49,17 @@ import org.apache.impala.calcite.rel.util.PrunedPartitionHelper; import org.apache.impala.calcite.schema.ImpalaRelMetadataProvider; import org.apache.impala.calcite.schema.CalciteTable; import org.apache.impala.calcite.schema.FilterSelectivityEstimator; -import org.apache.impala.calcite.service.CalciteJniFrontend.QueryContext; -import org.apache.impala.calcite.service.CalciteMetadataHandler; -import org.apache.impala.calcite.service.CalciteQueryParser; +import org.apache.impala.calcite.service.CalciteAnalysisResult; +import org.apache.impala.calcite.service.CalciteCompilerFactory; import org.apache.impala.calcite.service.CalciteRelNodeConverter; -import org.apache.impala.calcite.service.CalciteValidator; import org.apache.impala.calcite.type.ImpalaTypeSystemImpl; import org.apache.impala.catalog.BuiltinsDb; import org.apache.impala.catalog.FeFsPartition; +import org.apache.impala.common.ImpalaException; +import org.apache.impala.common.RuntimeEnv; +import org.apache.impala.planner.PlannerTestBase; +import org.apache.impala.thrift.TQueryOptions; +import org.apache.impala.thrift.TSessionState; import com.google.common.base.Preconditions; import static org.junit.Assert.assertEquals; @@ -106,20 +104,11 @@ public class TestCalciteStats extends PlannerTestBase { } private RelNode getRelNodeForQuery(String query) throws ImpalaException { - QueryContext queryCtx = new QueryContext(options, frontend_, query); - TSessionState session = new TSessionState(); - session.setConnected_user("dummy"); - queryCtx.getTQueryCtx().setSession(session); - CalciteQueryParser queryParser = new CalciteQueryParser(queryCtx); - SqlNode parsedSqlNode = queryParser.parse(); - - // Make sure the metadata cache has all the info for the query. - CalciteMetadataHandler mdHandler = - new CalciteMetadataHandler(parsedSqlNode, queryCtx); - CalciteValidator validator = new CalciteValidator(mdHandler, queryCtx); - SqlNode validatedNode = validator.validate(parsedSqlNode); - CalciteRelNodeConverter relNodeConverter = new CalciteRelNodeConverter(validator); - return relNodeConverter.convert(validatedNode); + CalciteAnalysisResult analysisResult = (CalciteAnalysisResult) parseAndAnalyze(query, + feFixture_.createAnalysisCtx(), new CalciteCompilerFactory()); + CalciteRelNodeConverter relNodeConverter = + new CalciteRelNodeConverter(analysisResult); + return relNodeConverter.convert(analysisResult.getValidatedNode()); } private RelMetadataQuery getMQ() { diff --git a/java/calcite-planner/src/test/java/org/apache/impala/planner/TestReduceExprShuttle.java b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TestReduceExprShuttle.java similarity index 85% rename from java/calcite-planner/src/test/java/org/apache/impala/planner/TestReduceExprShuttle.java rename to java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TestReduceExprShuttle.java index 554740553..6a2d5ba88 100644 --- a/java/calcite-planner/src/test/java/org/apache/impala/planner/TestReduceExprShuttle.java +++ b/java/calcite-planner/src/test/java/org/apache/impala/calcite/planner/TestReduceExprShuttle.java @@ -15,13 +15,8 @@ // specific language governing permissions and limitations // under the License. -package org.apache.impala.planner; +package org.apache.impala.calcite.planner; - -import org.apache.impala.common.ImpalaException; -import org.apache.impala.common.RuntimeEnv; -import org.apache.impala.thrift.TQueryOptions; -import org.apache.impala.thrift.TSessionState; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -48,15 +43,18 @@ import org.apache.impala.analysis.Analyzer; import org.apache.impala.analysis.Expr; import org.apache.impala.calcite.operators.ImpalaOperatorTable; import org.apache.impala.calcite.rules.ImpalaRexExecutor; -import org.apache.impala.calcite.service.CalciteJniFrontend.QueryContext; -import org.apache.impala.calcite.service.CalciteMetadataHandler; -import org.apache.impala.calcite.service.CalciteQueryParser; +import org.apache.impala.calcite.service.CalciteAnalysisResult; +import org.apache.impala.calcite.service.CalciteCompilerFactory; import org.apache.impala.calcite.service.CalciteRelNodeConverter; -import org.apache.impala.calcite.service.CalciteValidator; import org.apache.impala.calcite.type.ImpalaTypeSystemImpl; import org.apache.impala.catalog.BuiltinsDb; -import org.apache.impala.thrift.TColumnValue; +import org.apache.impala.common.ImpalaException; +import org.apache.impala.common.RuntimeEnv; +import org.apache.impala.planner.PlannerTestBase; import org.apache.impala.thrift.TQueryCtx; +import org.apache.impala.thrift.TQueryOptions; +import org.apache.impala.thrift.TSessionState; +import org.apache.impala.thrift.TColumnValue; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -84,7 +82,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { @Test public void testFoldAddTinyInt() { try { - ReduceShuttleObjects queryObj = new ReduceShuttleObjects("SELECT 1 + 1"); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT 1 + 1"); TColumnValue reducedValue = new TColumnValue(); reducedValue.setShort_val((short)2); @@ -107,7 +105,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { try { String expr = "CAST(1 AS SMALLINT) + CAST(2 AS SMALLINT)"; ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + createReduceShuttleObjects("SELECT " + expr); TColumnValue reducedCast1 = new TColumnValue(); reducedCast1.setShort_val((short)1); TColumnValue reducedCast2 = new TColumnValue(); @@ -139,8 +137,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { public void testFoldAddInt() { try { String expr = "CAST(1 AS INT) + CAST(2 AS INT)"; - ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT " + expr); TColumnValue reducedCast1 = new TColumnValue(); reducedCast1.setInt_val(1); TColumnValue reducedCast2 = new TColumnValue(); @@ -171,8 +168,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { public void testFoldAddDecimal() { try { String expr = "1.1 + 2.2"; - ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT " + expr); TColumnValue reducedCast1 = new TColumnValue(); reducedCast1.setString_val("3.3"); @@ -197,8 +193,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { public void testFoldConcatString() { try { String expr = "CONCAT(cast('a' as string), cast('b' as string))"; - ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT " + expr); TColumnValue reducedCast1 = new TColumnValue(); reducedCast1.setBinary_val("a".getBytes()); TColumnValue reducedCast2 = new TColumnValue(); @@ -230,8 +225,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { public void testBoolean() { try { String expr = "istrue(false)"; - ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT " + expr); TColumnValue reducedValue = new TColumnValue(); reducedValue.setBool_val(false); @@ -252,7 +246,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { @Test public void testPartialExpr() { try { - ReduceShuttleObjects queryObj = new ReduceShuttleObjects( + ReduceShuttleObjects queryObj = createReduceShuttleObjects( "SELECT 1 + 1 + tinyint_col from functional.alltypestiny"); TColumnValue reducedValue = new TColumnValue(); reducedValue.setShort_val((short)2); @@ -275,8 +269,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { public void testNonDeterministic() { try { String expr = "rand()"; - ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT " + expr); TestReducerTmp testReducer = new TestReducerTmp(); List<RexNode> reducedExprs = new ArrayList<>(); RexExecutor executor = new ImpalaRexExecutor( @@ -295,8 +288,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { public void testFoldMultipleFields() { try { String expr = "1 + 2, 3 + 4"; - ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT " + expr); TColumnValue reducedAdd1 = new TColumnValue(); reducedAdd1.setShort_val((short)3); TColumnValue reducedAdd2 = new TColumnValue(); @@ -326,8 +318,7 @@ public class TestReduceExprShuttle extends PlannerTestBase { try { String expr = "add_months(cast('2012-07-01 00:00:00' as timestamp), " + "cast(2 as integer))"; - ReduceShuttleObjects queryObj = - new ReduceShuttleObjects("SELECT " + expr); + ReduceShuttleObjects queryObj = createReduceShuttleObjects("SELECT " + expr); TColumnValue reducedTime1 = new TColumnValue(); reducedTime1.setString_val("2012-07-01 00:00:00"); TColumnValue reducedTime2 = new TColumnValue(); @@ -356,36 +347,37 @@ public class TestReduceExprShuttle extends PlannerTestBase { } + private ReduceShuttleObjects createReduceShuttleObjects(String query) + throws ImpalaException { + CalciteAnalysisResult analysisResult = + (CalciteAnalysisResult) parseAndAnalyze(query, + feFixture_.createAnalysisCtx(), new CalciteCompilerFactory()); + Analyzer analyzer = analysisResult.getAnalyzer(); + TQueryCtx queryCtx = analyzer.getQueryCtx(); + CalciteRelNodeConverter relNodeConverter = + new CalciteRelNodeConverter(analysisResult); + RelNode rootNode = relNodeConverter.convert(analysisResult.getValidatedNode()); + + Preconditions.checkState(rootNode instanceof Project); + Project project = (Project) rootNode; + + RelDataTypeFactory typeFactory = + new JavaTypeFactoryImpl(new ImpalaTypeSystemImpl()); + RexBuilder rexBuilder = new RexBuilder(typeFactory); + return new ReduceShuttleObjects(analyzer, queryCtx, project, rexBuilder); + } private static class ReduceShuttleObjects { public final Analyzer analyzer_; public final TQueryCtx queryCtx_; - public final RexBuilder rexBuilder_; public final Project project_; + public final RexBuilder rexBuilder_; - public ReduceShuttleObjects(String query) throws ImpalaException { - QueryContext queryCtx = new QueryContext(options, frontend_, query); - queryCtx_ = queryCtx.getTQueryCtx(); - - TSessionState session = new TSessionState(); - session.setConnected_user("dummy"); - queryCtx.getTQueryCtx().setSession(session); - CalciteQueryParser queryParser = new CalciteQueryParser(queryCtx); - SqlNode parsedSqlNode = queryParser.parse(); - - CalciteMetadataHandler mdHandler = - new CalciteMetadataHandler(parsedSqlNode, queryCtx); - analyzer_ = mdHandler.getAnalyzer(); - - CalciteValidator validator = new CalciteValidator(mdHandler, queryCtx); - SqlNode validatedNode = validator.validate(parsedSqlNode); - CalciteRelNodeConverter relNodeConverter = new CalciteRelNodeConverter(validator); - RelNode rootNode = relNodeConverter.convert(validatedNode); - Preconditions.checkState(rootNode instanceof Project); - project_ = (Project) rootNode; - - RelDataTypeFactory typeFactory = - new JavaTypeFactoryImpl(new ImpalaTypeSystemImpl()); - rexBuilder_ = new RexBuilder(typeFactory); + public ReduceShuttleObjects(Analyzer analyzer, TQueryCtx queryCtx, Project project, + RexBuilder rexBuilder) { + analyzer_ = analyzer; + queryCtx_ = queryCtx; + project_ = project; + rexBuilder_ = rexBuilder;; } }
