This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cf6dba79c48 [Enhancement] (nereids)implement CreateCatalogCommand in
nereids (#45150)
cf6dba79c48 is described below
commit cf6dba79c48c40775a40749d730c69916c5c900a
Author: Sridhar R Manikarnike <[email protected]>
AuthorDate: Mon Dec 16 14:50:33 2024 +0530
[Enhancement] (nereids)implement CreateCatalogCommand in nereids (#45150)
Issue Number: close #42594
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 8 +-
.../apache/doris/datasource/CatalogFactory.java | 10 ++
.../org/apache/doris/datasource/CatalogMgr.java | 33 +++--
.../doris/nereids/parser/LogicalPlanBuilder.java | 14 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../trees/plans/commands/CreateCatalogCommand.java | 144 +++++++++++++++++++++
.../trees/plans/visitor/CommandVisitor.java | 5 +
.../nereids_p0/test_create_catalog_command.out | 4 +
.../nereids_p0/test_create_catalog_command.groovy | 43 ++++++
9 files changed, 249 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 044ed7771a4..bc91a667eda 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -185,7 +185,10 @@ supportedCreateStatement
(WITH ROLLUP (rollupNames=identifierList)?)?
#createTableLike
| CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?
#createRole
| CREATE WORKLOAD GROUP (IF NOT EXISTS)?
- name=identifierOrText properties=propertyClause?
#createWorkloadGroup
+ name=identifierOrText properties=propertyClause?
#createWorkloadGroup
+ | CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier
+ (WITH RESOURCE resourceName=identifier)?
+ (COMMENT STRING_LITERAL)? properties=propertyClause?
#createCatalog
| CREATE ROW POLICY (IF NOT EXISTS)? name=identifier
ON table=multipartIdentifier
AS type=(RESTRICTIVE | PERMISSIVE)
@@ -744,9 +747,6 @@ analyzeProperties
unsupportedCreateStatement
: CREATE (DATABASE | SCHEMA) (IF NOT EXISTS)? name=multipartIdentifier
properties=propertyClause?
#createDatabase
- | CREATE CATALOG (IF NOT EXISTS)? catalogName=identifier
- (WITH RESOURCE resourceName=identifier)?
- (COMMENT STRING_LITERAL)? properties=propertyClause?
#createCatalog
| CREATE (GLOBAL | SESSION | LOCAL)?
(TABLES | AGGREGATE)? FUNCTION (IF NOT EXISTS)?
functionIdentifier LEFT_PAREN functionArguments? RIGHT_PAREN
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
index 43ac17839c9..984d29cae49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java
@@ -37,6 +37,7 @@ import
org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
import org.apache.doris.datasource.paimon.PaimonExternalCatalogFactory;
import org.apache.doris.datasource.test.TestExternalCatalog;
import
org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalogFactory;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
@@ -83,6 +84,15 @@ public class CatalogFactory {
log.getComment(), log.getProps(), true);
}
+ /**
+ * create the catalog instance from CreateCatalogCommand.
+ */
+ public static CatalogIf createFromCommand(long catalogId,
CreateCatalogCommand cmd)
+ throws DdlException {
+ return createCatalog(catalogId, cmd.getCatalogName(),
cmd.getResource(),
+ cmd.getComment(), cmd.getProperties(), false);
+ }
+
/**
* create the catalog instance from creating statement.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 070d31c071e..0203aa7020b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -50,6 +50,7 @@ import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import org.apache.doris.persist.OperationType;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -235,20 +236,16 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
lock.readLock().unlock();
}
- /**
- * Create and hold the catalog instance and write the meta log.
- */
- public void createCatalog(CreateCatalogStmt stmt) throws UserException {
- long id = Env.getCurrentEnv().getNextId();
- CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt);
+ private void createCatalogImpl(CatalogIf catalog, String catalogName,
+ boolean ifNotExists) throws UserException {
writeLock();
try {
if (nameToCatalog.containsKey(catalog.getName())) {
- if (stmt.isSetIfNotExists()) {
- LOG.warn("Catalog {} is already exist.",
stmt.getCatalogName());
+ if (ifNotExists) {
+ LOG.warn("Catalog {} is already exist.", catalogName);
return;
}
- throw new DdlException("Catalog had already exist with name: "
+ stmt.getCatalogName());
+ throw new DdlException("Catalog had already exist with name: "
+ catalogName);
}
createCatalogInternal(catalog, false);
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_CREATE_CATALOG,
catalog.constructEditLog());
@@ -257,6 +254,24 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
}
+ /**
+ * Create and hold the catalog instance and write the meta log.
+ */
+ public void createCatalog(CreateCatalogCommand cmd) throws UserException {
+ long id = Env.getCurrentEnv().getNextId();
+ CatalogIf catalog = CatalogFactory.createFromCommand(id, cmd);
+ createCatalogImpl(catalog, cmd.getCatalogName(),
cmd.isSetIfNotExists());
+ }
+
+ /**
+ * Create and hold the catalog instance and write the meta log.
+ */
+ public void createCatalog(CreateCatalogStmt stmt) throws UserException {
+ long id = Env.getCurrentEnv().getNextId();
+ CatalogIf catalog = CatalogFactory.createFromStmt(id, stmt);
+ createCatalogImpl(catalog, stmt.getCatalogName(),
stmt.isSetIfNotExists());
+ }
+
/**
* Remove the catalog instance by name and write the meta log.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index e6fee894b41..94c2757e560 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -95,6 +95,7 @@ import
org.apache.doris.nereids.DorisParser.ComplexColTypeContext;
import org.apache.doris.nereids.DorisParser.ComplexColTypeListContext;
import org.apache.doris.nereids.DorisParser.ComplexDataTypeContext;
import org.apache.doris.nereids.DorisParser.ConstantContext;
+import org.apache.doris.nereids.DorisParser.CreateCatalogContext;
import org.apache.doris.nereids.DorisParser.CreateEncryptkeyContext;
import org.apache.doris.nereids.DorisParser.CreateFileContext;
import org.apache.doris.nereids.DorisParser.CreateMTMVContext;
@@ -500,6 +501,7 @@ import
org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand;
import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
import org.apache.doris.nereids.trees.plans.commands.Constraint;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
@@ -4843,6 +4845,18 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
return new ShowDynamicPartitionCommand(dbName);
}
+ @Override
+ public LogicalPlan visitCreateCatalog(CreateCatalogContext ctx) {
+ String catalogName = ctx.catalogName.getText();
+ boolean ifNotExists = ctx.IF() != null;
+ String resourceName = ctx.resourceName == null ? null :
(ctx.resourceName.getText());
+ String comment = ctx.STRING_LITERAL() == null ? null :
stripQuotes(ctx.STRING_LITERAL().getText());
+ Map<String, String> properties = ctx.propertyClause() != null
+ ?
Maps.newHashMap(visitPropertyClause(ctx.propertyClause())) : Maps.newHashMap();
+
+ return new CreateCatalogCommand(catalogName, ifNotExists,
resourceName, comment, properties);
+ }
+
@Override
public LogicalPlan visitRecoverDatabase(RecoverDatabaseContext ctx) {
String dbName = ctx.name.getText();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index e8ce4b3e2d8..26b8ed4c0c9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -252,6 +252,7 @@ public enum PlanType {
REPLAY_COMMAND,
CREATE_ENCRYPTKEY_COMMAND,
CREATE_WORKLOAD_GROUP_COMMAND,
+ CREATE_CATALOG_COMMAND,
CREATE_FILE_COMMAND,
CREATE_ROUTINE_LOAD_COMMAND,
SHOW_TABLE_CREATION_COMMAND,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java
new file mode 100644
index 00000000000..22c1285a055
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Command to create a catalog in the Nereids planner.
+ */
+public class CreateCatalogCommand extends Command implements ForwardWithSync,
NeedAuditEncryption {
+ private final String catalogName;
+ private final boolean ifNotExists;
+ private final String resourceName;
+ private final String comment;
+ private final Map<String, String> properties;
+
+ /**
+ * Constructor
+ */
+ public CreateCatalogCommand(String catalogName, boolean ifNotExists,
String resourceName, String comment,
+ Map<String, String> properties) {
+ super(PlanType.CREATE_CATALOG_COMMAND);
+ this.catalogName = Objects.requireNonNull(catalogName, "Catalog name
cannot be null");
+ this.ifNotExists = ifNotExists;
+ this.resourceName = resourceName;
+ this.comment = comment;
+ this.properties = properties == null ? Maps.newHashMap() : properties;
+ }
+
+ private void validate(ConnectContext ctx) throws AnalysisException {
+ Util.checkCatalogAllRules(catalogName);
+ if (catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ throw new AnalysisException("Internal catalog name can't be
create.");
+ }
+
+ if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv(
+ ConnectContext.get(), catalogName, PrivPredicate.CREATE)) {
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED,
+ ctx.getQualifiedUser(), catalogName);
+ }
+
+ if (Config.disallow_create_catalog_with_resource &&
!Strings.isNullOrEmpty(resourceName)) {
+ throw new AnalysisException("Create catalog with resource is
deprecated and is not allowed."
+ + " You can set
`disallow_create_catalog_with_resource=false` in fe.conf"
+ + " to enable it temporarily.");
+ }
+
+ String currentDateTime =
LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " ");
+ properties.put(ExternalCatalog.CREATE_TIME, currentDateTime);
+ PropertyAnalyzer.checkCatalogProperties(properties, false);
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws
Exception {
+ validate(ctx);
+ Env.getCurrentEnv().getCatalogMgr().createCatalog(this);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitCreateCatalogCommand(this, context);
+ }
+
+ public String getCatalogName() {
+ return catalogName;
+ }
+
+ public boolean isSetIfNotExists() {
+ return ifNotExists;
+ }
+
+ public String getResource() {
+ return resourceName;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ @Override
+ public boolean needAuditEncryption() {
+ return true;
+ }
+
+ @Override
+ public String toSql() {
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append("CREATE CATALOG
").append("`").append(catalogName).append("`");
+ if (!Strings.isNullOrEmpty(resourceName)) {
+ stringBuilder.append(" WITH RESOURCE
`").append(resourceName).append("`");
+ }
+ if (!Strings.isNullOrEmpty(comment)) {
+ stringBuilder.append("\nCOMMENT \"").append(comment).append("\"");
+ }
+ if (properties.size() > 0) {
+ stringBuilder.append("\nPROPERTIES (\n");
+ stringBuilder.append(new PrintableMap<>(properties, "=", true,
true, true));
+ stringBuilder.append("\n)");
+ }
+ return stringBuilder.toString();
+ }
+}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 0341ed2f57d..efb81ae7ee0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -38,6 +38,7 @@ import
org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelWarmUpJobCommand;
import org.apache.doris.nereids.trees.plans.commands.CleanAllProfileCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateEncryptkeyCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateFileCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateJobCommand;
@@ -305,6 +306,10 @@ public interface CommandVisitor<R, C> {
return visitCommand(showProcedureStatusCommand, context);
}
+ default R visitCreateCatalogCommand(CreateCatalogCommand
createCatalogCommand, C context) {
+ return visitCommand(createCatalogCommand, context);
+ }
+
default R visitShowWarningErrorsCommand(ShowWarningErrorsCommand
showWarningErrorsCommand, C context) {
return visitCommand(showWarningErrorsCommand, context);
}
diff --git a/regression-test/data/nereids_p0/test_create_catalog_command.out
b/regression-test/data/nereids_p0/test_create_catalog_command.out
new file mode 100644
index 00000000000..b3ff20b67db
--- /dev/null
+++ b/regression-test/data/nereids_p0/test_create_catalog_command.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !cmd --
+test_create_catalog \nCREATE CATALOG `test_create_catalog`\nCOMMENT "Test
catalog for regression"\n PROPERTIES (\n"type" = "es",\n"hosts" =
"http://127.0.0.1:9200"\n);
+
diff --git
a/regression-test/suites/nereids_p0/test_create_catalog_command.groovy
b/regression-test/suites/nereids_p0/test_create_catalog_command.groovy
new file mode 100644
index 00000000000..be28de0a246
--- /dev/null
+++ b/regression-test/suites/nereids_p0/test_create_catalog_command.groovy
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_create_catalog_command", "nereids_p0") {
+ def catalogName = "test_create_catalog"
+ def resourceName = "test_resource"
+ def catalogProperties = "\"type\"=\"es\",
\"hosts\"=\"http://127.0.0.1:9200\"";
+
+ try {
+ // Drop catalog if it already exists
+ sql("DROP CATALOG IF EXISTS ${catalogName}")
+
+ // Create a new catalog
+ checkNereidsExecute(
+ """
+ CREATE CATALOG ${catalogName}
+ COMMENT 'Test catalog for regression'
+ PROPERTIES (${catalogProperties})
+ """
+ )
+
+ // Verify the catalog was created
+ checkNereidsExecute("""show create catalog ${catalogName}""")
+ qt_cmd("""show create catalog ${catalogName}""")
+ } finally {
+ // Clean up
+ sql("DROP CATALOG IF EXISTS ${catalogName}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
