This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d01f7c546a [refactor](iceberg-hudi) disable iceberg and hudi table by
default (#13932)
d01f7c546a is described below
commit d01f7c546ab176384084f6dabe802b11b395f65a
Author: Mingyu Chen <[email protected]>
AuthorDate: Sat Nov 5 19:22:27 2022 +0800
[refactor](iceberg-hudi) disable iceberg and hudi table by default (#13932)
---
.../org/apache/doris/analysis/CreateTableStmt.java | 12 +++++++-----
.../java/org/apache/doris/analysis/GrantStmt.java | 5 -----
.../main/java/org/apache/doris/analysis/LoadStmt.java | 5 -----
.../java/org/apache/doris/analysis/RevokeStmt.java | 5 -----
.../org/apache/doris/catalog/DatabaseProperty.java | 10 +++++++++-
.../src/main/java/org/apache/doris/common/Config.java | 19 +++++++------------
.../org/apache/doris/planner/SingleNodePlanner.java | 6 +++---
.../apache/doris/analysis/CreateTableStmtTest.java | 4 ++++
.../java/org/apache/doris/analysis/GrantStmtTest.java | 4 ----
.../java/org/apache/doris/analysis/LoadStmtTest.java | 3 ---
.../org/apache/doris/mysql/privilege/AuthTest.java | 3 ---
11 files changed, 30 insertions(+), 46 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 2a431ff56a..bf3b06783c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -274,11 +274,6 @@ public class CreateTableStmt extends DdlStmt {
analyzeEngineName();
- // TODO(wyb): spark-load
- if (engineName.equals("hive") && !Config.enable_spark_load) {
- throw new AnalysisException("Spark Load from hive table is coming
soon");
- }
-
// `analyzeUniqueKeyMergeOnWrite` would modify `properties`, which
will be used later,
// so we just clone a properties map here.
boolean enableUniqueKeyMergeOnWrite = false;
@@ -531,6 +526,13 @@ public class CreateTableStmt extends DdlStmt {
throw new AnalysisException("Do not support external table
with engine name = olap");
}
}
+
+ if (Config.disable_iceberg_hudi_table && (engineName.equals("iceberg")
|| engineName.equals("hudi"))) {
+ throw new AnalysisException(
+ "iceberg and hudi table is no longer supported. Use multi
catalog feature instead."
+ + ". Or you can temporarily set
'disable_iceberg_hudi_table=false'"
+ + " in fe.conf to reopen this feature.");
+ }
}
public static CreateTableStmt read(DataInput in) throws IOException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
index 3dcdec07cb..d44f0f51d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/GrantStmt.java
@@ -21,7 +21,6 @@ import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.catalog.Env;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeNameFormat;
@@ -109,10 +108,6 @@ public class GrantStmt extends DdlStmt {
if (tblPattern != null) {
tblPattern.analyze(analyzer);
} else {
- // TODO(wyb): spark-load
- if (!Config.enable_spark_load) {
- throw new AnalysisException("GRANT ON RESOURCE is coming
soon");
- }
resourcePattern.analyze();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
index 3b522d5198..98a5760267 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -22,7 +22,6 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PrintableMap;
@@ -365,10 +364,6 @@ public class LoadStmt extends DdlStmt {
if (resourceDesc != null) {
resourceDesc.analyze();
etlJobType = resourceDesc.getEtlJobType();
- // TODO(wyb): spark-load
- if (!Config.enable_spark_load) {
- throw new AnalysisException("Spark Load is coming soon");
- }
// check resource usage privilege
if
(!Env.getCurrentEnv().getAuth().checkResourcePriv(ConnectContext.get(),
resourceDesc.getName(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java
index a2239d520c..0c576cdbc0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RevokeStmt.java
@@ -20,7 +20,6 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.mysql.privilege.PaloPrivilege;
import org.apache.doris.mysql.privilege.PrivBitSet;
@@ -99,10 +98,6 @@ public class RevokeStmt extends DdlStmt {
if (tblPattern != null) {
tblPattern.analyze(analyzer);
} else {
- // TODO(wyb): spark-load
- if (!Config.enable_spark_load) {
- throw new AnalysisException("REVOKE ON RESOURCE is coming
soon");
- }
resourcePattern.analyze();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseProperty.java
index b5bdcee979..6b08070121 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseProperty.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
@@ -85,7 +86,14 @@ public class DatabaseProperty implements Writable {
Map<String, String> icebergProperties = new HashMap<>();
for (Map.Entry<String, String> entry : this.properties.entrySet()) {
if (entry.getKey().startsWith(ICEBERG_PROPERTY_PREFIX)) {
- icebergProperties.put(entry.getKey(), entry.getValue());
+ if (Config.disable_iceberg_hudi_table) {
+ throw new DdlException(
+ "database for iceberg is no longer supported. Use
multi catalog feature instead."
+ + ". Or you can temporarily set
'disable_iceberg_hudi_table=false'"
+ + " in fe.conf to reopen this feature.");
+ } else {
+ icebergProperties.put(entry.getKey(), entry.getValue());
+ }
}
}
if (icebergProperties.size() > 0) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index de5197ace4..49237bb0da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -1314,18 +1314,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean drop_backend_after_decommission = true;
- /**
- * enable spark load for temporary use
- */
- @ConfField(mutable = true, masterOnly = true)
- public static boolean enable_spark_load = true;
-
- /**
- * enable use odbc table
- */
- @ConfField(mutable = true, masterOnly = true)
- public static boolean enable_odbc_table = true;
-
/**
* Define thrift server's server model, default is TThreadPoolServer model
*/
@@ -1829,4 +1817,11 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = true, masterOnly = true)
public static long max_backend_heartbeat_failure_tolerance_count = 1;
+
+ /**
+ * The iceberg and hudi table will be removed in v1.3
+ * Use multi catalog instead.
+ */
+ @ConfField(mutable = true, masterOnly = false)
+ public static boolean disable_iceberg_hudi_table = true;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 55a8207b02..488b8901c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -1925,8 +1925,7 @@ public class SingleNodePlanner {
null, -1);
break;
case ICEBERG:
- scanNode = new IcebergScanNode(ctx.getNextNodeId(),
tblRef.getDesc(), "IcebergScanNode",
- null, -1);
+ scanNode = new ExternalFileScanNode(ctx.getNextNodeId(),
tblRef.getDesc());
break;
case HUDI:
scanNode = new HudiScanNode(ctx.getNextNodeId(),
tblRef.getDesc(), "HudiScanNode",
@@ -1948,7 +1947,8 @@ public class SingleNodePlanner {
default:
break;
}
- if (scanNode instanceof OlapScanNode || scanNode instanceof EsScanNode
|| scanNode instanceof HiveScanNode) {
+ if (scanNode instanceof OlapScanNode || scanNode instanceof EsScanNode
|| scanNode instanceof HiveScanNode
+ || scanNode instanceof ExternalFileScanNode) {
if (analyzer.enableInferPredicate()) {
PredicatePushDown.visitScanNode(scanNode, tblRef.getJoinOp(),
analyzer);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
index 2b2a3142f3..ec8e8d8811 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PropertyAnalyzer;
@@ -309,6 +310,7 @@ public class CreateTableStmtTest {
@Test
public void testCreateIcebergTable() throws UserException {
+ Config.disable_iceberg_hudi_table = false;
Map<String, String> properties = new HashMap<>();
properties.put("iceberg.database", "doris");
properties.put("iceberg.table", "test");
@@ -324,6 +326,7 @@ public class CreateTableStmtTest {
@Test
public void testCreateHudiTable() throws UserException {
+ Config.disable_iceberg_hudi_table = false;
Map<String, String> properties = new HashMap<>();
properties.put("hudi.database", "doris");
properties.put("hudi.table", "test");
@@ -340,6 +343,7 @@ public class CreateTableStmtTest {
@Test
public void testCreateHudiTableWithSchema() throws UserException {
+ Config.disable_iceberg_hudi_table = false;
Map<String, String> properties = new HashMap<>();
properties.put("hudi.database", "doris");
properties.put("hudi.table", "test");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/GrantStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/GrantStmtTest.java
index 20c0a9bfc9..46eddb2fe7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/GrantStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/GrantStmtTest.java
@@ -20,7 +20,6 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.mysql.privilege.PaloAuth;
import org.apache.doris.qe.ConnectContext;
@@ -95,9 +94,6 @@ public class GrantStmtTest {
@Test
public void testResourceNormal() throws UserException {
- // TODO(wyb): spark-load
- Config.enable_spark_load = true;
-
String resourceName = "spark0";
List<AccessPrivilege> privileges =
Lists.newArrayList(AccessPrivilege.USAGE_PRIV);
GrantStmt stmt = new GrantStmt(new UserIdentity("testUser", "%"),
null, new ResourcePattern(resourceName), privileges);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java
index aabfb5d9ec..192d7c5c1f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/LoadStmtTest.java
@@ -21,7 +21,6 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ResourceMgr;
import org.apache.doris.catalog.SparkResource;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.load.EtlJobType;
@@ -115,8 +114,6 @@ public class LoadStmtTest {
+ "(XXX)", stmt.toString());
// test ResourceDesc
- // TODO(wyb): spark-load
- Config.enable_spark_load = true;
stmt = new LoadStmt(new LabelName("testDb", "testLabel"),
dataDescriptionList,
new ResourceDesc(resourceName, null), null);
stmt.analyze(analyzer);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
index ffc1ee9662..3df1b4d1ed 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/AuthTest.java
@@ -32,7 +32,6 @@ import org.apache.doris.catalog.AccessPrivilege;
import org.apache.doris.catalog.DomainResolver;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
@@ -1400,8 +1399,6 @@ public class AuthTest {
ResourcePattern anyResourcePattern = new ResourcePattern(anyResource);
List<AccessPrivilege> usagePrivileges =
Lists.newArrayList(AccessPrivilege.USAGE_PRIV);
UserDesc userDesc = new UserDesc(userIdentity, "12345", true);
- // TODO(wyb): spark-load
- Config.enable_spark_load = true;
// ------ grant|revoke resource to|from user ------
// 1. create user with no role
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
