This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit ebfbe0c8dd2879a462c8a604242fccc11d897e4c Author: Mingyu Chen <[email protected]> AuthorDate: Thu Jan 11 17:59:40 2024 +0800 [opt](information_schema) support information_schema in external catalog (#28919) Add `information_schema` database for all catalog. This is useful when using BI tools to connect to Doris, the tools can get meta info from `information_schema`. This PR mainly changes: 1. There will be a `information_schema` db in each catalog. 2. Each `information_schema` db only store the meta info of the catalog it belongs to. 3. For `information_schema`, the `TABLE_SCHEMA` column's value is the database name. 4. There is a new global variable `show_full_dbname_in_info_schema_db`, default is false, if set to true, The `TABLE_SCHEMA` column's value is the like `ctl.db`, because: When connect to Doris, the `database` info in connection url will be: `xxx?db=ctl.db`. And then some BI will try to query `information_schema` with sql like: `select * from information_schema.columns where TABLE_SCHEMA = "ctl.db"` So it has to be format as `ctl.db` eg, the `information_schema.columns` table in external catalog `doris` is like: ``` mysql> select * from information_schema.columns limit 1\G *************************** 1. row *************************** TABLE_CATALOG: doris TABLE_SCHEMA: doris.__internal_schema TABLE_NAME: column_statistics COLUMN_NAME: id ORDINAL_POSITION: 1 COLUMN_DEFAULT: NULL IS_NULLABLE: NO DATA_TYPE: varchar CHARACTER_MAXIMUM_LENGTH: 4096 CHARACTER_OCTET_LENGTH: 16384 NUMERIC_PRECISION: NULL NUMERIC_SCALE: NULL DATETIME_PRECISION: NULL CHARACTER_SET_NAME: NULL COLLATION_NAME: NULL COLUMN_TYPE: varchar(4096) COLUMN_KEY: EXTRA: PRIVILEGES: COLUMN_COMMENT: COLUMN_SIZE: 4096 DECIMAL_DIGITS: NULL GENERATION_EXPRESSION: NULL SRS_ID: NULL ``` 6. Modify the behavior of - show tables - shwo databases - show columns - show table status The above statements may query the `information_schema` db if there is `where` predicate after them --- be/src/exec/schema_scanner.cpp | 9 + be/src/exec/schema_scanner.h | 4 + .../main/java/org/apache/doris/common/Config.java | 9 - fe/fe-core/src/main/cup/sql_parser.cup | 32 ++- .../java/org/apache/doris/analysis/Analyzer.java | 13 +- .../apache/doris/analysis/ExprSubstitutionMap.java | 6 + .../org/apache/doris/analysis/SchemaTableType.java | 1 - .../org/apache/doris/analysis/ShowColumnStmt.java | 52 ++--- .../java/org/apache/doris/analysis/ShowDbStmt.java | 17 +- .../org/apache/doris/analysis/ShowIndexStmt.java | 11 +- .../apache/doris/analysis/ShowTableStatusStmt.java | 46 ++--- .../org/apache/doris/analysis/ShowTableStmt.java | 15 +- .../apache/doris/analysis/ShowVariablesStmt.java | 2 +- .../java/org/apache/doris/catalog/TableIf.java | 8 +- .../doris/catalog/external/EsExternalTable.java | 5 - .../doris/catalog/external/ExternalDatabase.java | 27 ++- .../doris/catalog/external/HMSExternalTable.java | 5 - .../doris/catalog/external/JdbcExternalTable.java | 5 - .../catalog/external/MaxComputeExternalTable.java | 5 - .../doris/catalog/external/TestExternalTable.java | 5 - .../org/apache/doris/common/profile/Profile.java | 1 - .../org/apache/doris/datasource/CatalogMgr.java | 1 + .../apache/doris/datasource/ExternalCatalog.java | 40 +++- .../apache/doris/datasource/InitDatabaseLog.java | 1 + .../apache/doris/datasource/InternalCatalog.java | 6 +- .../infoschema/ExternalInfoSchemaDatabase.java | 55 +++++ .../infoschema/ExternalInfoSchemaTable.java} | 47 ++--- .../glue/translator/PhysicalPlanTranslator.java | 3 +- .../trees/plans/logical/LogicalSchemaScan.java | 5 +- .../trees/plans/physical/PhysicalSchemaScan.java | 10 +- .../org/apache/doris/persist/gson/GsonUtils.java | 17 +- .../org/apache/doris/planner/SchemaScanNode.java | 15 +- .../java/org/apache/doris/qe/GlobalVariable.java | 7 + .../java/org/apache/doris/qe/ShowExecutor.java | 33 +-- .../java/org/apache/doris/qe/StmtExecutor.java | 2 +- .../apache/doris/service/FrontendServiceImpl.java | 109 +++++----- .../apache/doris/analysis/ShowIndexStmtTest.java | 2 +- .../org/apache/doris/catalog/RefreshTableTest.java | 12 ++ .../doris/datasource/RefreshCatalogTest.java | 25 ++- .../info_schema_db/test_info_schema_db.out | 226 +++++++++++++++++++++ .../jdbc/test_mariadb_jdbc_catalog.out | 106 +++------- .../jdbc/test_mysql_jdbc_catalog.out | 91 +++------ .../jdbc/test_mysql_jdbc_catalog_nereids.out | 88 +++----- .../jdbc/test_oracle_jdbc_catalog.out | 2 + .../jdbc/test_pg_jdbc_catalog.out | 3 + .../jdbc/test_sqlserver_jdbc_catalog.out | 1 + .../hive/test_show_columns.groovy | 62 ------ .../info_schema_db/test_info_schema_db.groovy | 134 ++++++++++++ 48 files changed, 818 insertions(+), 563 deletions(-) diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index a905f69336d..b700d36f209 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -44,6 +44,7 @@ #include "exec/schema_scanner/schema_views_scanner.h" #include "olap/hll.h" #include "runtime/define_primitive_type.h" +#include "util/string_util.h" #include "util/types.h" #include "vec/columns/column.h" #include "vec/columns/column_complex.h" @@ -312,4 +313,12 @@ Status SchemaScanner::fill_dest_column_for_range(vectorized::Block* block, size_ return Status::OK(); } +std::string SchemaScanner::get_db_from_full_name(const std::string& full_name) { + std::vector<std::string> part = split(full_name, "."); + if (part.size() == 2) { + return part[1]; + } + return full_name; +} + } // namespace doris diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h index d1d11ba9358..50954999e2a 100644 --- a/be/src/exec/schema_scanner.h +++ b/be/src/exec/schema_scanner.h @@ -105,6 +105,10 @@ protected: Status fill_dest_column_for_range(vectorized::Block* block, size_t pos, const std::vector<void*>& datas); + // get dbname from catalogname.dbname + // if full_name does not have catalog part, just return origin name. + std::string get_db_from_full_name(const std::string& full_name); + bool _is_init; // this is used for sub class SchemaScannerParam* _param = nullptr; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index af9d9bee4f4..2494465e05e 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2062,15 +2062,6 @@ public class Config extends ConfigBase { @ConfField public static long lock_reporting_threshold_ms = 500L; - /** - * If false, when select from tables in information_schema database, - * the result will not contain the information of the table in external catalog. - * This is to avoid query time when external catalog is not reachable. - * TODO: this is a temp solution, we should support external catalog in the future. - */ - @ConfField(mutable = true) - public static boolean infodb_support_ext_catalog = false; - /** * If true, auth check will be disabled. The default value is false. * This is to solve the case that user forgot the password. diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index b7464052113..aaadfb4f563 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -3989,11 +3989,11 @@ show_param ::= /* Database */ | KW_DATABASES opt_wild_where {: - RESULT = new ShowDbStmt(parser.wild, parser.where); + RESULT = new ShowDbStmt(parser.wild, parser.where, null); :} - | KW_DATABASES KW_FROM ident:catalogName + | KW_DATABASES KW_FROM ident:catalogName opt_wild_where {: - RESULT = new ShowDbStmt(null, null, catalogName); + RESULT = new ShowDbStmt(parser.wild, parser.where, catalogName); :} /* show database id */ | KW_DATABASE INTEGER_LITERAL:dbId @@ -4007,7 +4007,7 @@ show_param ::= :} | KW_SCHEMAS opt_wild_where {: - RESULT = new ShowDbStmt(parser.wild, parser.where); + RESULT = new ShowDbStmt(parser.wild, parser.where, null); :} | KW_SCHEMAS KW_FROM ident:catalogName {: @@ -4033,6 +4033,10 @@ show_param ::= {: RESULT = new ShowColumnStmt(table, db, parser.wild, parser.isVerbose, parser.where); :} + | opt_full KW_FIELDS from_or_in table_name:table opt_db:db opt_wild_where + {: + RESULT = new ShowColumnStmt(table, db, parser.wild, parser.isVerbose, parser.where); + :} /* collation */ | KW_COLLATION opt_wild_where {: @@ -4469,6 +4473,10 @@ opt_full ::= {: parser.isVerbose = true; :} + | KW_EXTENDED + {: + parser.isVerbose = true; + :} ; opt_wild_where ::= @@ -4543,14 +4551,6 @@ describe_stmt ::= {: RESULT = new DescribeStmt(table, false, partitionNames); :} - | KW_SHOW KW_FIELDS KW_FROM table_name:table - {: - RESULT = new DescribeStmt(table, false); - :} - | KW_SHOW KW_COLUMNS KW_FROM table_name:table - {: - RESULT = new DescribeStmt(table, false); - :} | describe_command KW_FUNCTION table_valued_function_ref:tvf {: RESULT = new DescribeStmt(tvf); @@ -4559,14 +4559,6 @@ describe_stmt ::= {: RESULT = new DescribeStmt(table, true); :} - | KW_SHOW KW_EXTENDED KW_FIELDS KW_FROM table_name:table - {: - RESULT = new DescribeStmt(table, true); - :} - | KW_SHOW KW_EXTENDED KW_COLUMNS KW_FROM table_name:table - {: - RESULT = new DescribeStmt(table, true); - :} | describe_command opt_explain_options:options query_stmt:query {: query.setIsExplain(options); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index bd7b6bd7290..5dcb9c193e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -151,9 +151,11 @@ public class Analyzer { // map from tuple id to the current output column index private final Map<TupleId, Integer> currentOutputColumn = Maps.newHashMap(); // used for Information Schema Table Scan - private String schemaDb; + // This 3 fields is used for optimize the data fetching from FE, + // for stmt such as `show columns like`, `show databases like`, `show tables like` + // if can pre-filter the data in FrontendServiceImpl to reduce the data fetching from FE. private String schemaCatalog; - private String schemaWild; + private String schemaDb; private String schemaTable; // table used in DESCRIBE Table // Current depth of nested analyze() calls. Used for enforcing a @@ -2354,19 +2356,14 @@ public class Analyzer { return globalState.context; } - public String getSchemaWild() { - return schemaWild; - } - public TQueryGlobals getQueryGlobals() { return new TQueryGlobals(); } // for Schema Table Schema like SHOW TABLES LIKE "abc%" - public void setSchemaInfo(String db, String table, String wild, String catalog) { + public void setSchemaInfo(String db, String table, String catalog) { schemaDb = db; schemaTable = table; - schemaWild = wild; schemaCatalog = catalog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java index e6e50f7321b..f7ef598b758 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExprSubstitutionMap.java @@ -77,6 +77,11 @@ public final class ExprSubstitutionMap { rhs.add(rhsExpr); } + public void putNoAnalyze(Expr lhsExpr, Expr rhsExpr) { + lhs.add(lhsExpr); + rhs.add(rhsExpr); + } + /** * Returns the expr mapped to lhsExpr or null if no mapping to lhsExpr exists. */ @@ -357,3 +362,4 @@ public final class ExprSubstitutionMap { } } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index 10c958c47b0..59bae154d57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -20,7 +20,6 @@ package org.apache.doris.analysis; import org.apache.doris.thrift.TSchemaTableType; public enum SchemaTableType { - // defination SCH_AUTHORS("AUTHORS", "AUTHORS", TSchemaTableType.SCH_AUTHORS), SCH_CHARSETS("CHARSETS", "CHARACTER_SETS", TSchemaTableType.SCH_CHARSETS), SCH_COLLATIONS("COLLATIONS", "COLLATIONS", TSchemaTableType.SCH_COLLATIONS), diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java index e4b103b4152..dfa5c3f2cb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStmt.java @@ -21,8 +21,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -30,8 +28,6 @@ import com.google.common.collect.Lists; // SHOW COLUMNS public class ShowColumnStmt extends ShowStmt { - private static final TableName TABLE_NAME = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, - InfoSchemaDb.DATABASE_NAME, "COLUMNS"); private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Field", ScalarType.createVarchar(20))) .addColumn(new Column("Type", ScalarType.createVarchar(20))) @@ -76,6 +72,10 @@ public class ShowColumnStmt extends ShowStmt { this.where = where; } + public String getCtl() { + return tableName.getCtl(); + } + public String getDb() { return tableName.getDb(); } @@ -98,8 +98,6 @@ public class ShowColumnStmt extends ShowStmt { tableName.setDb(db); } tableName.analyze(analyzer); - // disallow external catalog - Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName()); if (isVerbose) { metaData = META_DATA_VERBOSE; } else { @@ -116,55 +114,56 @@ public class ShowColumnStmt extends ShowStmt { return selectStmt; } analyze(analyzer); + TableName columnsTableName = new TableName(tableName.getCtl(), InfoSchemaDb.DATABASE_NAME, "columns"); // Columns SelectList selectList = new SelectList(); ExprSubstitutionMap aliasMap = new ExprSubstitutionMap(); // Field - SelectListItem item = new SelectListItem(new SlotRef(TABLE_NAME, "COLUMN_NAME"), "Field"); + SelectListItem item = new SelectListItem(new SlotRef(columnsTableName, "COLUMN_NAME"), "Field"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Field"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Field"), item.getExpr().clone(null)); // Type - item = new SelectListItem(new SlotRef(TABLE_NAME, "DATA_TYPE"), "Type"); + item = new SelectListItem(new SlotRef(columnsTableName, "DATA_TYPE"), "Type"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Type"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Type"), item.getExpr().clone(null)); // Collation if (isVerbose) { - item = new SelectListItem(new SlotRef(TABLE_NAME, "COLLATION_NAME"), "Collation"); + item = new SelectListItem(new SlotRef(columnsTableName, "COLLATION_NAME"), "Collation"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Collation"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Collation"), item.getExpr().clone(null)); } // Null - item = new SelectListItem(new SlotRef(TABLE_NAME, "IS_NULLABLE"), "Null"); + item = new SelectListItem(new SlotRef(columnsTableName, "IS_NULLABLE"), "Null"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Null"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Null"), item.getExpr().clone(null)); // Key - item = new SelectListItem(new SlotRef(TABLE_NAME, "COLUMN_KEY"), "Key"); + item = new SelectListItem(new SlotRef(columnsTableName, "COLUMN_KEY"), "Key"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Key"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Key"), item.getExpr().clone(null)); // Default - item = new SelectListItem(new SlotRef(TABLE_NAME, "COLUMN_DEFAULT"), "Default"); + item = new SelectListItem(new SlotRef(columnsTableName, "COLUMN_DEFAULT"), "Default"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Default"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Default"), item.getExpr().clone(null)); // Extra - item = new SelectListItem(new SlotRef(TABLE_NAME, "EXTRA"), "Extra"); + item = new SelectListItem(new SlotRef(columnsTableName, "EXTRA"), "Extra"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Extra"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Extra"), item.getExpr().clone(null)); if (isVerbose) { // Privileges - item = new SelectListItem(new SlotRef(TABLE_NAME, "PRIVILEGES"), "Privileges"); + item = new SelectListItem(new SlotRef(columnsTableName, "PRIVILEGES"), "Privileges"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Privileges"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Privileges"), item.getExpr().clone(null)); // Comment - item = new SelectListItem(new SlotRef(TABLE_NAME, "COLUMN_COMMENT"), "Comment"); + item = new SelectListItem(new SlotRef(columnsTableName, "COLUMN_COMMENT"), "Comment"); selectList.addItem(item); - aliasMap.put(new SlotRef(null, "Comment"), item.getExpr().clone(null)); + aliasMap.putNoAnalyze(new SlotRef(null, "Comment"), item.getExpr().clone(null)); } where = where.substitute(aliasMap); selectStmt = new SelectStmt(selectList, - new FromClause(Lists.newArrayList(new TableRef(TABLE_NAME, null))), + new FromClause(Lists.newArrayList(new TableRef(columnsTableName, null))), where, null, null, null, LimitElement.NO_LIMIT); - analyzer.setSchemaInfo(tableName.getDb(), tableName.getTbl(), null, tableName.getCtl()); + analyzer.setSchemaInfo(tableName.getDb(), tableName.getTbl(), tableName.getCtl()); return selectStmt; } @@ -174,3 +173,4 @@ public class ShowColumnStmt extends ShowStmt { return metaData; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java index f38199042e5..f88040f4ba0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDbStmt.java @@ -46,11 +46,6 @@ public class ShowDbStmt extends ShowStmt { this.pattern = pattern; } - public ShowDbStmt(String pattern, Expr where) { - this.pattern = pattern; - this.where = where; - } - public ShowDbStmt(String pattern, Expr where, String catalogName) { this.pattern = pattern; this.where = where; @@ -82,18 +77,15 @@ public class ShowDbStmt extends ShowStmt { // Columns SelectList selectList = new SelectList(); ExprSubstitutionMap aliasMap = new ExprSubstitutionMap(false); - SelectListItem item = new SelectListItem(new SlotRef(TABLE_NAME, "SCHEMA_NAME"), DB_COL); + TableName tableName = new TableName(catalogName, InfoSchemaDb.DATABASE_NAME, "schemata"); + SelectListItem item = new SelectListItem(new SlotRef(tableName, "SCHEMA_NAME"), DB_COL); selectList.addItem(item); aliasMap.put(new SlotRef(null, DB_COL), item.getExpr().clone(null)); where = where.substitute(aliasMap); selectStmt = new SelectStmt(selectList, - new FromClause(Lists.newArrayList(new TableRef(TABLE_NAME, null))), + new FromClause(Lists.newArrayList(new TableRef(tableName, null))), where, null, null, null, LimitElement.NO_LIMIT); - if (catalogName != null) { - analyzer.setSchemaInfo(null, null, null, catalogName); - } else { - analyzer.setSchemaInfo(null, null, null, analyzer.getDefaultCatalog()); - } + analyzer.setSchemaInfo(null, null, catalogName); return selectStmt; } @@ -119,3 +111,4 @@ public class ShowDbStmt extends ShowStmt { return META_DATA; } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java index bacea1300da..b35f2e77287 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowIndexStmt.java @@ -24,7 +24,6 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; @@ -70,13 +69,11 @@ public class ShowIndexStmt extends ShowStmt { tableName.setDb(dbName); } tableName.analyze(analyzer); - // disallow external catalog - Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName()); - if (!Env.getCurrentEnv().getAccessManager().checkTblPriv( - ConnectContext.get(), tableName.getDb(), tableName.getTbl(), PrivPredicate.SHOW)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, analyzer.getQualifiedUser(), - tableName.getDb() + ": " + tableName.toString()); + ConnectContext.get(), tableName.getCtl(), tableName.getDb(), tableName.getTbl(), PrivPredicate.SHOW)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SHOW INDEX", + analyzer.getQualifiedUser(), ConnectContext.get().getRemoteIP(), + tableName.toSql()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java index 4de10180caf..20e87908268 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatusStmt.java @@ -26,7 +26,6 @@ import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; @@ -36,8 +35,7 @@ import com.google.common.collect.Lists; // SHOW TABLE STATUS public class ShowTableStatusStmt extends ShowStmt { - private static final TableName TABLE_NAME = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, - InfoSchemaDb.DATABASE_NAME, "tables"); + private static final ShowResultSetMetaData META_DATA = ShowResultSetMetaData.builder() .addColumn(new Column("Name", ScalarType.createVarchar(64))) .addColumn(new Column("Engine", ScalarType.createVarchar(10))) @@ -115,87 +113,87 @@ public class ShowTableStatusStmt extends ShowStmt { } analyze(analyzer); - + TableName tablesTableName = new TableName(catalog, InfoSchemaDb.DATABASE_NAME, "tables"); // Columns SelectList selectList = new SelectList(); ExprSubstitutionMap aliasMap = new ExprSubstitutionMap(false); // Name - SelectListItem item = new SelectListItem(new SlotRef(TABLE_NAME, "TABLE_NAME"), "Name"); + SelectListItem item = new SelectListItem(new SlotRef(tablesTableName, "TABLE_NAME"), "Name"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Name"), item.getExpr().clone(null)); // Engine - item = new SelectListItem(new SlotRef(TABLE_NAME, "ENGINE"), "Engine"); + item = new SelectListItem(new SlotRef(tablesTableName, "ENGINE"), "Engine"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Engine"), item.getExpr().clone(null)); // Version - item = new SelectListItem(new SlotRef(TABLE_NAME, "VERSION"), "Version"); + item = new SelectListItem(new SlotRef(tablesTableName, "VERSION"), "Version"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Version"), item.getExpr().clone(null)); // Version - item = new SelectListItem(new SlotRef(TABLE_NAME, "ROW_FORMAT"), "Row_format"); + item = new SelectListItem(new SlotRef(tablesTableName, "ROW_FORMAT"), "Row_format"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Row_format"), item.getExpr().clone(null)); // Rows - item = new SelectListItem(new SlotRef(TABLE_NAME, "TABLE_ROWS"), "Rows"); + item = new SelectListItem(new SlotRef(tablesTableName, "TABLE_ROWS"), "Rows"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Rows"), item.getExpr().clone(null)); // Avg_row_length - item = new SelectListItem(new SlotRef(TABLE_NAME, "AVG_ROW_LENGTH"), "Avg_row_length"); + item = new SelectListItem(new SlotRef(tablesTableName, "AVG_ROW_LENGTH"), "Avg_row_length"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Avg_row_length"), item.getExpr().clone(null)); // Data_length - item = new SelectListItem(new SlotRef(TABLE_NAME, "DATA_LENGTH"), "Data_length"); + item = new SelectListItem(new SlotRef(tablesTableName, "DATA_LENGTH"), "Data_length"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Data_length"), item.getExpr().clone(null)); // Max_data_length - item = new SelectListItem(new SlotRef(TABLE_NAME, "MAX_DATA_LENGTH"), "Max_data_length"); + item = new SelectListItem(new SlotRef(tablesTableName, "MAX_DATA_LENGTH"), "Max_data_length"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Max_data_length"), item.getExpr().clone(null)); // Index_length - item = new SelectListItem(new SlotRef(TABLE_NAME, "INDEX_LENGTH"), "Index_length"); + item = new SelectListItem(new SlotRef(tablesTableName, "INDEX_LENGTH"), "Index_length"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Index_length"), item.getExpr().clone(null)); // Data_free - item = new SelectListItem(new SlotRef(TABLE_NAME, "DATA_FREE"), "Data_free"); + item = new SelectListItem(new SlotRef(tablesTableName, "DATA_FREE"), "Data_free"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Data_free"), item.getExpr().clone(null)); // Data_free - item = new SelectListItem(new SlotRef(TABLE_NAME, "AUTO_INCREMENT"), "Auto_increment"); + item = new SelectListItem(new SlotRef(tablesTableName, "AUTO_INCREMENT"), "Auto_increment"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Auto_increment"), item.getExpr().clone(null)); // Create_time - item = new SelectListItem(new SlotRef(TABLE_NAME, "CREATE_TIME"), "Create_time"); + item = new SelectListItem(new SlotRef(tablesTableName, "CREATE_TIME"), "Create_time"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Create_time"), item.getExpr().clone(null)); // Update_time - item = new SelectListItem(new SlotRef(TABLE_NAME, "UPDATE_TIME"), "Update_time"); + item = new SelectListItem(new SlotRef(tablesTableName, "UPDATE_TIME"), "Update_time"); selectList.addItem(item); // Check_time - item = new SelectListItem(new SlotRef(TABLE_NAME, "CHECK_TIME"), "Check_time"); + item = new SelectListItem(new SlotRef(tablesTableName, "CHECK_TIME"), "Check_time"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Check_time"), item.getExpr().clone(null)); // Collation - item = new SelectListItem(new SlotRef(TABLE_NAME, "TABLE_COLLATION"), "Collation"); + item = new SelectListItem(new SlotRef(tablesTableName, "TABLE_COLLATION"), "Collation"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Collation"), item.getExpr().clone(null)); // Checksum - item = new SelectListItem(new SlotRef(TABLE_NAME, "CHECKSUM"), "Checksum"); + item = new SelectListItem(new SlotRef(tablesTableName, "CHECKSUM"), "Checksum"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Checksum"), item.getExpr().clone(null)); // Create_options - item = new SelectListItem(new SlotRef(TABLE_NAME, "CREATE_OPTIONS"), "Create_options"); + item = new SelectListItem(new SlotRef(tablesTableName, "CREATE_OPTIONS"), "Create_options"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Create_options"), item.getExpr().clone(null)); // Comment - item = new SelectListItem(new SlotRef(TABLE_NAME, "TABLE_COMMENT"), "Comment"); + item = new SelectListItem(new SlotRef(tablesTableName, "TABLE_COMMENT"), "Comment"); selectList.addItem(item); aliasMap.put(new SlotRef(null, "Comment"), item.getExpr().clone(null)); where = where.substitute(aliasMap); selectStmt = new SelectStmt(selectList, - new FromClause(Lists.newArrayList(new TableRef(TABLE_NAME, null))), + new FromClause(Lists.newArrayList(new TableRef(tablesTableName, null))), where, null, null, null, LimitElement.NO_LIMIT); - analyzer.setSchemaInfo(ClusterNamespace.getNameFromFullName(db), null, null, catalog); + analyzer.setSchemaInfo(ClusterNamespace.getNameFromFullName(db), null, catalog); return selectStmt; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java index 76a6e5d684d..3ed694b998e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java @@ -24,7 +24,6 @@ import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -37,9 +36,7 @@ public class ShowTableStmt extends ShowStmt { private static final Logger LOG = LogManager.getLogger(ShowTableStmt.class); private static final String NAME_COL_PREFIX = "Tables_in_"; private static final String TYPE_COL = "Table_type"; - private static final String STORAGE_FORMAT_COL = "StorageFormat"; - private static final TableName TABLE_NAME = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, - InfoSchemaDb.DATABASE_NAME, "tables"); + private static final String STORAGE_FORMAT_COL = "Storage_format"; private String db; private String catalog; private boolean isVerbose; @@ -107,25 +104,27 @@ public class ShowTableStmt extends ShowStmt { return selectStmt; } analyze(analyzer); + TableName tablesTableName = new TableName(catalog, InfoSchemaDb.DATABASE_NAME, "tables"); + // Columns SelectList selectList = new SelectList(); ExprSubstitutionMap aliasMap = new ExprSubstitutionMap(false); - SelectListItem item = new SelectListItem(new SlotRef(TABLE_NAME, "TABLE_NAME"), + SelectListItem item = new SelectListItem(new SlotRef(tablesTableName, "TABLE_NAME"), NAME_COL_PREFIX + ClusterNamespace.getNameFromFullName(db)); selectList.addItem(item); aliasMap.put(new SlotRef(null, NAME_COL_PREFIX + ClusterNamespace.getNameFromFullName(db)), item.getExpr().clone(null)); if (isVerbose) { - item = new SelectListItem(new SlotRef(TABLE_NAME, "TABLE_TYPE"), TYPE_COL); + item = new SelectListItem(new SlotRef(tablesTableName, "TABLE_TYPE"), TYPE_COL); selectList.addItem(item); aliasMap.put(new SlotRef(null, TYPE_COL), item.getExpr().clone(null)); } where = where.substitute(aliasMap); selectStmt = new SelectStmt(selectList, - new FromClause(Lists.newArrayList(new TableRef(TABLE_NAME, null))), + new FromClause(Lists.newArrayList(new TableRef(tablesTableName, null))), where, null, null, null, LimitElement.NO_LIMIT); - analyzer.setSchemaInfo(ClusterNamespace.getNameFromFullName(db), null, null, catalog); + analyzer.setSchemaInfo(ClusterNamespace.getNameFromFullName(db), null, catalog); return selectStmt; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java index dcdfc33d57b..68d04faefc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowVariablesStmt.java @@ -110,7 +110,7 @@ public class ShowVariablesStmt extends ShowStmt { // DB: type // table: thread id - analyzer.setSchemaInfo(type.toSql(), null, null, null); + analyzer.setSchemaInfo(null, null, null); return selectStmt; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index e6feb6b4a59..bb6abcdd509 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -351,7 +351,7 @@ public interface TableIf { case OLAP: return "Doris"; case SCHEMA: - return "MEMORY"; + return "SYSTEM VIEW"; case INLINE_VIEW: return "InlineView"; case VIEW: @@ -391,16 +391,16 @@ public interface TableIf { } } + // Refer to https://dev.mysql.com/doc/refman/8.0/en/information-schema-tables-table.html public String toMysqlType() { switch (this) { - case OLAP: - return "BASE TABLE"; case SCHEMA: return "SYSTEM VIEW"; case INLINE_VIEW: case VIEW: case MATERIALIZED_VIEW: return "VIEW"; + case OLAP: case MYSQL: case ODBC: case BROKER: @@ -414,7 +414,7 @@ public interface TableIf { case ES_EXTERNAL_TABLE: case ICEBERG_EXTERNAL_TABLE: case PAIMON_EXTERNAL_TABLE: - return "EXTERNAL TABLE"; + return "BASE TABLE"; default: return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java index 464fb79b8da..8516bccc1d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java @@ -60,11 +60,6 @@ public class EsExternalTable extends ExternalTable { return esTable; } - @Override - public String getMysqlType() { - return type.name(); - } - @Override public TTableDescriptor toThrift() { List<Column> schema = getFullSchema(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java index 3559637b8f3..e6584d4a0b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java @@ -20,6 +20,7 @@ package org.apache.doris.catalog.external; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DatabaseProperty; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; @@ -29,15 +30,19 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.InitDatabaseLog; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterCatalogExecutor; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; +import com.google.gson.internal.LinkedTreeMap; import org.apache.commons.lang3.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -160,7 +165,12 @@ public abstract class ExternalDatabase<T extends ExternalTable> initDatabaseLog.setType(dbLogType); initDatabaseLog.setCatalogId(extCatalog.getId()); initDatabaseLog.setDbId(id); - List<String> tableNames = extCatalog.listTableNames(null, name); + List<String> tableNames; + if (name.equals(InfoSchemaDb.DATABASE_NAME)) { + tableNames = ExternalInfoSchemaDatabase.listTableNames(); + } else { + tableNames = extCatalog.listTableNames(null, name); + } if (tableNames != null) { Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap(); Map<Long, T> tmpIdToTbl = Maps.newHashMap(); @@ -338,9 +348,20 @@ public abstract class ExternalDatabase<T extends ExternalTable> @Override public void gsonPostProcess() throws IOException { tableNameToId = Maps.newConcurrentMap(); - for (T tbl : idToTbl.values()) { - tableNameToId.put(tbl.getName(), tbl.getId()); + Map<Long, T> tmpIdToTbl = Maps.newConcurrentMap(); + for (Object obj : idToTbl.values()) { + if (obj instanceof LinkedTreeMap) { + ExternalInfoSchemaTable table = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(obj), + ExternalInfoSchemaTable.class); + tmpIdToTbl.put(table.getId(), (T) table); + tableNameToId.put(table.getName(), table.getId()); + } else { + Preconditions.checkState(obj instanceof ExternalTable); + tmpIdToTbl.put(((ExternalTable) obj).getId(), (T) obj); + tableNameToId.put(((ExternalTable) obj).getName(), ((ExternalTable) obj).getId()); + } } + idToTbl = tmpIdToTbl; rwLock = new ReentrantReadWriteLock(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index 0528194b64f..8c353320ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -263,11 +263,6 @@ public class HMSExternalTable extends ExternalTable { return remoteTable.isSetViewOriginalText() || remoteTable.isSetViewExpandedText(); } - @Override - public String getMysqlType() { - return type.name(); - } - @Override public String getEngine() { switch (type) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java index 7d584477f4f..100fa228098 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java @@ -66,11 +66,6 @@ public class JdbcExternalTable extends ExternalTable { return jdbcTable; } - @Override - public String getMysqlType() { - return type.name(); - } - @Override public TTableDescriptor toThrift() { makeSureInitialized(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java index 5c25cf6cce0..ecb896e1970 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java @@ -285,10 +285,5 @@ public class MaxComputeExternalTable extends ExternalTable { makeSureInitialized(); return odpsTable; } - - @Override - public String getMysqlType() { - return "BASE TABLE"; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java index 45d6cda9037..212f0511740 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java @@ -43,11 +43,6 @@ public class TestExternalTable extends ExternalTable { this.objectCreated = true; } - @Override - public String getMysqlType() { - return type.name(); - } - @Override public TTableDescriptor toThrift() { makeSureInitialized(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index 3b3464d96df..383880b2c82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -110,7 +110,6 @@ public class Profile { try { builder.append("\n"); executionProfile.getExecutionProfile().prettyPrint(builder, ""); - LOG.info(builder.toString()); } catch (Throwable aggProfileException) { LOG.warn("build merged simple profile failed", aggProfileException); builder.append("build merged simple profile failed"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 3604d1c628e..d727ee620f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -108,6 +108,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { public static CatalogMgr read(DataInput in) throws IOException { String json = Text.readString(in); + LOG.debug("debug: read json: {}", json); return GsonUtils.GSON.fromJson(json, CatalogMgr.class); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 0e12e942c9b..3ff599d0ab4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -20,6 +20,7 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.external.EsExternalDatabase; @@ -36,6 +37,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -253,16 +255,21 @@ public abstract class ExternalCatalog InitCatalogLog initCatalogLog = new InitCatalogLog(); initCatalogLog.setCatalogId(id); initCatalogLog.setType(logType); - List<String> allDatabases = listDatabaseNames(); + List<String> allDatabases = Lists.newArrayList(listDatabaseNames()); + + allDatabases.remove(InfoSchemaDb.DATABASE_NAME); + allDatabases.add(InfoSchemaDb.DATABASE_NAME); Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap(); Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap(); for (String dbName : allDatabases) { - // Exclude database map take effect with higher priority over include database map - if (!excludeDatabaseMap.isEmpty() && excludeDatabaseMap.containsKey(dbName)) { - continue; - } - if (!includeDatabaseMap.isEmpty() && !includeDatabaseMap.containsKey(dbName)) { - continue; + if (!dbName.equals(InfoSchemaDb.DATABASE_NAME)) { + // Exclude database map take effect with higher priority over include database map + if (!excludeDatabaseMap.isEmpty() && excludeDatabaseMap.containsKey(dbName)) { + continue; + } + if (!includeDatabaseMap.isEmpty() && !includeDatabaseMap.containsKey(dbName)) { + continue; + } } long dbId; if (dbNameToId != null && dbNameToId.containsKey(dbName)) { @@ -280,6 +287,7 @@ public abstract class ExternalCatalog initCatalogLog.addCreateDb(dbId, dbName); } } + dbNameToId = tmpDbNameToId; idToDb = tmpIdToDb; lastUpdateTime = System.currentTimeMillis(); @@ -379,10 +387,18 @@ public abstract class ExternalCatalog return null; } String realDbName = ClusterNamespace.getNameFromFullName(dbName); - if (!dbNameToId.containsKey(realDbName)) { - return null; + if (dbNameToId.containsKey(realDbName)) { + return idToDb.get(dbNameToId.get(realDbName)); + } else { + // This maybe a information_schema db request, and information_schema db name is case insensitive. + // So, we first extract db name to check if it is information_schema. + // Then we reassemble the origin cluster name with lower case db name, + // and finally get information_schema db from the name map. + if (realDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { + return idToDb.get(dbNameToId.get(InfoSchemaDb.DATABASE_NAME)); + } } - return idToDb.get(dbNameToId.get(realDbName)); + return null; } @Nullable @@ -479,6 +495,9 @@ public abstract class ExternalCatalog protected ExternalDatabase<? extends ExternalTable> getDbForInit(String dbName, long dbId, InitCatalogLog.Type logType) { + if (dbName.equals(InfoSchemaDb.DATABASE_NAME)) { + return new ExternalInfoSchemaDatabase(this, dbId); + } switch (logType) { case HMS: return new HMSExternalDatabase(this, dbId, dbName); @@ -625,3 +644,4 @@ public abstract class ExternalCatalog return new ConcurrentHashMap<>(idToDb); } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 3a85fb1edc5..0731181291c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -41,6 +41,7 @@ public class InitDatabaseLog implements Writable { HUDI, PAIMON, TEST, + INFO_SCHEMA_DB, UNKNOWN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 677292c7ca2..b5e68de5b2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -265,10 +265,8 @@ public class InternalCatalog implements CatalogIf<Database> { // So, we first extract db name to check if it is information_schema. // Then we reassemble the origin cluster name with lower case db name, // and finally get information_schema db from the name map. - String fullName = ClusterNamespace.getNameFromFullName(dbName); - if (fullName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { - fullName = fullName.toLowerCase(); - return fullNameToDb.get(fullName); + if (dbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { + return fullNameToDb.get(InfoSchemaDb.DATABASE_NAME); } } return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java new file mode 100644 index 00000000000..4c7c200c5df --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.infoschema; + +import org.apache.doris.catalog.InfoSchemaDb; +import org.apache.doris.catalog.SchemaTable; +import org.apache.doris.catalog.external.ExternalDatabase; +import org.apache.doris.catalog.external.ExternalTable; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.InitDatabaseLog.Type; + +import com.google.common.collect.Lists; + +import java.util.List; + +public class ExternalInfoSchemaDatabase extends ExternalDatabase { + /** + * Create external database. + * + * @param extCatalog The catalog this database belongs to. + * @param dbId The id of this database. + */ + public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) { + super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + } + + public static List<String> listTableNames() { + return Lists.newArrayList(SchemaTable.TABLE_MAP.keySet()); + } + + @Override + protected ExternalTable getExternalTable(String tableName, long tblId, ExternalCatalog catalog) { + return new ExternalInfoSchemaTable(tblId, tableName, catalog); + } + + @Override + public ExternalTable getTableNullable(String name) { + return super.getTableNullable(name.toLowerCase()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java similarity index 55% copy from fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java copy to fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java index 45d6cda9037..3551f050600 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java @@ -15,50 +15,39 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.catalog.external; +package org.apache.doris.datasource.infoschema; +import org.apache.doris.analysis.SchemaTableType; import org.apache.doris.catalog.Column; -import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.catalog.InfoSchemaDb; +import org.apache.doris.catalog.SchemaTable; +import org.apache.doris.catalog.external.ExternalTable; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.thrift.TSchemaTable; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.List; -/** - * TestExternalTable is a table for unit test. - */ -public class TestExternalTable extends ExternalTable { - private static final Logger LOG = LogManager.getLogger(TestExternalTable.class); - - public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE); - } +public class ExternalInfoSchemaTable extends ExternalTable { - @Override - public synchronized void makeSureInitialized() { - super.makeSureInitialized(); - this.objectCreated = true; + public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) { + super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA); } @Override - public String getMysqlType() { - return type.name(); + public List<Column> initSchema() { + makeSureInitialized(); + List<Column> columns = SchemaTable.TABLE_MAP.get(name).getFullSchema(); + return columns; } @Override public TTableDescriptor toThrift() { - makeSureInitialized(); - TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.TEST_EXTERNAL_TABLE, - getFullSchema().size(), - 0, getName(), ""); + TSchemaTable tSchemaTable = new TSchemaTable(SchemaTableType.getThriftType(this.name)); + TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.SCHEMA_TABLE, + getFullSchema().size(), 0, this.name, ""); + tTableDescriptor.setSchemaTable(tSchemaTable); return tTableDescriptor; } - - @Override - public List<Column> initSchema() { - return ((TestExternalCatalog) catalog).mockedSchema(dbName, name); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 713fcc1bdfe..9f8f82ad679 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -43,7 +43,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.external.ExternalTable; @@ -743,7 +742,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla @Override public PlanFragment visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, PlanTranslatorContext context) { - Table table = schemaScan.getTable(); + TableIf table = schemaScan.getTable(); List<Slot> slots = ImmutableList.copyOf(schemaScan.getOutput()); TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java index 24cabfc04ee..b71f6d5c432 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSchemaScan.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.plans.logical; -import org.apache.doris.catalog.SchemaTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; @@ -45,8 +44,8 @@ public class LogicalSchemaScan extends LogicalCatalogRelation { } @Override - public SchemaTable getTable() { - return (SchemaTable) table; + public TableIf getTable() { + return table; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java index fdd7dd1cc66..eab40dca980 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSchemaScan.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.trees.plans.physical; -import org.apache.doris.catalog.SchemaTable; +import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -36,12 +36,12 @@ import java.util.Optional; */ public class PhysicalSchemaScan extends PhysicalCatalogRelation { - public PhysicalSchemaScan(RelationId id, SchemaTable table, List<String> qualifier, + public PhysicalSchemaScan(RelationId id, TableIf table, List<String> qualifier, Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties) { super(id, PlanType.PHYSICAL_SCHEMA_SCAN, table, qualifier, groupExpression, logicalProperties); } - public PhysicalSchemaScan(RelationId id, SchemaTable table, List<String> qualifier, + public PhysicalSchemaScan(RelationId id, TableIf table, List<String> qualifier, Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_SCHEMA_SCAN, table, qualifier, groupExpression, @@ -49,8 +49,8 @@ public class PhysicalSchemaScan extends PhysicalCatalogRelation { } @Override - public SchemaTable getTable() { - return (SchemaTable) table; + public TableIf getTable() { + return table; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 5e4792cf4f0..db49ecdc992 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -58,6 +58,8 @@ import org.apache.doris.catalog.external.MaxComputeExternalDatabase; import org.apache.doris.catalog.external.MaxComputeExternalTable; import org.apache.doris.catalog.external.PaimonExternalDatabase; import org.apache.doris.catalog.external.PaimonExternalTable; +import org.apache.doris.catalog.external.TestExternalDatabase; +import org.apache.doris.catalog.external.TestExternalTable; import org.apache.doris.common.util.RangeUtils; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.EsExternalCatalog; @@ -70,10 +72,13 @@ import org.apache.doris.datasource.iceberg.IcebergGlueExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergHMSExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergHadoopExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergRestExternalCatalog; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog; import org.apache.doris.datasource.paimon.PaimonHMSExternalCatalog; +import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.job.base.AbstractJob; import org.apache.doris.job.extensions.insert.InsertJob; import org.apache.doris.job.extensions.mtmv.MTMVJob; @@ -212,7 +217,9 @@ public class GsonUtils { .registerSubtype(PaimonExternalCatalog.class, PaimonExternalCatalog.class.getSimpleName()) .registerSubtype(PaimonHMSExternalCatalog.class, PaimonHMSExternalCatalog.class.getSimpleName()) .registerSubtype(PaimonFileExternalCatalog.class, PaimonFileExternalCatalog.class.getSimpleName()) - .registerSubtype(MaxComputeExternalCatalog.class, MaxComputeExternalCatalog.class.getSimpleName()); + .registerSubtype(MaxComputeExternalCatalog.class, MaxComputeExternalCatalog.class.getSimpleName()) + .registerSubtype(TestExternalCatalog.class, TestExternalCatalog.class.getSimpleName()); + // routine load data source private static RuntimeTypeAdapterFactory<AbstractDataSourceProperties> rdsTypeAdapterFactory = RuntimeTypeAdapterFactory.of( @@ -231,7 +238,9 @@ public class GsonUtils { .registerSubtype(JdbcExternalDatabase.class, JdbcExternalDatabase.class.getSimpleName()) .registerSubtype(IcebergExternalDatabase.class, IcebergExternalDatabase.class.getSimpleName()) .registerSubtype(PaimonExternalDatabase.class, PaimonExternalDatabase.class.getSimpleName()) - .registerSubtype(MaxComputeExternalDatabase.class, MaxComputeExternalDatabase.class.getSimpleName()); + .registerSubtype(MaxComputeExternalDatabase.class, MaxComputeExternalDatabase.class.getSimpleName()) + .registerSubtype(ExternalInfoSchemaDatabase.class, ExternalInfoSchemaDatabase.class.getSimpleName()) + .registerSubtype(TestExternalDatabase.class, TestExternalDatabase.class.getSimpleName()); private static RuntimeTypeAdapterFactory<TableIf> tblTypeAdapterFactory = RuntimeTypeAdapterFactory.of( TableIf.class, "clazz").registerSubtype(ExternalTable.class, ExternalTable.class.getSimpleName()) @@ -241,7 +250,9 @@ public class GsonUtils { .registerSubtype(JdbcExternalTable.class, JdbcExternalTable.class.getSimpleName()) .registerSubtype(IcebergExternalTable.class, IcebergExternalTable.class.getSimpleName()) .registerSubtype(PaimonExternalTable.class, PaimonExternalTable.class.getSimpleName()) - .registerSubtype(MaxComputeExternalTable.class, MaxComputeExternalTable.class.getSimpleName()); + .registerSubtype(MaxComputeExternalTable.class, MaxComputeExternalTable.class.getSimpleName()) + .registerSubtype(ExternalInfoSchemaTable.class, ExternalInfoSchemaTable.class.getSimpleName()) + .registerSubtype(TestExternalTable.class, TestExternalTable.class.getSimpleName()); // runtime adapter for class "PartitionInfo" private static RuntimeTypeAdapterFactory<PartitionInfo> partitionInfoTypeAdapterFactory diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java index 4a8a488dfc3..fc50c7843f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.SchemaTable; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.planner.external.FederationBackendPolicy; import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.FrontendOptions; @@ -51,8 +50,6 @@ public class SchemaScanNode extends ScanNode { private String schemaDb; private String schemaTable; private String schemaWild; - private String user; - private String userIp; private String frontendIP; private int frontendPort; private String schemaCatalog; @@ -74,19 +71,15 @@ public class SchemaScanNode extends ScanNode { @Override public void finalize(Analyzer analyzer) throws UserException { // Convert predicates to MySQL columns and filters. + schemaCatalog = analyzer.getSchemaCatalog(); schemaDb = analyzer.getSchemaDb(); schemaTable = analyzer.getSchemaTable(); - schemaWild = analyzer.getSchemaWild(); - user = analyzer.getQualifiedUser(); - userIp = analyzer.getContext().getRemoteIP(); frontendIP = FrontendOptions.getLocalHostAddress(); frontendPort = Config.rpc_port; - schemaCatalog = analyzer.getSchemaCatalog(); } @Override public void finalizeForNereids() throws UserException { - // Convert predicates to MySQL columns and filters. frontendIP = FrontendOptions.getLocalHostAddress(); frontendPort = Config.rpc_port; } @@ -104,11 +97,7 @@ public class SchemaScanNode extends ScanNode { msg.schema_scan_node.setDb("SESSION"); } } - if (schemaCatalog != null) { - msg.schema_scan_node.setCatalog(schemaCatalog); - } else if (!Config.infodb_support_ext_catalog) { - msg.schema_scan_node.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME); - } + msg.schema_scan_node.setCatalog(desc.getTable().getDatabase().getCatalog().getName()); msg.schema_scan_node.show_hidden_cloumns = Util.showHiddenColumns(); if (schemaTable != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java index 1e6fff13829..419bb1377a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java @@ -45,6 +45,7 @@ public final class GlobalVariable { public static final String DEFAULT_PASSWORD_LIFETIME = "default_password_lifetime"; public static final String PASSWORD_HISTORY = "password_history"; public static final String VALIDATE_PASSWORD_POLICY = "validate_password_policy"; + public static final String SHOW_FULL_DBNAME_IN_INFO_SCHEMA_DB = "show_full_dbname_in_info_schema_db"; public static final long VALIDATE_PASSWORD_POLICY_DISABLED = 0; public static final long VALIDATE_PASSWORD_POLICY_STRONG = 2; @@ -97,6 +98,12 @@ public final class GlobalVariable { @VariableMgr.VarAttr(name = VALIDATE_PASSWORD_POLICY, flag = VariableMgr.GLOBAL) public static long validatePasswordPolicy = 0; + // If set to true, the db name of TABLE_SCHEMA column in tables in information_schema + // database will be shown as `ctl.db`. Otherwise, show only `db`. + // This is used to compatible with some MySQL tools. + @VariableMgr.VarAttr(name = SHOW_FULL_DBNAME_IN_INFO_SCHEMA_DB, flag = VariableMgr.GLOBAL) + public static boolean showFullDbNameInInfoSchemaDb = false; + // Don't allow to create instance. private GlobalVariable() { } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index bb294b969f1..e006f3761f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -242,6 +242,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -1016,7 +1017,9 @@ public class ShowExecutor { private void handleShowColumn() throws AnalysisException { ShowColumnStmt showStmt = (ShowColumnStmt) stmt; List<List<String>> rows = Lists.newArrayList(); - DatabaseIf db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException(showStmt.getDb()); + String ctl = showStmt.getCtl(); + DatabaseIf db = Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(ctl) + .getDbOrAnalysisException(showStmt.getDb()); TableIf table = db.getTableOrAnalysisException(showStmt.getTable()); PatternMatcher matcher = null; if (showStmt.getPattern() != null) { @@ -1031,7 +1034,7 @@ public class ShowExecutor { continue; } final String columnName = col.getName(); - final String columnType = col.getOriginType().toString(); + final String columnType = col.getOriginType().toString().toLowerCase(Locale.ROOT); final String isAllowNull = col.isAllowNull() ? "YES" : "NO"; final String isKey = col.isKey() ? "YES" : "NO"; final String defaultValue = col.getDefaultValue(); @@ -1068,18 +1071,22 @@ public class ShowExecutor { private void handleShowIndex() throws AnalysisException { ShowIndexStmt showStmt = (ShowIndexStmt) stmt; List<List<String>> rows = Lists.newArrayList(); - DatabaseIf db = ctx.getCurrentCatalog().getDbOrAnalysisException(showStmt.getDbName()); - OlapTable table = db.getOlapTableOrAnalysisException(showStmt.getTableName().getTbl()); - table.readLock(); - try { - List<Index> indexes = table.getIndexes(); - for (Index index : indexes) { - rows.add(Lists.newArrayList(showStmt.getTableName().toString(), "", index.getIndexName(), - "", String.join(",", index.getColumns()), "", "", "", "", - "", index.getIndexType().name(), index.getComment(), index.getPropertiesString())); + DatabaseIf db = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrAnalysisException(showStmt.getTableName().getCtl()) + .getDbOrAnalysisException(showStmt.getDbName()); + if (db instanceof Database) { + OlapTable table = db.getOlapTableOrAnalysisException(showStmt.getTableName().getTbl()); + table.readLock(); + try { + List<Index> indexes = table.getIndexes(); + for (Index index : indexes) { + rows.add(Lists.newArrayList(showStmt.getTableName().toString(), "", index.getIndexName(), + "", String.join(",", index.getColumns()), "", "", "", "", + "", index.getIndexType().name(), index.getComment(), index.getPropertiesString())); + } + } finally { + table.readUnlock(); } - } finally { - table.readUnlock(); } resultSet = new ShowResultSet(showStmt.getMetaData(), rows); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 9738cde7643..d97bf54c502 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -2809,7 +2809,7 @@ public class StmtExecutor { if (coord != null) { coord.close(); } - AuditLogHelper.logAuditLog(context, originStmt.toString(), parsedStmt, getQueryStatisticsForAuditLog(), + AuditLogHelper.logAuditLog(context, originStmt.originStmt, parsedStmt, getQueryStatisticsForAuditLog(), true); if (Config.enable_collect_internal_query_profile) { updateProfile(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index c69862d9984..7b0fef4a3d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -20,7 +20,6 @@ package org.apache.doris.service; import org.apache.doris.analysis.AbstractBackupTableRefClause; import org.apache.doris.analysis.AddPartitionClause; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.ColumnDef; import org.apache.doris.analysis.LabelName; import org.apache.doris.analysis.NativeInsertStmt; import org.apache.doris.analysis.PartitionExprUtil; @@ -30,7 +29,6 @@ import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; -import org.apache.doris.analysis.TypeDef; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.backup.Snapshot; import org.apache.doris.catalog.AutoIncrementGenerator; @@ -85,6 +83,7 @@ import org.apache.doris.qe.ConnectContext.ConnectType; import org.apache.doris.qe.ConnectProcessor; import org.apache.doris.qe.Coordinator; import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.MasterCatalogExecutor; import org.apache.doris.qe.MysqlConnectProcessor; import org.apache.doris.qe.OriginStatement; @@ -382,21 +381,13 @@ public class FrontendServiceImpl implements FrontendService.Iface { Env env = Env.getCurrentEnv(); List<CatalogIf> catalogIfs = Lists.newArrayList(); - // If infodb_support_ext_catalog is true, we will list all catalogs or the specified catalog. - // Otherwise, we will only list internal catalog, or if the specified catalog is internal catalog. - if (Config.infodb_support_ext_catalog) { - if (Strings.isNullOrEmpty(params.catalog)) { - catalogIfs = env.getCatalogMgr().listCatalogs(); - } else { - catalogIfs.add(env.getCatalogMgr() - .getCatalogOrException(params.catalog, - catalog -> new TException("Unknown catalog " + catalog))); - } + // list all catalogs or the specified catalog. + if (Strings.isNullOrEmpty(params.catalog)) { + catalogIfs = env.getCatalogMgr().listCatalogs(); } else { - if (Strings.isNullOrEmpty(params.catalog) - || params.catalog.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) { - catalogIfs.add(env.getInternalCatalog()); - } + catalogIfs.add(env.getCatalogMgr() + .getCatalogOrException(params.catalog, + catalog -> new TException("Unknown catalog " + catalog))); } for (CatalogIf catalog : catalogIfs) { @@ -425,17 +416,17 @@ public class FrontendServiceImpl implements FrontendService.Iface { currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } for (DatabaseIf db : dbs) { - String fullName = db.getFullName(); - if (!env.getAccessManager().checkDbPriv(currentUser, fullName, PrivPredicate.SHOW)) { + String dbName = db.getFullName(); + if (!env.getAccessManager().checkDbPriv(currentUser, dbName, PrivPredicate.SHOW)) { continue; } - if (matcher != null && !matcher.match(ClusterNamespace.getNameFromFullName(fullName))) { + if (matcher != null && !matcher.match(dbName)) { continue; } catalogNames.add(catalog.getName()); - dbNames.add(fullName); + dbNames.add(getMysqlTableSchema(catalog.getName(), dbName)); catalogIds.add(catalog.getId()); dbIds.add(db.getId()); } @@ -448,16 +439,25 @@ public class FrontendServiceImpl implements FrontendService.Iface { return result; } - private static ColumnDef initColumnfromThrift(TColumnDesc tColumnDesc, String comment) { - TypeDef typeDef = TypeDef.createTypeDef(tColumnDesc); - boolean isAllowNull = tColumnDesc.isIsAllowNull(); - ColumnDef.DefaultValue defaultVal = ColumnDef.DefaultValue.NOT_SET; - // Dynamic table's Array default value should be '[]' - if (typeDef.getType().isArrayType()) { - defaultVal = ColumnDef.DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE; + private String getMysqlTableSchema(String ctl, String db) { + if (!GlobalVariable.showFullDbNameInInfoSchemaDb) { + return db; + } + if (ctl.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { + return db; + } + return ctl + "." + db; + } + + private String getDbNameFromMysqlTableSchema(String ctl, String db) { + if (ctl.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { + return db; + } + String[] parts = db.split("\\."); + if (parts.length == 2) { + return parts[1]; } - return new ColumnDef(tColumnDesc.getColumnName(), typeDef, false, null, isAllowNull, false, - defaultVal, comment, true); + return db; } @LogException @@ -486,15 +486,10 @@ public class FrontendServiceImpl implements FrontendService.Iface { } String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME : params.catalog; - if (!Config.infodb_support_ext_catalog - && !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) { - throw new TException("Not support getting external catalog info when " - + "infodb_support_ext_catalog is false"); - } - + String dbName = getDbNameFromMysqlTableSchema(catalogName, params.db); DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr() - .getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog)) - .getDbNullable(params.db); + .getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog: " + catalog)) + .getDbNullable(dbName); if (db != null) { Set<String> tableNames; @@ -503,7 +498,7 @@ public class FrontendServiceImpl implements FrontendService.Iface { for (String tableName : tableNames) { LOG.debug("get table: {}, wait to check", tableName); if (!Env.getCurrentEnv().getAccessManager() - .checkTblPriv(currentUser, params.db, tableName, PrivPredicate.SHOW)) { + .checkTblPriv(currentUser, dbName, tableName, PrivPredicate.SHOW)) { continue; } if (matcher != null && !matcher.match(tableName)) { @@ -546,15 +541,10 @@ public class FrontendServiceImpl implements FrontendService.Iface { if (params.isSetCatalog()) { catalogName = params.catalog; } - if (!Config.infodb_support_ext_catalog - && !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) { - throw new TException("Not support getting external catalog info when " - + "infodb_support_ext_catalog is false"); - } - + String dbName = getDbNameFromMysqlTableSchema(catalogName, params.db); CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); if (catalog != null) { - DatabaseIf db = catalog.getDbNullable(params.db); + DatabaseIf db = catalog.getDbNullable(dbName); if (db != null) { try { List<TableIf> tables; @@ -570,7 +560,7 @@ public class FrontendServiceImpl implements FrontendService.Iface { } } for (TableIf table : tables) { - if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, params.db, + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, dbName, table.getName(), PrivPredicate.SHOW)) { continue; } @@ -644,11 +634,12 @@ public class FrontendServiceImpl implements FrontendService.Iface { CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName); if (catalog != null) { - DatabaseIf db = catalog.getDbNullable(params.db); + String dbName = getDbNameFromMysqlTableSchema(catalogName, params.db); + DatabaseIf db = catalog.getDbNullable(dbName); if (db != null) { List<TableIf> tables = db.getTables(); for (TableIf table : tables) { - if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, params.db, + if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(currentUser, dbName, table.getName(), PrivPredicate.SHOW)) { continue; } @@ -758,21 +749,17 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else { currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } + String dbName = getDbNameFromMysqlTableSchema(params.catalog, params.db); if (!Env.getCurrentEnv().getAccessManager() - .checkTblPriv(currentUser, params.db, params.getTableName(), PrivPredicate.SHOW)) { + .checkTblPriv(currentUser, params.catalog, dbName, params.getTableName(), PrivPredicate.SHOW)) { return result; } String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME : params.catalog; - if (!Config.infodb_support_ext_catalog - && !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) { - throw new TException("Not support getting external catalog info when " - + "infodb_support_ext_catalog is false"); - } DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr() .getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog)) - .getDbNullable(params.db); + .getDbNullable(dbName); if (db != null) { TableIf table = db.getTableNullableIfException(params.getTableName()); if (table != null) { @@ -831,23 +818,19 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else { currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } + String dbName = getDbNameFromMysqlTableSchema(params.catalog, params.db); for (String tableName : tables) { if (!Env.getCurrentEnv().getAccessManager() - .checkTblPriv(currentUser, params.db, tableName, PrivPredicate.SHOW)) { + .checkTblPriv(currentUser, params.catalog, dbName, tableName, PrivPredicate.SHOW)) { return result; } } String catalogName = Strings.isNullOrEmpty(params.catalog) ? InternalCatalog.INTERNAL_CATALOG_NAME : params.catalog; - if (!Config.infodb_support_ext_catalog - && !catalogName.equalsIgnoreCase(InternalCatalog.INTERNAL_CATALOG_NAME)) { - throw new TException("Not support getting external catalog info when " - + "infodb_support_ext_catalog is false"); - } DatabaseIf<TableIf> db = Env.getCurrentEnv().getCatalogMgr() .getCatalogOrException(catalogName, catalog -> new TException("Unknown catalog " + catalog)) - .getDbNullable(params.db); + .getDbNullable(dbName); if (db != null) { for (String tableName : tables) { TableIf table = db.getTableNullableIfException(tableName); @@ -879,7 +862,7 @@ public class FrontendServiceImpl implements FrontendService.Iface { return result; } - public TColumnDesc getColumnDesc(Column column) { + private TColumnDesc getColumnDesc(Column column) { final TColumnDesc desc = new TColumnDesc(column.getName(), column.getDataType().toThrift()); final Integer precision = column.getOriginType().getPrecision(); if (precision != null) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowIndexStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowIndexStmtTest.java index 8ad2ab53960..3d3350aaa31 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowIndexStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowIndexStmtTest.java @@ -43,7 +43,7 @@ public class ShowIndexStmtTest { public void setUp() { analyzer = AccessTestUtil.fetchAdminAnalyzer(true); MockedAuth.mockedAccess(accessManager); - MockedAuth.mockedConnectContext(ctx, "root", "192.168.1.1"); + MockedAuth.mockedConnectContext(ctx, "root", "%"); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 1c61d6b5ad4..c6389f2f764 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -29,6 +29,8 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaTable; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.qe.ConnectContext; @@ -97,6 +99,16 @@ public class RefreshTableTest extends TestWithFeService { // updateTime is equal to schema update time as default long l5 = table.getUpdateTime(); Assertions.assertTrue(l5 == l4); + + // external info schema db + ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase) test1.getDbNullable(InfoSchemaDb.DATABASE_NAME); + Assertions.assertNotNull(infoDb); + for (String tblName : SchemaTable.TABLE_MAP.keySet()) { + ExternalInfoSchemaTable infoTbl = (ExternalInfoSchemaTable) infoDb.getTableNullable(tblName); + Assertions.assertNotNull(infoTbl); + List<Column> schema = infoTbl.getFullSchema(); + Assertions.assertEquals(SchemaTable.TABLE_MAP.get(tblName).getColumns().size(), schema.size()); + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java index 8b64df074e9..999fda60b4a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java @@ -21,10 +21,14 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.external.TestExternalDatabase; import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.utframe.TestWithFeService; @@ -73,15 +77,28 @@ public class RefreshCatalogTest extends TestWithFeService { public void testRefreshCatalog() throws Exception { CatalogIf test1 = env.getCatalogMgr().getCatalog("test1"); List<String> dbNames1 = test1.getDbNames(); - // there are test1.db1 , test1.db2 - Assertions.assertEquals(2, dbNames1.size()); + // there are test1.db1 , test1.db2, information_schema + Assertions.assertEquals(3, dbNames1.size()); // 1.simulate ExternalCatalog adds a new table RefreshCatalogProvider.addData(); // 2.wait for the refresh time of the catalog Thread.sleep(5000); - // there are test1.db1 , test1.db2 , test1.db3 + // there are test1.db1 , test1.db2 , test1.db3, information_schema List<String> dbNames2 = test1.getDbNames(); - Assertions.assertEquals(3, dbNames2.size()); + Assertions.assertEquals(4, dbNames2.size()); + ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase) test1.getDb(InfoSchemaDb.DATABASE_NAME).get(); + Assertions.assertEquals(27, infoDb.getTables().size()); + TestExternalDatabase testDb = (TestExternalDatabase) test1.getDb("db1").get(); + Assertions.assertEquals(2, testDb.getTables().size()); + + String json = GsonUtils.GSON.toJson(env.getCatalogMgr()); + System.out.println(json); + CatalogMgr mgr2 = GsonUtils.GSON.fromJson(json, CatalogMgr.class); + test1 = mgr2.getCatalog("test1"); + infoDb = (ExternalInfoSchemaDatabase) test1.getDb(InfoSchemaDb.DATABASE_NAME).get(); + Assertions.assertEquals(27, infoDb.getTables().size()); + testDb = (TestExternalDatabase) test1.getDb("db1").get(); + Assertions.assertEquals(2, testDb.getTables().size()); } public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider { diff --git a/regression-test/data/external_table_p0/info_schema_db/test_info_schema_db.out b/regression-test/data/external_table_p0/info_schema_db/test_info_schema_db.out new file mode 100644 index 00000000000..72ed194b27d --- /dev/null +++ b/regression-test/data/external_table_p0/info_schema_db/test_info_schema_db.out @@ -0,0 +1,226 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql11 -- +id int NO YES \N +name varchar(20) NO YES \N + +-- !sql12 -- +id int NO YES \N +name varchar(20) NO YES \N + +-- !sql13 -- +id int \N NO DUP \N + +-- !sql14 -- +id int NO DUP \N + +-- !sql15 -- +id int NO YES \N + +-- !sql21 -- +id varchar(50) YES YES \N +name varchar(50) YES YES \N +age int YES YES \N +gender varchar(50) YES YES \N +addr varchar(50) YES YES \N +phone varchar(50) YES YES \N + +-- !sql22 -- +id varchar(50) YES YES \N \N +name varchar(50) YES YES \N \N +age int YES YES \N \N +gender varchar(50) YES YES \N \N +addr varchar(50) YES YES \N \N +phone varchar(50) YES YES \N \N + +-- !sql23 -- +id varchar \N YES \N + +-- !sql24 -- +id varchar YES \N + +-- !sql25 -- +id varchar(50) YES YES \N + +-- !sql31 -- +innertbl + +-- !sql32 -- +innertbl + +-- !sql33 -- +innertbl + +-- !sql41 -- +account_fund + +-- !sql42 -- +account_fund + +-- !sql51 -- +innertbl + +-- !sql52 -- +innertbl + +-- !sql52 -- +innertbl + +-- !sql61 -- +innerdb + +-- !sql62 -- +innerdb + +-- !sql71 -- +tpch1_orc +tpch1_parquet + +-- !sql72 -- + +-- !sql81 -- +innerdb + +-- !sql82 -- +innerdb + +-- !sql91 -- + +-- !sql111 -- + +-- !sql112 -- + +-- !sql113 -- + +-- !sql114 -- + +-- !sql115 -- + +-- !sql116 -- +internal innerdb innertbl + +-- !sql117 -- +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet region +hive_test_infodb tpch1_parquet region +hive_test_infodb tpch1_parquet region +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer + +-- !sql118 -- +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet lineitem +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet partsupp +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet nation +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet part +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet supplier +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet orders +hive_test_infodb tpch1_parquet region +hive_test_infodb tpch1_parquet region +hive_test_infodb tpch1_parquet region +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer +hive_test_infodb tpch1_parquet customer + diff --git a/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out index f6641395575..0ad76d0db4c 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mariadb_jdbc_catalog.out @@ -23,85 +23,33 @@ mariadb_jdbc_catalog 115 abg -- !information_schema -- -ALL_PLUGINS -APPLICABLE_ROLES -CHARACTER_SETS -CHECK_CONSTRAINTS -CLIENT_STATISTICS -COLLATIONS -COLLATION_CHARACTER_SET_APPLICABILITY -COLUMNS -COLUMN_PRIVILEGES -ENABLED_ROLES -ENGINES -EVENTS -FILES -GEOMETRY_COLUMNS -GLOBAL_STATUS -GLOBAL_VARIABLES -INDEX_STATISTICS -INNODB_BUFFER_PAGE -INNODB_BUFFER_PAGE_LRU -INNODB_BUFFER_POOL_STATS -INNODB_CMP -INNODB_CMPMEM -INNODB_CMPMEM_RESET -INNODB_CMP_PER_INDEX -INNODB_CMP_PER_INDEX_RESET -INNODB_CMP_RESET -INNODB_FT_BEING_DELETED -INNODB_FT_CONFIG -INNODB_FT_DEFAULT_STOPWORD -INNODB_FT_DELETED -INNODB_FT_INDEX_CACHE -INNODB_FT_INDEX_TABLE -INNODB_LOCKS -INNODB_LOCK_WAITS -INNODB_METRICS -INNODB_MUTEXES -INNODB_SYS_COLUMNS -INNODB_SYS_DATAFILES -INNODB_SYS_FIELDS -INNODB_SYS_FOREIGN -INNODB_SYS_FOREIGN_COLS -INNODB_SYS_INDEXES -INNODB_SYS_SEMAPHORE_WAITS -INNODB_SYS_TABLES -INNODB_SYS_TABLESPACES -INNODB_SYS_TABLESTATS -INNODB_SYS_VIRTUAL -INNODB_TABLESPACES_ENCRYPTION -INNODB_TABLESPACES_SCRUBBING -INNODB_TRX -KEYWORDS -KEY_CACHES -KEY_COLUMN_USAGE -OPTIMIZER_TRACE -PARAMETERS -PARTITIONS -PLUGINS -PROCESSLIST -PROFILING -REFERENTIAL_CONSTRAINTS -ROUTINES -SCHEMATA -SCHEMA_PRIVILEGES -SESSION_STATUS -SESSION_VARIABLES -SPATIAL_REF_SYS -SQL_FUNCTIONS -STATISTICS -SYSTEM_VARIABLES -TABLES -TABLESPACES -TABLE_CONSTRAINTS -TABLE_PRIVILEGES -TABLE_STATISTICS -TRIGGERS -USER_PRIVILEGES -USER_STATISTICS -VIEWS -user_variables +character_sets +collations +column_privileges +column_statistics +columns +engines +events +files +global_variables +key_column_usage +metadata_name_ids +parameters +partitions +profiling +referential_constraints +routines +rowsets +schema_privileges +schemata +session_variables +statistics +table_constraints +table_privileges +tables +triggers +user_privileges +views -- !auto_default_t -- 0 diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out index c81d86828c6..70957b94e2e 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog.out @@ -169,67 +169,33 @@ bca 2022-11-02 2022-11-02 8012 vivo 2 2 -- !information_schema -- -CHARACTER_SETS -COLLATIONS -COLLATION_CHARACTER_SET_APPLICABILITY -COLUMNS -COLUMN_PRIVILEGES -ENGINES -EVENTS -FILES -GLOBAL_STATUS -GLOBAL_VARIABLES -INNODB_BUFFER_PAGE -INNODB_BUFFER_PAGE_LRU -INNODB_BUFFER_POOL_STATS -INNODB_CMP -INNODB_CMPMEM -INNODB_CMPMEM_RESET -INNODB_CMP_PER_INDEX -INNODB_CMP_PER_INDEX_RESET -INNODB_CMP_RESET -INNODB_FT_BEING_DELETED -INNODB_FT_CONFIG -INNODB_FT_DEFAULT_STOPWORD -INNODB_FT_DELETED -INNODB_FT_INDEX_CACHE -INNODB_FT_INDEX_TABLE -INNODB_LOCKS -INNODB_LOCK_WAITS -INNODB_METRICS -INNODB_SYS_COLUMNS -INNODB_SYS_DATAFILES -INNODB_SYS_FIELDS -INNODB_SYS_FOREIGN -INNODB_SYS_FOREIGN_COLS -INNODB_SYS_INDEXES -INNODB_SYS_TABLES -INNODB_SYS_TABLESPACES -INNODB_SYS_TABLESTATS -INNODB_SYS_VIRTUAL -INNODB_TEMP_TABLE_INFO -INNODB_TRX -KEY_COLUMN_USAGE -OPTIMIZER_TRACE -PARAMETERS -PARTITIONS -PLUGINS -PROCESSLIST -PROFILING -REFERENTIAL_CONSTRAINTS -ROUTINES -SCHEMATA -SCHEMA_PRIVILEGES -SESSION_STATUS -SESSION_VARIABLES -STATISTICS -TABLES -TABLESPACES -TABLE_CONSTRAINTS -TABLE_PRIVILEGES -TRIGGERS -USER_PRIVILEGES -VIEWS +character_sets +collations +column_privileges +column_statistics +columns +engines +events +files +global_variables +key_column_usage +metadata_name_ids +parameters +partitions +profiling +referential_constraints +routines +rowsets +schema_privileges +schemata +session_variables +statistics +table_constraints +table_privileges +tables +triggers +user_privileges +views -- !auto_default_t -- 0 @@ -281,9 +247,11 @@ doris3 20 -- !specified_database_1 -- doris_test +information_schema -- !specified_database_2 -- doris_test +information_schema -- !specified_database_3 -- information_schema @@ -294,6 +262,7 @@ show_test_do_not_modify sys -- !specified_database_4 -- +information_schema -- !ex_tb1 -- {"k1":"v1", "k2":"v2"} diff --git a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out index 51f6d728eaa..a66c418fede 100644 --- a/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out +++ b/regression-test/data/external_table_p0/jdbc/test_mysql_jdbc_catalog_nereids.out @@ -160,67 +160,33 @@ bca 2022-11-02 2022-11-02 8012 vivo 123456789012345678901234567890123.12345 12345678901234567890123456789012.12345 1234567890123456789012345678901234.12345 123456789012345678901234567890123.12345 123456789012345678901234567890123456789012345678901234567890.12345 123456789012345678901234567890123456789012345678901234567890.12345 -- !information_schema -- -CHARACTER_SETS -COLLATIONS -COLLATION_CHARACTER_SET_APPLICABILITY -COLUMNS -COLUMN_PRIVILEGES -ENGINES -EVENTS -FILES -GLOBAL_STATUS -GLOBAL_VARIABLES -INNODB_BUFFER_PAGE -INNODB_BUFFER_PAGE_LRU -INNODB_BUFFER_POOL_STATS -INNODB_CMP -INNODB_CMPMEM -INNODB_CMPMEM_RESET -INNODB_CMP_PER_INDEX -INNODB_CMP_PER_INDEX_RESET -INNODB_CMP_RESET -INNODB_FT_BEING_DELETED -INNODB_FT_CONFIG -INNODB_FT_DEFAULT_STOPWORD -INNODB_FT_DELETED -INNODB_FT_INDEX_CACHE -INNODB_FT_INDEX_TABLE -INNODB_LOCKS -INNODB_LOCK_WAITS -INNODB_METRICS -INNODB_SYS_COLUMNS -INNODB_SYS_DATAFILES -INNODB_SYS_FIELDS -INNODB_SYS_FOREIGN -INNODB_SYS_FOREIGN_COLS -INNODB_SYS_INDEXES -INNODB_SYS_TABLES -INNODB_SYS_TABLESPACES -INNODB_SYS_TABLESTATS -INNODB_SYS_VIRTUAL -INNODB_TEMP_TABLE_INFO -INNODB_TRX -KEY_COLUMN_USAGE -OPTIMIZER_TRACE -PARAMETERS -PARTITIONS -PLUGINS -PROCESSLIST -PROFILING -REFERENTIAL_CONSTRAINTS -ROUTINES -SCHEMATA -SCHEMA_PRIVILEGES -SESSION_STATUS -SESSION_VARIABLES -STATISTICS -TABLES -TABLESPACES -TABLE_CONSTRAINTS -TABLE_PRIVILEGES -TRIGGERS -USER_PRIVILEGES -VIEWS +character_sets +collations +column_privileges +column_statistics +columns +engines +events +files +global_variables +key_column_usage +metadata_name_ids +parameters +partitions +profiling +referential_constraints +routines +rowsets +schema_privileges +schemata +session_variables +statistics +table_constraints +table_privileges +tables +triggers +user_privileges +views -- !test_insert1 -- doris1 18 diff --git a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out index afb3a35b994..06a38d57512 100644 --- a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out @@ -214,9 +214,11 @@ TINYINT_VALUE2 SMALLINT Yes false \N NONE -- !specified_database -- DORIS_TEST +information_schema -- !specified_database -- DORIS_TEST +information_schema -- !lower_case_table_names1 -- 1 111 123 7456123.89 573 34 673.43 34.1264 60.0 23.231 diff --git a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out index d63ee2c0bad..124393ea0e4 100644 --- a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out @@ -2266,9 +2266,11 @@ varchar_value TEXT Yes false \N NONE -- !specified_database_1 -- doris_test +information_schema -- !specified_database_2 -- doris_test +information_schema -- !specified_database_3 -- catalog_pg_test @@ -2278,6 +2280,7 @@ pg_toast public -- !specified_database_4 -- +information_schema -- !test_old -- 123 abc diff --git a/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out index 7624abc4589..b7becd6cdf0 100644 --- a/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out @@ -120,5 +120,6 @@ db_owner db_securityadmin dbo guest +information_schema sys diff --git a/regression-test/suites/external_table_p0/hive/test_show_columns.groovy b/regression-test/suites/external_table_p0/hive/test_show_columns.groovy deleted file mode 100644 index e7c751839a4..00000000000 --- a/regression-test/suites/external_table_p0/hive/test_show_columns.groovy +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_show_colums", "p0,external,hive,external_docker,external_docker_hive") { - - sql """create database if not exists db1""" - sql """drop table if exists db1.test_full_column""" - sql '''create table db1.test_full_column ( - id int not null, - name varchar(20) not null - ) - distributed by hash(id) buckets 4 - properties ( - "replication_num"="1" - ); - ''' - result = sql """show full columns from db1.test_full_column""" - assertEquals(result.size(), 2) - assertEquals(result[0][0], "id") - - String enabled = context.config.otherConfigs.get("enableHiveTest") - if (enabled != null && enabled.equalsIgnoreCase("true")) { - String hms_port = context.config.otherConfigs.get("hms_port") - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") - - String catalog_name = "hive_test_other" - - sql """drop catalog if exists ${catalog_name}""" - sql """create catalog if not exists ${catalog_name} properties ( - "type"="hms", - 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' - );""" - - sql """switch ${catalog_name}""" - - result = sql """show full columns from internal.db1.test_full_column""" - assertEquals(result.size(), 2) - assertEquals(result[0][0], "id") - - sql """switch internal""" - - result = sql """show full columns from internal.db1.test_full_column""" - assertEquals(result.size(), 2) - assertEquals(result[1][0], "name") - - sql """drop catalog if exists ${catalog_name}""" - } -} diff --git a/regression-test/suites/external_table_p0/info_schema_db/test_info_schema_db.groovy b/regression-test/suites/external_table_p0/info_schema_db/test_info_schema_db.groovy new file mode 100644 index 00000000000..7ab7b6d6c56 --- /dev/null +++ b/regression-test/suites/external_table_p0/info_schema_db/test_info_schema_db.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_info_schema_db", "p0,external,hive,external_docker,external_docker_hive") { + + String catalog_name = "hive_test_infodb"; + String innerdb = "innerdb"; + String innertbl = "innertbl"; + sql """drop database if exists ${innerdb}"""; + sql """create database if not exists ${innerdb}""" + sql """create table ${innerdb}.${innertbl} ( + id int not null, + name varchar(20) not null + ) + distributed by hash(id) buckets 4 + properties ( + "replication_num"="1" + ); + """ + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return; + } + + String hms_port = context.config.otherConfigs.get("hms_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + // 1. test show columns + // test internal catalog + sql "switch internal" + qt_sql11 """show columns from ${innerdb}.${innertbl}""" + qt_sql12 """show full columns from ${innerdb}.${innertbl}""" + qt_sql13 """show full columns from ${innerdb}.${innertbl} where field = 'id'""" + qt_sql14 """show columns from ${innerdb}.${innertbl} where field = 'id'""" + qt_sql15 """show columns from ${innerdb}.${innertbl} like '%i%'""" + + // test external catalog + sql "switch ${catalog_name}" + qt_sql21 """show columns from `default`.student;""" + qt_sql22 """show full columns from `default`.student;""" + qt_sql23 """show full columns from `default`.student where field like '%i%'""" + qt_sql24 """show columns from `default`.student where field = 'id'""" + qt_sql25 """show columns from `default`.student like '%i%'""" + + // 2. test show tables + // test internal catalog + sql "use internal.${innerdb}" + qt_sql31 """show tables""" + qt_sql32 """show tables like 'inner%'""" + qt_sql33 """show tables from ${innerdb} where table_name = '${innertbl}'""" + + // test external catalog + sql "use ${catalog_name}.`default`" + qt_sql41 """show tables like '%account%'""" + qt_sql42 """show tables where table_name = 'account_fund';""" + + // test cross catalog + qt_sql51 """show tables from internal.${innerdb}""" + qt_sql52 """show tables from internal.${innerdb} like 'inner%'""" + qt_sql52 """show tables from internal.${innerdb} where table_name = '${innertbl}'""" + + // 3. test show databases + // test internal catalog + sql "switch internal" + qt_sql61 """show databases like '%${innerdb}%'""" + qt_sql62 """show databases where schema_name='${innerdb}'""" + + // test external catalog + sql "switch ${catalog_name}" + qt_sql71 """show databases like 'tpch%'""" + qt_sql72 """show databases where schema_name='hive.tpch1_orc'""" + + // test cross catalog + qt_sql81 """show databases from internal where schema_name='${innerdb}'""" + qt_sql82 """show databases from internal like '${innerdb}'""" + + // 4. test show index for external catalog + qt_sql91 """show index from ${catalog_name}.tpch1_parquet.lineitem""" + + // 5. test show table status + sql "use ${catalog_name}.tpch1_parquet" + def result101 = order_sql """show table status""" + assertEquals(8, result101.size()); + assertEquals("customer", result101[0][0]); + assertEquals("supplier", result101[7][0]); + + def result102 = order_sql """show table status like '%line%'""" + assertEquals(1, result102.size(), 1); + assertEquals("lineitem", result102[0][0]); + + def result103 = order_sql """show table status where name='lineitem'""" + assertEquals(1, result103.size()); + assertEquals("lineitem", result103[0][0]); + + def result104 = order_sql """show table status from ${catalog_name}.tpch1_parquet where name='lineitem'""" + assertEquals(result104.size(), 1); + assertEquals("lineitem", result104[0][0]); + + def result105 = order_sql """show table status from internal.${innerdb} where name='${innertbl}'""" + assertEquals(1, result105.size()); + assertEquals("innertbl", result105[0][0]); + + // 6. test info db + sql "switch internal" + qt_sql111 """select * from information_schema.tables where table_catalog!='internal'""" + qt_sql112 """select * from INFORMATION_SCHEMA.tables where table_catalog!='internal'""" + qt_sql113 """select * from ${catalog_name}.information_schema.tables where table_catalog!='${catalog_name}'""" + qt_sql114 """select * from information_schema.columns where table_catalog!='internal'""" + qt_sql115 """select * from ${catalog_name}.information_schema.columns where table_catalog!='${catalog_name}'""" + qt_sql116 """select table_catalog, table_schema, table_name from information_schema.tables where table_schema='${innerdb}'""" + qt_sql117 """select table_catalog, table_schema, table_name from ${catalog_name}.information_schema.columns where table_schema='tpch1_parquet'""" + qt_sql118 """select table_catalog, table_schema, table_name from ${catalog_name}.INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='tpch1_parquet'""" +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
