This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 5b3b2cec80b [feat](metatable) support table$partitions for hive table
(#40774) (#41230)
5b3b2cec80b is described below
commit 5b3b2cec80b717956ac50050c1cbdc68f130b750
Author: Mingyu Chen <[email protected]>
AuthorDate: Wed Sep 25 09:52:07 2024 +0800
[feat](metatable) support table$partitions for hive table (#40774) (#41230)
bp #40774
and pick part of #34552, add `isPartitionedTable()` interface in `TableIf`
---
be/src/vec/exec/scan/vmeta_scanner.cpp | 170 +++++++++++--------
be/src/vec/exec/scan/vmeta_scanner.h | 2 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 4 +-
.../org/apache/doris/analysis/DescribeStmt.java | 28 +++-
.../doris/catalog/BuiltinTableValuedFunctions.java | 4 +-
.../main/java/org/apache/doris/catalog/Table.java | 6 -
.../java/org/apache/doris/catalog/TableIf.java | 6 +-
.../org/apache/doris/common/util/TimeUtils.java | 35 ++++
.../org/apache/doris/datasource/CatalogIf.java | 23 +++
.../doris/datasource/hive/HMSExternalCatalog.java | 92 +++++++++++
.../doris/datasource/hive/HMSExternalTable.java | 6 +
.../doris/nereids/parser/LogicalPlanBuilder.java | 7 +-
.../doris/nereids/rules/analysis/BindRelation.java | 24 ++-
.../functions/table/PartitionValues.java | 76 +++++++++
.../visitor/TableValuedFunctionVisitor.java | 5 +
.../apache/doris/nereids/util/RelationUtil.java | 6 +-
.../doris/tablefunction/MetadataGenerator.java | 143 ++++++++++++++--
.../PartitionValuesTableValuedFunction.java | 180 +++++++++++++++++++++
.../PartitionsTableValuedFunction.java | 2 +-
.../doris/tablefunction/TableValuedFunctionIf.java | 2 +
.../apache/doris/common/util/TimeUtilsTest.java | 51 ++++++
.../nereids/rules/analysis/BindRelationTest.java | 9 ++
gensrc/thrift/Data.thrift | 1 +
gensrc/thrift/FrontendService.thrift | 1 +
gensrc/thrift/PlanNodes.thrift | 8 +
gensrc/thrift/Types.thrift | 3 +-
.../hive/test_hive_partition_values_tvf.out | 120 ++++++++++++++
.../hive/test_hive_partition_values_tvf.groovy | 144 +++++++++++++++++
28 files changed, 1062 insertions(+), 96 deletions(-)
diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp
b/be/src/vec/exec/scan/vmeta_scanner.cpp
index 02cf15e1af3..64981d0f483 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.cpp
+++ b/be/src/vec/exec/scan/vmeta_scanner.cpp
@@ -85,16 +85,6 @@ Status VMetaScanner::prepare(RuntimeState* state, const
VExprContextSPtrs& conju
VLOG_CRITICAL << "VMetaScanner::prepare";
RETURN_IF_ERROR(VScanner::prepare(_state, conjuncts));
_tuple_desc = state->desc_tbl().get_tuple_descriptor(_tuple_id);
- bool has_col_nullable =
- std::any_of(std::begin(_tuple_desc->slots()),
std::end(_tuple_desc->slots()),
- [](SlotDescriptor* slot_desc) { return
slot_desc->is_nullable(); });
-
- if (has_col_nullable) {
- // We do not allow any columns to be Nullable here, since FE can not
- // transmit a NULL value to BE, so we can not distinguish a empty
string
- // from a NULL value.
- return Status::InternalError("Logical error, VMetaScanner do not allow
ColumnNullable");
- }
return Status::OK();
}
@@ -157,60 +147,90 @@ Status VMetaScanner::_fill_block_with_remote_data(const
std::vector<MutableColum
}
for (int _row_idx = 0; _row_idx < _batch_data.size(); _row_idx++) {
- // No need to check nullable column since no nullable column is
- // guaranteed in VMetaScanner::prepare
vectorized::IColumn* col_ptr = columns[col_idx].get();
- switch (slot_desc->type().type) {
- case TYPE_BOOLEAN: {
- bool data =
_batch_data[_row_idx].column_value[col_idx].boolVal;
-
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt8>*>(col_ptr)
- ->insert_value((uint8_t)data);
- break;
- }
- case TYPE_INT: {
- int64_t data =
_batch_data[_row_idx].column_value[col_idx].intVal;
-
reinterpret_cast<vectorized::ColumnVector<vectorized::Int32>*>(col_ptr)
- ->insert_value(data);
- break;
- }
- case TYPE_BIGINT: {
- int64_t data =
_batch_data[_row_idx].column_value[col_idx].longVal;
-
reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)
- ->insert_value(data);
- break;
- }
- case TYPE_FLOAT: {
- double data =
_batch_data[_row_idx].column_value[col_idx].doubleVal;
-
reinterpret_cast<vectorized::ColumnVector<vectorized::Float32>*>(col_ptr)
- ->insert_value(data);
- break;
- }
- case TYPE_DOUBLE: {
- double data =
_batch_data[_row_idx].column_value[col_idx].doubleVal;
-
reinterpret_cast<vectorized::ColumnVector<vectorized::Float64>*>(col_ptr)
- ->insert_value(data);
- break;
- }
- case TYPE_DATETIMEV2: {
- uint64_t data =
_batch_data[_row_idx].column_value[col_idx].longVal;
-
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt64>*>(col_ptr)
- ->insert_value(data);
- break;
- }
- case TYPE_STRING:
- case TYPE_CHAR:
- case TYPE_VARCHAR: {
- std::string data =
_batch_data[_row_idx].column_value[col_idx].stringVal;
-
reinterpret_cast<vectorized::ColumnString*>(col_ptr)->insert_data(data.c_str(),
-
data.length());
- break;
- }
- default: {
- std::string error_msg =
- fmt::format("Invalid column type {} on column: {}.",
- slot_desc->type().debug_string(),
slot_desc->col_name());
- return Status::InternalError(std::string(error_msg));
- }
+ TCell& cell = _batch_data[_row_idx].column_value[col_idx];
+ if (cell.__isset.isNull && cell.isNull) {
+ DCHECK(slot_desc->is_nullable())
+ << "cell is null but column is not nullable: " <<
slot_desc->col_name();
+ auto& null_col = reinterpret_cast<ColumnNullable&>(*col_ptr);
+ null_col.get_nested_column().insert_default();
+ null_col.get_null_map_data().push_back(1);
+ } else {
+ if (slot_desc->is_nullable()) {
+ auto& null_col =
reinterpret_cast<ColumnNullable&>(*col_ptr);
+ null_col.get_null_map_data().push_back(0);
+ col_ptr = null_col.get_nested_column_ptr();
+ }
+ switch (slot_desc->type().type) {
+ case TYPE_BOOLEAN: {
+ bool data = cell.boolVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt8>*>(col_ptr)
+ ->insert_value((uint8_t)data);
+ break;
+ }
+ case TYPE_TINYINT: {
+ int8_t data = (int8_t)cell.intVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::Int8>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_SMALLINT: {
+ int16_t data = (int16_t)cell.intVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::Int16>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_INT: {
+ int32_t data = cell.intVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::Int32>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_BIGINT: {
+ int64_t data = cell.longVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_FLOAT: {
+ double data = cell.doubleVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::Float32>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_DOUBLE: {
+ double data = cell.doubleVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::Float64>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_DATEV2: {
+ uint32_t data = (uint32_t)cell.longVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt32>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_DATETIMEV2: {
+ uint64_t data = cell.longVal;
+
reinterpret_cast<vectorized::ColumnVector<vectorized::UInt64>*>(col_ptr)
+ ->insert_value(data);
+ break;
+ }
+ case TYPE_STRING:
+ case TYPE_CHAR:
+ case TYPE_VARCHAR: {
+ std::string data = cell.stringVal;
+
reinterpret_cast<vectorized::ColumnString*>(col_ptr)->insert_data(
+ data.c_str(), data.length());
+ break;
+ }
+ default: {
+ std::string error_msg =
+ fmt::format("Invalid column type {} on column:
{}.",
+ slot_desc->type().debug_string(),
slot_desc->col_name());
+ return Status::InternalError(std::string(error_msg));
+ }
+ }
}
}
}
@@ -252,6 +272,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange&
meta_scan_range) {
case TMetadataType::TASKS:
RETURN_IF_ERROR(_build_tasks_metadata_request(meta_scan_range,
&request));
break;
+ case TMetadataType::PARTITION_VALUES:
+
RETURN_IF_ERROR(_build_partition_values_metadata_request(meta_scan_range,
&request));
+ break;
default:
_meta_eos = true;
return Status::OK();
@@ -472,6 +495,27 @@ Status VMetaScanner::_build_tasks_metadata_request(const
TMetaScanRange& meta_sc
return Status::OK();
}
+Status VMetaScanner::_build_partition_values_metadata_request(
+ const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest*
request) {
+ VLOG_CRITICAL << "VMetaScanner::_build_partition_values_metadata_request";
+ if (!meta_scan_range.__isset.partition_values_params) {
+ return Status::InternalError(
+ "Can not find TPartitionValuesMetadataParams from
meta_scan_range.");
+ }
+
+ // create request
+ request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE);
+
+ // create TMetadataTableRequestParams
+ TMetadataTableRequestParams metadata_table_params;
+ metadata_table_params.__set_metadata_type(TMetadataType::PARTITION_VALUES);
+ metadata_table_params.__set_partition_values_metadata_params(
+ meta_scan_range.partition_values_params);
+
+ request->__set_metada_table_params(metadata_table_params);
+ return Status::OK();
+}
+
Status VMetaScanner::close(RuntimeState* state) {
VLOG_CRITICAL << "VMetaScanner::close";
RETURN_IF_ERROR(VScanner::close(state));
diff --git a/be/src/vec/exec/scan/vmeta_scanner.h
b/be/src/vec/exec/scan/vmeta_scanner.h
index b44511826f2..600624ee636 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.h
+++ b/be/src/vec/exec/scan/vmeta_scanner.h
@@ -93,6 +93,8 @@ private:
TFetchSchemaTableDataRequest*
request);
Status _build_queries_metadata_request(const TMetaScanRange&
meta_scan_range,
TFetchSchemaTableDataRequest*
request);
+ Status _build_partition_values_metadata_request(const TMetaScanRange&
meta_scan_range,
+
TFetchSchemaTableDataRequest* request);
bool _meta_eos;
TupleId _tuple_id;
TUserIdentity _user_identity;
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index a0e3a4582d9..9c78ab4e997 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -538,8 +538,8 @@ relationPrimary
| LEFT_PAREN query RIGHT_PAREN tableAlias lateralView*
#aliasedQuery
| tvfName=identifier LEFT_PAREN
(properties=propertyItemList)?
- RIGHT_PAREN tableAlias
#tableValuedFunction
- | LEFT_PAREN relations RIGHT_PAREN
#relationList
+ RIGHT_PAREN tableAlias
#tableValuedFunction
+ | LEFT_PAREN relations RIGHT_PAREN
#relationList
;
materializedViewName
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
index 45883fd6f46..67155c8591e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
@@ -32,6 +32,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.proc.IndexSchemaProcNode;
import org.apache.doris.common.proc.ProcNodeInterface;
@@ -44,6 +45,7 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
@@ -55,6 +57,7 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
public class DescribeStmt extends ShowStmt {
@@ -123,6 +126,25 @@ public class DescribeStmt extends ShowStmt {
@Override
public void analyze(Analyzer analyzer) throws UserException {
+ // First handle meta table.
+ // It will convert this to corresponding table valued functions
+ // eg: DESC table$partitions -> partition_values(...)
+ if (dbTableName != null) {
+ dbTableName.analyze(analyzer);
+ CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl());
+ Pair<String, String> sourceTableNameWithMetaName =
catalog.getSourceTableNameWithMetaTableName(
+ dbTableName.getTbl());
+ if (!Strings.isNullOrEmpty(sourceTableNameWithMetaName.second)) {
+ isTableValuedFunction = true;
+ Optional<TableValuedFunctionRef> optTvfRef =
catalog.getMetaTableFunctionRef(
+ dbTableName.getDb(), dbTableName.getTbl());
+ if (!optTvfRef.isPresent()) {
+ throw new AnalysisException("meta table not found: " +
sourceTableNameWithMetaName.second);
+ }
+ tableValuedFunctionRef = optTvfRef.get();
+ }
+ }
+
if (!isAllTables && isTableValuedFunction) {
tableValuedFunctionRef.analyze(analyzer);
List<Column> columns =
tableValuedFunctionRef.getTable().getBaseSchema();
@@ -148,8 +170,6 @@ public class DescribeStmt extends ShowStmt {
}
}
- dbTableName.analyze(analyzer);
-
if (!Env.getCurrentEnv().getAccessManager()
.checkTblPriv(ConnectContext.get(), dbTableName,
PrivPredicate.SHOW)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR,
"DESCRIBE",
@@ -159,8 +179,7 @@ public class DescribeStmt extends ShowStmt {
CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalogOrAnalysisException(dbTableName.getCtl());
DatabaseIf db = catalog.getDbOrAnalysisException(dbTableName.getDb());
- TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl());
-
+ TableIf table = db.getTableOrDdlException(dbTableName.getTbl());
table.readLock();
try {
if (!isAllTables) {
@@ -387,3 +406,4 @@ public class DescribeStmt extends ShowStmt {
return emptyRow;
}
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
index db66c260e56..88c98162093 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
@@ -29,6 +29,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.table.Jobs;
import org.apache.doris.nereids.trees.expressions.functions.table.Local;
import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos;
import org.apache.doris.nereids.trees.expressions.functions.table.Numbers;
+import
org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues;
import org.apache.doris.nereids.trees.expressions.functions.table.Partitions;
import org.apache.doris.nereids.trees.expressions.functions.table.Query;
import org.apache.doris.nereids.trees.expressions.functions.table.S3;
@@ -59,7 +60,8 @@ public class BuiltinTableValuedFunctions implements
FunctionHelper {
tableValued(Partitions.class, "partitions"),
tableValued(Jobs.class, "jobs"),
tableValued(Tasks.class, "tasks"),
- tableValued(Query.class, "query")
+ tableValued(Query.class, "query"),
+ tableValued(PartitionValues.class, "partition_values")
);
public static final BuiltinTableValuedFunctions INSTANCE = new
BuiltinTableValuedFunctions();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index 8cfe52356cf..862d6c1878e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -500,12 +500,6 @@ public abstract class Table extends MetaObject implements
Writable, TableIf {
this.createTime = in.readLong();
}
- // return if this table is partitioned.
- // For OlapTable, return true only if its partition type is RANGE or HASH
- public boolean isPartitionedTable() {
- return false;
- }
-
// return if this table is partitioned, for planner.
// For OlapTable ture when is partitioned, or distributed by hash when no
partition
public boolean isPartitionDistributed() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index be8f1fe6f94..d42a32ef8d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -66,8 +66,6 @@ public interface TableIf {
default void readUnlock() {
}
- ;
-
default void writeLock() {
}
@@ -556,4 +554,8 @@ public interface TableIf {
default Set<String> getDistributionColumnNames() {
return Sets.newHashSet();
}
+
+ default boolean isPartitionedTable() {
+ return false;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
index 55b2ed1c58d..e7066846c30 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java
@@ -340,4 +340,39 @@ public class TimeUtils {
parts.length > 3 ? String.format(" %02d:%02d:%02d",
Integer.parseInt(parts[3]),
Integer.parseInt(parts[4]),
Integer.parseInt(parts[5])) : "");
}
+
+
+ // Refer to be/src/vec/runtime/vdatetime_value.h
+ public static long convertToDateTimeV2(
+ int year, int month, int day, int hour, int minute, int second,
int microsecond) {
+ return (long) microsecond | (long) second << 20 | (long) minute << 26
| (long) hour << 32
+ | (long) day << 37 | (long) month << 42 | (long) year << 46;
+ }
+
+ // Refer to be/src/vec/runtime/vdatetime_value.h
+ public static long convertToDateV2(
+ int year, int month, int day) {
+ return (long) day | (long) month << 5 | (long) year << 9;
+ }
+
+ public static long convertStringToDateTimeV2(String dateTimeStr, int
scale) {
+ String format = "yyyy-MM-dd HH:mm:ss";
+ if (scale > 0) {
+ format += ".";
+ for (int i = 0; i < scale; i++) {
+ format += "S";
+ }
+ }
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format);
+ LocalDateTime dateTime =
TimeUtils.formatDateTimeAndFullZero(dateTimeStr, formatter);
+ return convertToDateTimeV2(dateTime.getYear(),
dateTime.getMonthValue(), dateTime.getDayOfMonth(),
+ dateTime.getHour(), dateTime.getMinute(),
dateTime.getSecond(), dateTime.getNano() / 1000);
+ }
+
+ public static long convertStringToDateV2(String dateStr) {
+ DateTimeFormatter formatter =
DateTimeFormatter.ofPattern("yyyy-MM-dd");
+ LocalDateTime dateTime = TimeUtils.formatDateTimeAndFullZero(dateStr,
formatter);
+ return convertToDateV2(dateTime.getYear(), dateTime.getMonthValue(),
dateTime.getDayOfMonth());
+ }
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index a79897c67df..41cb44ef0b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DropDbStmt;
import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TableValuedFunctionRef;
import org.apache.doris.analysis.TruncateTableStmt;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
@@ -30,7 +31,9 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
+import
org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -197,4 +200,24 @@ public interface CatalogIf<T extends DatabaseIf> {
void dropTable(DropTableStmt stmt) throws DdlException;
void truncateTable(TruncateTableStmt truncateTableStmt) throws
DdlException;
+
+ /**
+ * Try to parse meta table name from table name.
+ * Some catalog allow querying meta table like "table_name$partitions".
+ * Catalog can override this method to parse meta table name from table
name.
+ *
+ * @param tableName table name like "table_name" or "table_name$partitions"
+ * @return pair of source table name and meta table name
+ */
+ default Pair<String, String> getSourceTableNameWithMetaTableName(String
tableName) {
+ return Pair.of(tableName, "");
+ }
+
+ default Optional<TableValuedFunction> getMetaTableFunction(String dbName,
String sourceNameWithMetaName) {
+ return Optional.empty();
+ }
+
+ default Optional<TableValuedFunctionRef> getMetaTableFunctionRef(String
dbName, String sourceNameWithMetaName) {
+ return Optional.empty();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 5faf1f2bb6e..5173b414b8a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -17,11 +17,13 @@
package org.apache.doris.datasource.hive;
+import org.apache.doris.analysis.TableValuedFunctionRef;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.common.security.authentication.HadoopAuthenticator;
@@ -39,10 +41,15 @@ import
org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.fs.FileSystemProvider;
import org.apache.doris.fs.FileSystemProviderImpl;
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import
org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues;
+import
org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
import org.apache.doris.transaction.TransactionManagerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import lombok.Getter;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -52,6 +59,7 @@ import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.ThreadPoolExecutor;
/**
@@ -277,6 +285,36 @@ public class HMSExternalCatalog extends ExternalCatalog {
}
}
+ @Override
+ public Pair<String, String> getSourceTableNameWithMetaTableName(String
tableName) {
+ for (MetaTableFunction metaFunction : MetaTableFunction.values()) {
+ if (metaFunction.containsMetaTable(tableName)) {
+ return Pair.of(metaFunction.getSourceTableName(tableName),
metaFunction.name().toLowerCase());
+ }
+ }
+ return Pair.of(tableName, "");
+ }
+
+ @Override
+ public Optional<TableValuedFunction> getMetaTableFunction(String dbName,
String sourceNameWithMetaName) {
+ for (MetaTableFunction metaFunction : MetaTableFunction.values()) {
+ if (metaFunction.containsMetaTable(sourceNameWithMetaName)) {
+ return Optional.of(metaFunction.createFunction(name, dbName,
sourceNameWithMetaName));
+ }
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<TableValuedFunctionRef> getMetaTableFunctionRef(String
dbName, String sourceNameWithMetaName) {
+ for (MetaTableFunction metaFunction : MetaTableFunction.values()) {
+ if (metaFunction.containsMetaTable(sourceNameWithMetaName)) {
+ return Optional.of(metaFunction.createFunctionRef(name,
dbName, sourceNameWithMetaName));
+ }
+ }
+ return Optional.empty();
+ }
+
public String getHiveMetastoreUris() {
return catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS,
"");
}
@@ -292,4 +330,58 @@ public class HMSExternalCatalog extends ExternalCatalog {
public boolean isEnableHmsEventsIncrementalSync() {
return enableHmsEventsIncrementalSync;
}
+
+ /**
+ * Enum for meta tables in hive catalog.
+ * eg: tbl$partitions
+ */
+ private enum MetaTableFunction {
+ PARTITIONS("partition_values");
+
+ private final String suffix;
+ private final String tvfName;
+
+ MetaTableFunction(String tvfName) {
+ this.suffix = "$" + name().toLowerCase();
+ this.tvfName = tvfName;
+ }
+
+ boolean containsMetaTable(String tableName) {
+ return tableName.endsWith(suffix) && (tableName.length() >
suffix.length());
+ }
+
+ String getSourceTableName(String tableName) {
+ return tableName.substring(0, tableName.length() -
suffix.length());
+ }
+
+ public TableValuedFunction createFunction(String ctlName, String
dbName, String sourceNameWithMetaName) {
+ switch (this) {
+ case PARTITIONS:
+ List<String> nameParts = Lists.newArrayList(ctlName,
dbName,
+ getSourceTableName(sourceNameWithMetaName));
+ return PartitionValues.create(nameParts);
+ default:
+ throw new AnalysisException("Unsupported meta function
type: " + this);
+ }
+ }
+
+ public TableValuedFunctionRef createFunctionRef(String ctlName, String
dbName, String sourceNameWithMetaName) {
+ switch (this) {
+ case PARTITIONS:
+ Map<String, String> params = Maps.newHashMap();
+ params.put("catalog", ctlName);
+ params.put("database", dbName);
+ params.put("table",
getSourceTableName(sourceNameWithMetaName));
+ try {
+ return new TableValuedFunctionRef(tvfName, null,
params);
+ } catch (org.apache.doris.common.AnalysisException e) {
+ LOG.warn("should not happen. {}.{}.{}", ctlName,
dbName, sourceNameWithMetaName);
+ return null;
+ }
+ default:
+ throw new AnalysisException("Unsupported meta function
type: " + this);
+ }
+ }
+ }
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 52070953279..a486e286c6e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -920,4 +920,10 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
String bindBrokerName = catalog.bindBrokerName();
return cache.getFilesByPartitionsWithoutCache(hivePartitions,
bindBrokerName);
}
+
+ @Override
+ public boolean isPartitionedTable() {
+ makeSureInitialized();
+ return !isView() && remoteTable.getPartitionKeysSize() > 0;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index fae6ddb137b..61521bd7681 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -1377,7 +1377,7 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
@Override
public LogicalPlan visitTableName(TableNameContext ctx) {
- List<String> tableId =
visitMultipartIdentifier(ctx.multipartIdentifier());
+ List<String> nameParts =
visitMultipartIdentifier(ctx.multipartIdentifier());
List<String> partitionNames = new ArrayList<>();
boolean isTempPart = false;
if (ctx.specifiedPartition() != null) {
@@ -1425,8 +1425,9 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
TableSample tableSample = ctx.sample() == null ? null : (TableSample)
visit(ctx.sample());
UnboundRelation relation = new
UnboundRelation(StatementScopeIdGenerator.newRelationId(),
- tableId, partitionNames, isTempPart, tabletIdLists,
relationHints,
- Optional.ofNullable(tableSample), indexName,
scanParams, Optional.ofNullable(tableSnapshot));
+ nameParts, partitionNames, isTempPart, tabletIdLists,
relationHints,
+ Optional.ofNullable(tableSample), indexName, scanParams,
Optional.ofNullable(tableSnapshot));
+
LogicalPlan checkedRelation =
LogicalPlanBuilderAssistant.withCheckPolicy(relation);
LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias());
for (LateralViewContext lateralViewContext : ctx.lateralView()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 4e18039c4b1..b418a33bae0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -65,6 +65,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.agg.Max;
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion;
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
+import
org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@@ -81,6 +82,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalTestScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalView;
import org.apache.doris.nereids.util.RelationUtil;
@@ -358,14 +360,32 @@ public class BindRelation extends OneAnalysisRuleFactory {
}
}
+ private Optional<LogicalPlan> handleMetaTable(TableIf table,
UnboundRelation unboundRelation,
+ List<String> qualifiedTableName) {
+ Optional<TableValuedFunction> tvf =
table.getDatabase().getCatalog().getMetaTableFunction(
+ qualifiedTableName.get(1), qualifiedTableName.get(2));
+ if (tvf.isPresent()) {
+ return Optional.of(new
LogicalTVFRelation(unboundRelation.getRelationId(), tvf.get()));
+ }
+ return Optional.empty();
+ }
+
private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation
unboundRelation,
- List<String>
qualifiedTableName, CascadesContext cascadesContext) {
- // for create view stmt replace tablNname to ctl.db.tableName
+ List<String> qualifiedTableName,
CascadesContext cascadesContext) {
+ // for create view stmt replace tableName to ctl.db.tableName
unboundRelation.getIndexInSqlString().ifPresent(pair -> {
StatementContext statementContext =
cascadesContext.getStatementContext();
statementContext.addIndexInSqlToString(pair,
Utils.qualifiedNameWithBackquote(qualifiedTableName));
});
+
+ // Handle meta table like "table_name$partitions"
+ // qualifiedTableName should be like "ctl.db.tbl$partitions"
+ Optional<LogicalPlan> logicalPlan = handleMetaTable(table,
unboundRelation, qualifiedTableName);
+ if (logicalPlan.isPresent()) {
+ return logicalPlan.get();
+ }
+
List<String> qualifierWithoutTableName = Lists.newArrayList();
qualifierWithoutTableName.addAll(qualifiedTableName.subList(0,
qualifiedTableName.size() - 1));
boolean isView = false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java
new file mode 100644
index 00000000000..0389bb62637
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/PartitionValues.java
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.table;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Properties;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.coercion.AnyDataType;
+import org.apache.doris.tablefunction.PartitionValuesTableValuedFunction;
+import org.apache.doris.tablefunction.TableValuedFunctionIf;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * partition_values
+ */
+public class PartitionValues extends TableValuedFunction {
+ public PartitionValues(Properties properties) {
+ super("partition_values", properties);
+ }
+
+ /**
+ * Create PartitionValues function.
+ * @param qualifiedTableName ctl.db.tbl
+ * @return PartitionValues function
+ */
+ public static TableValuedFunction create(List<String> qualifiedTableName) {
+ Preconditions.checkArgument(qualifiedTableName != null &&
qualifiedTableName.size() == 3);
+ Map<String, String> parameters = Maps.newHashMap();
+ parameters.put(PartitionValuesTableValuedFunction.CATALOG,
qualifiedTableName.get(0));
+ parameters.put(PartitionValuesTableValuedFunction.DB,
qualifiedTableName.get(1));
+ parameters.put(PartitionValuesTableValuedFunction.TABLE,
qualifiedTableName.get(2));
+ return new PartitionValues(new Properties(parameters));
+ }
+
+ @Override
+ public FunctionSignature customSignature() {
+ return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX,
getArgumentsTypes());
+ }
+
+ @Override
+ protected TableValuedFunctionIf toCatalogFunction() {
+ try {
+ Map<String, String> arguments = getTVFProperties().getMap();
+ return new PartitionValuesTableValuedFunction(arguments);
+ } catch (Throwable t) {
+ throw new AnalysisException("Can not build
PartitionsTableValuedFunction by "
+ + this + ": " + t.getMessage(), t);
+ }
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitPartitionValues(this, context);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
index ca14edd87de..0b4b57e11dc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
@@ -29,6 +29,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.table.Jobs;
import org.apache.doris.nereids.trees.expressions.functions.table.Local;
import org.apache.doris.nereids.trees.expressions.functions.table.MvInfos;
import org.apache.doris.nereids.trees.expressions.functions.table.Numbers;
+import
org.apache.doris.nereids.trees.expressions.functions.table.PartitionValues;
import org.apache.doris.nereids.trees.expressions.functions.table.Partitions;
import org.apache.doris.nereids.trees.expressions.functions.table.Query;
import org.apache.doris.nereids.trees.expressions.functions.table.S3;
@@ -59,6 +60,10 @@ public interface TableValuedFunctionVisitor<R, C> {
return visitTableValuedFunction(partitions, context);
}
+ default R visitPartitionValues(PartitionValues partitionValues, C context)
{
+ return visitTableValuedFunction(partitionValues, context);
+ }
+
default R visitJobs(Jobs jobs, C context) {
return visitTableValuedFunction(jobs, context);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java
index b145338ff81..b72498a2227 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/RelationUtil.java
@@ -100,8 +100,10 @@ public class RelationUtil {
try {
DatabaseIf<TableIf> db = catalog.getDb(dbName).orElseThrow(() ->
new AnalysisException(
"Database [" + dbName + "] does not exist."));
- TableIf table = db.getTable(tableName).orElseThrow(() -> new
AnalysisException(
- "Table [" + tableName + "] does not exist in database [" +
dbName + "]."));
+ Pair<String, String> sourceTblNameWithMetaTblName =
catalog.getSourceTableNameWithMetaTableName(tableName);
+ String sourceTableName = sourceTblNameWithMetaTblName.first;
+ TableIf table = db.getTable(sourceTableName).orElseThrow(() -> new
AnalysisException(
+ "Table [" + sourceTableName + "] does not exist in
database [" + dbName + "]."));
return Pair.of(db, table);
} catch (Throwable e) {
throw new AnalysisException(e.getMessage(), e.getCause());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 746519082ae..ac0443bbf52 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -30,10 +30,13 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.TableProperty;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Pair;
@@ -42,11 +45,14 @@ import org.apache.doris.common.proc.FrontendsProcNode;
import org.apache.doris.common.proc.PartitionsProcDir;
import org.apache.doris.common.util.NetUtils;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.datasource.TablePartitionValues;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
import org.apache.doris.datasource.hudi.source.HudiCachedPartitionProcessor;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
@@ -78,6 +84,7 @@ import
org.apache.doris.thrift.TMaterializedViewsMetadataParams;
import org.apache.doris.thrift.TMetadataTableRequestParams;
import org.apache.doris.thrift.TMetadataType;
import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TPartitionValuesMetadataParams;
import org.apache.doris.thrift.TPartitionsMetadataParams;
import org.apache.doris.thrift.TPipelineWorkloadGroup;
import org.apache.doris.thrift.TRow;
@@ -204,7 +211,8 @@ public class MetadataGenerator {
}
TFetchSchemaTableDataResult result;
TMetadataTableRequestParams params = request.getMetadaTableParams();
- switch (request.getMetadaTableParams().getMetadataType()) {
+ TMetadataType metadataType =
request.getMetadaTableParams().getMetadataType();
+ switch (metadataType) {
case ICEBERG:
result = icebergMetadataResult(params);
break;
@@ -232,11 +240,17 @@ public class MetadataGenerator {
case TASKS:
result = taskMetadataResult(params);
break;
+ case PARTITION_VALUES:
+ result = partitionValuesMetadataResult(params);
+ break;
default:
return errorResult("Metadata table params is not set.");
}
if (result.getStatus().getStatusCode() == TStatusCode.OK) {
- filterColumns(result, params.getColumnsName(),
params.getMetadataType(), params);
+ if (metadataType != TMetadataType.PARTITION_VALUES) {
+ // partition_values' result already sorted by column names
+ filterColumns(result, params.getColumnsName(),
params.getMetadataType(), params);
+ }
}
if (LOG.isDebugEnabled()) {
LOG.debug("getMetadataTable() end.");
@@ -329,7 +343,8 @@ public class MetadataGenerator {
TRow trow = new TRow();
LocalDateTime committedAt =
LocalDateTime.ofInstant(Instant.ofEpochMilli(
snapshot.timestampMillis()),
TimeUtils.getTimeZone().toZoneId());
- long encodedDatetime =
convertToDateTimeV2(committedAt.getYear(), committedAt.getMonthValue(),
+ long encodedDatetime =
TimeUtils.convertToDateTimeV2(committedAt.getYear(),
+ committedAt.getMonthValue(),
committedAt.getDayOfMonth(),
committedAt.getHour(), committedAt.getMinute(),
committedAt.getSecond(), committedAt.getNano() /
1000);
@@ -785,12 +800,6 @@ public class MetadataGenerator {
result.setDataBatch(filterColumnsRows);
}
- private static long convertToDateTimeV2(
- int year, int month, int day, int hour, int minute, int second,
int microsecond) {
- return (long) microsecond | (long) second << 20 | (long) minute << 26
| (long) hour << 32
- | (long) day << 37 | (long) month << 42 | (long) year << 46;
- }
-
private static TFetchSchemaTableDataResult
mtmvMetadataResult(TMetadataTableRequestParams params) {
if (LOG.isDebugEnabled()) {
LOG.debug("mtmvMetadataResult() start");
@@ -1477,4 +1486,120 @@ public class MetadataGenerator {
}
}
}
+
+ private static TFetchSchemaTableDataResult
partitionValuesMetadataResult(TMetadataTableRequestParams params) {
+ if (!params.isSetPartitionValuesMetadataParams()) {
+ return errorResult("partition values metadata params is not set.");
+ }
+
+ TPartitionValuesMetadataParams metaParams =
params.getPartitionValuesMetadataParams();
+ String ctlName = metaParams.getCatalog();
+ String dbName = metaParams.getDatabase();
+ String tblName = metaParams.getTable();
+ List<TRow> dataBatch;
+ try {
+ TableIf table =
PartitionValuesTableValuedFunction.analyzeAndGetTable(ctlName, dbName, tblName,
false);
+ TableType tableType = table.getType();
+ switch (tableType) {
+ case HMS_EXTERNAL_TABLE:
+ dataBatch =
partitionValuesMetadataResultForHmsTable((HMSExternalTable) table,
+ params.getColumnsName());
+ break;
+ default:
+ return errorResult("not support table type " +
tableType.name());
+ }
+ TFetchSchemaTableDataResult result = new
TFetchSchemaTableDataResult();
+ result.setDataBatch(dataBatch);
+ result.setStatus(new TStatus(TStatusCode.OK));
+ return result;
+ } catch (Throwable t) {
+ LOG.warn("error when get partition values metadata. {}.{}.{}",
ctlName, dbName, tblName, t);
+ return errorResult("error when get partition values metadata: " +
Util.getRootCauseMessage(t));
+ }
+ }
+
+ private static List<TRow>
partitionValuesMetadataResultForHmsTable(HMSExternalTable tbl, List<String>
colNames)
+ throws AnalysisException {
+ List<Column> partitionCols = tbl.getPartitionColumns();
+ List<Integer> colIdxs = Lists.newArrayList();
+ List<Type> types = Lists.newArrayList();
+ for (String colName : colNames) {
+ for (int i = 0; i < partitionCols.size(); ++i) {
+ if (partitionCols.get(i).getName().equalsIgnoreCase(colName)) {
+ colIdxs.add(i);
+ types.add(partitionCols.get(i).getType());
+ }
+ }
+ }
+ if (colIdxs.size() != colNames.size()) {
+ throw new AnalysisException(
+ "column " + colNames + " does not match partition columns
of table " + tbl.getName());
+ }
+
+ HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
+ .getMetaStoreCache((HMSExternalCatalog) tbl.getCatalog());
+ HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
+ tbl.getDbName(), tbl.getName(), tbl.getPartitionColumnTypes());
+ Map<Long, List<String>> valuesMap =
hivePartitionValues.getPartitionValuesMap();
+ List<TRow> dataBatch = Lists.newArrayList();
+ for (Map.Entry<Long, List<String>> entry : valuesMap.entrySet()) {
+ TRow trow = new TRow();
+ List<String> values = entry.getValue();
+ if (values.size() != partitionCols.size()) {
+ continue;
+ }
+
+ for (int i = 0; i < colIdxs.size(); ++i) {
+ int idx = colIdxs.get(i);
+ String partitionValue = values.get(idx);
+ if (partitionValue == null ||
partitionValue.equals(TablePartitionValues.HIVE_DEFAULT_PARTITION)) {
+ trow.addToColumnValue(new TCell().setIsNull(true));
+ } else {
+ Type type = types.get(i);
+ switch (type.getPrimitiveType()) {
+ case BOOLEAN:
+ trow.addToColumnValue(new
TCell().setBoolVal(Boolean.valueOf(partitionValue)));
+ break;
+ case TINYINT:
+ case SMALLINT:
+ case INT:
+ trow.addToColumnValue(new
TCell().setIntVal(Integer.valueOf(partitionValue)));
+ break;
+ case BIGINT:
+ trow.addToColumnValue(new
TCell().setLongVal(Long.valueOf(partitionValue)));
+ break;
+ case FLOAT:
+ trow.addToColumnValue(new
TCell().setDoubleVal(Float.valueOf(partitionValue)));
+ break;
+ case DOUBLE:
+ trow.addToColumnValue(new
TCell().setDoubleVal(Double.valueOf(partitionValue)));
+ break;
+ case VARCHAR:
+ case CHAR:
+ case STRING:
+ trow.addToColumnValue(new
TCell().setStringVal(partitionValue));
+ break;
+ case DATE:
+ case DATEV2:
+ trow.addToColumnValue(
+ new
TCell().setLongVal(TimeUtils.convertStringToDateV2(partitionValue)));
+ break;
+ case DATETIME:
+ case DATETIMEV2:
+ trow.addToColumnValue(
+ new
TCell().setLongVal(TimeUtils.convertStringToDateTimeV2(partitionValue,
+ ((ScalarType)
type).getScalarScale())));
+ break;
+ default:
+ throw new AnalysisException(
+ "Unsupported partition column type for
$partitions sys table " + type);
+ }
+ }
+ }
+ dataBatch.add(trow);
+ }
+ return dataBatch;
+ }
+
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java
new file mode 100644
index 00000000000..59efe9fcefd
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionValuesTableValuedFunction.java
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.tablefunction;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.thrift.TMetaScanRange;
+import org.apache.doris.thrift.TMetadataType;
+import org.apache.doris.thrift.TPartitionValuesMetadataParams;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * The Implement of table valued function
+ * partition_values("catalog"="ctl1", "database" = "db1","table" = "table1").
+ */
+public class PartitionValuesTableValuedFunction extends
MetadataTableValuedFunction {
+ private static final Logger LOG =
LogManager.getLogger(PartitionValuesTableValuedFunction.class);
+
+ public static final String NAME = "partition_values";
+
+ public static final String CATALOG = "catalog";
+ public static final String DB = "database";
+ public static final String TABLE = "table";
+
+ private static final ImmutableSet<String> PROPERTIES_SET =
ImmutableSet.of(CATALOG, DB, TABLE);
+
+ private final String catalogName;
+ private final String databaseName;
+ private final String tableName;
+ private TableIf table;
+ private List<Column> schema;
+
+ public PartitionValuesTableValuedFunction(Map<String, String> params)
throws AnalysisException {
+ Map<String, String> validParams = Maps.newHashMap();
+ for (String key : params.keySet()) {
+ if (!PROPERTIES_SET.contains(key.toLowerCase())) {
+ throw new AnalysisException("'" + key + "' is invalid
property");
+ }
+ // check ctl, db, tbl
+ validParams.put(key.toLowerCase(), params.get(key));
+ }
+ String catalogName = validParams.get(CATALOG);
+ String dbName = validParams.get(DB);
+ String tableName = validParams.get(TABLE);
+ if (StringUtils.isEmpty(catalogName) || StringUtils.isEmpty(dbName) ||
StringUtils.isEmpty(tableName)) {
+ throw new AnalysisException("catalog, database and table are
required");
+ }
+ this.table = analyzeAndGetTable(catalogName, dbName, tableName, true);
+ this.catalogName = catalogName;
+ this.databaseName = dbName;
+ this.tableName = tableName;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("PartitionsTableValuedFunction() end");
+ }
+ }
+
+ public static TableIf analyzeAndGetTable(String catalogName, String
dbName, String tableName, boolean checkAuth) {
+ if (checkAuth) {
+ // This method will be called at 2 places:
+ // One is when planing the query, which should check the privilege
of the user.
+ // the other is when BE call FE to fetch partition values, which
should not check the privilege.
+ if (!Env.getCurrentEnv().getAccessManager()
+ .checkTblPriv(ConnectContext.get(), catalogName, dbName,
+ tableName, PrivPredicate.SHOW)) {
+ String message =
ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("SHOW PARTITIONS",
+ ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(),
+ catalogName + ": " + dbName + ": " + tableName);
+ throw new AnalysisException(message);
+ }
+ }
+ CatalogIf catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
+ if (catalog == null) {
+ throw new AnalysisException("can not find catalog: " +
catalogName);
+ }
+ // disallow unsupported catalog
+ if (!(catalog.isInternalCatalog() || catalog instanceof
HMSExternalCatalog
+ || catalog instanceof MaxComputeExternalCatalog)) {
+ throw new AnalysisException(String.format("Catalog of type '%s' is
not allowed in ShowPartitionsStmt",
+ catalog.getType()));
+ }
+
+ Optional<DatabaseIf> db = catalog.getDb(dbName);
+ if (!db.isPresent()) {
+ throw new AnalysisException("can not find database: " + dbName);
+ }
+ TableIf table;
+ try {
+ table = db.get().getTableOrMetaException(tableName, TableType.OLAP,
+ TableType.HMS_EXTERNAL_TABLE,
TableType.MAX_COMPUTE_EXTERNAL_TABLE);
+ } catch (MetaNotFoundException e) {
+ throw new AnalysisException(e.getMessage(), e);
+ }
+
+ if (!(table instanceof HMSExternalTable)) {
+ throw new AnalysisException("Currently only support hive table's
partition values meta table");
+ }
+ HMSExternalTable hmsTable = (HMSExternalTable) table;
+ if (!hmsTable.isPartitionedTable()) {
+ throw new AnalysisException("Table " + tableName + " is not a
partitioned table");
+ }
+ return table;
+ }
+
+ @Override
+ public TMetadataType getMetadataType() {
+ return TMetadataType.PARTITION_VALUES;
+ }
+
+ @Override
+ public TMetaScanRange getMetaScanRange() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getMetaScanRange() start");
+ }
+ TMetaScanRange metaScanRange = new TMetaScanRange();
+ metaScanRange.setMetadataType(TMetadataType.PARTITION_VALUES);
+ TPartitionValuesMetadataParams partitionParam = new
TPartitionValuesMetadataParams();
+ partitionParam.setCatalog(catalogName);
+ partitionParam.setDatabase(databaseName);
+ partitionParam.setTable(tableName);
+ metaScanRange.setPartitionValuesParams(partitionParam);
+ return metaScanRange;
+ }
+
+ @Override
+ public String getTableName() {
+ return "PartitionsValuesTableValuedFunction";
+ }
+
+ @Override
+ public List<Column> getTableColumns() throws AnalysisException {
+ Preconditions.checkNotNull(table);
+ // TODO: support other type of sys tables
+ if (schema == null) {
+ List<Column> partitionColumns = ((HMSExternalTable)
table).getPartitionColumns();
+ schema = Lists.newArrayList();
+ for (Column column : partitionColumns) {
+ schema.add(new Column(column));
+ }
+ }
+ return schema;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java
index 1ceddeb89cf..00169ad555f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java
@@ -55,7 +55,7 @@ import java.util.Optional;
/**
* The Implement of table valued function
- * partitions("database" = "db1","table" = "table1").
+ * partitions("catalog"="ctl1", "database" = "db1","table" = "table1").
*/
public class PartitionsTableValuedFunction extends MetadataTableValuedFunction
{
private static final Logger LOG =
LogManager.getLogger(PartitionsTableValuedFunction.class);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
index 6b6fda088a8..d4faa460195 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
@@ -77,6 +77,8 @@ public abstract class TableValuedFunctionIf {
return new GroupCommitTableValuedFunction(params);
case QueryTableValueFunction.NAME:
return
QueryTableValueFunction.createQueryTableValueFunction(params);
+ case PartitionValuesTableValuedFunction.NAME:
+ return new PartitionValuesTableValuedFunction(params);
default:
throw new AnalysisException("Could not find table function " +
funcName);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
index 114ddada9cd..f57e8fe2fe6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/TimeUtilsTest.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import mockit.Expectations;
@@ -31,6 +32,7 @@ import org.junit.Before;
import org.junit.Test;
import java.time.ZoneId;
+import java.time.format.DateTimeParseException;
import java.util.Calendar;
import java.util.Date;
import java.util.LinkedList;
@@ -201,4 +203,53 @@ public class TimeUtilsTest {
Assert.assertNull(TimeUtils.getHourAsDate("111"));
Assert.assertNull(TimeUtils.getHourAsDate("-1"));
}
+
+ @Test
+ public void testConvertToBEDateType() {
+ long result = TimeUtils.convertStringToDateV2("2021-01-01");
+ Assert.assertEquals(1034785, result);
+ result = TimeUtils.convertStringToDateV2("1900-01-01");
+ Assert.assertEquals(972833, result);
+ result = TimeUtils.convertStringToDateV2("1899-12-31");
+ Assert.assertEquals(972703, result);
+ result = TimeUtils.convertStringToDateV2("9999-12-31");
+ Assert.assertEquals(5119903, result);
+
+ ExceptionChecker.expectThrows(DateTimeParseException.class, () ->
TimeUtils.convertStringToDateV2("2021-1-1"));
+ ExceptionChecker.expectThrows(DateTimeParseException.class, () ->
TimeUtils.convertStringToDateV2("1900-01-1"));
+ ExceptionChecker.expectThrows(DateTimeParseException.class, () ->
TimeUtils.convertStringToDateV2("20210101"));
+ ExceptionChecker.expectThrows(DateTimeParseException.class, () ->
TimeUtils.convertStringToDateV2(""));
+ ExceptionChecker.expectThrows(NullPointerException.class, () ->
TimeUtils.convertStringToDateV2(null));
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateV2("2024:12:31"));
+ }
+
+ @Test
+ public void testConvertToBEDatetimeV2Type() {
+ long result = TimeUtils.convertStringToDateTimeV2("2021-01-01
10:10:10", 0);
+ Assert.assertEquals(142219811099770880L, result);
+ result = TimeUtils.convertStringToDateTimeV2("1900-01-01 00:00:00.12",
2);
+ Assert.assertEquals(133705149423146176L, result);
+ result = TimeUtils.convertStringToDateTimeV2("1899-12-31
23:59:59.000", 3);
+ Assert.assertEquals(133687385164611584L, result);
+ result = TimeUtils.convertStringToDateTimeV2("9999-12-31
23:59:59.123456", 6);
+ Assert.assertEquals(703674213003812984L, result);
+
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateTimeV2("2021-1-1", 0));
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateTimeV2("1900-01-1", 0));
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateTimeV2("20210101", 0));
+ ExceptionChecker.expectThrows(DateTimeParseException.class, () ->
TimeUtils.convertStringToDateTimeV2("", 0));
+ ExceptionChecker.expectThrows(NullPointerException.class, () ->
TimeUtils.convertStringToDateTimeV2(null, 0));
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateTimeV2("2024-10-10", 0));
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateTimeV2("2024-10-10 10", 0));
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateTimeV2("2024:12:31", 0));
+ ExceptionChecker.expectThrows(DateTimeParseException.class,
+ () -> TimeUtils.convertStringToDateTimeV2("2024-10-10
11:11:11", 6));
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java
index 67115e67687..369a57017cb 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java
@@ -18,6 +18,8 @@
package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionInfo;
@@ -98,6 +100,8 @@ class BindRelationTest extends TestWithFeService implements
GeneratedPlanPattern
new Column("name", Type.VARCHAR)
);
+ Database externalDatabase = new Database(10000, DEFAULT_CLUSTER_PREFIX
+ DB1);
+
OlapTable externalOlapTable = new OlapTable(1, tableName,
externalTableColumns, KeysType.DUP_KEYS,
new PartitionInfo(), new RandomDistributionInfo(10)) {
@Override
@@ -109,6 +113,11 @@ class BindRelationTest extends TestWithFeService
implements GeneratedPlanPattern
public boolean hasDeleteSign() {
return false;
}
+
+ @Override
+ public DatabaseIf getDatabase() {
+ return externalDatabase;
+ }
};
CustomTableResolver customTableResolver = qualifiedTable -> {
diff --git a/gensrc/thrift/Data.thrift b/gensrc/thrift/Data.thrift
index d1163821076..dc1190c6e42 100644
--- a/gensrc/thrift/Data.thrift
+++ b/gensrc/thrift/Data.thrift
@@ -54,6 +54,7 @@ struct TCell {
3: optional i64 longVal
4: optional double doubleVal
5: optional string stringVal
+ 6: optional bool isNull
// add type: date datetime
}
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index dac90824fa2..ad3828e392e 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -981,6 +981,7 @@ struct TMetadataTableRequestParams {
10: optional PlanNodes.TTasksMetadataParams tasks_metadata_params
11: optional PlanNodes.TPartitionsMetadataParams partitions_metadata_params
12: optional PlanNodes.TMetaCacheStatsParams meta_cache_stats_params
+ 13: optional PlanNodes.TPartitionValuesMetadataParams
partition_values_metadata_params
}
struct TSchemaTableRequestParams {
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index d6690fe77a1..6e328a93e15 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -509,6 +509,12 @@ struct TPartitionsMetadataParams {
3: optional string table
}
+struct TPartitionValuesMetadataParams {
+ 1: optional string catalog
+ 2: optional string database
+ 3: optional string table
+}
+
struct TJobsMetadataParams {
1: optional string type
2: optional Types.TUserIdentity current_user_ident
@@ -526,6 +532,7 @@ struct TQueriesMetadataParams {
4: optional TJobsMetadataParams jobs_params
5: optional TTasksMetadataParams tasks_params
6: optional TPartitionsMetadataParams partitions_params
+ 7: optional TPartitionValuesMetadataParams partition_values_params
}
struct TMetaCacheStatsParams {
@@ -542,6 +549,7 @@ struct TMetaScanRange {
8: optional TTasksMetadataParams tasks_params
9: optional TPartitionsMetadataParams partitions_params
10: optional TMetaCacheStatsParams meta_cache_stats_params
+ 11: optional TPartitionValuesMetadataParams partition_values_params
}
// Specification of an individual data range which is held in its entirety
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index 9290768e0db..fd37adb8654 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -712,7 +712,8 @@ enum TMetadataType {
JOBS,
TASKS,
WORKLOAD_SCHED_POLICY,
- PARTITIONS;
+ PARTITIONS,
+ PARTITION_VALUES;
}
enum TIcebergQueryType {
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out
b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out
new file mode 100644
index 00000000000..bc69c716277
--- /dev/null
+++
b/regression-test/data/external_table_p0/hive/test_hive_partition_values_tvf.out
@@ -0,0 +1,120 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql01 --
+\N 0.1 test1
+\N 0.2 test2
+100 0.3 test3
+
+-- !sql02 --
+\N 0.1 test1
+\N 0.2 test2
+100 0.3 test3
+
+-- !sql03 --
+\N 0.1 test1
+\N 0.2 test2
+100 0.3 test3
+
+-- !sql11 --
+\N 0.1
+\N 0.2
+100 0.3
+
+-- !sql12 --
+0.1 \N
+0.2 \N
+0.3 100
+
+-- !sql13 --
+test1 \N
+test2 \N
+test3 100
+
+-- !sql21 --
+test3 100 0.3
+
+-- !sql22 --
+test1
+test2
+test3
+
+-- !sql22 --
+3
+
+-- !sql22 --
+3
+
+-- !sql31 --
+0.1 \N
+0.2 \N
+0.3 100
+
+-- !sql41 --
+100
+
+-- !sql42 --
+test3
+
+-- !sql51 --
+test3
+
+-- !sql61 --
+
+-- !sql71 --
+\N 0.2 test2
+100 0.3 test3
+\N 0.1 test1
+
+-- !sql72 --
+
+-- !sql73 --
+test3
+
+-- !sql81 --
+100 0.3 test3 100 0.3 test3
+
+-- !sql91 --
+t_int int Yes true \N NONE
+t_float float Yes true \N NONE
+t_string varchar(65533) Yes true \N NONE
+
+-- !sql92 --
+t_int int Yes true \N NONE
+t_float float Yes true \N NONE
+t_string varchar(65533) Yes true \N NONE
+
+-- !sql93 --
+t_int int Yes true \N NONE
+
+-- !sql94 --
+t_int int Yes true \N NONE
+
+-- !sql95 --
+\N 0.1 test1
+\N 0.2 test2
+100 0.3 test3
+
+-- !sql101 --
+k1 int Yes true \N
+
+-- !sql102 --
+
+-- !sql111 --
+p1 boolean Yes true \N NONE
+p2 tinyint Yes true \N NONE
+p3 smallint Yes true \N NONE
+p4 int Yes true \N NONE
+p5 bigint Yes true \N NONE
+p6 date Yes true \N NONE
+p7 datetime(6) Yes true \N NONE
+p8 varchar(65533) Yes true \N NONE
+
+-- !sql112 --
+1 test1 true -128 -32768 -2147483648 -9223372036854775808
1900-01-01 1899-01-01T23:59:59 \N
+2 \N false 127 32767 2147483647 9223372036854775807
9999-12-31 0001-01-01T00:00:01.321 boston
+3 \N \N \N \N \N \N \N \N
+
+-- !sql113 --
+\N \N \N \N \N \N \N \N
+false -128 -32768 -2147483648 -9223372036854775808 1900-01-01
1899-01-01T23:59:59 \N
+false 127 32767 2147483647 9223372036854775807 9999-12-31
0001-01-01T00:00:01.321 boston
+
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy
new file mode 100644
index 00000000000..cbdb9f8ff35
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/test_hive_partition_values_tvf.groovy
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_partition_values_tvf",
"p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable Hive test.")
+ return;
+ }
+ sql """set enable_nereids_planner=true"""
+ sql """set enable_fallback_to_original_planner=false"""
+ for (String hivePrefix : ["hive3"]) {
+ String extHiveHmsHost =
context.config.otherConfigs.get("externalEnvIp")
+ String extHiveHmsPort = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String catalog_name =
"${hivePrefix}_test_external_catalog_hive_partition"
+
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+ );
+ """
+
+ // 1. test qualifier
+ qt_sql01 """ select * from
${catalog_name}.multi_catalog.orc_partitioned_columns\$partitions order by
t_int, t_float, t_string"""
+ sql """ switch ${catalog_name} """
+ qt_sql02 """ select * from
multi_catalog.orc_partitioned_columns\$partitions order by t_int, t_float,
t_string"""
+ sql """ use multi_catalog"""
+ qt_sql03 """ select * from orc_partitioned_columns\$partitions order
by t_int, t_float, t_string"""
+
+ // 2. test select order
+ qt_sql11 """ select * except(t_string) from
orc_partitioned_columns\$partitions order by t_int, t_float, t_string"""
+ qt_sql12 """ select t_float, t_int from
orc_partitioned_columns\$partitions order by t_int, t_float, t_string"""
+ qt_sql13 """ select t_string, t_int from
orc_partitioned_columns\$partitions order by t_int, t_float, t_string"""
+
+ // 3. test agg
+ qt_sql21 """ select max(t_string), max(t_int), max(t_float) from
orc_partitioned_columns\$partitions"""
+ qt_sql22 """ select max(t_string) from
orc_partitioned_columns\$partitions group by t_int, t_float order by t_int,
t_float"""
+ qt_sql22 """ select count(*) from
orc_partitioned_columns\$partitions;"""
+ qt_sql22 """ select count(1) from
orc_partitioned_columns\$partitions;"""
+
+ // 4. test alias
+ qt_sql31 """ select pv.t_float, pv.t_int from
orc_partitioned_columns\$partitions as pv group by t_int, t_float order by
t_int, t_float"""
+
+ // 5. test CTE
+ qt_sql41 """ with v1 as (select t_string, t_int from
orc_partitioned_columns\$partitions order by t_int, t_float, t_string) select
max(t_int) from v1; """
+ qt_sql42 """ with v1 as (select t_string, t_int from
orc_partitioned_columns\$partitions order by t_int, t_float, t_string) select
c1 from (select max(t_string) as c1 from v1) x; """
+
+ // 6. test subquery
+ qt_sql51 """select c1 from (select max(t_string) as c1 from (select *
from multi_catalog.orc_partitioned_columns\$partitions)x)y;"""
+
+ // 7. test where
+ qt_sql61 """select * from orc_partitioned_columns\$partitions where
t_int != "__HIVE_DEFAULT_PARTITION__" order by t_int, t_float, t_string; """
+
+ // 8. test view
+ sql """drop database if exists internal.partition_values_db"""
+ sql """create database if not exists internal.partition_values_db"""
+ sql """create view internal.partition_values_db.v1 as select * from
${catalog_name}.multi_catalog.orc_partitioned_columns\$partitions"""
+ qt_sql71 """select * from internal.partition_values_db.v1"""
+ qt_sql72 """select t_string, t_int from
internal.partition_values_db.v1 where t_int != "__HIVE_DEFAULT_PARTITION__""""
+ qt_sql73 """with v1 as (select t_string, t_int from
internal.partition_values_db.v1 order by t_int, t_float, t_string) select c1
from (select max(t_string) as c1 from v1) x;"""
+
+ // 9. test join
+ qt_sql81 """select * from orc_partitioned_columns\$partitions p1 join
orc_partitioned_columns\$partitions p2 on p1.t_int = p2.t_int order by
p1.t_int, p1.t_float"""
+
+ // 10. test desc
+ qt_sql91 """desc orc_partitioned_columns\$partitions"""
+ qt_sql92 """desc function partition_values("catalog" =
"${catalog_name}", "database" = "multi_catalog", "table" =
"orc_partitioned_columns");"""
+ qt_sql93 """desc orc_partitioned_one_column\$partitions"""
+ qt_sql94 """desc function partition_values("catalog" =
"${catalog_name}", "database" = "multi_catalog", "table" =
"orc_partitioned_one_column");"""
+ qt_sql95 """select * from partition_values("catalog" =
"${catalog_name}", "database" = "multi_catalog", "table" =
"orc_partitioned_columns") order by t_int, t_float"""
+
+ // 11. test non partition table
+ test {
+ sql """select * from hive_text_complex_type\$partitions"""
+ exception "is not a partitioned table"
+ }
+ test {
+ sql """desc hive_text_complex_type\$partitions"""
+ exception "is not a partitioned table"
+ }
+
+ // 12. test inner table
+ sql """create table internal.partition_values_db.pv_inner1 (k1 int)
distributed by hash (k1) buckets 1 properties("replication_num" = "1")"""
+ qt_sql101 """desc internal.partition_values_db.pv_inner1"""
+ qt_sql102 """select * from internal.partition_values_db.pv_inner1"""
+ test {
+ sql """desc internal.partition_values_db.pv_inner1\$partitions"""
+ exception """Unknown table 'pv_inner1\$partitions'"""
+ }
+
+ test {
+ sql """select * from
internal.partition_values_db.pv_inner1\$partitions"""
+ exception """Table [pv_inner1\$partitions] does not exist in
database [partition_values_db]"""
+ }
+
+ // 13. test all types of partition columns
+ sql """switch ${catalog_name}"""
+ sql """drop database if exists partition_values_db""";
+ sql """create database partition_values_db"""
+ sql """use partition_values_db"""
+
+ sql """create table partition_values_all_types (
+ k1 int,
+ k2 string,
+ p1 boolean,
+ p2 tinyint,
+ p3 smallint,
+ p4 int,
+ p5 bigint,
+ p6 date,
+ p7 datetime,
+ p8 string
+ ) partition by list(p1, p2, p3, p4, p5, p6, p7, p8)();
+ """
+
+ qt_sql111 """desc partition_values_all_types\$partitions;"""
+
+ sql """insert into partition_values_all_types values
+ (1, "test1", true, -128, -32768, -2147483648,
-9223372036854775808, "1900-01-01", "1899-01-01 23:59:59", ""),
+ (2, null, false, 127, 32767, 2147483647, 9223372036854775807,
"9999-12-31", "0001-01-01 00:00:01.321", "boston"),
+ (3, "", null, null, null, null, null, null, null, null);
+ """
+
+ qt_sql112 """select * from partition_values_all_types order by k1;"""
+ qt_sql113 """select * from partition_values_all_types\$partitions
order by p1,p2,p3;"""
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
