This is an automated email from the ASF dual-hosted git repository. wzhou pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9cc0aa886cbbaac3f69c7ea91b397ebbea739986 Author: Tamas Mate <[email protected]> AuthorDate: Wed Nov 8 13:07:32 2023 +0100 IMPALA-12495: Describe statement for Iceberg metadata tables Iceberg metadata tables are virtual tables and their schemata are predefined by the Iceberg library. This commit extends the DESCRIBE <table> statement, so the users can print the table description of these tables. Metadata tables do not exist in the HMS, therefore DESCRIBE FORMATTED|EXTENDED statements are not permitted. Testing: - Added E2E tests Change-Id: Ibe22f271a59a6885035991c09b5101193ade6e97 Reviewed-on: http://gerrit.cloudera.org:8080/20695 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/service/frontend.cc | 3 + common/thrift/Frontend.thrift | 7 +- .../java/org/apache/impala/analysis/Analyzer.java | 22 +- .../apache/impala/analysis/DescribeTableStmt.java | 14 ++ .../org/apache/impala/analysis/FromClause.java | 3 - .../java/org/apache/impala/analysis/TableName.java | 3 +- .../impala/service/DescribeResultFactory.java | 18 +- .../java/org/apache/impala/service/Frontend.java | 70 ++++-- .../org/apache/impala/service/JniFrontend.java | 11 +- .../authorization/AuthorizationTestBase.java | 8 +- .../queries/QueryTest/iceberg-metadata-tables.test | 268 ++++++++++++++++++++- 11 files changed, 377 insertions(+), 50 deletions(-) diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc index e8f53f3d7..0a9ce18dc 100644 --- a/be/src/service/frontend.cc +++ b/be/src/service/frontend.cc @@ -186,6 +186,9 @@ Status Frontend::DescribeTable(const TDescribeTableParams& params, tparams.__set_output_style(params.output_style); if (params.__isset.table_name) tparams.__set_table_name(params.table_name); if (params.__isset.result_struct) tparams.__set_result_struct(params.result_struct); + if (params.__isset.metadata_table_name) { + tparams.__set_metadata_table_name(params.metadata_table_name); + } tparams.__set_session(session); return JniUtil::CallJniMethod(fe_, describe_table_id_, tparams, response); } diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift index e7118ed19..f19f532c2 100644 --- a/common/thrift/Frontend.thrift +++ b/common/thrift/Frontend.thrift @@ -187,11 +187,14 @@ struct TDescribeTableParams { // Set when describing a table. 2: optional CatalogObjects.TTableName table_name + // Set for metadata tables + 3: optional string metadata_table_name + // Set when describing a path to a nested collection. - 3: optional Types.TColumnType result_struct + 4: optional Types.TColumnType result_struct // Session state for the user who initiated this request. - 4: optional Query.TSessionState session + 5: optional Query.TSessionState session } // Results of a call to describeDb() and describeTable() diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java index 91e3a0151..f6ce7b2bd 100644 --- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java @@ -3563,6 +3563,9 @@ public class Analyzer { */ public FeTable getTable(TableName tblName, boolean mustExist) throws AnalysisException, TableLoadingException { + if (IcebergMetadataTable.isIcebergMetadataTable(tblName.toPath())) { + return getMetadataVirtualTable(tblName.toPath()); + } FeTable table = globalState_.stmtTableCache.tables.get(tblName); if (table == null) { if (!mustExist) { @@ -3603,13 +3606,15 @@ public class Analyzer { } /** - * Adds a new Iceberg metadata table to the stmt table cache. At this point it is - * unknown if the base table is loaded for scanning as well, therefore the original - * table is kept. The metadata table will have its vTbl field filled, while the original - * table gets a new key without the vTbl field. - * 'tblRefPath' parameter has to be an IcebergMetadataTable reference path. + * Retrieves the Iceberg metadata table from the stmtTableCache if the Iceberg metadata + * table exists or creates and adds it to the stmtTableCache if it does not exist. At + * this point it is unknown if the base table is loaded for scanning as well, therefore + * the original table is kept. The metadata table will have its vTbl field filled, while + * the original table gets a new key without the vTbl field. 'tblRefPath' parameter has + * to be an IcebergMetadataTable reference path. Returns the the Iceberg metadata table. */ - public void addMetadataVirtualTable(List<String> tblRefPath) throws AnalysisException { + public FeTable getMetadataVirtualTable(List<String> tblRefPath) + throws AnalysisException { Preconditions.checkArgument(IcebergMetadataTable.isIcebergMetadataTable(tblRefPath)); try { TableName catalogTableName = new TableName(tblRefPath.get(0), @@ -3619,11 +3624,14 @@ public class Analyzer { // The catalog table (the base of the virtual table) has been loaded and cached // under the name of the virtual table. FeTable catalogTable = getStmtTableCache().tables.get(virtualTableName); - if (catalogTable instanceof IcebergMetadataTable || catalogTable == null) return; + if (catalogTable instanceof IcebergMetadataTable || catalogTable == null) { + return catalogTable; + } IcebergMetadataTable virtualTable = new IcebergMetadataTable(catalogTable, tblRefPath.get(2)); getStmtTableCache().tables.put(catalogTableName, catalogTable); getStmtTableCache().tables.put(virtualTableName, virtualTable); + return virtualTable; } catch (ImpalaRuntimeException e) { throw new AnalysisException("Could not create metadata table for table " + "reference: " + StringUtils.join(tblRefPath, "."), e); diff --git a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java index ec99f6f62..4830354df 100644 --- a/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/DescribeTableStmt.java @@ -25,6 +25,7 @@ import org.apache.impala.authorization.Privilege; import org.apache.impala.catalog.FeTable; import org.apache.impala.catalog.StructType; import org.apache.impala.catalog.TableLoadingException; +import org.apache.impala.catalog.iceberg.IcebergMetadataTable; import org.apache.impala.common.AnalysisException; import org.apache.impala.thrift.TDescribeOutputStyle; import org.apache.impala.thrift.TDescribeTableParams; @@ -124,6 +125,7 @@ public class DescribeTableStmt extends StatementBase { // all columns returning an empty result due to insufficient VIEW_METADATA privilege. analyzer.getTable(table_.getTableName(), /* add column-level privilege */ true, Privilege.ANY); + checkMinimalForIcebergMetadataTable(); // Describing a table. if (path_.destTable() != null) return; @@ -150,6 +152,14 @@ public class DescribeTableStmt extends StatementBase { } } + private void checkMinimalForIcebergMetadataTable() throws AnalysisException { + if (table_ instanceof IcebergMetadataTable && + outputStyle_ != TDescribeOutputStyle.MINIMAL) { + throw new AnalysisException("DESCRIBE FORMATTED|EXTENDED cannot refer to a " + + "metadata table."); + } + } + public TDescribeTableParams toThrift() { TDescribeTableParams params = new TDescribeTableParams(); params.setOutput_style(outputStyle_); @@ -158,6 +168,10 @@ public class DescribeTableStmt extends StatementBase { } else { Preconditions.checkNotNull(table_); params.setTable_name(table_.getTableName().toThrift()); + if (table_ instanceof IcebergMetadataTable) { + params.setMetadata_table_name(((IcebergMetadataTable)table_). + getMetadataTableName()); + } } return params; } diff --git a/fe/src/main/java/org/apache/impala/analysis/FromClause.java b/fe/src/main/java/org/apache/impala/analysis/FromClause.java index 656c609c4..306b5af86 100644 --- a/fe/src/main/java/org/apache/impala/analysis/FromClause.java +++ b/fe/src/main/java/org/apache/impala/analysis/FromClause.java @@ -84,9 +84,6 @@ public class FromClause extends StmtNode implements Iterable<TableRef> { boolean hasJoiningUnnest = false; for (int i = 0; i < tableRefs_.size(); ++i) { TableRef tblRef = tableRefs_.get(i); - if (IcebergMetadataTable.isIcebergMetadataTable(tblRef.getPath())) { - analyzer.addMetadataVirtualTable(tblRef.getPath()); - } tblRef = analyzer.resolveTableRef(tblRef); tableRefs_.set(i, Preconditions.checkNotNull(tblRef)); tblRef.setLeftTblRef(leftTblRef); diff --git a/fe/src/main/java/org/apache/impala/analysis/TableName.java b/fe/src/main/java/org/apache/impala/analysis/TableName.java index d7cdcc6c6..1f0efc67e 100644 --- a/fe/src/main/java/org/apache/impala/analysis/TableName.java +++ b/fe/src/main/java/org/apache/impala/analysis/TableName.java @@ -140,9 +140,10 @@ public class TableName { } public List<String> toPath() { - List<String> result = Lists.newArrayListWithCapacity(2); + List<String> result = Lists.newArrayListWithCapacity(3); if (db_ != null) result.add(db_); result.add(tbl_); + if (vTbl_ != null && !vTbl_.isEmpty()) result.add(vTbl_); return result; } diff --git a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java index 888df7b65..6ce58e5c1 100644 --- a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java +++ b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java @@ -33,6 +33,7 @@ import org.apache.impala.catalog.IcebergColumn; import org.apache.impala.catalog.KuduColumn; import org.apache.impala.catalog.StructField; import org.apache.impala.catalog.StructType; +import org.apache.impala.catalog.iceberg.IcebergMetadataTable; import org.apache.impala.common.ImpalaRuntimeException; import org.apache.impala.compat.MetastoreShim; import org.apache.impala.thrift.TColumnValue; @@ -351,4 +352,19 @@ public class DescribeResultFactory { } return descResult; } -} + + /** + * Builds a TDescribeResult for an Iceberg metadata table from an IcebergTable and a + * metadata table name. + * + * This describe request is issued against a VirtualTable which only exists in the + * Analyzer's StmtTableCache. Therefore, to get the columns of an IcebergMetadataTable + * it is simpler to re-create this object than to extract those from a new + * org.apache.iceberg.Table object or to send it over. + */ + public static TDescribeResult buildIcebergMetadataDescribeMinimalResult(FeTable table, + String vTableName) throws ImpalaRuntimeException { + IcebergMetadataTable metadataTable = new IcebergMetadataTable(table, vTableName); + return buildIcebergDescribeMinimalResult(metadataTable.getColumns()); + } +} \ No newline at end of file diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java index 31f9a570d..d030655fd 100644 --- a/fe/src/main/java/org/apache/impala/service/Frontend.java +++ b/fe/src/main/java/org/apache/impala/service/Frontend.java @@ -127,9 +127,11 @@ import org.apache.impala.catalog.ImpaladTableUsageTracker; import org.apache.impala.catalog.MaterializedViewHdfsTable; import org.apache.impala.catalog.MetaStoreClientPool; import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient; +import org.apache.impala.catalog.StructType; import org.apache.impala.catalog.TableLoadingException; import org.apache.impala.catalog.Type; import org.apache.impala.catalog.local.InconsistentMetadataFetchException; +import org.apache.impala.catalog.iceberg.IcebergMetadataTable; import org.apache.impala.common.AnalysisException; import org.apache.impala.common.FileSystemUtil; import org.apache.impala.common.ImpalaException; @@ -169,6 +171,7 @@ import org.apache.impala.thrift.TDescribeHistoryParams; import org.apache.impala.thrift.TDescribeOutputStyle; import org.apache.impala.thrift.TDescribeResult; import org.apache.impala.thrift.TImpalaTableType; +import org.apache.impala.thrift.TDescribeTableParams; import org.apache.impala.thrift.TIcebergDmlFinalizeParams; import org.apache.impala.thrift.TIcebergOperation; import org.apache.impala.thrift.TExecRequest; @@ -664,7 +667,8 @@ public class Frontend { columns.add(new TColumn("encoding", Type.STRING.toThrift())); columns.add(new TColumn("compression", Type.STRING.toThrift())); columns.add(new TColumn("block_size", Type.STRING.toThrift())); - } else if (descStmt.getTable() instanceof FeIcebergTable + } else if ((descStmt.getTable() instanceof FeIcebergTable + || descStmt.getTable() instanceof IcebergMetadataTable) && descStmt.getOutputStyle() == TDescribeOutputStyle.MINIMAL) { columns.add(new TColumn("nullable", Type.STRING.toThrift())); } @@ -1656,24 +1660,34 @@ public class Frontend { * Throws an exception if the table or db is not found or if there is an error loading * the table metadata. */ - public TDescribeResult describeTable(TTableName tableName, - TDescribeOutputStyle outputStyle, User user) throws ImpalaException { - RetryTracker retries = new RetryTracker( - String.format("fetching table %s.%s", tableName.db_name, tableName.table_name)); - while (true) { - try { - return doDescribeTable(tableName, outputStyle, user); - } catch(InconsistentMetadataFetchException e) { - retries.handleRetryOrThrow(e); + public TDescribeResult describeTable(TDescribeTableParams params, User user) + throws ImpalaException { + if (params.isSetTable_name()) { + RetryTracker retries = new RetryTracker( + String.format("fetching table %s.%s", params.table_name.db_name, + params.table_name.table_name)); + while (true) { + try { + return doDescribeTable(params.table_name, params.output_style, user, + params.metadata_table_name); + } catch(InconsistentMetadataFetchException e) { + retries.handleRetryOrThrow(e); + } } + } else { + Preconditions.checkState(params.output_style == TDescribeOutputStyle.MINIMAL); + Preconditions.checkNotNull(params.result_struct); + StructType structType = (StructType)Type.fromThrift(params.result_struct); + return DescribeResultFactory.buildDescribeMinimalResult(structType); } } - private TDescribeResult doDescribeTable(TTableName tableName, - TDescribeOutputStyle outputStyle, User user) throws ImpalaException { - FeTable table = getCatalog().getTable(tableName.db_name, - tableName.table_name); - List<Column> filteredColumns; + /** + * Filters out columns that the user is not authorized to see. + */ + private List<Column> filterAuthorizedColumnsForDescribeTable(FeTable table, User user) + throws InternalException { + List<Column> authFilteredColumns; if (authzFactory_.getAuthorizationConfig().isEnabled()) { // First run a table check PrivilegeRequest privilegeRequest = new PrivilegeRequestBuilder( @@ -1681,7 +1695,7 @@ public class Frontend { .allOf(Privilege.VIEW_METADATA).onTable(table).build(); if (!authzChecker_.get().hasAccess(user, privilegeRequest)) { // Filter out columns that the user is not authorized to see. - filteredColumns = new ArrayList<Column>(); + authFilteredColumns = new ArrayList<Column>(); for (Column col: table.getColumnsInHiveOrder()) { String colName = col.getName(); privilegeRequest = new PrivilegeRequestBuilder( @@ -1690,22 +1704,37 @@ public class Frontend { .onColumn(table.getDb().getName(), table.getName(), colName, table.getOwnerUser()).build(); if (authzChecker_.get().hasAccess(user, privilegeRequest)) { - filteredColumns.add(col); + authFilteredColumns.add(col); } } } else { // User has table-level access - filteredColumns = table.getColumnsInHiveOrder(); + authFilteredColumns = table.getColumnsInHiveOrder(); } } else { // Authorization is disabled - filteredColumns = table.getColumnsInHiveOrder(); + authFilteredColumns = table.getColumnsInHiveOrder(); } + return authFilteredColumns; + } + + private TDescribeResult doDescribeTable(TTableName tableName, + TDescribeOutputStyle outputStyle, User user, String vTableName) + throws ImpalaException { + FeTable table = getCatalog().getTable(tableName.db_name, + tableName.table_name); + List<Column> filteredColumns = filterAuthorizedColumnsForDescribeTable(table, user); if (outputStyle == TDescribeOutputStyle.MINIMAL) { if (table instanceof FeKuduTable) { return DescribeResultFactory.buildKuduDescribeMinimalResult(filteredColumns); } else if (table instanceof FeIcebergTable) { - return DescribeResultFactory.buildIcebergDescribeMinimalResult(filteredColumns); + if (vTableName == null) { + return DescribeResultFactory.buildIcebergDescribeMinimalResult(filteredColumns); + } else { + Preconditions.checkArgument(vTableName != null); + return DescribeResultFactory.buildIcebergMetadataDescribeMinimalResult(table, + vTableName); + } } else { return DescribeResultFactory.buildDescribeMinimalResult( Column.columnsToStruct(filteredColumns)); @@ -1713,6 +1742,7 @@ public class Frontend { } else { Preconditions.checkArgument(outputStyle == TDescribeOutputStyle.FORMATTED || outputStyle == TDescribeOutputStyle.EXTENDED); + Preconditions.checkArgument(vTableName == null); TDescribeResult result = DescribeResultFactory.buildDescribeFormattedResult(table, filteredColumns); // Filter out LOCATION text diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java index 1f2213e15..4c6d05f7c 100644 --- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java +++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java @@ -495,18 +495,9 @@ public class JniFrontend { Preconditions.checkNotNull(frontend_); TDescribeTableParams params = new TDescribeTableParams(); JniUtil.deserializeThrift(protocolFactory_, params, thriftDescribeTableParams); - Preconditions.checkState(params.isSetTable_name() ^ params.isSetResult_struct()); User user = new User(TSessionStateUtil.getEffectiveUser(params.getSession())); - TDescribeResult result = null; - if (params.isSetTable_name()) { - result = frontend_.describeTable(params.getTable_name(), params.output_style, user); - } else { - Preconditions.checkState(params.output_style == TDescribeOutputStyle.MINIMAL); - StructType structType = (StructType)Type.fromThrift(params.result_struct); - result = DescribeResultFactory.buildDescribeMinimalResult(structType); - } - + TDescribeResult result = frontend_.describeTable(params, user); try { TSerializer serializer = new TSerializer(protocolFactory_); return serializer.serialize(result); diff --git a/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java index b9b34c6c0..3e551227a 100644 --- a/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java +++ b/fe/src/test/java/org/apache/impala/authorization/AuthorizationTestBase.java @@ -38,6 +38,7 @@ import org.apache.impala.testutil.ImpaladTestCatalog; import org.apache.impala.thrift.TColumnValue; import org.apache.impala.thrift.TDescribeOutputStyle; import org.apache.impala.thrift.TDescribeResult; +import org.apache.impala.thrift.TDescribeTableParams; import org.apache.impala.thrift.TFunctionBinaryType; import org.apache.impala.thrift.TPrivilege; import org.apache.impala.thrift.TPrivilegeLevel; @@ -272,8 +273,11 @@ public abstract class AuthorizationTestBase extends FrontendTestBase { Preconditions.checkArgument(includedStrings_.length != 0 || excludedStrings_.length != 0, "One or both of included or excluded strings must be defined."); - List<String> result = resultToStringList(authzFrontend_.describeTable(table, - outputStyle_, user_)); + TDescribeTableParams testParams = new TDescribeTableParams(); + testParams.setTable_name(table); + testParams.setOutput_style(outputStyle_); + List<String> result = resultToStringList(authzFrontend_.describeTable(testParams, + user_)); for (String str: includedStrings_) { assertTrue(String.format("\"%s\" is not in the describe output.\n" + "Expected : %s\n Actual : %s", str, Arrays.toString(includedStrings_), diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test index c34f3aacc..0f58dae99 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-metadata-tables.test @@ -420,9 +420,9 @@ INT,STRING,BIGINT #### ==== ---- QUERY -describe functional_parquet.iceberg_query_metadata.snapshots; +describe formatted functional_parquet.iceberg_query_metadata.snapshots; ---- CATCH -AnalysisException: Could not resolve path: 'functional_parquet.iceberg_query_metadata.snapshots' +AnalysisException: DESCRIBE FORMATTED|EXTENDED cannot refer to a metadata table. ==== ---- QUERY show create table functional_parquet.iceberg_query_metadata.snapshots; @@ -456,7 +456,7 @@ ParseException: Syntax error in line 1 ==== #### -# Test 9 : Query STRUCT type columns +# Test 9 : Query nested type columns #### ==== ---- QUERY @@ -550,4 +550,264 @@ as select equality_ids from functional_parquet.iceberg_query_metadata.all_files; select item from iceberg_query_metadata_all_files a, a.equality_ids e, e.delete_ids; ---- CATCH AnalysisException: Querying collection types (ARRAY/MAP) is not supported for Iceberg Metadata tables. (IMPALA-12610, IMPALA-12611) -==== \ No newline at end of file +==== + +#### +# Test 10 : Describe all the metadata tables once +#### +---- QUERY +describe functional_parquet.iceberg_query_metadata.snapshots; +---- RESULTS +'committed_at','timestamp','','false' +'snapshot_id','bigint','','false' +'parent_id','bigint','','true' +'operation','string','','true' +'manifest_list','string','','true' +'summary','map<string,string>','','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.`files`; +---- RESULTS +'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true' +'file_path','string','Location URI with FS scheme','false' +'file_format','string','File format name: avro, orc, or parquet','false' +'spec_id','int','Partition spec ID','true' +'record_count','bigint','Number of records in the file','false' +'file_size_in_bytes','bigint','Total file size in bytes','false' +'column_sizes','map<int,bigint>','Map of column id to total size on disk','true' +'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true' +'null_value_counts','map<int,bigint>','Map of column id to null value count','true' +'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true' +'lower_bounds','map<int,binary>','Map of column id to lower bound','true' +'upper_bounds','map<int,binary>','Map of column id to upper bound','true' +'key_metadata','binary','Encryption key metadata blob','true' +'split_offsets','array<bigint>','Splittable offsets','true' +'equality_ids','array<int>','Equality comparison field IDs','true' +'sort_order_id','int','Sort order ID','true' +'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.data_files; +---- RESULTS +'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true' +'file_path','string','Location URI with FS scheme','false' +'file_format','string','File format name: avro, orc, or parquet','false' +'spec_id','int','Partition spec ID','true' +'record_count','bigint','Number of records in the file','false' +'file_size_in_bytes','bigint','Total file size in bytes','false' +'column_sizes','map<int,bigint>','Map of column id to total size on disk','true' +'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true' +'null_value_counts','map<int,bigint>','Map of column id to null value count','true' +'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true' +'lower_bounds','map<int,binary>','Map of column id to lower bound','true' +'upper_bounds','map<int,binary>','Map of column id to upper bound','true' +'key_metadata','binary','Encryption key metadata blob','true' +'split_offsets','array<bigint>','Splittable offsets','true' +'equality_ids','array<int>','Equality comparison field IDs','true' +'sort_order_id','int','Sort order ID','true' +'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.delete_files; +---- RESULTS +'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true' +'file_path','string','Location URI with FS scheme','false' +'file_format','string','File format name: avro, orc, or parquet','false' +'spec_id','int','Partition spec ID','true' +'record_count','bigint','Number of records in the file','false' +'file_size_in_bytes','bigint','Total file size in bytes','false' +'column_sizes','map<int,bigint>','Map of column id to total size on disk','true' +'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true' +'null_value_counts','map<int,bigint>','Map of column id to null value count','true' +'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true' +'lower_bounds','map<int,binary>','Map of column id to lower bound','true' +'upper_bounds','map<int,binary>','Map of column id to upper bound','true' +'key_metadata','binary','Encryption key metadata blob','true' +'split_offsets','array<bigint>','Splittable offsets','true' +'equality_ids','array<int>','Equality comparison field IDs','true' +'sort_order_id','int','Sort order ID','true' +'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.history; +---- RESULTS +'made_current_at','timestamp','','false' +'snapshot_id','bigint','','false' +'parent_id','bigint','','true' +'is_current_ancestor','boolean','','false' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.metadata_log_entries; +---- RESULTS +'timestamp','timestamp','','false' +'file','string','','false' +'latest_snapshot_id','bigint','','true' +'latest_schema_id','int','','true' +'latest_sequence_number','bigint','','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.snapshots; +---- RESULTS +'committed_at','timestamp','','false' +'snapshot_id','bigint','','false' +'parent_id','bigint','','true' +'operation','string','','true' +'manifest_list','string','','true' +'summary','map<string,string>','','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.refs; +---- RESULTS +'name','string','','false' +'type','string','','false' +'snapshot_id','bigint','','false' +'max_reference_age_in_ms','bigint','','true' +'min_snapshots_to_keep','int','','true' +'max_snapshot_age_in_ms','bigint','','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.manifests; +---- RESULTS +'content','int','','false' +'path','string','','false' +'length','bigint','','false' +'partition_spec_id','int','','false' +'added_snapshot_id','bigint','','false' +'added_data_files_count','int','','false' +'existing_data_files_count','int','','false' +'deleted_data_files_count','int','','false' +'added_delete_files_count','int','','false' +'existing_delete_files_count','int','','false' +'deleted_delete_files_count','int','','false' +'partition_summaries','array<struct<\n contains_null:boolean,\n contains_nan:boolean,\n lower_bound:string,\n upper_bound:string\n>>','','false' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.`partitions`; +---- RESULTS +'record_count','bigint','Count of records in data files','false' +'file_count','int','Count of data files','false' +'position_delete_record_count','bigint','Count of records in position delete files','false' +'position_delete_file_count','int','Count of position delete files','false' +'equality_delete_record_count','bigint','Count of records in equality delete files','false' +'equality_delete_file_count','int','Count of equality delete files','false' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.all_data_files; +---- RESULTS +'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true' +'file_path','string','Location URI with FS scheme','false' +'file_format','string','File format name: avro, orc, or parquet','false' +'spec_id','int','Partition spec ID','true' +'record_count','bigint','Number of records in the file','false' +'file_size_in_bytes','bigint','Total file size in bytes','false' +'column_sizes','map<int,bigint>','Map of column id to total size on disk','true' +'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true' +'null_value_counts','map<int,bigint>','Map of column id to null value count','true' +'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true' +'lower_bounds','map<int,binary>','Map of column id to lower bound','true' +'upper_bounds','map<int,binary>','Map of column id to upper bound','true' +'key_metadata','binary','Encryption key metadata blob','true' +'split_offsets','array<bigint>','Splittable offsets','true' +'equality_ids','array<int>','Equality comparison field IDs','true' +'sort_order_id','int','Sort order ID','true' +'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.all_delete_files; +---- RESULTS +'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true' +'file_path','string','Location URI with FS scheme','false' +'file_format','string','File format name: avro, orc, or parquet','false' +'spec_id','int','Partition spec ID','true' +'record_count','bigint','Number of records in the file','false' +'file_size_in_bytes','bigint','Total file size in bytes','false' +'column_sizes','map<int,bigint>','Map of column id to total size on disk','true' +'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true' +'null_value_counts','map<int,bigint>','Map of column id to null value count','true' +'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true' +'lower_bounds','map<int,binary>','Map of column id to lower bound','true' +'upper_bounds','map<int,binary>','Map of column id to upper bound','true' +'key_metadata','binary','Encryption key metadata blob','true' +'split_offsets','array<bigint>','Splittable offsets','true' +'equality_ids','array<int>','Equality comparison field IDs','true' +'sort_order_id','int','Sort order ID','true' +'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.all_files; +---- RESULTS +'content','int','Contents of the file: 0=data, 1=position deletes, 2=equality deletes','true' +'file_path','string','Location URI with FS scheme','false' +'file_format','string','File format name: avro, orc, or parquet','false' +'spec_id','int','Partition spec ID','true' +'record_count','bigint','Number of records in the file','false' +'file_size_in_bytes','bigint','Total file size in bytes','false' +'column_sizes','map<int,bigint>','Map of column id to total size on disk','true' +'value_counts','map<int,bigint>','Map of column id to total count, including null and NaN','true' +'null_value_counts','map<int,bigint>','Map of column id to null value count','true' +'nan_value_counts','map<int,bigint>','Map of column id to number of NaN values in the column','true' +'lower_bounds','map<int,binary>','Map of column id to lower bound','true' +'upper_bounds','map<int,binary>','Map of column id to upper bound','true' +'key_metadata','binary','Encryption key metadata blob','true' +'split_offsets','array<bigint>','Splittable offsets','true' +'equality_ids','array<int>','Equality comparison field IDs','true' +'sort_order_id','int','Sort order ID','true' +'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.all_manifests; +---- RESULTS +'content','int','','false' +'path','string','','false' +'length','bigint','','false' +'partition_spec_id','int','','true' +'added_snapshot_id','bigint','','true' +'added_data_files_count','int','','true' +'existing_data_files_count','int','','true' +'deleted_data_files_count','int','','true' +'added_delete_files_count','int','','false' +'existing_delete_files_count','int','','false' +'deleted_delete_files_count','int','','false' +'partition_summaries','array<struct<\n contains_null:boolean,\n contains_nan:boolean,\n lower_bound:string,\n upper_bound:string\n>>','','true' +'reference_snapshot_id','bigint','','false' +---- TYPES +STRING,STRING,STRING,STRING +==== +---- QUERY +describe functional_parquet.iceberg_query_metadata.all_entries; +---- RESULTS +'status','int','','false' +'snapshot_id','bigint','','true' +'sequence_number','bigint','','true' +'file_sequence_number','bigint','','true' +'data_file','struct<\n content:int comment ''contents of the file: 0=data, 1=position deletes, 2=equality deletes'',\n file_path:string comment ''location uri with fs scheme'',\n file_format:string comment ''file format name: avro, orc, or parquet'',\n spec_id:int comment ''partition spec id'',\n record_count:bigint comment ''number of records in the file'',\n file_size_in_bytes:bigint comment ''total file size in bytes'',\n column_sizes:map<int,bigint> comment ''map of column id [...] +'readable_metrics','struct<\n i:struct<\n column_size:bigint comment ''total size on disk'',\n value_count:bigint comment ''total count, including null and nan'',\n null_value_count:bigint comment ''null value count'',\n nan_value_count:bigint comment ''nan value count'',\n lower_bound:int comment ''lower bound'',\n upper_bound:int comment ''upper bound''\n > comment ''metrics for column i''\n>','Column metrics in readable form','true' +---- TYPES +STRING,STRING,STRING,STRING +====
