This is an automated email from the ASF dual-hosted git repository.
szehon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 73b03777b8 Docs: Add all types of files metadata tables in Spark
queries (#8316)
73b03777b8 is described below
commit 73b03777b86a84b20e85189e652bb134145856e1
Author: Manu Zhang <[email protected]>
AuthorDate: Fri Sep 1 00:51:37 2023 +0800
Docs: Add all types of files metadata tables in Spark queries (#8316)
---
docs/spark-queries.md | 22 +++++++++++++++++-----
1 file changed, 17 insertions(+), 5 deletions(-)
diff --git a/docs/spark-queries.md b/docs/spark-queries.md
index 4629b2e7ea..2923638306 100644
--- a/docs/spark-queries.md
+++ b/docs/spark-queries.md
@@ -270,17 +270,29 @@ order by made_current_at
### Files
-To show a table's current data files:
+To show a table's current files:
```sql
SELECT * FROM prod.db.table.files;
```
-|content|file_path
|file_format|spec_id|partition|record_count|file_size_in_bytes|column_sizes
|value_counts |null_value_counts|nan_value_counts|lower_bounds
|upper_bounds |key_metadata|split_offsets|equality_ids|sort_order_id|
+| content | file_path | file_format | spec_id | record_count |
file_size_in_bytes | column_sizes | value_counts | null_value_counts |
nan_value_counts | lower_bounds | upper_bounds | key_metadata | split_offsets |
equality_ids | sort_order_id | readable_metrics |
| -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- |
-- | -- |
-| 0 | s3:/.../table/data/00000-3-8d6d60e8-d427-4809-bcf0-f5d45a4aad96.parquet
| PARQUET | 0 | {1999-01-01, 01} | 1 | 597 | [1 ->
90, 2 -> 62] | [1 -> 1, 2 -> 1] | [1 -> 0, 2 -> 0] | [] | [1 ->
, 2 -> c] | [1 -> , 2 -> c] | null | [4] | null | null |
-| 0 | s3:/.../table/data/00001-4-8d6d60e8-d427-4809-bcf0-f5d45a4aad96.parquet
| PARQUET | 0 | {1999-01-01, 02} | 1 | 597 | [1 ->
90, 2 -> 62] | [1 -> 1, 2 -> 1] | [1 -> 0, 2 -> 0] | [] | [1 ->
, 2 -> b] | [1 -> , 2 -> b] | null | [4] | null | null |
-| 0 | s3:/.../table/data/00002-5-8d6d60e8-d427-4809-bcf0-f5d45a4aad96.parquet
| PARQUET | 0 | {1999-01-01, 03} | 1 | 597 | [1 ->
90, 2 -> 62] | [1 -> 1, 2 -> 1] | [1 -> 0, 2 -> 0] | [] | [1 ->
, 2 -> a] | [1 -> , 2 -> a] | null | [4] | null | null |
+| 0 |
s3:/.../table/data/00042-3-a9aa8b24-20bc-4d56-93b0-6b7675782bb5-00001.parquet |
PARQUET | 0 | 1 | 652 | {1:52,2:48} | {1:1,2:1} | {1:0,2:0} | {} | {1:,2:d} |
{1:,2:d} | NULL | [4] | NULL | 0 |
{"data":{"column_size":48,"value_count":1,"null_value_count":0,"nan_value_count":null,"lower_bound":"d","upper_bound":"d"},"id":{"column_size":52,"value_count":1,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":1}}
|
+| 0 |
s3:/.../table/data/00000-0-f9709213-22ca-4196-8733-5cb15d2afeb9-00001.parquet |
PARQUET | 0 | 1 | 643 | {1:46,2:48} | {1:1,2:1} | {1:0,2:0} | {} | {1:,2:a} |
{1:,2:a} | NULL | [4] | NULL | 0 |
{"data":{"column_size":48,"value_count":1,"null_value_count":0,"nan_value_count":null,"lower_bound":"a","upper_bound":"a"},"id":{"column_size":46,"value_count":1,"null_value_count":0,"nan_value_count":null,"lower_bound":1,"upper_bound":1}}
|
+| 0 |
s3:/.../table/data/00001-1-f9709213-22ca-4196-8733-5cb15d2afeb9-00001.parquet |
PARQUET | 0 | 2 | 644 | {1:49,2:51} | {1:2,2:2} | {1:0,2:0} | {} | {1:,2:b} |
{1:,2:c} | NULL | [4] | NULL | 0 |
{"data":{"column_size":51,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":"b","upper_bound":"c"},"id":{"column_size":49,"value_count":2,"null_value_count":0,"nan_value_count":null,"lower_bound":2,"upper_bound":3}}
|
+| 1 |
s3:/.../table/data/00081-4-a9aa8b24-20bc-4d56-93b0-6b7675782bb5-00001-deletes.parquet
| PARQUET | 0 | 1 | 1560 | {2147483545:46,2147483546:152} |
{2147483545:1,2147483546:1} | {2147483545:0,2147483546:0} | {} |
{2147483545:,2147483546:s3:/.../table/data/00000-0-f9709213-22ca-4196-8733-5cb15d2afeb9-00001.parquet}
|
{2147483545:,2147483546:s3:/.../table/data/00000-0-f9709213-22ca-4196-8733-5cb15d2afeb9-00001.parquet}
| NULL | [4] | NULL | NULL | {"data":{"column_size":null,"value_cou [...]
+| 2 |
s3:/.../table/data/00047-25-833044d0-127b-415c-b874-038a4f978c29-00612.parquet
| PARQUET | 0 | 126506 | 28613985 | {100:135377,101:11314} |
{100:126506,101:126506} | {100:105434,101:11} | {} | {100:0,101:17} |
{100:404455227527,101:23} | NULL | NULL | [1] | 0 |
{"id":{"column_size":135377,"value_count":126506,"null_value_count":105434,"nan_value_count":null,"lower_bound":0,"upper_bound":404455227527},"data":{"column_size":11314,"value_count":126506,"null_value_count":
11,"nan_value [...]
+
+{{< hint info >}}
+Content refers to type of content stored by the data file:
+ 0 Data
+ 1 Position Deletes
+ 2 Equality Deletes
+{{< /hint >}}
+
+To show only data files or delete files, query `prod.db.table.data_files` and
`prod.db.table.delete_files` respectively.
+To show all files, data files and delete files across all tracked snapshots,
query `prod.db.table.all_files`, `prod.db.table.all_data_files` and
`prod.db.table.all_delete_files` respectively.
### Manifests
