This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new f3d9141c Add policies for metadata compaction, orphan file removal and
snapshot retention (#969)
f3d9141c is described below
commit f3d9141c9708940523aa8d206a0bb32465398a7f
Author: Yufei Gu <yu...@apache.org>
AuthorDate: Tue Feb 25 10:59:06 2025 -0800
Add policies for metadata compaction, orphan file removal and snapshot
retention (#969)
---
.../system/data-compaction/2025-02-03.json | 6 +--
.../2025-02-03.json | 16 +++----
.../system/orphan-file-removal/2025-02-03.json | 50 ++++++++++++++++++++++
.../2025-02-03.json | 18 ++++----
4 files changed, 69 insertions(+), 21 deletions(-)
diff --git
a/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
b/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
index b452eb80..f222d7c1 100644
---
a/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
+++
b/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
@@ -2,13 +2,13 @@
"license": "Licensed under the Apache License, Version 2.0
(http://www.apache.org/licenses/LICENSE-2.0)",
"$id":
"https://polaris.apache.org/schemas/policies/system/data-compaction/2025-02-03.json";,
"title": "Data Compaction Policy",
- "description": "Inheritable Polaris policy schema for Iceberg table data
compaction.",
+ "description": "Inheritable Polaris policy schema for Iceberg table data
compaction",
"type": "object",
"properties": {
"version": {
"type": "string",
"const": "2025-02-03",
- "description": "Schema version."
+ "description": "Schema version"
},
"enable": {
"type": "boolean",
@@ -32,7 +32,7 @@
"target_file_size_bytes": 134217728,
"compaction_strategy": "bin-pack",
"max-concurrent-file-group-rewrites": 5,
- "my-key": "my-value"
+ "key1": "value1"
}
}
]
diff --git
a/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
b/polaris-core/src/main/resources/schemas/policies/system/metadata-compaction/2025-02-03.json
similarity index 64%
copy from
polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
copy to
polaris-core/src/main/resources/schemas/policies/system/metadata-compaction/2025-02-03.json
index b452eb80..d99dc396 100644
---
a/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
+++
b/polaris-core/src/main/resources/schemas/policies/system/metadata-compaction/2025-02-03.json
@@ -1,18 +1,18 @@
{
"license": "Licensed under the Apache License, Version 2.0
(http://www.apache.org/licenses/LICENSE-2.0)",
- "$id":
"https://polaris.apache.org/schemas/policies/system/data-compaction/2025-02-03.json";,
- "title": "Data Compaction Policy",
- "description": "Inheritable Polaris policy schema for Iceberg table data
compaction.",
+ "$id":
"https://polaris.apache.org/schemas/policies/system/metadata-compaction/2025-02-03.json";,
+ "title": "Metadata Compaction Policy",
+ "description": "Inheritable Polaris policy schema for Iceberg table metadata
compaction",
"type": "object",
"properties": {
"version": {
"type": "string",
"const": "2025-02-03",
- "description": "Schema version."
+ "description": "Schema version"
},
"enable": {
"type": "boolean",
- "description": "Enable or disable data compaction."
+ "description": "Enable or disable metadata compaction."
},
"config": {
"type": "object",
@@ -29,10 +29,8 @@
"version": "2025-02-03",
"enable": true,
"config": {
- "target_file_size_bytes": 134217728,
- "compaction_strategy": "bin-pack",
- "max-concurrent-file-group-rewrites": 5,
- "my-key": "my-value"
+ "spec_id": 1,
+ "key1": "value1"
}
}
]
diff --git
a/polaris-core/src/main/resources/schemas/policies/system/orphan-file-removal/2025-02-03.json
b/polaris-core/src/main/resources/schemas/policies/system/orphan-file-removal/2025-02-03.json
new file mode 100644
index 00000000..19f35dec
--- /dev/null
+++
b/polaris-core/src/main/resources/schemas/policies/system/orphan-file-removal/2025-02-03.json
@@ -0,0 +1,50 @@
+{
+ "license": "Licensed under the Apache License, Version 2.0
(http://www.apache.org/licenses/LICENSE-2.0)",
+ "$id":
"https://polaris.apache.org/schemas/policies/system/orphan-file-removal/2025-02-03.json";,
+ "title": "Orphan File Removal Policy",
+ "description": "Inheritable Polaris policy schema for Iceberg table orphan
file removal",
+ "type": "object",
+ "properties": {
+ "version": {
+ "type": "string",
+ "const": "2025-02-03",
+ "description": "Schema version"
+ },
+ "enable": {
+ "type": "boolean",
+ "description": "Enable or disable orphan file removal."
+ },
+ "max_orphan_file_age_in_days": {
+ "type": "number",
+ "description": "Specifies the maximum age (in days) for orphaned files
before they are eligible for removal."
+ },
+ "locations": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "Specifies a list of custom directories to search for
files instead of the default table location. Use with caution—if set to a broad
location (e.g., s3://my-bucket instead of s3://my-bucket/my-table-location),
all unreferenced files in that path may be permanently deleted, including files
from other tables. Following best practices, tables should be stored in
separate locations to avoid accidental data loss."
+ },
+ "config": {
+ "type": "object",
+ "description": "A map containing custom configuration properties. Note
that interoperability is not guaranteed.",
+ "additionalProperties": {
+ "type": ["string", "number", "boolean"]
+ }
+ }
+ },
+ "required": ["enable"],
+ "additionalProperties": false,
+ "examples": [
+ {
+ "version": "2025-02-03",
+ "enable": true,
+ "max_orphan_file_age_in_days": 30,
+ "location": "s3://my-bucket/my-table-location",
+ "config": {
+ "prefix_mismatch_mode": "ignore",
+ "key1": "value1"
+ }
+ }
+ ]
+}
diff --git
a/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
b/polaris-core/src/main/resources/schemas/policies/system/snapshot-retention/2025-02-03.json
similarity index 64%
copy from
polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
copy to
polaris-core/src/main/resources/schemas/policies/system/snapshot-retention/2025-02-03.json
index b452eb80..dfd79e19 100644
---
a/polaris-core/src/main/resources/schemas/policies/system/data-compaction/2025-02-03.json
+++
b/polaris-core/src/main/resources/schemas/policies/system/snapshot-retention/2025-02-03.json
@@ -1,18 +1,18 @@
{
"license": "Licensed under the Apache License, Version 2.0
(http://www.apache.org/licenses/LICENSE-2.0)",
- "$id":
"https://polaris.apache.org/schemas/policies/system/data-compaction/2025-02-03.json";,
- "title": "Data Compaction Policy",
- "description": "Inheritable Polaris policy schema for Iceberg table data
compaction.",
+ "$id":
"https://polaris.apache.org/schemas/policies/system/snapshot-retention/2025-02-03.json";,
+ "title": "Snapshot Retention Policy",
+ "description": "Inheritable Polaris policy schema for Iceberg table snapshot
retention",
"type": "object",
"properties": {
"version": {
"type": "string",
"const": "2025-02-03",
- "description": "Schema version."
+ "description": "Schema version"
},
"enable": {
"type": "boolean",
- "description": "Enable or disable data compaction."
+ "description": "Enable or disable snapshot retention."
},
"config": {
"type": "object",
@@ -29,10 +29,10 @@
"version": "2025-02-03",
"enable": true,
"config": {
- "target_file_size_bytes": 134217728,
- "compaction_strategy": "bin-pack",
- "max-concurrent-file-group-rewrites": 5,
- "my-key": "my-value"
+ "min_snapshot_to_keep": 1,
+ "max_snapshot_age_days": 2,
+ "max_ref_age_days": 3,
+ "key1": "value1"
}
}
]
