This is an automated email from the ASF dual-hosted git repository. aokolnychyi pushed a commit to branch snapshot-flag in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 5b271df6ae5738f4710c83fdc1e9e918651f3bfe Author: Anton Okolnychyi <[email protected]> AuthorDate: Fri Nov 20 08:35:59 2020 -0800 Core: Add snapshot table property --- .../java/org/apache/iceberg/RemoveSnapshots.java | 7 +++++++ .../java/org/apache/iceberg/TableProperties.java | 3 +++ .../java/org/apache/iceberg/TestRemoveSnapshots.java | 16 ++++++++++++++++ .../iceberg/actions/TestExpireSnapshotsAction.java | 20 ++++++++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/RemoveSnapshots.java b/core/src/main/java/org/apache/iceberg/RemoveSnapshots.java index 734e209..f843b89 100644 --- a/core/src/main/java/org/apache/iceberg/RemoveSnapshots.java +++ b/core/src/main/java/org/apache/iceberg/RemoveSnapshots.java @@ -30,6 +30,7 @@ import org.apache.iceberg.avro.Avro; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.relocated.com.google.common.base.Joiner; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -51,6 +52,8 @@ import static org.apache.iceberg.TableProperties.COMMIT_NUM_RETRIES; import static org.apache.iceberg.TableProperties.COMMIT_NUM_RETRIES_DEFAULT; import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS; import static org.apache.iceberg.TableProperties.COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT; +import static org.apache.iceberg.TableProperties.SNAPSHOT; +import static org.apache.iceberg.TableProperties.SNAPSHOT_DEFAULT; @SuppressWarnings("UnnecessaryAnonymousClass") class RemoveSnapshots implements ExpireSnapshots { @@ -78,6 +81,10 @@ class RemoveSnapshots implements ExpireSnapshots { RemoveSnapshots(TableOperations ops) { this.ops = ops; this.base = ops.current(); + + ValidationException.check( + !PropertyUtil.propertyAsBoolean(base.properties(), SNAPSHOT, SNAPSHOT_DEFAULT), + "Not allowed to expire snapshots in a snapshot table as this may remove files in the original table"); } @Override diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 985abe3..987435c 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -134,4 +134,7 @@ public class TableProperties { public static final String ENGINE_HIVE_ENABLED = "engine.hive.enabled"; public static final boolean ENGINE_HIVE_ENABLED_DEFAULT = false; + + public static final String SNAPSHOT = "snapshot"; + public static final boolean SNAPSHOT_DEFAULT = false; } diff --git a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java index e9a8862..546f5de 100644 --- a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java +++ b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java @@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import org.apache.iceberg.ManifestEntry.Status; +import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -1044,4 +1045,19 @@ public class TestRemoveSnapshots extends TableTestBase { }); }); } + + @Test + public void testExpireSnapshotsInSnapshotTable() { + table.updateProperties() + .set(TableProperties.SNAPSHOT, "true") + .commit(); + + table.newAppend() + .appendFile(FILE_A) + .commit(); + + AssertHelpers.assertThrows("Should complain about expiring snapshots", + ValidationException.class, "Not allowed to expire snapshots", + () -> table.expireSnapshots()); + } } diff --git a/spark/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java b/spark/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java index 558c54b..35665c9 100644 --- a/spark/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java +++ b/spark/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java @@ -40,6 +40,7 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; @@ -383,6 +384,25 @@ public abstract class TestExpireSnapshotsAction extends SparkTestBase { } @Test + public void testExpireSnapshotsInSnapshotTable() { + table.updateProperties() + .set(TableProperties.SNAPSHOT, "true") + .commit(); + + table.newAppend() + .appendFile(FILE_A) + .commit(); + + ExpireSnapshotsAction action = Actions.forTable(table).expireSnapshots() + .expireOlderThan(System.currentTimeMillis()) + .retainLast(2); + + AssertHelpers.assertThrows("Should complain about expiring snapshots", + ValidationException.class, "Not allowed to expire snapshots", + action::execute); + } + + @Test public void testExpireOlderThanMultipleCalls() { table.newAppend() .appendFile(FILE_A) // data_bucket=0
