stevenzwu commented on code in PR #17523:
URL: https://github.com/apache/iceberg/pull/17523#discussion_r4557375386
##########
api/src/main/java/org/apache/iceberg/Snapshot.java:
##########
@@ -166,10 +166,26 @@ default Iterable<DeleteFile> removedDeleteFiles(FileIO
io) {
/**
* Return the location of this snapshot's manifest list, or null if it is
not separate.
*
- * @return the location of the manifest list for this Snapshot
+ * <p>Only set for tables with format version 1 through 3. Snapshots in v4
tables use {@link
Review Comment:
Removed the version-specific paragraph.
##########
core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java:
##########
@@ -241,8 +241,8 @@ private static List<Snapshot> updatePathInSnapshots(
TableMetadata metadata, String sourcePrefix, String targetPrefix) {
List<Snapshot> newSnapshots =
Lists.newArrayListWithCapacity(metadata.snapshots().size());
for (Snapshot snapshot : metadata.snapshots()) {
- String newManifestListLocation =
- newPath(snapshot.manifestListLocation(), sourcePrefix, targetPrefix);
+ String manifestList = snapshot.manifestListLocation();
Review Comment:
I'd keep that out of this PR since nothing writes root manifests yet. Both
cleanup paths read through allManifests, which now throws
UnsupportedOperationException for root-manifest snapshots, so they fail loudly
instead of silently skipping files.
##########
core/src/main/java/org/apache/iceberg/BaseSnapshot.java:
##########
@@ -65,9 +66,14 @@ class BaseSnapshot implements Snapshot {
Map<String, String> summary,
Integer schemaId,
String manifestList,
+ String rootManifest,
Long firstRowId,
Long addedRows,
String keyId) {
+ Preconditions.checkArgument(
Review Comment:
Added a second check so exactly one of manifest list or root manifest is
set. The v1 embedded-manifests constructor is unaffected.
##########
api/src/main/java/org/apache/iceberg/Snapshot.java:
##########
@@ -166,10 +166,26 @@ default Iterable<DeleteFile> removedDeleteFiles(FileIO
io) {
/**
* Return the location of this snapshot's manifest list, or null if it is
not separate.
*
- * @return the location of the manifest list for this Snapshot
+ * <p>Only set for tables with format version 1 through 3. Snapshots in v4
tables use {@link
+ * #rootManifestLocation()} instead.
+ *
+ * @return the location of the manifest list for this snapshot
*/
String manifestListLocation();
+ /**
+ * Return the location of this snapshot's root manifest, or null if this
snapshot has no root
+ * manifest.
+ *
+ * <p>v4 snapshots store this in {@code root-manifest} instead of a manifest
list. At most one of
Review Comment:
Removed this paragraph too.
##########
api/src/main/java/org/apache/iceberg/Snapshot.java:
##########
@@ -166,10 +166,26 @@ default Iterable<DeleteFile> removedDeleteFiles(FileIO
io) {
/**
* Return the location of this snapshot's manifest list, or null if it is
not separate.
*
- * @return the location of the manifest list for this Snapshot
+ * <p>Only set for tables with format version 1 through 3. Snapshots in v4
tables use {@link
+ * #rootManifestLocation()} instead.
+ *
+ * @return the location of the manifest list for this snapshot
*/
String manifestListLocation();
+ /**
+ * Return the location of this snapshot's root manifest, or null if this
snapshot has no root
+ * manifest.
+ *
+ * <p>v4 snapshots store this in {@code root-manifest} instead of a manifest
list. At most one of
Review Comment:
Removed that paragraph. The first sentence now describes the null case
directly.
##########
core/src/test/java/org/apache/iceberg/TestSnapshot.java:
##########
@@ -34,22 +34,88 @@ public class TestSnapshot extends TestBase {
public void snapshotValueMethodsIncludeMetadataFields() {
Snapshot snapshot =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot same =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot differentFirstRowId =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 11L, 5L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 11L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot differentAddedRows =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 6L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 6L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot differentKeyId =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L,
"key-2");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-2" /* keyId */);
Snapshot withoutMetadataFields =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null,
null, null);
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
Review Comment:
Added tests for a root-manifest snapshot covering the both/neither
validation and allManifests rejecting root manifests.
##########
core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java:
##########
@@ -252,7 +252,8 @@ private static List<Snapshot> updatePathInSnapshots(
snapshot.operation(),
snapshot.summary(),
snapshot.schemaId(),
- newManifestListLocation,
+ manifestList == null ? null : newPath(manifestList,
sourcePrefix, targetPrefix),
+ rootManifest == null ? null : newPath(rootManifest,
sourcePrefix, targetPrefix),
Review Comment:
added `replacePathsRewritesRootManifestLocation`, which asserts the
rewritten location.
##########
core/src/test/java/org/apache/iceberg/TestSnapshotJson.java:
##########
@@ -156,13 +164,16 @@ public void testJsonConversionWithRowLineage() throws
IOException {
null,
null,
manifestList,
+ null /* rootManifest */,
firstRowId,
addedRows,
null);
String json = SnapshotParser.toJson(expected);
Snapshot snapshot = SnapshotParser.fromJson(json);
assertThat(snapshot.snapshotId()).isEqualTo(expected.snapshotId());
+
assertThat(snapshot.manifestListLocation()).isEqualTo(expected.manifestListLocation());
+
assertThat(snapshot.rootManifestLocation()).isEqualTo(expected.rootManifestLocation());
Review Comment:
Done.
##########
core/src/main/java/org/apache/iceberg/SnapshotProducer.java:
##########
@@ -372,6 +372,7 @@ public Snapshot apply() {
summary(base),
base.currentSchemaId(),
manifestList.location(),
+ null /* rootManifest */,
Review Comment:
Done, now a line comment: `null, // rootManifest`
##########
core/src/test/java/org/apache/iceberg/TestSnapshot.java:
##########
@@ -34,22 +34,88 @@ public class TestSnapshot extends TestBase {
public void snapshotValueMethodsIncludeMetadataFields() {
Snapshot snapshot =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot same =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot differentFirstRowId =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 11L, 5L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 11L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot differentAddedRows =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 6L,
"key-1");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 6L /* addedRows */,
+ "key-1" /* keyId */);
Snapshot differentKeyId =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L,
"key-2");
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ 10L /* firstRowId */,
+ 5L /* addedRows */,
+ "key-2" /* keyId */);
Snapshot withoutMetadataFields =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null,
null, null);
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ null /* firstRowId */,
+ null /* addedRows */,
+ null /* keyId */);
Review Comment:
Reverted the wrapping that was only for names. Only the null argument is
added now. The sites that still overflow are one argument per line without
names.
##########
core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java:
##########
@@ -70,40 +70,108 @@ public void cleanup() {
public void testSnapshotRowIDValidation() {
Snapshot snapshot =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null,
null, null);
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "ml.avro" /* manifestListLocation */,
+ null /* rootManifest */,
+ null /* firstRowId */,
+ null /* addedRows */,
+ null /* keyId */);
Review Comment:
Reverted the wrapping that was only for names. Only the null argument is
added now. The sites that still overflow are one argument per line without
names.
##########
core/src/test/java/org/apache/iceberg/TestRowLineageMetadata.java:
##########
@@ -158,12 +250,34 @@ public void testInvalidSnapshotAddition() {
// add rows to check TableMetadata validation; Snapshot rejects negative
next-row-id
Snapshot addRows =
new BaseSnapshot(
- 0, 1, null, 0, DataOperations.APPEND, null, 1, "foo",
base.nextRowId(), newRows, null);
+ 0 /* sequenceNumber */,
+ 1 /* snapshotId */,
+ null /* parentId */,
+ 0 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "foo" /* manifestListLocation */,
+ null /* rootManifest */,
+ base.nextRowId() /* firstRowId */,
+ newRows,
+ null /* keyId */);
TableMetadata added =
TableMetadata.buildFrom(base).addSnapshot(addRows).build();
Snapshot invalidNewRows =
new BaseSnapshot(
- 1, 2, 1L, 1, DataOperations.APPEND, null, 1, "foo",
added.nextRowId() - 1, 10L, null);
+ 1 /* sequenceNumber */,
+ 2 /* snapshotId */,
+ 1L /* parentId */,
+ 1 /* timestampMillis */,
+ DataOperations.APPEND,
+ null /* summary */,
+ 1 /* schemaId */,
+ "foo" /* manifestListLocation */,
+ null /* rootManifest */,
+ added.nextRowId() - 1 /* firstRowId */,
+ 10L /* addedRows */,
+ null /* keyId */);
Review Comment:
Reverted the wrapping that was only for names. Only the null argument is
added now. The sites that still overflow are one argument per line without
names.
##########
api/src/main/java/org/apache/iceberg/Snapshot.java:
##########
@@ -166,10 +166,26 @@ default Iterable<DeleteFile> removedDeleteFiles(FileIO
io) {
/**
* Return the location of this snapshot's manifest list, or null if it is
not separate.
*
- * @return the location of the manifest list for this Snapshot
+ * <p>Only set for tables with format version 1 through 3. Snapshots in v4
tables use {@link
+ * #rootManifestLocation()} instead.
+ *
+ * @return the location of the manifest list for this snapshot
*/
String manifestListLocation();
+ /**
+ * Return the location of this snapshot's root manifest, or null if this
snapshot has no root
+ * manifest.
+ *
+ * <p>v4 snapshots store this in {@code root-manifest} instead of a manifest
list. At most one of
+ * this and {@link #manifestListLocation()} is set.
Review Comment:
Done, kept the first sentence with "or null if it has no root manifest." and
dropped the second paragraph.
##########
api/src/main/java/org/apache/iceberg/Snapshot.java:
##########
@@ -166,10 +166,26 @@ default Iterable<DeleteFile> removedDeleteFiles(FileIO
io) {
/**
* Return the location of this snapshot's manifest list, or null if it is
not separate.
*
- * @return the location of the manifest list for this Snapshot
+ * <p>Only set for tables with format version 1 through 3. Snapshots in v4
tables use {@link
+ * #rootManifestLocation()} instead.
Review Comment:
Used a shorter first sentence:
> Return the location of this snapshot's manifest list, or null if this
snapshot has no manifest list.
It drops the paragraph and the "not separate" phrasing.
##########
api/src/main/java/org/apache/iceberg/Snapshot.java:
##########
@@ -166,10 +166,26 @@ default Iterable<DeleteFile> removedDeleteFiles(FileIO
io) {
/**
* Return the location of this snapshot's manifest list, or null if it is
not separate.
Review Comment:
Covered by the updated manifestListLocation() Javadoc, which returns null
when this snapshot has no manifest list.
##########
core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java:
##########
@@ -423,6 +425,37 @@ public void testAddSnapshotFromJson() throws IOException {
assertEquals(action, expected, MetadataUpdateParser.fromJson(json));
}
+ @Test
+ public void testAddSnapshotWithRootManifest() {
+ String action = MetadataUpdateParser.ADD_SNAPSHOT;
+ long snapshotId = 2;
+ int schemaId = 3;
Review Comment:
Done.
##########
core/src/main/java/org/apache/iceberg/BaseSnapshot.java:
##########
@@ -181,7 +189,11 @@ private void cacheManifests(FileIO fileIO) {
}
if (allManifests == null) {
- // if manifests isn't set, then the snapshotFile is set and should be
read to get the list
+ // if manifests isn't set, then the manifest list must be read
+ Preconditions.checkArgument(
Review Comment:
Done. cacheManifests throws UnsupportedOperationException up front for
root-manifest snapshots, and the rest is unchanged.
##########
core/src/main/java/org/apache/iceberg/SnapshotParser.java:
##########
@@ -176,9 +180,11 @@ static Snapshot fromJson(JsonNode node) {
String keyId = JsonUtil.getStringOrNull(KEY_ID, node);
- if (node.has(MANIFEST_LIST)) {
- // the manifest list is stored in a manifest list file
- String manifestList = JsonUtil.getString(MANIFEST_LIST, node);
+ String manifestList = JsonUtil.getStringOrNull(MANIFEST_LIST, node);
+ String rootManifest = JsonUtil.getStringOrNull(ROOT_MANIFEST, node);
+
+ if (manifestList != null || rootManifest != null) {
+ // manifest-list is used by v1-v3 snapshots, root-manifest by v4
snapshots
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]