stevenzwu commented on code in PR #18248:
URL: https://github.com/apache/iceberg/pull/18248#discussion_r4098889193
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java:
##########
@@ -318,49 +321,143 @@ public void readManifestFile(FileFormat format) throws
IOException {
@FieldSource("MANIFEST_FORMATS")
public void statusFilter(FileFormat format) throws IOException {
List<TrackedFile> files =
- ImmutableList.of(
- unpartitionedFileWithStatus(EntryStatus.ADDED,
"s3://bucket/added.parquet"),
- unpartitionedFileWithStatus(EntryStatus.MODIFIED,
"s3://bucket/modified.parquet"),
- unpartitionedFileWithStatus(EntryStatus.DELETED,
"s3://bucket/deleted.parquet"),
- unpartitionedFileWithStatus(EntryStatus.EXISTING,
"s3://bucket/existing.parquet"),
- unpartitionedFileWithStatus(EntryStatus.REPLACED,
"s3://bucket/replaced.parquet"));
+ List.of(
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.ADDED, 42L, null, null, null,
null, null, null),
+ "s3://bucket/added.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.MODIFIED, 40L, 5L, 5L, 42L,
5_000L, null, null),
+ "s3://bucket/modified.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.DELETED, 42L, 2L, 2L, null,
1_000L, null, null),
+ "s3://bucket/deleted.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.EXISTING, 38L, 4L, 4L, null,
3_000L, null, null),
+ "s3://bucket/existing.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.REPLACED, 42L, 2L, 2L, 40L,
2_000L, null, null),
+ "s3://bucket/replaced.parquet"));
ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, files);
+ when(manifest.firstRowId()).thenReturn(10_000L);
- List<TrackedFile> liveFiles =
- read(
- V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
UNPARTITIONED_SPECS)
- .metricsConfig(METRICS_CONFIG));
+ List<TrackedFile> expectedFiles =
+ List.of(
+ unpartitionedDataFile( // inherits seq number and assigned first
row ID
+ new TrackingStruct(
+ EntryStatus.ADDED, 42L, MANIFEST_SEQ, MANIFEST_SEQ, null,
10_000L, null, null),
+ "s3://bucket/added.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.MODIFIED, 40L, 5L, 5L, 42L,
5_000L, null, null),
+ "s3://bucket/modified.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.DELETED, 42L, 2L, 2L, null,
1_000L, null, null),
+ "s3://bucket/deleted.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.EXISTING, 38L, 4L, 4L, null,
3_000L, null, null),
+ "s3://bucket/existing.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.REPLACED, 42L, 2L, 2L, 40L,
2_000L, null, null),
+ "s3://bucket/replaced.parquet"));
+
+ V4ManifestReader.Builder builder =
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
UNPARTITIONED_SPECS)
+ .metricsConfig(METRICS_CONFIG);
+
+ List<TrackedFile> liveFiles = read(builder);
assertThat(liveFiles)
- .usingComparatorForType(FILE_COMPARATOR, TrackedFile.class)
- .containsExactly(files.get(0), files.get(1), files.get(3));
+ .usingComparatorForType(FILE_AND_TRACKING_COMPARATOR,
TrackedFile.class)
+ .containsExactly(expectedFiles.get(0), expectedFiles.get(1),
expectedFiles.get(3));
- List<TrackedFile> allFiles =
- read(
- V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
UNPARTITIONED_SPECS)
- .metricsConfig(METRICS_CONFIG)
- .includeAll());
+ List<TrackedFile> allFiles = read(builder.includeAll());
assertThat(allFiles)
- .usingComparatorForType(FILE_COMPARATOR, TrackedFile.class)
- .containsExactlyElementsOf(files);
+ .usingComparatorForType(FILE_AND_TRACKING_COMPARATOR,
TrackedFile.class)
+ .isEqualTo(expectedFiles);
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void mdvFilter(FileFormat format) throws IOException {
+ List<TrackedFile> files =
+ List.of(
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.ADDED, 42L, null, null, null,
null, null, null),
+ "s3://bucket/added.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.MODIFIED, 40L, 5L, 5L, 42L,
5_000L, null, null),
+ "s3://bucket/modified.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.DELETED, 42L, 2L, 2L, null,
1_000L, null, null),
+ "s3://bucket/deleted.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.EXISTING, 38L, 4L, 4L, null,
3_000L, null, null),
+ "s3://bucket/existing.parquet"),
+ unpartitionedDataFile(
+ new TrackingStruct(EntryStatus.REPLACED, 42L, 2L, 2L, 40L,
2_000L, null, null),
+ "s3://bucket/replaced.parquet"));
+
+ ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, files);
+
+ // this bitmap deletes a MODIFIED entry and an already DELETED entry
+ when(manifest.manifestDeletionVector())
+ .thenReturn(MumblingTestUtil.bitmap(MumblingTestUtil.sparse(1, 2)));
+ when(manifest.firstRowId()).thenReturn(10_000L);
+
+ List<TrackedFile> expectedFiles =
+ List.of(
+ unpartitionedDataFile( // inherits seq number and assigned first
row ID
+ new TrackingStruct(
+ EntryStatus.ADDED, 42L, MANIFEST_SEQ, MANIFEST_SEQ, null,
10_000L, null, null),
+ "s3://bucket/added.parquet"),
+ unpartitionedDataFile( // status changed to DELETED, delete
snapshot ID is unknown
Review Comment:
got it. this is for `includeAll` scenario where manifest DV marked entries
are converted to deleted status.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
