0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4495266584
##########
paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java:
##########
@@ -171,6 +182,237 @@ private static List<IcebergManifestEntry>
readFromIterator(
}
}
+ /**
+ * Decodes the whole file one entry at a time, drops each entry at once,
and compares the number
+ * of files and rows that it found with the counters of its manifest list
entry. Memory use does
+ * not depend on the size of the file.
+ *
+ * <p>A manifest list records the counters when it is written, and a step
that only adds files
+ * trusts them, so the length of a file says nothing about its contents.
+ *
+ * @param betweenEntries runs every {@link #VALIDATE_CHECK_INTERVAL}
entries, and may throw to
+ * stop the scan. What it throws is not taken for damage.
+ * @throws CorruptManifestException when the storage delivered the whole
file and it does not
+ * decode, or when the counts differ
+ * @throws IOException when the storage failed, or ended the read before
the end of the file, so
+ * that a failing read is never reported as damage
+ */
+ public void validate(IcebergManifestFileMeta meta, long fileSize, Runnable
betweenEntries)
+ throws IOException {
+ Path path = pathFactory.toPath(new
Path(meta.manifestPath()).getName());
+ ReadWatch watch = new ReadWatch(fileIO, fileSize);
+ RuntimeException[] fromCheck = new RuntimeException[1];
+ long[] counts = new long[6];
+ try (CloseableIterator<InternalRow> iterator =
+ FileUtils.createFormatReader(watch, readerFactory, path,
fileSize)
+ .toCloseableIterator()) {
+ IcebergManifestEntrySerializer entrySerializer =
+ (IcebergManifestEntrySerializer) serializer;
+ long entries = 0;
+ while (iterator.hasNext()) {
+ IcebergManifestEntry entry =
entrySerializer.fromRow(iterator.next(), meta);
+ long rows = entry.file().recordCount();
+ switch (entry.status()) {
+ case ADDED:
+ counts[0]++;
+ counts[3] += rows;
+ break;
+ case EXISTING:
+ counts[1]++;
+ counts[4] += rows;
+ break;
+ case DELETED:
+ counts[2]++;
+ counts[5] += rows;
+ break;
+ default:
+ throw new IllegalStateException("Unknown status " +
entry.status());
+ }
+ if (++entries % VALIDATE_CHECK_INTERVAL == 0) {
+ try {
+ betweenEntries.run();
+ } catch (RuntimeException e) {
+ fromCheck[0] = e;
+ throw e;
+ }
+ }
+ }
+ } catch (Exception e) {
+ if (e == fromCheck[0]) {
+ throw (RuntimeException) e;
+ }
+ watch.throwIfStorageFailed(path, e);
+ throw new CorruptManifestException(path + " cannot be decoded: " +
e, e);
+ }
+ // A reader can take a read that ended early, at the end of a block,
for the end of the
+ // file, and then the counters below differ. That is the storage
failing, not damage.
+ watch.throwIfStorageFailed(path, null);
+
+ long[] expected =
+ new long[] {
+ meta.addedFilesCount(),
+ meta.existingFilesCount(),
+ meta.deletedFilesCount(),
+ meta.addedRowsCount(),
+ meta.existingRowsCount(),
+ meta.deletedRowsCount()
+ };
+ if (!Arrays.equals(counts, expected)) {
+ throw new CorruptManifestException(
+ path
+ + " holds (added, existing, deleted files and
rows) "
+ + Arrays.toString(counts)
+ + ", and its manifest list records "
+ + Arrays.toString(expected));
+ }
+ }
+
+ /** The bytes of a manifest were delivered by the storage, and they are
not a valid manifest. */
+ public static class CorruptManifestException extends RuntimeException {
+
+ public CorruptManifestException(String message) {
+ super(message);
+ }
+
+ public CorruptManifestException(String message, Throwable cause) {
+ super(message, cause);
+ }
+ }
+
+ /**
+ * Passes every call on to the file system and notes whether a read failed
or ended early, so
+ * that a decoder error can be told from a storage error.
+ */
+ private static class ReadWatch implements FileIO {
+
+ private final FileIO delegate;
+ private final long fileSize;
+ private volatile IOException failure;
+ private volatile boolean endedShort;
+
+ private ReadWatch(FileIO delegate, long fileSize) {
+ this.delegate = delegate;
+ this.fileSize = fileSize;
+ }
+
+ /** Fails when a read failed, or ended before the end of the file,
whatever was decoded. */
+ private void throwIfStorageFailed(Path path, @Nullable Throwable
cause) throws IOException {
+ if (failure != null) {
+ throw failure;
+ }
+ if (endedShort) {
+ throw new IOException(
+ "A read of " + path + " ended before the end of the
file", cause);
+ }
+ }
+
+ @Override
+ public SeekableInputStream newInputStream(Path path) throws
IOException {
+ SeekableInputStream in;
+ try {
+ in = delegate.newInputStream(path);
+ } catch (IOException e) {
+ failure = e;
+ throw e;
+ }
+ return new SeekableInputStream() {
+ @Override
+ public void seek(long desired) throws IOException {
+ try {
+ in.seek(desired);
+ } catch (IOException e) {
+ failure = e;
+ throw e;
+ }
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return in.getPos();
+ }
+
+ @Override
+ public int read() throws IOException {
+ try {
+ int b = in.read();
+ if (b < 0 && in.getPos() < fileSize) {
+ endedShort = true;
+ }
+ return b;
+ } catch (IOException e) {
+ failure = e;
+ throw e;
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException
{
+ try {
+ int n = in.read(b, off, len);
+ if (n < 0 && in.getPos() < fileSize) {
+ endedShort = true;
+ }
+ return n;
+ } catch (IOException e) {
+ failure = e;
+ throw e;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
Review Comment:
Fixed. A failure when closing a stream is now recorded like a failure to
open, seek or read. When a manifest read fails, a recorded storage failure is
rethrown as is and only a real format error counts as damage. A copy that reads
fine and fails only on close now propagates the error without a rebuild. This
is tested for a failing read, a failing close, and a stream that ends early.
The same rule now applies to every read of a base's metadata and lists.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]