0dunay0 commented on code in PR #10995:
URL: https://github.com/apache/paimon/pull/10995#discussion_r4495266584


##########
paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergManifestFile.java:
##########
@@ -171,6 +182,237 @@ private static List<IcebergManifestEntry> 
readFromIterator(
         }
     }
 
+    /**
+     * Decodes the whole file one entry at a time, drops each entry at once, 
and compares the number
+     * of files and rows that it found with the counters of its manifest list 
entry. Memory use does
+     * not depend on the size of the file.
+     *
+     * <p>A manifest list records the counters when it is written, and a step 
that only adds files
+     * trusts them, so the length of a file says nothing about its contents.
+     *
+     * @param betweenEntries runs every {@link #VALIDATE_CHECK_INTERVAL} 
entries, and may throw to
+     *     stop the scan. What it throws is not taken for damage.
+     * @throws CorruptManifestException when the storage delivered the whole 
file and it does not
+     *     decode, or when the counts differ
+     * @throws IOException when the storage failed, or ended the read before 
the end of the file, so
+     *     that a failing read is never reported as damage
+     */
+    public void validate(IcebergManifestFileMeta meta, long fileSize, Runnable 
betweenEntries)
+            throws IOException {
+        Path path = pathFactory.toPath(new 
Path(meta.manifestPath()).getName());
+        ReadWatch watch = new ReadWatch(fileIO, fileSize);
+        RuntimeException[] fromCheck = new RuntimeException[1];
+        long[] counts = new long[6];
+        try (CloseableIterator<InternalRow> iterator =
+                FileUtils.createFormatReader(watch, readerFactory, path, 
fileSize)
+                        .toCloseableIterator()) {
+            IcebergManifestEntrySerializer entrySerializer =
+                    (IcebergManifestEntrySerializer) serializer;
+            long entries = 0;
+            while (iterator.hasNext()) {
+                IcebergManifestEntry entry = 
entrySerializer.fromRow(iterator.next(), meta);
+                long rows = entry.file().recordCount();
+                switch (entry.status()) {
+                    case ADDED:
+                        counts[0]++;
+                        counts[3] += rows;
+                        break;
+                    case EXISTING:
+                        counts[1]++;
+                        counts[4] += rows;
+                        break;
+                    case DELETED:
+                        counts[2]++;
+                        counts[5] += rows;
+                        break;
+                    default:
+                        throw new IllegalStateException("Unknown status " + 
entry.status());
+                }
+                if (++entries % VALIDATE_CHECK_INTERVAL == 0) {
+                    try {
+                        betweenEntries.run();
+                    } catch (RuntimeException e) {
+                        fromCheck[0] = e;
+                        throw e;
+                    }
+                }
+            }
+        } catch (Exception e) {
+            if (e == fromCheck[0]) {
+                throw (RuntimeException) e;
+            }
+            watch.throwIfStorageFailed(path, e);
+            throw new CorruptManifestException(path + " cannot be decoded: " + 
e, e);
+        }
+        // A reader can take a read that ended early, at the end of a block, 
for the end of the
+        // file, and then the counters below differ. That is the storage 
failing, not damage.
+        watch.throwIfStorageFailed(path, null);
+
+        long[] expected =
+                new long[] {
+                    meta.addedFilesCount(),
+                    meta.existingFilesCount(),
+                    meta.deletedFilesCount(),
+                    meta.addedRowsCount(),
+                    meta.existingRowsCount(),
+                    meta.deletedRowsCount()
+                };
+        if (!Arrays.equals(counts, expected)) {
+            throw new CorruptManifestException(
+                    path
+                            + " holds (added, existing, deleted files and 
rows) "
+                            + Arrays.toString(counts)
+                            + ", and its manifest list records "
+                            + Arrays.toString(expected));
+        }
+    }
+
+    /** The bytes of a manifest were delivered by the storage, and they are 
not a valid manifest. */
+    public static class CorruptManifestException extends RuntimeException {
+
+        public CorruptManifestException(String message) {
+            super(message);
+        }
+
+        public CorruptManifestException(String message, Throwable cause) {
+            super(message, cause);
+        }
+    }
+
+    /**
+     * Passes every call on to the file system and notes whether a read failed 
or ended early, so
+     * that a decoder error can be told from a storage error.
+     */
+    private static class ReadWatch implements FileIO {
+
+        private final FileIO delegate;
+        private final long fileSize;
+        private volatile IOException failure;
+        private volatile boolean endedShort;
+
+        private ReadWatch(FileIO delegate, long fileSize) {
+            this.delegate = delegate;
+            this.fileSize = fileSize;
+        }
+
+        /** Fails when a read failed, or ended before the end of the file, 
whatever was decoded. */
+        private void throwIfStorageFailed(Path path, @Nullable Throwable 
cause) throws IOException {
+            if (failure != null) {
+                throw failure;
+            }
+            if (endedShort) {
+                throw new IOException(
+                        "A read of " + path + " ended before the end of the 
file", cause);
+            }
+        }
+
+        @Override
+        public SeekableInputStream newInputStream(Path path) throws 
IOException {
+            SeekableInputStream in;
+            try {
+                in = delegate.newInputStream(path);
+            } catch (IOException e) {
+                failure = e;
+                throw e;
+            }
+            return new SeekableInputStream() {
+                @Override
+                public void seek(long desired) throws IOException {
+                    try {
+                        in.seek(desired);
+                    } catch (IOException e) {
+                        failure = e;
+                        throw e;
+                    }
+                }
+
+                @Override
+                public long getPos() throws IOException {
+                    return in.getPos();
+                }
+
+                @Override
+                public int read() throws IOException {
+                    try {
+                        int b = in.read();
+                        if (b < 0 && in.getPos() < fileSize) {
+                            endedShort = true;
+                        }
+                        return b;
+                    } catch (IOException e) {
+                        failure = e;
+                        throw e;
+                    }
+                }
+
+                @Override
+                public int read(byte[] b, int off, int len) throws IOException 
{
+                    try {
+                        int n = in.read(b, off, len);
+                        if (n < 0 && in.getPos() < fileSize) {
+                            endedShort = true;
+                        }
+                        return n;
+                    } catch (IOException e) {
+                        failure = e;
+                        throw e;
+                    }
+                }
+
+                @Override
+                public void close() throws IOException {
+                    in.close();

Review Comment:
   Fixed. A failure when closing a stream is now recorded like a failure to 
open, seek or read. When a manifest read fails, a recorded storage failure is 
rethrown as is and only a real format error counts as damage. A copy that reads 
fine and fails only on close now propagates the error without a rebuild. This 
is tested for a failing read, a failing close, and a stream that ends early. 
The same rule now applies to every read of a base's metadata and lists.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to