wombatu-kun commented on code in PR #20647: URL: https://github.com/apache/hudi/pull/20647#discussion_r4573655563
########## hudi-trino/src/test/java/io/trino/plugin/hudi/benchmark/LatencyInjectingFileSystemFactory.java: ########## @@ -0,0 +1,377 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.benchmark; + +import io.trino.filesystem.FileIterator; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.filesystem.TrinoInput; +import io.trino.filesystem.TrinoInputFile; +import io.trino.filesystem.TrinoInputStream; +import io.trino.filesystem.TrinoOutputFile; +import io.trino.spi.security.ConnectorIdentity; + +import java.io.IOException; +import java.time.Instant; +import java.util.Comparator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; + +/** + * Wraps a local {@link TrinoFileSystemFactory} and charges an object-store-like latency on each request a + * reader would send to S3: opening a stream (one GET), each positioned or tail read (one ranged GET), a + * length or existence probe (one HEAD) and each listing call (one LIST). Local disk answers these in + * microseconds, which hides any per-split metadata read; the injected latency makes those reads cost what + * they cost on S3, so a local run shows their effect on wall time and on dynamic-filter timing. + * + * <p>The counters are environment-independent: they count requests regardless of the latency setting, so + * a change that removes per-split metadata reads shows up as a lower count even with zero latency. + */ +final class LatencyInjectingFileSystemFactory + implements TrinoFileSystemFactory +{ + private final TrinoFileSystemFactory delegate; + private final Latency latency; + private final Counters counters; + + LatencyInjectingFileSystemFactory(TrinoFileSystemFactory delegate, Latency latency, Counters counters) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + this.latency = requireNonNull(latency, "latency is null"); + this.counters = requireNonNull(counters, "counters is null"); + } + + @Override + public TrinoFileSystem create(ConnectorIdentity identity) + { + return new LatencyInjectingFileSystem(delegate.create(identity)); + } + + /** + * Per-request latencies in milliseconds. Defaults approximate same-region S3 from EC2: a first-byte GET of + * ~20ms, a HEAD of ~15ms, a LIST page of ~40ms. + */ + record Latency(long getMillis, long headMillis, long listMillis) + { + static Latency none() + { + return new Latency(0, 0, 0); + } + + static Latency fromSystemProperties() + { + long scale = Long.getLong("bench.s3.latency.enabled", 1) == 0 ? 0 : 1; + return new Latency( + scale * Long.getLong("bench.s3.getMillis", 20), + scale * Long.getLong("bench.s3.headMillis", 15), + scale * Long.getLong("bench.s3.listMillis", 40)); + } + } + + /** Request counts since the last {@link #reset()}, in total and per request type and Hudi file type. */ + static final class Counters + { + final AtomicLong gets = new AtomicLong(); + final AtomicLong heads = new AtomicLong(); + final AtomicLong lists = new AtomicLong(); + private final Map<String, AtomicLong> byKind = new ConcurrentHashMap<>(); + private final Map<String, String> samples = new ConcurrentHashMap<>(); + + void reset() + { + gets.set(0); + heads.set(0); + lists.set(0); + byKind.clear(); + samples.clear(); + } + + void record(AtomicLong counter, Location location) + { + counter.incrementAndGet(); + String request = counter == gets ? "GET" : counter == heads ? "HEAD" : "LIST"; + String kind = request + ":" + fileType(location); + byKind.computeIfAbsent(kind, key -> new AtomicLong()).incrementAndGet(); + samples.putIfAbsent(kind, location.toString()); + } + + /** One example location per request and file type, to see which files a request type hits. */ + String samples() + { + return samples.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(entry -> entry.getKey() + " -> " + entry.getValue()) + .collect(Collectors.joining("\n")); + } + + /** Per request type and file type, e.g. {@code HEAD:LOG=96}, largest first. */ + String breakdown() + { + return byKind.entrySet().stream() + .sorted(Map.Entry.<String, AtomicLong>comparingByValue(Comparator.comparingLong(AtomicLong::get)).reversed()) + .map(entry -> entry.getKey() + "=" + entry.getValue().get()) + .collect(Collectors.joining(" ")); + } + + private static String fileType(Location location) + { + String path = location.toString(); + if (path.contains("/.hoodie")) { + if (path.contains("/.hoodie/timeline")) { + return "TIMELINE"; + } + if (path.endsWith("/.hoodie/hoodie.properties")) { + return "TABLE_PROPERTIES"; + } + return path.endsWith("/.hoodie") ? "HOODIE_DIR" : "OTHER_HOODIE"; + } + if (path.contains(".log.")) { + return "LOG"; + } + if (path.endsWith(".parquet")) { + return "BASE"; + } + if (path.contains(".hoodie_partition_metadata")) { + return "PARTITION_METADATA"; + } + return path.contains("=") ? "PARTITION_DIR" : "TABLE_DIR"; + } + + @Override + public String toString() + { + return "GET=" + gets.get() + " HEAD=" + heads.get() + " LIST=" + lists.get(); + } + } + + private void charge(Location location, AtomicLong counter, long millis) + { + // The file metastore keeps its table definitions on the same filesystem; on a cluster the metastore + // is a separate service (Glue, HMS), so its files are not charged as table storage requests. + String path = location.toString(); + if (path.contains("/.trino") || path.contains("/managed/")) { + return; + } + counters.record(counter, location); + if (millis > 0) { + try { + TimeUnit.MILLISECONDS.sleep(millis); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + private final class LatencyInjectingFileSystem + implements TrinoFileSystem + { + private final TrinoFileSystem fileSystem; + + private LatencyInjectingFileSystem(TrinoFileSystem fileSystem) + { + this.fileSystem = fileSystem; + } + + @Override + public TrinoInputFile newInputFile(Location location) + { + return new LatencyInjectingInputFile(fileSystem.newInputFile(location)); + } + + @Override + public TrinoInputFile newInputFile(Location location, long length) + { + return new LatencyInjectingInputFile(fileSystem.newInputFile(location, length)); + } + + @Override + public TrinoInputFile newInputFile(Location location, long length, Instant lastModified) + { + return new LatencyInjectingInputFile(fileSystem.newInputFile(location, length, lastModified)); + } + + @Override + public TrinoOutputFile newOutputFile(Location location) + { + return fileSystem.newOutputFile(location); + } + + @Override + public void deleteFile(Location location) + throws IOException + { + fileSystem.deleteFile(location); + } + + @Override + public void deleteDirectory(Location location) + throws IOException + { + fileSystem.deleteDirectory(location); + } + + @Override + public void renameFile(Location source, Location target) + throws IOException + { + fileSystem.renameFile(source, target); + } + + @Override + public FileIterator listFiles(Location location) + throws IOException + { + charge(location, counters.lists, latency.listMillis()); + return fileSystem.listFiles(location); + } + + @Override + public Optional<Boolean> directoryExists(Location location) + throws IOException + { + charge(location, counters.lists, latency.listMillis()); + return fileSystem.directoryExists(location); + } + + @Override + public void createDirectory(Location location) + throws IOException + { + fileSystem.createDirectory(location); + } + + @Override + public void renameDirectory(Location source, Location target) + throws IOException + { + fileSystem.renameDirectory(source, target); + } + + @Override + public Set<Location> listDirectories(Location location) + throws IOException + { + charge(location, counters.lists, latency.listMillis()); + return fileSystem.listDirectories(location); + } + + @Override + public Optional<Location> createTemporaryDirectory(Location targetPath, String temporaryPrefix, String relativePrefix) + throws IOException + { + return fileSystem.createTemporaryDirectory(targetPath, temporaryPrefix, relativePrefix); + } + } + + private final class LatencyInjectingInputFile + implements TrinoInputFile + { + private final TrinoInputFile inputFile; + + private LatencyInjectingInputFile(TrinoInputFile inputFile) + { + this.inputFile = inputFile; + } + + @Override + public TrinoInput newInput() + throws IOException + { + return new LatencyInjectingInput(inputFile.location(), inputFile.newInput()); + } + + @Override + public TrinoInputStream newStream() + throws IOException + { + charge(inputFile.location(), counters.gets, latency.getMillis()); + return inputFile.newStream(); + } + + @Override + public long length() Review Comment: `LatencyInjectingInputFile.length()` and `lastModified()` charge a HEAD on every call, while Trino's `S3InputFile` sends none when the length came through `newInputFile(location, length)` (every base file here) and caches both values after its first HEAD or `exists()`, so each base file pays a HEAD that S3 skips and `HudiTrinoStorage.getPathInfo` pays three where S3 sends one. Could the wrapper remember a supplied or already fetched length and lastModified and charge only when S3 would actually send the request? ########## hudi-trino/src/test/java/io/trino/plugin/hudi/benchmark/LatencyInjectingFileSystemFactory.java: ########## @@ -0,0 +1,377 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.benchmark; + +import io.trino.filesystem.FileIterator; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.filesystem.TrinoInput; +import io.trino.filesystem.TrinoInputFile; +import io.trino.filesystem.TrinoInputStream; +import io.trino.filesystem.TrinoOutputFile; +import io.trino.spi.security.ConnectorIdentity; + +import java.io.IOException; +import java.time.Instant; +import java.util.Comparator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNull; + +/** + * Wraps a local {@link TrinoFileSystemFactory} and charges an object-store-like latency on each request a + * reader would send to S3: opening a stream (one GET), each positioned or tail read (one ranged GET), a + * length or existence probe (one HEAD) and each listing call (one LIST). Local disk answers these in + * microseconds, which hides any per-split metadata read; the injected latency makes those reads cost what + * they cost on S3, so a local run shows their effect on wall time and on dynamic-filter timing. + * + * <p>The counters are environment-independent: they count requests regardless of the latency setting, so + * a change that removes per-split metadata reads shows up as a lower count even with zero latency. + */ +final class LatencyInjectingFileSystemFactory + implements TrinoFileSystemFactory +{ + private final TrinoFileSystemFactory delegate; + private final Latency latency; + private final Counters counters; + + LatencyInjectingFileSystemFactory(TrinoFileSystemFactory delegate, Latency latency, Counters counters) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + this.latency = requireNonNull(latency, "latency is null"); + this.counters = requireNonNull(counters, "counters is null"); + } + + @Override + public TrinoFileSystem create(ConnectorIdentity identity) + { + return new LatencyInjectingFileSystem(delegate.create(identity)); + } + + /** + * Per-request latencies in milliseconds. Defaults approximate same-region S3 from EC2: a first-byte GET of + * ~20ms, a HEAD of ~15ms, a LIST page of ~40ms. + */ + record Latency(long getMillis, long headMillis, long listMillis) + { + static Latency none() + { + return new Latency(0, 0, 0); + } + + static Latency fromSystemProperties() + { + long scale = Long.getLong("bench.s3.latency.enabled", 1) == 0 ? 0 : 1; + return new Latency( + scale * Long.getLong("bench.s3.getMillis", 20), + scale * Long.getLong("bench.s3.headMillis", 15), + scale * Long.getLong("bench.s3.listMillis", 40)); + } + } + + /** Request counts since the last {@link #reset()}, in total and per request type and Hudi file type. */ + static final class Counters + { + final AtomicLong gets = new AtomicLong(); + final AtomicLong heads = new AtomicLong(); + final AtomicLong lists = new AtomicLong(); + private final Map<String, AtomicLong> byKind = new ConcurrentHashMap<>(); + private final Map<String, String> samples = new ConcurrentHashMap<>(); + + void reset() + { + gets.set(0); + heads.set(0); + lists.set(0); + byKind.clear(); + samples.clear(); + } + + void record(AtomicLong counter, Location location) + { + counter.incrementAndGet(); + String request = counter == gets ? "GET" : counter == heads ? "HEAD" : "LIST"; + String kind = request + ":" + fileType(location); + byKind.computeIfAbsent(kind, key -> new AtomicLong()).incrementAndGet(); + samples.putIfAbsent(kind, location.toString()); + } + + /** One example location per request and file type, to see which files a request type hits. */ + String samples() + { + return samples.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(entry -> entry.getKey() + " -> " + entry.getValue()) + .collect(Collectors.joining("\n")); + } + + /** Per request type and file type, e.g. {@code HEAD:LOG=96}, largest first. */ + String breakdown() + { + return byKind.entrySet().stream() + .sorted(Map.Entry.<String, AtomicLong>comparingByValue(Comparator.comparingLong(AtomicLong::get)).reversed()) + .map(entry -> entry.getKey() + "=" + entry.getValue().get()) + .collect(Collectors.joining(" ")); + } + + private static String fileType(Location location) + { + String path = location.toString(); + if (path.contains("/.hoodie")) { + if (path.contains("/.hoodie/timeline")) { + return "TIMELINE"; + } + if (path.endsWith("/.hoodie/hoodie.properties")) { + return "TABLE_PROPERTIES"; + } + return path.endsWith("/.hoodie") ? "HOODIE_DIR" : "OTHER_HOODIE"; + } + if (path.contains(".log.")) { Review Comment: With `-Dbench.nativeLogs=true` the delete round writes `*.deletes.parquet` native files, which `Counters.fileType` counts as `BASE` because only `.log.` is matched before `.parquet`, and the `fileOps` column (`FileOperationUtils.FileType.fromFilePath`) counts every native log file as `DATA`. Could `fileType` test `FSUtils.isLogFile(location.fileName())` first, which matches inline and native log files alike, and `Measurement.of` classify spans with it too? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
