voonhous commented on code in PR #20647:
URL: https://github.com/apache/hudi/pull/20647#discussion_r4641230612


##########
hudi-trino/src/test/java/io/trino/plugin/hudi/benchmark/LatencyInjectingFileSystemFactory.java:
##########
@@ -0,0 +1,377 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.hudi.benchmark;
+
+import io.trino.filesystem.FileIterator;
+import io.trino.filesystem.Location;
+import io.trino.filesystem.TrinoFileSystem;
+import io.trino.filesystem.TrinoFileSystemFactory;
+import io.trino.filesystem.TrinoInput;
+import io.trino.filesystem.TrinoInputFile;
+import io.trino.filesystem.TrinoInputStream;
+import io.trino.filesystem.TrinoOutputFile;
+import io.trino.spi.security.ConnectorIdentity;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Wraps a local {@link TrinoFileSystemFactory} and charges an 
object-store-like latency on each request a
+ * reader would send to S3: opening a stream (one GET), each positioned or 
tail read (one ranged GET), a
+ * length or existence probe (one HEAD) and each listing call (one LIST). 
Local disk answers these in
+ * microseconds, which hides any per-split metadata read; the injected latency 
makes those reads cost what
+ * they cost on S3, so a local run shows their effect on wall time and on 
dynamic-filter timing.
+ *
+ * <p>The counters are environment-independent: they count requests regardless 
of the latency setting, so
+ * a change that removes per-split metadata reads shows up as a lower count 
even with zero latency.
+ */
+final class LatencyInjectingFileSystemFactory
+        implements TrinoFileSystemFactory
+{
+    private final TrinoFileSystemFactory delegate;
+    private final Latency latency;
+    private final Counters counters;
+
+    LatencyInjectingFileSystemFactory(TrinoFileSystemFactory delegate, Latency 
latency, Counters counters)
+    {
+        this.delegate = requireNonNull(delegate, "delegate is null");
+        this.latency = requireNonNull(latency, "latency is null");
+        this.counters = requireNonNull(counters, "counters is null");
+    }
+
+    @Override
+    public TrinoFileSystem create(ConnectorIdentity identity)
+    {
+        return new LatencyInjectingFileSystem(delegate.create(identity));
+    }
+
+    /**
+     * Per-request latencies in milliseconds. Defaults approximate same-region 
S3 from EC2: a first-byte GET of
+     * ~20ms, a HEAD of ~15ms, a LIST page of ~40ms.
+     */
+    record Latency(long getMillis, long headMillis, long listMillis)
+    {
+        static Latency none()
+        {
+            return new Latency(0, 0, 0);
+        }
+
+        static Latency fromSystemProperties()
+        {
+            long scale = Long.getLong("bench.s3.latency.enabled", 1) == 0 ? 0 
: 1;
+            return new Latency(
+                    scale * Long.getLong("bench.s3.getMillis", 20),
+                    scale * Long.getLong("bench.s3.headMillis", 15),
+                    scale * Long.getLong("bench.s3.listMillis", 40));
+        }
+    }
+
+    /** Request counts since the last {@link #reset()}, in total and per 
request type and Hudi file type. */
+    static final class Counters
+    {
+        final AtomicLong gets = new AtomicLong();
+        final AtomicLong heads = new AtomicLong();
+        final AtomicLong lists = new AtomicLong();
+        private final Map<String, AtomicLong> byKind = new 
ConcurrentHashMap<>();
+        private final Map<String, String> samples = new ConcurrentHashMap<>();
+
+        void reset()
+        {
+            gets.set(0);
+            heads.set(0);
+            lists.set(0);
+            byKind.clear();
+            samples.clear();
+        }
+
+        void record(AtomicLong counter, Location location)
+        {
+            counter.incrementAndGet();
+            String request = counter == gets ? "GET" : counter == heads ? 
"HEAD" : "LIST";
+            String kind = request + ":" + fileType(location);
+            byKind.computeIfAbsent(kind, key -> new 
AtomicLong()).incrementAndGet();
+            samples.putIfAbsent(kind, location.toString());
+        }
+
+        /** One example location per request and file type, to see which files 
a request type hits. */
+        String samples()
+        {
+            return samples.entrySet().stream()
+                    .sorted(Map.Entry.comparingByKey())
+                    .map(entry -> entry.getKey() + " -> " + entry.getValue())
+                    .collect(Collectors.joining("\n"));
+        }
+
+        /** Per request type and file type, e.g. {@code HEAD:LOG=96}, largest 
first. */
+        String breakdown()
+        {
+            return byKind.entrySet().stream()
+                    .sorted(Map.Entry.<String, 
AtomicLong>comparingByValue(Comparator.comparingLong(AtomicLong::get)).reversed())
+                    .map(entry -> entry.getKey() + "=" + 
entry.getValue().get())
+                    .collect(Collectors.joining(" "));
+        }
+
+        private static String fileType(Location location)
+        {
+            String path = location.toString();
+            if (path.contains("/.hoodie")) {
+                if (path.contains("/.hoodie/timeline")) {
+                    return "TIMELINE";
+                }
+                if (path.endsWith("/.hoodie/hoodie.properties")) {
+                    return "TABLE_PROPERTIES";
+                }
+                return path.endsWith("/.hoodie") ? "HOODIE_DIR" : 
"OTHER_HOODIE";
+            }
+            if (path.contains(".log.")) {

Review Comment:
   Follow-up: 5b57745e drops the span-based `fileOps` column, so 
`Counters.fileType` is the only file-type classifier now, and native logs show 
as `LOG` in the `storage[...]` breakdown.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to