yandrey321 commented on code in PR #11118: URL: https://github.com/apache/ozone/pull/11118#discussion_r3863831195
########## hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestOmMixedWorkloadUnderDeletionBench.java: ########## @@ -0,0 +1,686 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.service; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.utils.db.CodecBuffer; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * End-to-end benchmark reproducing the interactive-workload degradation seen when a large background deletion backlog + * is being reclaimed on the same bucket that clients are actively writing to. + * + * <p>Every OM write is applied on a single serial state-machine apply thread under a per-bucket write lock. Reclaiming + * a deletion backlog drives {@code PurgeDirectories} transactions through that thread; when the deleted directories are + * densely populated, a single purge batch moves a large number of sub-files/sub-dirs under one write-lock hold, so the + * apply thread — and the bucket write lock — is occupied for the whole batch. Concurrent user {@code create}, + * {@code mkdir} and {@code rename} contend for that same lock and thread, and reads contend for the bucket read lock, + * so their latency degrades while the backlog drains. + * + * <p>This benchmark stages two sets under a single volume and FSO bucket: a densely-populated <em>backlog</em> subtree + * that is recursively deleted and drained, and a separate stable <em>workload</em> set that is never deleted during the + * run (a pre-staged dataset the read ops resolve against, plus a per-thread scratch area the write ops create into). + * Sharing the bucket is intentional — deletion and the client workload contend on the same bucket lock, while the + * workload always hits live paths. It then compares a mixed client workload on that bucket — create, mkdir, rename and + * a data-plane file write (create + write a block), plus a data-plane file read and the metadata read RPCs that take + * the bucket read lock (getFileStatus, listStatus, getBucketInfo, lookupKey) — in two conditions: + * <ul> + * <li><b>control</b> — no deletion running, and</li> + * <li><b>under load</b> — the same workload while the backlog subtree is recursively deleted and fully purged from + * OM in the background (both FSO phases: moved into the deletedTable, then purged back out),</li> + * </ul> + * reporting per-operation p50/p99 latency and the under-load degradation, so the two code versions can be compared on + * how much apply-thread purge work bleeds into interactive latency. It also reports the phase-1 drain time — the + * {@code DirectoryDeletingService} move into the deletedTable via {@code OMDirectoriesPurgeRequestWithFSO}, the apply + * path this change optimizes — separately from the full both-phase drain, so pure apply-thread throughput can be + * compared alongside the interactive degradation. + * + * <p>Deletion is configured with production-representative per-task limits so batches are large. The {@code benchmark} + * tag is excluded from {@code mvn test} and CI by default, so it must be re-enabled explicitly to run on demand + * (rebuild the reactor first to avoid stale-class errors): + * <pre> + * mvn -pl :ozone-integration-test test -DskipShade -DskipRecon \ + * -Dtest=TestOmMixedWorkloadUnderDeletionBench -Dgroups=benchmark -Dexcluded-test-groups= \ + * -Dsurefire.failIfNoSpecifiedTests=false + * </pre> + * Tunables: {@code bench.backlogDirs} (default 80), {@code bench.backlogFilesPerDir} (default 1000), + * {@code bench.backlogNonEmptyEvery} (default 3 — every 3rd backlog file is written with a block, the rest are + * empty so a large backlog stays cheap to stage), {@code bench.workloadDirs} (default 20) and + * {@code bench.workloadFilesPerDir} (default 100) sizing the stable dataset the read ops resolve against, + * {@code bench.fileBytes} (default 1 MiB) sizing the data-plane file write/read payload (and the block-bearing + * staged files the reads pull), {@code bench.clientThreads} (default 4), + * {@code bench.opsPerThread} (default 400), {@code bench.pathDeletingLimitPerTask} (default 2000) and + * {@code bench.keyDeletingLimitPerTask} (default 40000). The last two size how much a single deletion round gathers; + * with the Ratis appender byte limit non-binding at these entry sizes, a round's paths pack into one purge + * transaction, so raising them makes each apply move far more entries under a single bucket write-lock hold — the + * regime where the apply-thread per-entry cost dominates interactive latency. + * + * <p>Adding {@code -Dbench.profile.event=<cpu|lock|wall|alloc>} profiles only the under-load window with + * async-profiler, loaded reflectively from a local install whose paths must be supplied via + * {@code -Dbench.profiler.jar} (the async-profiler jar) and {@code -Dbench.profiler.lib} (its native library); the + * JFR is written under {@code -Dbench.profile.out}, default {@code /tmp}. For accurate leaf frames also pass + * {@code -DargLine="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints"}. + */ +@Tag("benchmark") +public class TestOmMixedWorkloadUnderDeletionBench { + + private static final Logger LOG = LoggerFactory.getLogger(TestOmMixedWorkloadUnderDeletionBench.class); + + private static final String OP_CREATE = "create"; + private static final String OP_MKDIR = "mkdir"; + private static final String OP_RENAME = "rename"; + private static final String OP_FILEWRITE = "filewrite"; + private static final String OP_FILEREAD = "fileread"; + private static final String OP_GETFILESTATUS = "getfilestatus"; + private static final String OP_LISTSTATUS = "liststatus"; + private static final String OP_INFOBUCKET = "infobucket"; + private static final String OP_GETKEYINFO = "getkeyinfo"; + private static final String[] OPS = + {OP_CREATE, OP_MKDIR, OP_RENAME, OP_FILEWRITE, OP_FILEREAD, + OP_GETFILESTATUS, OP_LISTSTATUS, OP_INFOBUCKET, OP_GETKEYINFO}; + + // The three sandboxes share the parent /workload/bucket but are separate subtrees; only the backlog is deleted. + // The deletion set (built, recursively deleted, then drained through the apply thread). + private static final String BACKLOG_ROOT = "workload/bucket/backlog"; + // The workload set — never deleted during the test: a stable pre-staged dataset the read ops resolve against, plus + // a scratch area the write ops create into. Keeping this separate from the deletion set means the concurrent + // operations always hit live paths while the backlog drains. + private static final String WORKLOAD_DATA_ROOT = "workload/bucket/data"; + private static final String WORKLOAD_SCRATCH_ROOT = "workload/bucket/scratch"; + + // Every bench.backlogNonEmptyEvery-th backlog file is written with a single block so its KeyInfo carries a + // key-location list, exercising the block-metadata parse/serialize the purge apply and flush paths hit in + // production. The rest are left empty because block-bearing files are far more expensive to stage (block + // allocation + datanode write + commit), and a large backlog is what actually stresses the apply thread. + private static final byte[] FILE_CONTENT = new byte[4]; + + /** + * Removes test-harness-only overhead that would otherwise distort the apply/flush cost under measurement: the + * mini-cluster unconditionally enables {@link CodecBuffer} leak detection (a per-allocation finalizer), and the test + * log config runs the {@code CodecBuffer}/managed-RocksDB loggers at DEBUG/TRACE, which capture a full stack trace on + * every buffer allocation. Neither happens in a production OM running at INFO. + */ + private static void stripTestOnlyOverhead() { + CodecBuffer.disableLeakDetection(); + org.apache.log4j.Logger.getLogger("org.apache.hadoop.hdds.utils.db.CodecBuffer") + .setLevel(org.apache.log4j.Level.INFO); + org.apache.log4j.Logger.getLogger("org.apache.hadoop.hdds.utils.db.managed") + .setLevel(org.apache.log4j.Level.INFO); + } + + @Test + @Timeout(value = 120, unit = TimeUnit.MINUTES) Review Comment: I can't drop it entirely: HDDS-12575 set a global junit.jupiter.execution.timeout.default = 5m, so removing the annotation would bound the test to 5 minutes — shorter than the benchmark's own 900s (15 min) phase-drain deadline, so it'd be killed mid-drain before that deadline ever fires (and the drain deadline only bounds one phase's wait, not backlog staging). So this @Timeout is an intentional override, its safe to reduce it to 20 mins. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
