This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 21c7219707f HDDS-15762. Remove mini-chaos-tests (#10682)
21c7219707f is described below
commit 21c7219707f1004077686e4067f673115f059dc8
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jul 8 17:02:07 2026 +0200
HDDS-15762. Remove mini-chaos-tests (#10682)
---
dev-support/ci/selective_ci_checks.sh | 1 -
hadoop-hdds/docs/content/tools/TestTools.md | 4 -
hadoop-hdds/docs/content/tools/TestTools.zh.md | 5 -
hadoop-ozone/dev-support/checks/unit.sh | 2 +-
.../dev-support/findbugsExcludeFile.xml | 24 --
.../fault-injection-test/mini-chaos-tests/pom.xml | 156 --------
.../mini-chaos-tests/src/test/bin/start-chaos.sh | 57 ---
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 401 ---------------------
.../hadoop/ozone/MiniOzoneLoadGenerator.java | 148 --------
.../org/apache/hadoop/ozone/OzoneChaosCluster.java | 43 ---
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 55 ---
.../ozone/TestDatanodeMiniChaosOzoneCluster.java | 57 ---
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 222 ------------
.../TestOzoneManagerMiniChaosOzoneCluster.java | 62 ----
...orageContainerManagerMiniChaosOzoneCluster.java | 62 ----
.../hadoop/ozone/failure/FailureManager.java | 99 -----
.../org/apache/hadoop/ozone/failure/Failures.java | 226 ------------
.../apache/hadoop/ozone/failure/package-info.java | 19 -
.../ozone/loadgenerators/AgedDirLoadGenerator.java | 48 ---
.../ozone/loadgenerators/AgedLoadGenerator.java | 77 ----
.../hadoop/ozone/loadgenerators/DataBuffer.java | 52 ---
.../loadgenerators/FilesystemLoadGenerator.java | 55 ---
.../hadoop/ozone/loadgenerators/LoadBucket.java | 320 ----------------
.../hadoop/ozone/loadgenerators/LoadExecutors.java | 109 ------
.../hadoop/ozone/loadgenerators/LoadGenerator.java | 67 ----
.../loadgenerators/NestedDirLoadGenerator.java | 55 ---
.../loadgenerators/RandomDirLoadGenerator.java | 44 ---
.../ozone/loadgenerators/RandomLoadGenerator.java | 53 ---
.../loadgenerators/ReadOnlyLoadGenerator.java | 52 ---
.../services/org.apache.hadoop.fs.FileSystem | 17 -
.../src/test/resources/log4j.properties | 40 --
hadoop-ozone/fault-injection-test/pom.xml | 1 -
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 -
33 files changed, 1 insertion(+), 2636 deletions(-)
diff --git a/dev-support/ci/selective_ci_checks.sh
b/dev-support/ci/selective_ci_checks.sh
index 1e99a625854..57e86347077 100755
--- a/dev-support/ci/selective_ci_checks.sh
+++ b/dev-support/ci/selective_ci_checks.sh
@@ -266,7 +266,6 @@ function get_count_integration_files() {
"^hadoop-ozone/dev-support/checks/junit.sh"
"^hadoop-ozone/integration-test"
"^hadoop-ozone/mini-cluster"
- "^hadoop-ozone/fault-injection-test/mini-chaos-tests"
"src/test/java"
"src/test/resources"
)
diff --git a/hadoop-hdds/docs/content/tools/TestTools.md
b/hadoop-hdds/docs/content/tools/TestTools.md
index e6ffb364a21..e678db47ad0 100644
--- a/hadoop-hdds/docs/content/tools/TestTools.md
+++ b/hadoop-hdds/docs/content/tools/TestTools.md
@@ -66,10 +66,6 @@ python -m pytest -s .
See the README in the blockade directory for more details.
-## MiniChaosOzoneCluster
-
-This is a way to get [chaos](https://en.wikipedia.org/wiki/Chaos_engineering)
in your machine. It can be started from the source code and a MiniOzoneCluster
(which starts real daemons) will be started and killed randomly.
-
## Freon
Freon is a command line application which is included in the Ozone
distribution. It's a load generator which is used in our stress tests.
diff --git a/hadoop-hdds/docs/content/tools/TestTools.zh.md
b/hadoop-hdds/docs/content/tools/TestTools.zh.md
index 05f2c7f07f4..a528c303764 100644
--- a/hadoop-hdds/docs/content/tools/TestTools.zh.md
+++ b/hadoop-hdds/docs/content/tools/TestTools.zh.md
@@ -66,11 +66,6 @@ python -m pytest -s .
更多细节查看 blockade 目录下的 README。
-## MiniChaosOzoneCluster
-
-这是一种在你的机器上获得[混沌](https://en.wikipedia.org/wiki/Chaos_engineering)的方法。它可以直接从源码启动一个
MiniOzoneCluster
-（会启动真实的守护进程），并随机杀死它。
-
## Freon
Freon 是 Ozone 发行包中包含的命令行应用，它是一个负载生成器，用于压力测试。
diff --git a/hadoop-ozone/dev-support/checks/unit.sh
b/hadoop-ozone/dev-support/checks/unit.sh
index 20eb4b955fc..3e990f6416f 100755
--- a/hadoop-ozone/dev-support/checks/unit.sh
+++ b/hadoop-ozone/dev-support/checks/unit.sh
@@ -17,5 +17,5 @@
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CHECK=unit
source "${DIR}/junit.sh" \
- -pl
\!:ozone-integration-test,\!:ozone-integration-test-recon,\!:ozone-integration-test-s3,\!:mini-chaos-tests
\
+ -pl
\!:ozone-integration-test,\!:ozone-integration-test-recon,\!:ozone-integration-test-s3
\
"$@"
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/dev-support/findbugsExcludeFile.xml
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/dev-support/findbugsExcludeFile.xml
deleted file mode 100644
index b6f6582f6c8..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/dev-support/findbugsExcludeFile.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
- <!-- Test -->
- <Match>
- <Class name="org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator"/>
- <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
- </Match>
-</FindBugsFilter>
diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml
deleted file mode 100644
index 643fedf1b00..00000000000
--- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml
+++ /dev/null
@@ -1,156 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-fault-injection-test</artifactId>
- <version>2.3.0-SNAPSHOT</version>
- </parent>
-
- <artifactId>mini-chaos-tests</artifactId>
- <version>2.3.0-SNAPSHOT</version>
- <name>Apache Ozone Mini Ozone Chaos Tests</name>
- <description>Apache Ozone Mini Ozone Chaos Tests</description>
-
- <dependencies>
- <dependency>
- <groupId>info.picocli</groupId>
- <artifactId>picocli</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-auth</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-cli-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-client</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-config</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-container-service</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-server-scm</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-server-scm</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>hdds-test-utils</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-client</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-filesystem</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-freon</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-integration-test</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-manager</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-mini-cluster</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.ozone</groupId>
- <artifactId>ozone-recon</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <proc>none</proc>
- </configuration>
- </plugin>
- <plugin>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-maven-plugin</artifactId>
- <configuration>
-
<excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/bin/start-chaos.sh
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/bin/start-chaos.sh
deleted file mode 100755
index d3f71f09b52..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/bin/start-chaos.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-date=$(date +"%Y-%m-%d-%H-%M-%S-%Z")
-logfiledirectory="/tmp/chaos-${date}/"
-completesuffix="complete.log"
-chaossuffix="chaos.log"
-problemsuffix="problem.log"
-compilesuffix="compile.log"
-heapformat="dump.hprof"
-
-#log goes to something like /tmp/2019-12-04--00-01-26-IST/complete.log
-logfilename="${logfiledirectory}${completesuffix}"
-#log goes to something like /tmp/2019-12-04--00-01-26-IST/chaos.log
-chaosfilename="${logfiledirectory}${chaossuffix}"
-#compilation log goes to something like
/tmp/2019-12-04--00-01-26-IST/compile.log
-compilefilename="${logfiledirectory}${compilesuffix}"
-#log goes to something like /tmp/2019-12-04--00-01-26-IST/dump.hprof
-heapdumpfile="${logfiledirectory}${heapformat}"
-#log goes to something like /tmp/2019-12-04--00-01-26-IST/problem.log
-problemfilename="${logfiledirectory}${problemsuffix}"
-
-#TODO: add gc log file details as well
-MVN_OPTS="-XX:+HeapDumpOnOutOfMemoryError "
-MVN_OPTS+="-XX:HeapDumpPath=${heapdumpfile} "
-MVN_OPTS+="-XX:NativeMemoryTracking=detail"
-export MAVEN_OPTS=$MVN_OPTS
-
-mkdir -p ${logfiledirectory}
-echo "logging chaos logs and heapdump to ${logfiledirectory}"
-
-echo "Starting MiniOzoneChaosCluster with ${MVN_OPTS}"
-mvn clean install -DskipTests > "${compilefilename}" 2>&1
-mvn exec:java \
- -Dexec.mainClass="org.apache.hadoop.ozone.OzoneChaosCluster" \
- -Dexec.classpathScope=test \
- -Dchaoslogfilename=${chaosfilename} \
- -Dproblemlogfilename=${problemfilename} \
- -Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false
\
- -Dio.netty.leakDetection.level=advanced \
- -Dio.netty.leakDetectionLevel=advanced \
- -Dtest.build.data="${logfiledirectory}" \
- -Dexec.args="$*" > "${logfilename}" 2>&1
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
deleted file mode 100644
index f8ef8e01c15..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
+++ /dev/null
@@ -1,401 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.io.IOException;
-import java.time.Duration;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.OzoneClientConfig;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import
org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration;
-import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.ozone.container.common.utils.DatanodeStoreCache;
-import org.apache.hadoop.ozone.failure.FailureManager;
-import org.apache.hadoop.ozone.failure.Failures;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import
org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.ozone.test.GenericTestUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class causes random failures in the chaos cluster.
- */
-public class MiniOzoneChaosCluster extends MiniOzoneHAClusterImpl {
-
- static final Logger LOG =
- LoggerFactory.getLogger(MiniOzoneChaosCluster.class);
-
- private final int numDatanodes;
- private final int numOzoneManagers;
- private final int numStorageContainerManagers;
-
- private final FailureManager failureManager;
-
- private static final int WAIT_FOR_CLUSTER_TO_BE_READY_TIMEOUT = 120000; // 2
min
-
- private final Set<OzoneManager> failedOmSet;
- private final Set<StorageContainerManager> failedScmSet;
- private final Set<DatanodeDetails> failedDnSet;
-
- @SuppressWarnings("parameternumber")
- public MiniOzoneChaosCluster(OzoneConfiguration conf,
- OMHAService omService, SCMHAService scmService,
- List<HddsDatanodeService> hddsDatanodes, String clusterPath,
- Set<Class<? extends Failures>> clazzes) {
- super(conf, new SCMConfigurator(), omService, scmService, hddsDatanodes,
- clusterPath, Collections.emptyList());
- this.numDatanodes = getHddsDatanodes().size();
- this.numOzoneManagers = omService.getServices().size();
- this.numStorageContainerManagers = scmService.getServices().size();
-
- this.failedOmSet = new HashSet<>();
- this.failedDnSet = new HashSet<>();
- this.failedScmSet = new HashSet<>();
-
- this.failureManager = new FailureManager(this, conf, clazzes);
- LOG.info("Starting MiniOzoneChaosCluster with {} OzoneManagers and {} " +
- "Datanodes", numOzoneManagers, numDatanodes);
- clazzes.forEach(c -> LOG.info("added failure:{}", c.getSimpleName()));
- }
-
- void startChaos(long initialDelay, long period, TimeUnit timeUnit) {
- LOG.info("Starting Chaos with failure period:{} unit:{} numDataNodes:{} " +
- "numOzoneManagers:{} numStorageContainerManagers:{}",
- period, timeUnit, numDatanodes,
- numOzoneManagers, numStorageContainerManagers);
- failureManager.start(initialDelay, period, timeUnit);
- }
-
- @Override
- public void shutdown() {
- try {
- failureManager.stop();
- } catch (Exception e) {
- LOG.error("failed to stop FailureManager", e);
- }
- //this should be called after failureManager.stop to be sure that the
- //datanode collection is not modified during the shutdown
- super.shutdown();
- }
-
- /**
- * Check if cluster is ready for a restart or shutdown of an OM node. If
- * yes, then set isClusterReady to false so that another thread cannot
- * restart/ shutdown OM till all OMs are up again.
- */
- @Override
- public void waitForClusterToBeReady()
- throws TimeoutException, InterruptedException {
- super.waitForClusterToBeReady();
- GenericTestUtils.waitFor(() -> {
- for (OzoneManager om : getOzoneManagersList()) {
- if (!om.isRunning()) {
- return false;
- }
- }
- return true;
- }, 1000, WAIT_FOR_CLUSTER_TO_BE_READY_TIMEOUT);
- }
-
- /**
- * Builder for configuring the MiniOzoneChaosCluster to run.
- */
- public static class Builder extends MiniOzoneHAClusterImpl.Builder {
-
- private final Set<Class<? extends Failures>> clazzes = new HashSet<>();
-
- /**
- * Creates a new Builder.
- *
- * @param conf configuration
- */
- public Builder(OzoneConfiguration conf) {
- super(conf);
- }
-
- /**
- * Sets the number of HddsDatanodes to be started as part of
- * MiniOzoneChaosCluster.
- * @param val number of datanodes
- * @return MiniOzoneChaosCluster.Builder
- */
- @Override
- public Builder setNumDatanodes(int val) {
- super.setNumDatanodes(val);
- return this;
- }
-
- /**
- * Sets the number of OzoneManagers to be started as part of
- * MiniOzoneChaosCluster.
- * @param val number of OzoneManagers
- * @return MiniOzoneChaosCluster.Builder
- */
- public Builder setNumOzoneManagers(int val) {
- super.setNumOfOzoneManagers(val);
- super.setNumOfActiveOMs(val);
- return this;
- }
-
- /**
- * Sets OM Service ID.
- */
- public Builder setOMServiceID(String omServiceID) {
- super.setOMServiceId(omServiceID);
- return this;
- }
-
- /**
- * Sets SCM Service ID.
- */
- public Builder setSCMServiceID(String scmServiceID) {
- super.setSCMServiceId(scmServiceID);
- return this;
- }
-
- public Builder setNumStorageContainerManagers(int val) {
- super.setNumOfStorageContainerManagers(val);
- super.setNumOfActiveSCMs(val);
- return this;
- }
-
- public Builder addFailures(Class<? extends Failures> clazz) {
- this.clazzes.add(clazz);
- return this;
- }
-
- @Override
- protected void initializeConfiguration() throws IOException {
- super.initializeConfiguration();
-
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setStreamBufferFlushSize(8 * 1024 * 1024);
- clientConfig.setStreamBufferMaxSize(16 * 1024 * 1024);
- clientConfig.setStreamBufferSize(4 * 1024);
- conf.setFromObject(clientConfig);
-
- conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- 4, StorageUnit.KB);
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
- 32, StorageUnit.KB);
- conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
- 1, StorageUnit.MB);
- conf.setStorageSize(
- ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
- 0, org.apache.hadoop.hdds.conf.StorageUnit.MB);
- conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL, 10,
- TimeUnit.SECONDS);
- conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL, 20,
- TimeUnit.SECONDS);
- conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1,
- TimeUnit.SECONDS);
- conf.setTimeDuration(HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL, 1,
- TimeUnit.SECONDS);
- conf.setTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
- 1, TimeUnit.SECONDS);
- conf.setTimeDuration(HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL, 1,
- TimeUnit.SECONDS);
- conf.setInt(
- OzoneConfigKeys
- .HDDS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_KEY,
- 4);
- conf.setInt(
- OzoneConfigKeys.HDDS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY,
- 2);
- conf.setInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE, 2);
- ReplicationManagerConfiguration replicationConf =
- conf.getObject(ReplicationManagerConfiguration.class);
- replicationConf.setInterval(Duration.ofSeconds(10));
- replicationConf.setEventTimeout(Duration.ofSeconds(20));
- replicationConf.setDatanodeTimeoutOffset(0);
- conf.setFromObject(replicationConf);
- conf.setInt(OzoneConfigKeys.HDDS_RATIS_SNAPSHOT_THRESHOLD_KEY, 100);
- conf.setInt(OzoneConfigKeys.HDDS_CONTAINER_RATIS_LOG_PURGE_GAP, 100);
- conf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP, 100);
-
- conf.setInt(OMConfigKeys.
- OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY, 100);
- }
-
- @Override
- public MiniOzoneChaosCluster build() throws IOException {
- DefaultMetricsSystem.setMiniClusterMode(true);
- DatanodeStoreCache.setMiniClusterMode();
-
- initializeConfiguration();
- if (numberOfOzoneManagers() > 1) {
- initOMRatisConf();
- }
-
- SCMHAService scmService;
- OMHAService omService;
- try {
- scmService = createSCMService();
- omService = createOMService();
- } catch (AuthenticationException ex) {
- throw new IOException("Unable to build MiniOzoneCluster. ", ex);
- }
-
- final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes();
-
- MiniOzoneChaosCluster cluster =
- new MiniOzoneChaosCluster(conf, omService, scmService, hddsDatanodes,
- path, clazzes);
-
- if (startDataNodes) {
- cluster.startHddsDatanodes();
- }
- prepareForNextBuild();
- return cluster;
- }
- }
-
- // OzoneManager specific
- public static int getNumberOfOmToFail() {
- return 1;
- }
-
- public Set<OzoneManager> omToFail() {
- int numNodesToFail = getNumberOfOmToFail();
- if (failedOmSet.size() >= numOzoneManagers / 2) {
- return Collections.emptySet();
- }
-
- int numOms = getOzoneManagersList().size();
- Set<OzoneManager> oms = new HashSet<>();
- for (int i = 0; i < numNodesToFail; i++) {
- int failedNodeIndex = FailureManager.getBoundedRandomIndex(numOms);
- oms.add(getOzoneManager(failedNodeIndex));
- }
- return oms;
- }
-
- @Override
- public void shutdownOzoneManager(OzoneManager om) {
- super.shutdownOzoneManager(om);
- failedOmSet.add(om);
- }
-
- @Override
- public void restartOzoneManager(OzoneManager om, boolean waitForOM)
- throws IOException, TimeoutException, InterruptedException {
- super.restartOzoneManager(om, waitForOM);
- failedOmSet.remove(om);
- }
-
- // Should the selected node be stopped or started.
- public boolean shouldStopOm() {
- if (failedOmSet.size() >= numOzoneManagers / 2) {
- return false;
- }
- return RandomUtils.secure().randomBoolean();
- }
-
- // Datanode specific
- private int getNumberOfDnToFail() {
- return RandomUtils.secure().randomBoolean() ? 1 : 2;
- }
-
- public Set<DatanodeDetails> dnToFail() {
- int numNodesToFail = getNumberOfDnToFail();
- int numDns = getHddsDatanodes().size();
- Set<DatanodeDetails> dns = new HashSet<>();
- for (int i = 0; i < numNodesToFail; i++) {
- int failedNodeIndex = FailureManager.getBoundedRandomIndex(numDns);
- dns.add(getHddsDatanodes().get(failedNodeIndex).getDatanodeDetails());
- }
- return dns;
- }
-
- @Override
- public void restartHddsDatanode(DatanodeDetails dn, boolean waitForDatanode)
- throws InterruptedException, TimeoutException, IOException {
- failedDnSet.add(dn);
- super.restartHddsDatanode(dn, waitForDatanode);
- failedDnSet.remove(dn);
- }
-
- @Override
- public void shutdownHddsDatanode(DatanodeDetails dn) throws IOException {
- failedDnSet.add(dn);
- super.shutdownHddsDatanode(dn);
- }
-
- // Should the selected node be stopped or started.
- public boolean shouldStop(DatanodeDetails dn) {
- return !failedDnSet.contains(dn);
- }
-
- // StorageContainerManager specific
- public static int getNumberOfScmToFail() {
- return 1;
- }
-
- public Set<StorageContainerManager> scmToFail() {
- int numNodesToFail = getNumberOfScmToFail();
- if (failedScmSet.size() >= numStorageContainerManagers / 2) {
- return Collections.emptySet();
- }
-
- int numSCMs = getStorageContainerManagersList().size();
- Set<StorageContainerManager> scms = new HashSet<>();
- for (int i = 0; i < numNodesToFail; i++) {
- int failedNodeIndex = FailureManager.getBoundedRandomIndex(numSCMs);
- scms.add(getStorageContainerManager(failedNodeIndex));
- }
- return scms;
- }
-
- @Override
- public void shutdownStorageContainerManager(StorageContainerManager scm) {
- super.shutdownStorageContainerManager(scm);
- failedScmSet.add(scm);
- }
-
- @Override
- public StorageContainerManager restartStorageContainerManager(
- StorageContainerManager scm, boolean waitForScm)
- throws IOException, TimeoutException, InterruptedException,
- AuthenticationException {
- failedScmSet.remove(scm);
- return super.restartStorageContainerManager(scm, waitForScm);
- }
-
- // Should the selected node be stopped or started.
- public boolean shouldStopScm() {
- if (failedScmSet.size() >= numStorageContainerManagers / 2) {
- return false;
- }
- return RandomUtils.secure().randomBoolean();
- }
-
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
deleted file mode 100644
index 2d704dbef9a..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.loadgenerators.DataBuffer;
-import org.apache.hadoop.ozone.loadgenerators.LoadBucket;
-import org.apache.hadoop.ozone.loadgenerators.LoadExecutors;
-import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A Simple Load generator for testing.
- */
-public final class MiniOzoneLoadGenerator {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(MiniOzoneLoadGenerator.class);
-
- private final List<LoadGenerator> loadGenerators;
- private final LoadExecutors loadExecutor;
-
- private final OzoneVolume volume;
- private final OzoneConfiguration conf;
- private final String omServiceID;
- private final BucketArgs bucketArgs;
-
- private MiniOzoneLoadGenerator(OzoneVolume volume, int numThreads,
- int numBuffers, OzoneConfiguration conf, String omServiceId,
- BucketArgs bucketArgs, Set<Class<? extends LoadGenerator>>
- loadGeneratorClazzes) throws Exception {
- DataBuffer buffer = new DataBuffer(numBuffers);
- loadGenerators = new ArrayList<>();
- this.volume = volume;
- this.conf = conf;
- this.omServiceID = omServiceId;
- this.bucketArgs = bucketArgs;
-
- for (Class<? extends LoadGenerator> clazz : loadGeneratorClazzes) {
- addLoads(clazz, buffer);
- }
-
- this.loadExecutor = new LoadExecutors(numThreads, loadGenerators);
- }
-
- private void addLoads(Class<? extends LoadGenerator> clazz,
- DataBuffer buffer) throws Exception {
- String bucketName =
RandomStringUtils.secure().nextAlphabetic(10).toLowerCase();
-
- volume.createBucket(bucketName, bucketArgs);
- LoadBucket ozoneBucket = new LoadBucket(volume.getBucket(bucketName),
- conf, omServiceID);
-
- LoadGenerator loadGenerator = clazz
- .getConstructor(DataBuffer.class, LoadBucket.class)
- .newInstance(buffer, ozoneBucket);
- loadGenerators.add(loadGenerator);
- }
-
- void startIO(long time, TimeUnit timeUnit) throws Exception {
- LOG.info("Starting MiniOzoneLoadGenerator for time {}:{}", time, timeUnit);
- long runTime = timeUnit.toMillis(time);
- // start and wait for executors to finish
- loadExecutor.startLoad(runTime);
- loadExecutor.waitForCompletion();
- }
-
- void shutdownLoadGenerator() {
- loadExecutor.shutdown();
- }
-
- /**
- * Builder to create Ozone load generator.
- */
- public static class Builder {
- private Set<Class<? extends LoadGenerator>> clazzes = new HashSet<>();
- private String omServiceId;
- private OzoneConfiguration conf;
- private int numBuffers;
- private int numThreads;
- private OzoneVolume volume;
- private BucketArgs bucketArgs;
-
- public Builder addLoadGenerator(Class<? extends LoadGenerator> clazz) {
- clazzes.add(clazz);
- return this;
- }
-
- public Builder setOMServiceId(String serviceId) {
- omServiceId = serviceId;
- return this;
- }
-
- public Builder setConf(OzoneConfiguration configuration) {
- this.conf = configuration;
- return this;
- }
-
- public Builder setNumBuffers(int buffers) {
- this.numBuffers = buffers;
- return this;
- }
-
- public Builder setNumThreads(int threads) {
- this.numThreads = threads;
- return this;
- }
-
- public Builder setVolume(OzoneVolume vol) {
- this.volume = vol;
- return this;
- }
-
- public Builder setBucketArgs(BucketArgs buckArgs) {
- this.bucketArgs = buckArgs;
- return this;
- }
-
- public MiniOzoneLoadGenerator build() throws Exception {
- return new MiniOzoneLoadGenerator(volume, numThreads, numBuffers,
- conf, omServiceId, bucketArgs, clazzes);
- }
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/OzoneChaosCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/OzoneChaosCluster.java
deleted file mode 100644
index 4688536eec2..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/OzoneChaosCluster.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.apache.hadoop.hdds.cli.GenericCli;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import picocli.CommandLine;
-
-/**
- * Main driver class for Ozone Chaos Cluster
- * This has multiple sub implementations of chaos cluster as options.
- */
[email protected](
- name = "chaos",
- description = "Starts IO with MiniOzoneChaosCluster",
- subcommands = {
- TestAllMiniChaosOzoneCluster.class,
- TestDatanodeMiniChaosOzoneCluster.class,
- TestOzoneManagerMiniChaosOzoneCluster.class,
- TestStorageContainerManagerMiniChaosOzoneCluster.class
- },
- versionProvider = HddsVersionProvider.class,
- mixinStandardHelpOptions = true)
-public class OzoneChaosCluster extends GenericCli {
- public static void main(String[] args) {
- new OzoneChaosCluster().run(args);
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java
deleted file mode 100644
index 4275beed02f..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.util.concurrent.Callable;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.ozone.failure.Failures;
-import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.TestInstance;
-import picocli.CommandLine;
-
-/**
- * Test all kinds of chaos.
- */
[email protected](
- name = "all",
- description = "run chaos cluster across all daemons",
- mixinStandardHelpOptions = true,
- versionProvider = HddsVersionProvider.class)
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestAllMiniChaosOzoneCluster extends TestMiniChaosOzoneCluster
- implements Callable<Void> {
-
- @BeforeAll
- void setup() {
- setNumManagers(3, 3, true);
-
- LoadGenerator.getClassList().forEach(this::addLoadClasses);
- Failures.getClassList().forEach(this::addFailureClasses);
- }
-
- @Override
- public Void call() throws Exception {
- setup();
- startChaosCluster();
- return null;
- }
-
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java
deleted file mode 100644
index 828d57e3db9..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.util.concurrent.Callable;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.ozone.failure.Failures;
-import org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.RandomLoadGenerator;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.TestInstance;
-import picocli.CommandLine;
-
-/**
- * Test Datanode with Chaos.
- */
[email protected](
- name = "dn",
- description = "run chaos cluster across Ozone Datanodes",
- mixinStandardHelpOptions = true,
- versionProvider = HddsVersionProvider.class)
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestDatanodeMiniChaosOzoneCluster extends
- TestMiniChaosOzoneCluster implements Callable<Void> {
-
- @BeforeAll
- void setup() {
- addLoadClasses(RandomLoadGenerator.class);
- addLoadClasses(AgedLoadGenerator.class);
-
- addFailureClasses(Failures.DatanodeStartStopFailure.class);
- addFailureClasses(Failures.DatanodeRestartFailure.class);
- }
-
- @Override
- public Void call() throws Exception {
- setup();
- startChaosCluster();
- return null;
- }
-
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
deleted file mode 100644
index a4c4ea0d324..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hdds.cli.GenericCli;
-import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.utils.IOUtils;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.ObjectStore;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.failure.Failures;
-import org.apache.hadoop.ozone.freon.FreonReplicationOptions;
-import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
-import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.apache.ozone.test.tag.Unhealthy;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import picocli.CommandLine;
-import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
-
-/**
- * Test Read Write with Mini Ozone Chaos Cluster.
- */
-@Command(description = "Starts IO with MiniOzoneChaosCluster",
- name = "chaos", mixinStandardHelpOptions = true)
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-@Unhealthy("HDDS-3131")
-public class TestMiniChaosOzoneCluster extends GenericCli {
-
- private final List<Class<? extends Failures>> failureClasses
- = new ArrayList<>();
-
- private final List<Class<? extends LoadGenerator>> loadClasses
- = new ArrayList<>();
-
- @Option(names = {"-d", "--num-datanodes", "--numDatanodes"},
- description = "num of datanodes. Full name --numDatanodes will be" +
- " removed in later versions.")
- private int numDatanodes = 20;
-
- @Option(names = {"-o", "--num-ozone-manager", "--numOzoneManager"},
- description = "num of ozoneManagers. Full name --numOzoneManager will" +
- " be removed in later versions.")
- private int numOzoneManagers = 1;
-
- @Option(names = {"-s", "--num-storage-container-manager",
- "--numStorageContainerManagers"},
- description = "num of storageContainerManagers." +
- "Full name --numStorageContainerManagers will" +
- " be removed in later versions.")
- private int numStorageContainerManagerss = 1;
-
- @Option(names = {"-t", "--num-threads", "--numThreads"},
- description = "num of IO threads. Full name --numThreads will be" +
- " removed in later versions.")
- private int numThreads = 5;
-
- @Option(names = {"-b", "--num-buffers", "--numBuffers"},
- description = "num of IO buffers. Full name --numBuffers will be" +
- " removed in later versions.")
- private int numBuffers = 16;
-
- @Option(names = {"-m", "--num-minutes", "--numMinutes"},
- description = "total run time. Full name --numMinutes will be " +
- "removed in later versions.")
- private int numMinutes = 1440; // 1 day by default
-
- @Option(names = {"-v", "--num-data-volume", "--numDataVolume"},
- description = "number of datanode volumes to create. Full name " +
- "--numDataVolume will be removed in later versions.")
- private int numDataVolumes = 3;
-
- @Option(names = {"--initial-delay"},
- description = "time (in seconds) before first failure event")
- private int initialDelay = 300; // seconds
-
- @Option(names = {"-i", "--failure-interval", "--failureInterval"},
- description = "time between failure events in seconds. Full name " +
- "--failureInterval will be removed in later versions.")
- private int failureInterval = 300; // 5 minute period between failures.
-
- @CommandLine.Mixin
- private FreonReplicationOptions freonReplication =
- new FreonReplicationOptions();
-
- @Option(names = {"-l", "--layout"},
- description = "Allowed Bucket Layouts: ${COMPLETION-CANDIDATES}")
- private AllowedBucketLayouts allowedBucketLayout =
- AllowedBucketLayouts.FILE_SYSTEM_OPTIMIZED;
-
- private MiniOzoneChaosCluster cluster;
- private OzoneClient client;
- private MiniOzoneLoadGenerator loadGenerator;
-
- private String omServiceId;
- private String scmServiceId;
-
- private static final String OM_SERVICE_ID = "ozoneChaosTest";
- private static final String SCM_SERVICE_ID = "scmChaosTest";
-
- private void init() throws Exception {
- OzoneConfiguration configuration = new OzoneConfiguration();
-
- MiniOzoneChaosCluster.Builder chaosBuilder =
- new MiniOzoneChaosCluster.Builder(configuration);
-
- chaosBuilder
- .setNumDatanodes(numDatanodes)
- .setNumOzoneManagers(numOzoneManagers)
- .setOMServiceID(omServiceId)
- .setNumStorageContainerManagers(numStorageContainerManagerss)
- .setSCMServiceID(scmServiceId)
- .setDatanodeFactory(UniformDatanodesFactory.newBuilder()
- .setNumDataVolumes(numDataVolumes)
- .build());
- failureClasses.forEach(chaosBuilder::addFailures);
-
- cluster = chaosBuilder.build();
- cluster.waitForClusterToBeReady();
-
- client = cluster.newClient();
- ObjectStore store = client.getObjectStore();
- String volumeName =
RandomStringUtils.secure().nextAlphabetic(10).toLowerCase();
- store.createVolume(volumeName);
- OzoneVolume volume = store.getVolume(volumeName);
-
- BucketLayout bucketLayout =
- BucketLayout.valueOf(allowedBucketLayout.toString());
- final BucketArgs.Builder builder = BucketArgs.newBuilder();
-
- freonReplication.fromParams(configuration).ifPresent(config ->
- builder.setDefaultReplicationConfig(
- new DefaultReplicationConfig(config)));
- builder.setBucketLayout(bucketLayout);
-
- MiniOzoneLoadGenerator.Builder loadBuilder =
- new MiniOzoneLoadGenerator.Builder()
- .setVolume(volume)
- .setConf(configuration)
- .setNumBuffers(numBuffers)
- .setNumThreads(numThreads)
- .setOMServiceId(omServiceId)
- .setBucketArgs(builder.build());
- loadClasses.forEach(loadBuilder::addLoadGenerator);
- loadGenerator = loadBuilder.build();
- }
-
- void addFailureClasses(Class<? extends Failures> clz) {
- failureClasses.add(clz);
- }
-
- void addLoadClasses(Class<? extends LoadGenerator> clz) {
- loadClasses.add(clz);
- }
-
- void setNumDatanodes(int nDns) {
- numDatanodes = nDns;
- }
-
- void setNumManagers(int nOms, int numScms, boolean enableHA) {
-
- if (nOms > 1 || enableHA) {
- omServiceId = OM_SERVICE_ID;
- }
- numOzoneManagers = nOms;
-
- if (numScms > 1 || enableHA) {
- scmServiceId = SCM_SERVICE_ID;
- }
- numStorageContainerManagerss = numScms;
- }
-
- private void shutdown() {
- if (loadGenerator != null) {
- loadGenerator.shutdownLoadGenerator();
- }
-
- IOUtils.closeQuietly(client, cluster);
- }
-
- public void startChaosCluster() throws Exception {
- try {
- init();
- cluster.startChaos(initialDelay, failureInterval, TimeUnit.SECONDS);
- loadGenerator.startIO(numMinutes, TimeUnit.MINUTES);
- } finally {
- shutdown();
- }
- }
-
- @Test
- void test() throws Exception {
- initialDelay = 5; // seconds
- failureInterval = 10; // seconds
- numMinutes = 2;
- startChaosCluster();
- }
-
- enum AllowedBucketLayouts { FILE_SYSTEM_OPTIMIZED, OBJECT_STORE }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java
deleted file mode 100644
index 662e3f4b7c6..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.util.concurrent.Callable;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.ozone.failure.Failures;
-import org.apache.hadoop.ozone.loadgenerators.AgedDirLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.NestedDirLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.RandomDirLoadGenerator;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.TestInstance;
-import picocli.CommandLine;
-
-/**
- * Chaos cluster for Ozone Manager.
- */
[email protected](
- name = "om",
- description = "run chaos cluster across Ozone Managers",
- mixinStandardHelpOptions = true,
- versionProvider = HddsVersionProvider.class)
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestOzoneManagerMiniChaosOzoneCluster extends
- TestMiniChaosOzoneCluster implements Callable<Void> {
-
- @BeforeAll
- void setup() {
- setNumManagers(3, 1, true);
- setNumDatanodes(3);
-
- addLoadClasses(AgedDirLoadGenerator.class);
- addLoadClasses(RandomDirLoadGenerator.class);
- addLoadClasses(NestedDirLoadGenerator.class);
-
- addFailureClasses(Failures.OzoneManagerRestartFailure.class);
- addFailureClasses(Failures.OzoneManagerStartStopFailure.class);
- }
-
- @Override
- public Void call() throws Exception {
- setup();
- startChaosCluster();
- return null;
- }
-
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerMiniChaosOzoneCluster.java
deleted file mode 100644
index bf0ea95663a..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerMiniChaosOzoneCluster.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import java.util.concurrent.Callable;
-import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.ozone.failure.Failures;
-import org.apache.hadoop.ozone.loadgenerators.AgedDirLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.NestedDirLoadGenerator;
-import org.apache.hadoop.ozone.loadgenerators.RandomDirLoadGenerator;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.TestInstance;
-import picocli.CommandLine;
-
-/**
- * Chaos cluster for Storage Container Manager.
- */
[email protected](
- name = "scm",
- description = "run chaos cluster across Storage Container Managers",
- mixinStandardHelpOptions = true,
- versionProvider = HddsVersionProvider.class)
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestStorageContainerManagerMiniChaosOzoneCluster extends
- TestMiniChaosOzoneCluster implements Callable<Void> {
-
- @BeforeAll
- void setup() {
- setNumManagers(3, 3, true);
- setNumDatanodes(3);
-
- addLoadClasses(AgedDirLoadGenerator.class);
- addLoadClasses(RandomDirLoadGenerator.class);
- addLoadClasses(NestedDirLoadGenerator.class);
-
- addFailureClasses(Failures.StorageContainerManagerRestartFailure.class);
- addFailureClasses(Failures.StorageContainerManagerStartStopFailure.class);
- }
-
- @Override
- public Void call() throws Exception {
- setup();
- startChaosCluster();
- return null;
- }
-
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/FailureManager.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/FailureManager.java
deleted file mode 100644
index 59fae25bbad..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/FailureManager.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.failure;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.MiniOzoneChaosCluster;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Manages all the failures in the MiniOzoneChaosCluster.
- */
-public class FailureManager {
-
- static final Logger LOG =
- LoggerFactory.getLogger(Failures.class);
-
- private final MiniOzoneChaosCluster cluster;
- private final List<Failures> failures;
- private ScheduledFuture scheduledFuture;
- private final ScheduledExecutorService executorService;
-
- public FailureManager(MiniOzoneChaosCluster cluster,
- Configuration conf,
- Set<Class<? extends Failures>> clazzes) {
- this.cluster = cluster;
- this.executorService = Executors.newSingleThreadScheduledExecutor();
-
- failures = new ArrayList<>();
- for (Class<? extends Failures> clazz : clazzes) {
- Failures f = ReflectionUtils.newInstance(clazz, conf);
- f.validateFailure(cluster);
- failures.add(f);
- }
-
- }
-
- // Fail nodes randomly at configured timeout period.
- private void fail() {
- Failures f = failures.get(getBoundedRandomIndex(failures.size()));
- try {
- LOG.info("time failure with {}", f.getName());
- f.fail(cluster);
- } catch (Throwable t) {
- LOG.info("Caught exception while inducing failure:{}", f.getName(), t);
- throw new RuntimeException();
- }
-
- }
-
- public void start(long initialDelay, long period, TimeUnit timeUnit) {
- LOG.info("starting failure manager {} {} {}", initialDelay,
- period, timeUnit);
- scheduledFuture = executorService.scheduleAtFixedRate(this::fail,
- initialDelay, period, timeUnit);
- }
-
- public void stop() throws Exception {
- if (scheduledFuture != null) {
- scheduledFuture.cancel(false);
- scheduledFuture.get();
- }
-
- executorService.shutdown();
- executorService.awaitTermination(1, TimeUnit.MINUTES);
- }
-
- public static boolean isFastRestart() {
- return RandomUtils.secure().randomBoolean();
- }
-
- public static int getBoundedRandomIndex(int size) {
- return RandomUtils.secure().randomInt(0, size);
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java
deleted file mode 100644
index 21a0303f32b..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/Failures.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.failure;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.ozone.MiniOzoneChaosCluster;
-import org.apache.hadoop.ozone.om.OzoneManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Implementation of all the failures.
- */
-public abstract class Failures {
- static final Logger LOG =
- LoggerFactory.getLogger(Failures.class);
-
- public String getName() {
- return this.getClass().getSimpleName();
- }
-
- public abstract void fail(MiniOzoneChaosCluster cluster);
-
- public abstract void validateFailure(MiniOzoneChaosCluster cluster);
-
- public static List<Class<? extends Failures>> getClassList() {
- List<Class<? extends Failures>> classList = new ArrayList<>();
-
- classList.add(OzoneManagerRestartFailure.class);
- classList.add(OzoneManagerStartStopFailure.class);
- classList.add(DatanodeRestartFailure.class);
- classList.add(DatanodeStartStopFailure.class);
- classList.add(StorageContainerManagerStartStopFailure.class);
- classList.add(StorageContainerManagerRestartFailure.class);
-
- return classList;
- }
-
- /**
- * Ozone Manager failures.
- */
- public abstract static class OzoneFailures extends Failures {
- @Override
- public void validateFailure(MiniOzoneChaosCluster cluster) {
- if (cluster.getOzoneManagersList().size() < 3) {
- throw new IllegalArgumentException("Not enough number of " +
- "OzoneManagers to test chaos on OzoneManagers. Set number of " +
- "OzoneManagers to at least 3");
- }
- }
- }
-
- /**
- * Restart Ozone Manager to induce failure.
- */
- public static class OzoneManagerRestartFailure extends OzoneFailures {
- @Override
- public void fail(MiniOzoneChaosCluster cluster) {
- boolean failureMode = FailureManager.isFastRestart();
- Set<OzoneManager> oms = cluster.omToFail();
- oms.parallelStream().forEach(om -> {
- try {
- cluster.shutdownOzoneManager(om);
- cluster.restartOzoneManager(om, failureMode);
- cluster.waitForClusterToBeReady();
- } catch (Throwable t) {
- LOG.error("Failed to restartNodes OM {}", om, t);
- }
- });
- }
- }
-
- /**
- * Start/Stop Ozone Manager to induce failure.
- */
- public static class OzoneManagerStartStopFailure extends OzoneFailures {
- @Override
- public void fail(MiniOzoneChaosCluster cluster) {
- // Get the number of OzoneManager to fail in the cluster.
- boolean shouldStop = cluster.shouldStopOm();
- Set<OzoneManager> oms = cluster.omToFail();
- oms.parallelStream().forEach(om -> {
- try {
- if (shouldStop) {
- // start another OM before failing the next one.
- cluster.shutdownOzoneManager(om);
- } else {
- cluster.restartOzoneManager(om, true);
- }
- } catch (Throwable t) {
- LOG.error("Failed to shutdown OM {}", om, t);
- }
- });
- }
- }
-
- /**
- * Ozone Manager failures.
- */
- public abstract static class ScmFailures extends Failures {
- @Override
- public void validateFailure(MiniOzoneChaosCluster cluster) {
- if (cluster.getStorageContainerManagersList().size() < 3) {
- throw new IllegalArgumentException("Not enough number of " +
- "StorageContainerManagers to test chaos on" +
- "StorageContainerManagers. Set number of " +
- "StorageContainerManagers to at least 3");
- }
- }
- }
-
- /**
- * Start/Stop Ozone Manager to induce failure.
- */
- public static class StorageContainerManagerStartStopFailure
- extends ScmFailures {
- @Override
- public void fail(MiniOzoneChaosCluster cluster) {
- // Get the number of OzoneManager to fail in the cluster.
- boolean shouldStop = cluster.shouldStopScm();
- Set<StorageContainerManager> scms = cluster.scmToFail();
- scms.parallelStream().forEach(scm -> {
- try {
- if (shouldStop) {
- // start another OM before failing the next one.
- cluster.shutdownStorageContainerManager(scm);
- } else {
- cluster.restartStorageContainerManager(scm, true);
- }
- } catch (Throwable t) {
- LOG.error("Failed to shutdown OM {}", scm, t);
- }
- });
- }
- }
-
- /**
- * Start/Stop Ozone Manager to induce failure.
- */
- public static class StorageContainerManagerRestartFailure
- extends ScmFailures {
- @Override
- public void fail(MiniOzoneChaosCluster cluster) {
- boolean failureMode = FailureManager.isFastRestart();
- Set<StorageContainerManager> scms = cluster.scmToFail();
- scms.parallelStream().forEach(scm -> {
- try {
- cluster.shutdownStorageContainerManager(scm);
- cluster.restartStorageContainerManager(scm, failureMode);
- cluster.waitForClusterToBeReady();
- } catch (Throwable t) {
- LOG.error("Failed to restartNodes SCM {}", scm, t);
- }
- });
- }
- }
-
- /**
- * Datanode failures.
- */
- public abstract static class DatanodeFailures extends Failures {
- @Override
- public void validateFailure(MiniOzoneChaosCluster cluster) {
- // Nothing to do here.
- }
- }
-
- /**
- * Restart Datanodes to induce failure.
- */
- public static class DatanodeRestartFailure extends DatanodeFailures {
- @Override
- public void fail(MiniOzoneChaosCluster cluster) {
- boolean failureMode = FailureManager.isFastRestart();
- Set<DatanodeDetails> dns = cluster.dnToFail();
- dns.parallelStream().forEach(dn -> {
- try {
- cluster.restartHddsDatanode(dn, failureMode);
- } catch (Throwable t) {
- LOG.error("Failed to restartNodes Datanode {}", dn.getID(), t);
- }
- });
- }
- }
-
- /**
- * Start/Stop Datanodes to induce failure.
- */
- public static class DatanodeStartStopFailure extends DatanodeFailures {
- @Override
- public void fail(MiniOzoneChaosCluster cluster) {
- // Get the number of datanodes to fail in the cluster.
- Set<DatanodeDetails> dns = cluster.dnToFail();
- dns.parallelStream().forEach(dn -> {
- try {
- if (cluster.shouldStop(dn)) {
- cluster.shutdownHddsDatanode(dn);
- } else {
- cluster.restartHddsDatanode(dn, true);
- }
- } catch (Throwable t) {
- LOG.error("Failed to shutdown Datanode {}", dn.getID(), t);
- }
- });
- }
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/package-info.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/package-info.java
deleted file mode 100644
index a14f0bc03f5..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/failure/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Failure manager tests. */
-package org.apache.hadoop.ozone.failure;
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
deleted file mode 100644
index 2187309839e..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import org.apache.commons.lang3.RandomUtils;
-
-/**
- * A load generator where directories are read multiple times.
- */
-public class AgedDirLoadGenerator extends LoadGenerator {
- private final LoadBucket fsBucket;
- private final int maxDirIndex;
-
- public AgedDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
- this.fsBucket = fsBucket;
- this.maxDirIndex = 100;
- }
-
- @Override
- public void generateLoad() throws Exception {
- int index = RandomUtils.secure().randomInt(0, maxDirIndex);
- String keyName = getKeyName(index);
- fsBucket.readDirectory(keyName);
- }
-
- @Override
- public void initialize() throws Exception {
- for (int i = 0; i < maxDirIndex; i++) {
- String keyName = getKeyName(i);
- fsBucket.createDirectory(keyName);
- }
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
deleted file mode 100644
index 4551eedea41..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.nio.ByteBuffer;
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.lang3.RandomUtils;
-
-/**
- * Aged Load Generator for Ozone.
- *
- * This Load Generator reads and write key to an Ozone bucket.
- *
- * The default writes to read ratio is 10:90.
- */
-public class AgedLoadGenerator extends LoadGenerator {
-
- private final AtomicInteger agedFileWrittenIndex;
- private final AtomicInteger agedFileAllocationIndex;
- private final LoadBucket agedLoadBucket;
- private final DataBuffer dataBuffer;
-
- public AgedLoadGenerator(DataBuffer data, LoadBucket agedLoadBucket) {
- this.dataBuffer = data;
- this.agedFileWrittenIndex = new AtomicInteger(0);
- this.agedFileAllocationIndex = new AtomicInteger(0);
- this.agedLoadBucket = agedLoadBucket;
- }
-
- @Override
- public void generateLoad() throws Exception {
- if (RandomUtils.secure().randomInt(0, 100) <= 10) {
- synchronized (agedFileAllocationIndex) {
- int index = agedFileAllocationIndex.getAndIncrement();
- ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index);
- agedLoadBucket.writeKey(buffer, keyName);
- agedFileWrittenIndex.getAndIncrement();
- }
- } else {
- Optional<Integer> index = randomKeyToRead();
- if (index.isPresent()) {
- ByteBuffer buffer = dataBuffer.getBuffer(index.get());
- String keyName = getKeyName(index.get());
- agedLoadBucket.readKey(buffer, keyName);
- }
- }
- }
-
- private Optional<Integer> randomKeyToRead() {
- int currentIndex = agedFileWrittenIndex.get();
- return currentIndex != 0
- ? Optional.of(RandomUtils.secure().randomInt(0, currentIndex))
- : Optional.empty();
- }
-
- @Override
- public void initialize() {
- // Nothing to do here
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java
deleted file mode 100644
index 9c5019e6986..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/DataBuffer.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.StorageUnit;
-
-/**
- * List of buffers used by the load generators.
- */
-public class DataBuffer {
- private List<ByteBuffer> buffers;
- // number of buffer to be allocated, each is allocated with length which
- // is multiple of 2, each buffer is populated with random data.
- private int numBuffers;
-
- public DataBuffer(int numBuffers) {
- // allocate buffers and populate random data.
- this.numBuffers = numBuffers;
- this.buffers = new ArrayList<>();
- for (int i = 0; i < numBuffers; i++) {
- int size = (int) StorageUnit.KB.toBytes(1 << i);
- ByteBuffer buffer = ByteBuffer.allocate(size);
- buffer.put(RandomUtils.secure().randomBytes(size));
- this.buffers.add(buffer);
- }
- // TODO: add buffers of sizes of prime numbers.
- }
-
- public ByteBuffer getBuffer(int keyIndex) {
- return buffers.get(keyIndex % numBuffers);
- }
-
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java
deleted file mode 100644
index 61ec463a5c0..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/FilesystemLoadGenerator.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.nio.ByteBuffer;
-import org.apache.commons.lang3.RandomUtils;
-
-/**
- * Filesystem load generator for Ozone.
- *
- * This load generator read, writes and deletes data using the filesystem
- * apis.
- */
-public class FilesystemLoadGenerator extends LoadGenerator {
-
- private final LoadBucket fsBucket;
- private final DataBuffer dataBuffer;
-
- public FilesystemLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
- this.dataBuffer = dataBuffer;
- this.fsBucket = fsBucket;
- }
-
- @Override
- public void generateLoad() throws Exception {
- int index = RandomUtils.secure().randomInt();
- ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index);
- fsBucket.writeKey(true, buffer, keyName);
-
- fsBucket.readKey(true, buffer, keyName);
-
- fsBucket.deleteKey(true, keyName);
- }
-
- @Override
- public void initialize() {
- // Nothing to do here
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadBucket.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadBucket.java
deleted file mode 100644
index 62a990a0086..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadBucket.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.HashMap;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.ozone.OzoneFileSystem;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Bucket to perform read/write & delete ops.
- */
-public class LoadBucket {
- private static final Logger LOG =
- LoggerFactory.getLogger(LoadBucket.class);
-
- private final OzoneBucket bucket;
- private final OzoneFileSystem fs;
-
- public LoadBucket(OzoneBucket bucket, OzoneConfiguration conf,
- String omServiceID) throws Exception {
- this.bucket = bucket;
- if (omServiceID == null) {
- this.fs = (OzoneFileSystem) FileSystem.get(getFSUri(bucket), conf);
- } else {
- this.fs = (OzoneFileSystem) FileSystem.get(getFSUri(bucket, omServiceID),
- conf);
- }
- }
-
- private boolean isFsOp() {
- return RandomUtils.secure().randomBoolean();
- }
-
- // Write ops.
- public void writeKey(ByteBuffer buffer,
- String keyName) throws Exception {
- writeKey(isFsOp(), buffer, keyName);
- }
-
- public void writeKey(boolean fsOp, ByteBuffer buffer,
- String keyName) throws Exception {
- Op writeOp = new WriteOp(fsOp, keyName, buffer);
- writeOp.execute();
- }
-
- public void createDirectory(String keyName) throws Exception {
- Op dirOp = new DirectoryOp(keyName, false);
- dirOp.execute();
- }
-
- public void readDirectory(String keyName) throws Exception {
- Op dirOp = new DirectoryOp(keyName, true);
- dirOp.execute();
- }
-
- // Read ops.
- public void readKey(ByteBuffer buffer, String keyName) throws Exception {
- readKey(isFsOp(), buffer, keyName);
- }
-
- public void readKey(boolean fsOp, ByteBuffer buffer,
- String keyName) throws Exception {
- Op readOp = new ReadOp(fsOp, keyName, buffer);
- readOp.execute();
- }
-
- // Delete ops.
- public void deleteKey(String keyName) throws Exception {
- deleteKey(isFsOp(), keyName);
- }
-
- public void deleteKey(boolean fsOp, String keyName) throws Exception {
- Op deleteOp = new DeleteOp(fsOp, keyName);
- deleteOp.execute();
- }
-
- private static URI getFSUri(OzoneBucket bucket) throws URISyntaxException {
- return new URI(String.format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME,
- bucket.getName(), bucket.getVolumeName()));
- }
-
- private static URI getFSUri(OzoneBucket bucket, String omServiceID)
- throws URISyntaxException {
- return new URI(String.format("%s://%s.%s.%s/",
OzoneConsts.OZONE_URI_SCHEME,
- bucket.getName(), bucket.getVolumeName(), omServiceID));
- }
-
- abstract class Op {
- private final boolean fsOp;
- private final String opName;
- private final String keyName;
-
- Op(boolean fsOp, String keyName) {
- this.fsOp = fsOp;
- this.keyName = keyName;
- this.opName = (fsOp ? "Filesystem" : "Bucket") + ":"
- + getClass().getSimpleName();
- }
-
- public void execute() throws Exception {
- LOG.info("Going to {}", this);
- try {
- if (fsOp) {
- Path p = new Path("/", keyName);
- doFsOp(p);
- } else {
- doBucketOp(keyName);
- }
- doPostOp();
- LOG.trace("Done: {}", this);
- } catch (Throwable t) {
- LOG.error("Unable to {}", this, t);
- throw t;
- }
- }
-
- abstract void doFsOp(Path p) throws IOException;
-
- abstract void doBucketOp(String key) throws IOException;
-
- abstract void doPostOp() throws IOException;
-
- @Override
- public String toString() {
- return "opType=" + opName + " keyName=" + keyName;
- }
- }
-
- /**
- * Create and Read Directories.
- */
- public class DirectoryOp extends Op {
- private final boolean readDir;
-
- DirectoryOp(String keyName, boolean readDir) {
- super(true, keyName);
- this.readDir = readDir;
- }
-
- @Override
- void doFsOp(Path p) throws IOException {
- if (readDir) {
- FileStatus status = fs.getFileStatus(p);
- assertTrue(status.isDirectory());
- assertEquals(p,
Path.getPathWithoutSchemeAndAuthority(status.getPath()));
- } else {
- assertTrue(fs.mkdirs(p));
- }
- }
-
- @Override
- void doBucketOp(String key) throws IOException {
- // nothing to do here
- }
-
- @Override
- void doPostOp() throws IOException {
- // Nothing to do here
- }
-
- @Override
- public String toString() {
- return super.toString() + " "
- + (readDir ? "readDirectory" : "writeDirectory");
- }
- }
-
- /**
- * Write file/key to bucket.
- */
- public class WriteOp extends Op {
- private OutputStream os;
- private final ByteBuffer buffer;
-
- WriteOp(boolean fsOp, String keyName, ByteBuffer buffer) {
- super(fsOp, keyName);
- this.buffer = buffer;
- }
-
- @Override
- void doFsOp(Path p) throws IOException {
- os = fs.create(p);
- }
-
- @Override
- void doBucketOp(String key) throws IOException {
- os = bucket.createKey(key, 0, ReplicationType.RATIS,
- ReplicationFactor.THREE, new HashMap<>());
- }
-
- @Override
- void doPostOp() throws IOException {
- try {
- os.write(buffer.array());
- } finally {
- os.close();
- }
- }
-
- @Override
- public String toString() {
- return super.toString() + " buffer:" + buffer.limit();
- }
- }
-
- /**
- * Read file/key from bucket.
- */
- public class ReadOp extends Op {
- private InputStream is;
- private final ByteBuffer buffer;
-
- ReadOp(boolean fsOp, String keyName, ByteBuffer buffer) {
- super(fsOp, keyName);
- this.buffer = buffer;
- this.is = null;
- }
-
- @Override
- void doFsOp(Path p) throws IOException {
- is = fs.open(p);
- }
-
- @Override
- void doBucketOp(String key) throws IOException {
- is = bucket.readKey(key);
- }
-
- @Override
- void doPostOp() throws IOException {
- int bufferCapacity = buffer.capacity();
- try {
- byte[] readBuffer = new byte[bufferCapacity];
- int readLen = is.read(readBuffer);
-
- if (readLen < bufferCapacity) {
- throw new IOException("Read mismatch, " +
- " read data length:" + readLen + " is smaller than excepted:"
- + bufferCapacity);
- }
-
- if (!Arrays.equals(readBuffer, buffer.array())) {
- throw new IOException("Read mismatch," +
- " read data does not match the written data");
- }
- } finally {
- is.close();
- }
- }
-
- @Override
- public String toString() {
- return super.toString() + " buffer:" + buffer.limit();
- }
- }
-
- /**
- * Delete file/key from bucket.
- */
- public class DeleteOp extends Op {
- DeleteOp(boolean fsOp, String keyName) {
- super(fsOp, keyName);
- }
-
- @Override
- void doFsOp(Path p) throws IOException {
- fs.delete(p, true);
- }
-
- @Override
- void doBucketOp(String key) throws IOException {
- bucket.deleteKey(key);
- }
-
- @Override
- void doPostOp() {
- // Nothing to do here
- }
-
- @Override
- public String toString() {
- return super.toString();
- }
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java
deleted file mode 100644
index 5e240186b7e..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadExecutors.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Load executors for Ozone, this class provides a plugable
- * executor for different load generators.
- */
-public class LoadExecutors {
- private static final Logger LOG =
- LoggerFactory.getLogger(LoadExecutors.class);
-
- private final List<LoadGenerator> generators;
- private final int numThreads;
- private final ExecutorService executor;
- private final int numGenerators;
- private final List<CompletableFuture<Void>> futures = new ArrayList<>();
-
- public LoadExecutors(int numThreads, List<LoadGenerator> generators) {
- this.numThreads = numThreads;
- this.generators = generators;
- this.numGenerators = generators.size();
- this.executor = Executors.newFixedThreadPool(numThreads);
- }
-
- private void load(long runTimeMillis) {
- long threadID = Thread.currentThread().getId();
- LOG.info("LOADGEN: Started IO Thread:{}.", threadID);
- long startTime = Time.monotonicNow();
-
- while (Time.monotonicNow() - startTime < runTimeMillis) {
- LoadGenerator gen =
- generators.get(RandomUtils.secure().randomInt(0, numGenerators));
-
- try {
- gen.generateLoad();
- } catch (Throwable t) {
- LOG.error("{} LOADGEN: Exiting due to exception", gen, t);
- ExitUtil.terminate(new ExitUtil.ExitException(1, t));
- break;
- }
- }
- }
-
- public void startLoad(long time) throws Exception {
- LOG.info("Starting {} threads for {} generators", numThreads,
- generators.size());
- for (LoadGenerator gen : generators) {
- try {
- LOG.info("Initializing {} generator", gen);
- gen.initialize();
- } catch (Throwable t) {
- LOG.error("Failed to initialize loadgen:{}", gen, t);
- throw t;
- }
- }
-
- for (int i = 0; i < numThreads; i++) {
- futures.add(CompletableFuture.runAsync(() -> load(time), executor));
- }
- }
-
- public void waitForCompletion() {
- // Wait for IO to complete
- for (CompletableFuture<Void> f : futures) {
- try {
- f.get();
- } catch (Throwable t) {
- LOG.error("startIO failed with exception", t);
- }
- }
- }
-
- public void shutdown() {
- try {
- executor.shutdown();
- executor.awaitTermination(1, TimeUnit.DAYS);
- } catch (Exception e) {
- LOG.error("error while closing ", e);
- }
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java
deleted file mode 100644
index 7e8c9138064..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/LoadGenerator.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Interface for load generator.
- */
-public abstract class LoadGenerator {
-
- private static final String KEY_NAME_DELIMITER = "_";
-
- public static List<Class<? extends LoadGenerator>> getClassList() {
- List<Class<? extends LoadGenerator>> classList = new ArrayList<>();
-
- classList.add(AgedDirLoadGenerator.class);
- classList.add(AgedLoadGenerator.class);
- classList.add(FilesystemLoadGenerator.class);
- classList.add(NestedDirLoadGenerator.class);
- classList.add(RandomDirLoadGenerator.class);
- classList.add(RandomLoadGenerator.class);
- classList.add(ReadOnlyLoadGenerator.class);
-
- return classList;
- }
-
- /*
- * The implemented LoadGenerators constructors should have the
- * constructor with the signature as following
- * class NewLoadGen implements LoadGenerator {
- *
- * NewLoadGen(DataBuffer buffer, LoadBucket bucket) {
- * // Add code here
- * }
- * }
- */
-
- public abstract void initialize() throws Exception;
-
- public abstract void generateLoad() throws Exception;
-
- String getKeyName(int keyIndex) {
- return toString() + KEY_NAME_DELIMITER + keyIndex;
- }
-
- @Override
- public String toString() {
- return this.getClass().getSimpleName();
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
deleted file mode 100644
index f1a82719b66..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.commons.lang3.RandomUtils;
-
-/**
- * A Load generator where nested directories are created and read them.
- */
-public class NestedDirLoadGenerator extends LoadGenerator {
- private final LoadBucket fsBucket;
- private final int maxDirDepth;
- private final Map<Integer, String> pathMap;
-
- public NestedDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
- this.fsBucket = fsBucket;
- this.maxDirDepth = 20;
- this.pathMap = new ConcurrentHashMap<>();
- }
-
- private String createNewPath(int i, String s) {
- String base = s != null ? s : "";
- return base + "/" + getKeyName(i);
- }
-
- @Override
- public void generateLoad() throws Exception {
- int index = RandomUtils.secure().randomInt(0, maxDirDepth);
- String str = this.pathMap.compute(index, this::createNewPath);
- fsBucket.createDirectory(str);
- fsBucket.readDirectory(str);
- }
-
- @Override
- public void initialize() throws Exception {
- // Nothing to do here
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
deleted file mode 100644
index 88ef4a60b31..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import org.apache.commons.lang3.RandomUtils;
-
-/**
- * A simple directory based load generator.
- */
-public class RandomDirLoadGenerator extends LoadGenerator {
- private final LoadBucket fsBucket;
-
- public RandomDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
- this.fsBucket = fsBucket;
- }
-
- @Override
- public void generateLoad() throws Exception {
- int index = RandomUtils.secure().randomInt();
- String keyName = getKeyName(index);
- fsBucket.createDirectory(keyName);
- fsBucket.readDirectory(keyName);
- }
-
- @Override
- public void initialize() {
- // Nothing to do here
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java
deleted file mode 100644
index f9cda3b5f6d..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomLoadGenerator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.nio.ByteBuffer;
-import org.apache.commons.lang3.RandomUtils;
-
-/**
- * Random load generator which writes, read and deletes keys from
- * the bucket.
- */
-public class RandomLoadGenerator extends LoadGenerator {
-
- private final LoadBucket ozoneBucket;
- private final DataBuffer dataBuffer;
-
- public RandomLoadGenerator(DataBuffer dataBuffer, LoadBucket bucket) {
- this.ozoneBucket = bucket;
- this.dataBuffer = dataBuffer;
- }
-
- @Override
- public void generateLoad() throws Exception {
- int index = RandomUtils.secure().randomInt();
- ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index);
- ozoneBucket.writeKey(buffer, keyName);
-
- ozoneBucket.readKey(buffer, keyName);
-
- ozoneBucket.deleteKey(keyName);
- }
-
- @Override
- public void initialize() {
- // Nothing to do here
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/ReadOnlyLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/ReadOnlyLoadGenerator.java
deleted file mode 100644
index 2e928cfb07a..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/ReadOnlyLoadGenerator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.loadgenerators;
-
-import java.nio.ByteBuffer;
-import org.apache.commons.lang3.RandomUtils;
-
-/**
- * This load generator writes some files and reads the same file multiple
times.
- */
-public class ReadOnlyLoadGenerator extends LoadGenerator {
- private final LoadBucket replBucket;
- private final DataBuffer dataBuffer;
- private static final int NUM_KEYS = 10;
-
- public ReadOnlyLoadGenerator(DataBuffer dataBuffer, LoadBucket replBucket) {
- this.dataBuffer = dataBuffer;
- this.replBucket = replBucket;
- }
-
- @Override
- public void generateLoad() throws Exception {
- int index = RandomUtils.secure().randomInt(0, NUM_KEYS);
- ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index);
- replBucket.readKey(buffer, keyName);
- }
-
- @Override
- public void initialize() throws Exception {
- for (int index = 0; index < NUM_KEYS; index++) {
- ByteBuffer buffer = dataBuffer.getBuffer(index);
- String keyName = getKeyName(index);
- replBucket.writeKey(buffer, keyName);
- }
- }
-}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
deleted file mode 100644
index e444f66e7ce..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.hadoop.fs.ozone.OzoneFileSystem
-org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/log4j.properties
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/log4j.properties
deleted file mode 100644
index 20d22627906..00000000000
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,40 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout,PROBLEM
-log4j.threshold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2}
(%F:%M(%L)) - %m%n
-
-log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-
-# Suppress info messages on every put key from Ratis
-log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN
-
-log4j.logger.org.apache.hadoop.ozone.utils=DEBUG,stdout,CHAOS
-log4j.logger.org.apache.hadoop.ozone.loadgenerators=WARN,stdout,CHAOS
-log4j.logger.org.apache.hadoop.ozone.failure=INFO, CHAOS
-log4j.appender.CHAOS.File=${chaoslogfilename}
-log4j.appender.CHAOS=org.apache.log4j.FileAppender
-log4j.appender.CHAOS.layout=org.apache.log4j.PatternLayout
-log4j.appender.CHAOS.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2}
(%F:%M(%L)) - %m%n
-
-log4j.appender.PROBLEM.File=${problemlogfilename}
-log4j.appender.PROBLEM.Threshold=WARN
-log4j.appender.PROBLEM=org.apache.log4j.FileAppender
-log4j.appender.PROBLEM.layout=org.apache.log4j.PatternLayout
-log4j.appender.PROBLEM.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2}
(%F:%M(%L)) - %m%n
-
-log4j.additivity.org.apache.hadoop.ozone.utils=false
diff --git a/hadoop-ozone/fault-injection-test/pom.xml
b/hadoop-ozone/fault-injection-test/pom.xml
index 79bfa1fec71..703755cc1d4 100644
--- a/hadoop-ozone/fault-injection-test/pom.xml
+++ b/hadoop-ozone/fault-injection-test/pom.xml
@@ -26,7 +26,6 @@
<description>Apache Ozone Fault Injection Tests</description>
<modules>
- <module>mini-chaos-tests</module>
<module>network-tests</module>
</modules>
</project>
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 1e6f43b88f7..303a16a1a63 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -514,10 +514,6 @@ public MiniOzoneHAClusterImpl build() throws IOException {
return cluster;
}
- protected int numberOfOzoneManagers() {
- return numOfOMs;
- }
-
protected void initOMRatisConf() {
// If test change the following config values we will respect,
// otherwise we will set lower timeout values.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
