This is an automated email from the ASF dual-hosted git repository.
binlijin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new d69ecf6 HBASE-23352: Allow chaos monkeys to access cmd line params,
and improve FillDiskCommandAction (#885)
d69ecf6 is described below
commit d69ecf6092c08323f49d8bb4131312b8f4981aa9
Author: BukrosSzabolcs <[email protected]>
AuthorDate: Mon Dec 2 03:29:06 2019 +0100
HBASE-23352: Allow chaos monkeys to access cmd line params, and improve
FillDiskCommandAction (#885)
Instead of using the default properties when checking for monkey
properties, now we use the ones already extended with command line
params.
Change FillDiskCommandAction to try to stop the remote process if the
command failed with an exception.
Signed-off-by: stack <[email protected]>
---
.../org/apache/hadoop/hbase/IntegrationTestBase.java | 3 ++-
.../hbase/chaos/actions/FillDiskCommandAction.java | 16 ++++++++++------
.../hadoop/hbase/chaos/factories/MonkeyConstants.java | 4 ++--
3 files changed, 14 insertions(+), 9 deletions(-)
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java
index 354c95a..3e60c2e 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBase.java
@@ -92,7 +92,7 @@ public abstract class IntegrationTestBase extends
AbstractHBaseTool {
// Add entries for the CM from hbase-site.xml as a convenience.
// Do this prior to loading from the properties file to make sure those in
the properties
// file are given precedence to those in hbase-site.xml (backwards
compatibility).
- loadMonkeyProperties(monkeyProps, HBaseConfiguration.create());
+ loadMonkeyProperties(monkeyProps, conf);
if (cmd.hasOption(CHAOS_MONKEY_PROPS)) {
String chaosMonkeyPropsFile = cmd.getOptionValue(CHAOS_MONKEY_PROPS);
if (StringUtils.isNotEmpty(chaosMonkeyPropsFile)) {
@@ -183,6 +183,7 @@ public abstract class IntegrationTestBase extends
AbstractHBaseTool {
if (fact == null) {
fact = getDefaultMonkeyFactory();
}
+ LOG.info("Using chaos monkey factory: {}", fact.getClass());
monkey = fact.setUtil(util)
.setTableName(getTablename())
.setProperties(monkeyProps)
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java
index b7af31f..f5ca1d7 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FillDiskCommandAction.java
@@ -58,15 +58,15 @@ public class FillDiskCommandAction extends
SudoCommandAction {
String hostname = server.getHostname();
try {
- clusterManager.execSudoWithRetries(hostname, timeout, getFillCommand());
- Thread.sleep(duration);
- } catch (InterruptedException e) {
- LOG.debug("Failed to run the command for the full duration", e);
+ clusterManager.execSudo(hostname, duration, getFillCommand());
+ } catch (IOException ex) {
+ LOG.info("Potential timeout. We try to stop the dd process on target
machine");
+ clusterManager.execSudoWithRetries(hostname, timeout, getStopCommand());
+ throw ex;
} finally {
clusterManager.execSudoWithRetries(hostname, timeout, getClearCommand());
+ LOG.info("Finished to execute FillDiskCommandAction");
}
-
- LOG.info("Finished to execute FillDiskCommandAction");
}
private String getFillCommand(){
@@ -80,4 +80,8 @@ public class FillDiskCommandAction extends SudoCommandAction {
private String getClearCommand(){
return String.format("rm -f %s/garbage", path);
}
+
+ private String getStopCommand() {
+ return String.format("killall dd");
+ }
}
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
index f4c34b5..7fbdd1f 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/MonkeyConstants.java
@@ -55,7 +55,7 @@ public interface MonkeyConstants {
String NETWORK_ISSUE_RATIO = "network.issue.ratio";
String NETWORK_ISSUE_DELAY = "network.issue.delay";
String NETWORK_ISSUE_INTERFACE = "network.issue.interface";
- //should be big enough to create the file
+ //should be higher than the usual timeout because the target machine might
respond slowly
String FILL_DISK_COMMAND_TIMEOUT = "fill.disk.command.timeout";
String FILL_DISK_PATH = "fill.disk.path";
String FILL_DISK_FILE_SIZE = "fill.disk.file.size";
@@ -101,7 +101,7 @@ public interface MonkeyConstants {
float DEFAULT_NETWORK_ISSUE_RATIO = 0.1f;
long DEFAULT_NETWORK_ISSUE_DELAY = 100;
String DEFAULT_NETWORK_ISSUE_INTERFACE = "eth0";
- long DEFAULT_FILL_DISK_COMMAND_TIMEOUT = 5 * 60 * 1000 + 30 *
1000;//duration + timeout
+ long DEFAULT_FILL_DISK_COMMAND_TIMEOUT = 60 * 1000;
String DEFAULT_FILL_DISK_PATH = "/tmp";
long DEFAULT_FILL_DISK_FILE_SIZE = 0;
long DEFAULT_FILL_DISK_ISSUE_DURATION = 5 * 60 * 1000;
