This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new ce361f6382f HBASE-27938 PE load any custom implementation of tests at
runtime (#5307) (#5899)
ce361f6382f is described below
commit ce361f6382fba8c4646b0fd8ea1f1eebbdabf568
Author: gvprathyusha6 <[email protected]>
AuthorDate: Thu May 16 01:28:32 2024 +0530
HBASE-27938 PE load any custom implementation of tests at runtime (#5307)
(#5899)
Signed-off-by: Duo Zhang <[email protected]>
Signed-off-by: Wellington Chevreuil <[email protected]>
Signed-off-by: Viraj Jasani <[email protected]>
---
.../apache/hadoop/hbase/PerformanceEvaluation.java | 52 +++++++++++++++++++---
.../hadoop/hbase/TestPerformanceEvaluation.java | 47 +++++++++++++++++++
2 files changed, 94 insertions(+), 5 deletions(-)
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 3c72cba9fa3..4c262fe07a8 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Properties;
import java.util.Queue;
import java.util.Random;
import java.util.TreeMap;
@@ -360,7 +361,8 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// {RegionSplitPolicy,replica count} does not match requested, or when the
// number of column families does not match requested.
if (
- (exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions)
+ (exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions
+ && opts.presplitRegions != admin.getRegions(tableName).size())
|| (!isReadCmd && desc != null
&& !StringUtils.equals(desc.getRegionSplitPolicyClassName(),
opts.splitPolicy))
|| (!isReadCmd && desc != null && desc.getRegionReplication() !=
opts.replicas)
@@ -721,6 +723,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
boolean cacheBlocks = true;
Scan.ReadType scanReadType = Scan.ReadType.DEFAULT;
long bufferSize = 2l * 1024l * 1024l;
+ Properties commandProperties;
public TestOptions() {
}
@@ -777,6 +780,11 @@ public class PerformanceEvaluation extends Configured
implements Tool {
this.cacheBlocks = that.cacheBlocks;
this.scanReadType = that.scanReadType;
this.bufferSize = that.bufferSize;
+ this.commandProperties = that.commandProperties;
+ }
+
+ public Properties getCommandProperties() {
+ return commandProperties;
}
public int getCaching() {
@@ -1142,10 +1150,10 @@ public class PerformanceEvaluation extends Configured
implements Tool {
protected final Configuration conf;
protected final TestOptions opts;
- private final Status status;
+ protected final Status status;
private String testName;
- private Histogram latencyHistogram;
+ protected Histogram latencyHistogram;
private Histogram replicaLatencyHistogram;
private Histogram valueSizeHistogram;
private Histogram rpcCallsHistogram;
@@ -2618,7 +2626,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
System.err.println(message);
}
System.err.print("Usage: hbase " + shortName);
- System.err.println(" <OPTIONS> [-D<property=value>]* <command>
<nclients>");
+ System.err.println(" <OPTIONS> [-D<property=value>]* <command|class>
<nclients>");
System.err.println();
System.err.println("General Options:");
System.err.println(
@@ -2719,6 +2727,13 @@ public class PerformanceEvaluation extends Configured
implements Tool {
System.err.println(String.format(" %-20s %s", command.getName(),
command.getDescription()));
}
System.err.println();
+ System.err.println("Class:");
+ System.err.println("To run any custom implementation of
PerformanceEvaluation.Test, "
+ + "provide the classname of the implementaion class in place of "
+ + "command name and it will be loaded at runtime from classpath.:");
+ System.err.println("Please consider to contribute back "
+ + "this custom test impl into a builtin PE command for the benefit of
the community");
+ System.err.println();
System.err.println("Args:");
System.err.println(" nclients Integer. Required. Total number of
clients "
+ "(and HRegionServers) running. 1 <= value <= 500");
@@ -3013,6 +3028,20 @@ public class PerformanceEvaluation extends Configured
implements Tool {
continue;
}
+ final String commandPropertiesFile = "--commandPropertiesFile=";
+ if (cmd.startsWith(commandPropertiesFile)) {
+ String fileName =
String.valueOf(cmd.substring(commandPropertiesFile.length()));
+ Properties properties = new Properties();
+ try {
+ properties
+
.load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName));
+ opts.commandProperties = properties;
+ } catch (IOException e) {
+ LOG.error("Failed to load metricIds from properties file", e);
+ }
+ continue;
+ }
+
validateParsedOpts(opts);
if (isCommandClass(cmd)) {
@@ -3126,7 +3155,20 @@ public class PerformanceEvaluation extends Configured
implements Tool {
}
private static boolean isCommandClass(String cmd) {
- return COMMANDS.containsKey(cmd);
+ return COMMANDS.containsKey(cmd) || isCustomTestClass(cmd);
+ }
+
+ private static boolean isCustomTestClass(String cmd) {
+ Class<? extends Test> cmdClass;
+ try {
+ cmdClass =
+ (Class<? extends Test>)
PerformanceEvaluation.class.getClassLoader().loadClass(cmd);
+ addCommandDescriptor(cmdClass, cmd, "custom command");
+ return true;
+ } catch (Throwable th) {
+ LOG.info("No class found for command: " + cmd, th);
+ return false;
+ }
}
private static Class<? extends TestBase> determineCommandClass(String cmd) {
diff --git
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
index f3718118030..7e4babe58b9 100644
---
a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
+++
b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
@@ -28,6 +28,8 @@ import com.codahale.metrics.Snapshot;
import com.codahale.metrics.UniformReservoir;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
@@ -35,6 +37,7 @@ import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
import java.util.LinkedList;
import java.util.NoSuchElementException;
+import java.util.Properties;
import java.util.Queue;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
@@ -42,7 +45,9 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.PerformanceEvaluation.RandomReadTest;
+import org.apache.hadoop.hbase.PerformanceEvaluation.Status;
import org.apache.hadoop.hbase.PerformanceEvaluation.TestOptions;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -357,4 +362,46 @@ public class TestPerformanceEvaluation {
assertEquals(true, options.valueRandom);
}
+ @Test
+ public void testCustomTestClassOptions() throws IOException {
+ Queue<String> opts = new LinkedList<>();
+ // create custom properties that can be used for a custom test class
+ Properties commandProps = new Properties();
+ commandProps.put("prop1", "val1");
+ String cmdPropsFilePath =
+ this.getClass().getClassLoader().getResource("").getPath() +
"cmd_properties.txt";
+ FileWriter writer = new FileWriter(new File(cmdPropsFilePath));
+ commandProps.store(writer, null);
+ // create opts for the custom test class - commandPropertiesFile,
testClassName
+ opts.offer("--commandPropertiesFile=" + "cmd_properties.txt");
+ String testClassName =
"org.apache.hadoop.hbase.TestPerformanceEvaluation$PESampleTestImpl";
+ opts.offer(testClassName);
+ opts.offer("1");
+ PerformanceEvaluation.TestOptions options =
PerformanceEvaluation.parseOpts(opts);
+ assertNotNull(options);
+ assertNotNull(options.getCmdName());
+ assertEquals(testClassName, options.getCmdName());
+ assertNotNull(options.getCommandProperties());
+ assertEquals("val1", options.getCommandProperties().get("prop1"));
+ }
+
+ class PESampleTestImpl extends PerformanceEvaluation.Test {
+
+ PESampleTestImpl(Connection con, TestOptions options, Status status) {
+ super(con, options, status);
+ }
+
+ @Override
+ void onStartup() throws IOException {
+ }
+
+ @Override
+ void onTakedown() throws IOException {
+ }
+
+ @Override
+ boolean testRow(int i, long startTime) throws IOException,
InterruptedException {
+ return false;
+ }
+ }
}
