This is an automated email from the ASF dual-hosted git repository.
smgoller pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode-benchmarks.git
The following commit(s) were added to refs/heads/develop by this push:
new 7d6dfc6 Refactor metadata production and management. (#57)
7d6dfc6 is described below
commit 7d6dfc6b44b6d0822ca8a39e3d4f2fe348fc6e78
Author: Sean Goller <[email protected]>
AuthorDate: Wed Feb 27 14:56:45 2019 -0800
Refactor metadata production and management. (#57)
* Tests now output metadata if the metadata file doesn't exist.
* benchmark run metadata.json is actually json.
* Create configuration directory to store cluster information.
* Move SSH private key for cluster to configuration directory.
* Create metadata for cluster, currently including an instance ID and
the list of IPs in the cluster.
* Delete metadata when the cluster is destroyed.
---
gradle/dependency-versions.properties | 1 +
harness/build.gradle | 1 +
.../geode/perftest/runner/DefaultTestRunner.java | 9 +++-
.../yardstick/hdrhistogram/HdrHistogramWriter.java | 7 +++
infrastructure/build.gradle | 1 +
.../geode/infrastructure/BenchmarkMetadata.java | 14 +++--
.../infrastructure/aws/AwsBenchmarkMetadata.java | 6 ++-
.../geode/infrastructure/aws/DestroyCluster.java | 11 ++++
.../geode/infrastructure/aws/KeyInstaller.java | 8 +--
.../geode/infrastructure/aws/LaunchCluster.java | 60 +++++++++++++++++-----
.../{KeyInstaller.java => MetadataInstaller.java} | 24 +++++----
11 files changed, 107 insertions(+), 35 deletions(-)
diff --git a/gradle/dependency-versions.properties
b/gradle/dependency-versions.properties
index 017d40b..e906204 100644
--- a/gradle/dependency-versions.properties
+++ b/gradle/dependency-versions.properties
@@ -28,3 +28,4 @@ awaitility.version = 3.0.0
sshd-core.version = 2.1.0
assertj-core.version = 3.11.1
software-amazon-awssdk.version = 2.1.4
+JSON.version = 20180813
diff --git a/harness/build.gradle b/harness/build.gradle
index a4c6cce..e0e323c 100644
--- a/harness/build.gradle
+++ b/harness/build.gradle
@@ -41,6 +41,7 @@ dependencies {
compile(group: 'commons-io', name: 'commons-io', version:
project.'commons-io.version')
compile(group: 'org.yardstickframework', name: 'yardstick', version:
project.'yardstick.version')
compile(group: 'org.hdrhistogram', name: 'HdrHistogram', version:
project.'HdrHistogram.version')
+ compile(group: 'org.json', name: 'json', version: project.'JSON.version')
testCompile(group: 'org.mockito', name: 'mockito-all', version:
project.'mockito-all.version')
testCompile(group: 'org.awaitility', name: 'awaitility', version:
project.'awaitility.version')
testCompile(group: 'org.slf4j', name: 'slf4j-simple', version:
project.'slf4j-simple.version')
diff --git
a/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
b/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
index 8c676fe..019cb18 100644
---
a/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
+++
b/harness/src/main/java/org/apache/geode/perftest/runner/DefaultTestRunner.java
@@ -22,6 +22,7 @@ import java.io.FileWriter;
import java.util.List;
import java.util.Map;
+import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,9 +77,15 @@ public class DefaultTestRunner implements TestRunner {
FileWriter metadataWriter = new
FileWriter(metadataOutput.getAbsoluteFile(), true);
String[] metadataEntries = metadata.split(",");
+ JSONObject JSONmetadata = new JSONObject();
+
for (String data : metadataEntries) {
- metadataWriter.write(data + "\n");
+ String[] kv = data.split(":");
+ if (kv.length == 2) {
+ JSONmetadata.put(kv[0], kv[1]);
+ }
}
+ metadataWriter.write(JSONmetadata.toString());
metadataWriter.flush();
}
diff --git
a/harness/src/main/java/org/apache/geode/perftest/yardstick/hdrhistogram/HdrHistogramWriter.java
b/harness/src/main/java/org/apache/geode/perftest/yardstick/hdrhistogram/HdrHistogramWriter.java
index 6233783..3ed4033 100644
---
a/harness/src/main/java/org/apache/geode/perftest/yardstick/hdrhistogram/HdrHistogramWriter.java
+++
b/harness/src/main/java/org/apache/geode/perftest/yardstick/hdrhistogram/HdrHistogramWriter.java
@@ -26,14 +26,17 @@ import org.HdrHistogram.HistogramLogWriter;
public class HdrHistogramWriter implements Consumer<Histogram> {
public static final String FILE_NAME = "latency.hlog";
+ public static final String FILE_NAME_CSV = "latency_csv";
public static final String FILE_NAME_HDR = "latency_hdr";
private final File outputFile;
private final File outputHDRFile;
+ private final File outputCSVFile;
public HdrHistogramWriter(File outputDir) {
this.outputFile = new File(outputDir, FILE_NAME);
this.outputHDRFile = new File(outputDir, FILE_NAME_HDR);
+ this.outputCSVFile = new File(outputDir, FILE_NAME_CSV);
}
@Override
@@ -51,6 +54,10 @@ public class HdrHistogramWriter implements
Consumer<Histogram> {
new HistogramLogProcessor(new String[] {"-i",
outputFile.getAbsolutePath(), "-o",
outputHDRFile.getAbsolutePath()});
histogramLogProcessor.run();
+ HistogramLogProcessor histogramLogProcessorCSV =
+ new HistogramLogProcessor(new String[] {"-csv", "-i",
outputFile.getAbsolutePath(), "-o",
+ outputCSVFile.getAbsolutePath()});
+ histogramLogProcessorCSV.run();
} catch (FileNotFoundException e) {
throw new UncheckedIOException(e);
}
diff --git a/infrastructure/build.gradle b/infrastructure/build.gradle
index bf66802..3901df3 100644
--- a/infrastructure/build.gradle
+++ b/infrastructure/build.gradle
@@ -28,6 +28,7 @@ repositories {
}
dependencies {
+ implementation(group: 'org.json', name: 'json', version:
project.'JSON.version')
implementation 'software.amazon.awssdk:ec2'
implementation(group: 'com.hierynomus', name: 'sshj', version:
project.'sshj.version')
runtime(group: 'org.slf4j', name: 'slf4j-simple', version:
project.'slf4j-simple.version')
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java
index dc920f5..550ef15 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/BenchmarkMetadata.java
@@ -21,7 +21,6 @@ package org.apache.geode.infrastructure;
*/
public class BenchmarkMetadata {
public static String PREFIX = "geode-benchmarks";
- public static String SSH_DIRECTORY = ".ssh/geode-benchmarks";
public static String benchmarkPrefix(String tag) {
return PREFIX + "-" + tag;
@@ -31,11 +30,16 @@ public class BenchmarkMetadata {
return benchmarkPrefix(tag) + "-" + suffix;
}
- public static String benchmarkKeyFileDirectory() {
- return System.getProperty("user.home") + "/" + SSH_DIRECTORY;
+ public static String benchmarkConfigDirectory() {
+ return System.getProperty("user.home") + "/." + PREFIX;
}
- public static String benchmarkKeyFileName(String tag) {
- return benchmarkKeyFileDirectory() + "/" + tag + ".pem";
+
+ public static String benchmarkPrivateKeyFileName(String tag) {
+ return benchmarkConfigDirectory() + "/" + tag + "-privkey.pem";
+ }
+
+ public static String benchmarkMetadataFileName(String tag) {
+ return benchmarkConfigDirectory() + "/" + tag + "-metadata.json";
}
}
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/AwsBenchmarkMetadata.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/AwsBenchmarkMetadata.java
index 90fb757..b274050 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/AwsBenchmarkMetadata.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/AwsBenchmarkMetadata.java
@@ -48,7 +48,11 @@ class AwsBenchmarkMetadata extends BenchmarkMetadata {
}
public static String keyPairFileName(String tag) {
- return BenchmarkMetadata.benchmarkKeyFileName(tag);
+ return BenchmarkMetadata.benchmarkPrivateKeyFileName(tag);
+ }
+
+ public static String metadataFileName(String tag) {
+ return BenchmarkMetadata.benchmarkMetadataFileName(tag);
}
public static InstanceType instanceType() {
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
index 137f8ab..1c7b668 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/DestroyCluster.java
@@ -55,6 +55,7 @@ public class DestroyCluster {
deleteSecurityGroup(benchmarkTag);
deletePlacementGroup(benchmarkTag);
deleteKeyPair(benchmarkTag);
+ deleteMetadata(benchmarkTag);
}
private static void deleteKeyPair(String benchmarkTag) {
@@ -71,6 +72,16 @@ public class DestroyCluster {
}
}
+ private static void deleteMetadata(String benchmarkTag) {
+ try {
+
Files.deleteIfExists(Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag)));
+ System.out.println("Metadata for cluster '" + benchmarkTag + "'
deleted.");
+ } catch (Exception e) {
+ System.out.println("We got an exception while deleting the Key pair");
+ System.out.println("Exception message: " + e);
+ }
+ }
+
private static void deleteInstances(String benchmarkTag) throws
InterruptedException {
// delete instances
try {
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/KeyInstaller.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/KeyInstaller.java
index d7b9362..b082e91 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/KeyInstaller.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/KeyInstaller.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.ConnectException;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collection;
import java.util.Collections;
@@ -39,12 +40,11 @@ public class KeyInstaller {
private final String user;
private final Path privateKey;
- public KeyInstaller(String user, Path privateKey) {
- this.user = user;
- this.privateKey = privateKey;
+ public KeyInstaller(String benchmarkTag) {
+ this.user = AwsBenchmarkMetadata.USER;
+ this.privateKey =
Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag));
}
-
public void installPrivateKey(Collection<String> hosts) {
hosts.forEach(this::installKey);
}
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
index d523fec..0e54a3c 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/LaunchCluster.java
@@ -29,8 +29,11 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
+import java.util.UUID;
import java.util.stream.Collectors;
+import org.json.JSONArray;
+import org.json.JSONObject;
import software.amazon.awssdk.services.ec2.Ec2Client;
import
software.amazon.awssdk.services.ec2.model.AuthorizeSecurityGroupIngressRequest;
import software.amazon.awssdk.services.ec2.model.CreateKeyPairRequest;
@@ -83,12 +86,20 @@ public class LaunchCluster {
List<String> instanceIds = launchInstances(benchmarkTag, tags, count);
DescribeInstancesResponse instances = waitForInstances(instanceIds);
-
- List<String> publicIps = installPrivateKey(benchmarkTag, instances);
+ List<String> publicIps = getPublicIps(instances);
+ createMetadata(benchmarkTag, publicIps);
+ installPrivateKey(benchmarkTag, publicIps);
+ installMetadata(benchmarkTag, publicIps);
System.out.println("Instances successfully launched! Public IPs: " +
publicIps);
}
+ private static List<String> getPublicIps(DescribeInstancesResponse
describeInstancesResponse) {
+ return describeInstancesResponse.reservations().stream()
+ .flatMap(reservation -> reservation.instances().stream())
+ .map(Instance::publicIpAddress).collect(Collectors.toList());
+ }
+
private static void usage(String s) {
throw new IllegalStateException(s);
}
@@ -132,19 +143,16 @@ public class LaunchCluster {
return describeInstancesResponse;
}
- private static List<String> installPrivateKey(String benchmarkTag,
- DescribeInstancesResponse describeInstancesResponse) {
- List<String> publicIps =
- describeInstancesResponse.reservations().stream()
- .flatMap(reservation -> reservation.instances().stream())
- .map(Instance::publicIpAddress).collect(Collectors.toList());
-
- new KeyInstaller(AwsBenchmarkMetadata.USER,
-
Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag))).installPrivateKey(publicIps);
-
+ private static void installPrivateKey(String benchmarkTag,
+ List<String> publicIps) {
+ new KeyInstaller(benchmarkTag).installPrivateKey(publicIps);
System.out.println("Private key installed on all instances for
passwordless ssh");
+ }
- return publicIps;
+ private static void installMetadata(String benchmarkTag,
+ List<String> publicIps) {
+ new MetadataInstaller(benchmarkTag).installMetadata(publicIps);
+ System.out.println("Instance ID information installed on all instances");
}
private static long instanceCount(DescribeInstancesResponse
describeInstancesResponse) {
@@ -164,14 +172,36 @@ public class LaunchCluster {
}
private static void createKeyPair(String benchmarkTag) throws IOException {
+ Path configDirectory =
Paths.get(BenchmarkMetadata.benchmarkConfigDirectory());
CreateKeyPairResponse ckpr = ec2.createKeyPair(
CreateKeyPairRequest.builder().keyName(AwsBenchmarkMetadata.keyPair(benchmarkTag)).build());
-
Files.createDirectories(Paths.get(BenchmarkMetadata.benchmarkKeyFileDirectory()));
+
+ if (!configDirectory.toFile().exists()) {
+
Files.createDirectories(Paths.get(BenchmarkMetadata.benchmarkConfigDirectory()));
+ }
Path privateKey =
Files.write(Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag)),
ckpr.keyMaterial().getBytes());
Files.setPosixFilePermissions(privateKey,
PosixFilePermissions.fromString("rw-------"));
}
+ private static void createMetadata(String benchmarkTag, List<String>
publicIps)
+ throws IOException {
+ UUID instanceId = UUID.randomUUID();
+ JSONObject metadataJSON = new JSONObject();
+
+ metadataJSON.put("instanceId", instanceId.toString());
+ metadataJSON.put("publicIps", new JSONArray(publicIps));
+ Path configDirectory =
Paths.get(BenchmarkMetadata.benchmarkConfigDirectory());
+
+ if (!configDirectory.toFile().exists()) {
+
Files.createDirectories(Paths.get(BenchmarkMetadata.benchmarkConfigDirectory()));
+ }
+
+ Path metadata =
Files.write(Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag)),
+ metadataJSON.toString().getBytes());
+ Files.setPosixFilePermissions(metadata,
PosixFilePermissions.fromString("rw-------"));
+ }
+
private static void createLaunchTemplate(String benchmarkTag, Image
newestImage) {
ArrayList<String> securityGroupList = new ArrayList<>();
securityGroupList.add(AwsBenchmarkMetadata.securityGroup(benchmarkTag));
@@ -233,6 +263,8 @@ public class LaunchCluster {
List<Tag> tags = new ArrayList<>();
tags.add(Tag.builder().key("purpose").value(BenchmarkMetadata.PREFIX).build());
tags.add(Tag.builder().key(BenchmarkMetadata.PREFIX).value(benchmarkTag).build());
+ tags.add(Tag.builder().key(BenchmarkMetadata.benchmarkString(benchmarkTag,
"instanceId"))
+ .value(UUID.randomUUID().toString()).build());
return tags;
}
diff --git
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/KeyInstaller.java
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/MetadataInstaller.java
similarity index 79%
copy from
infrastructure/src/main/java/org/apache/geode/infrastructure/aws/KeyInstaller.java
copy to
infrastructure/src/main/java/org/apache/geode/infrastructure/aws/MetadataInstaller.java
index d7b9362..07dc5e2 100644
---
a/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/KeyInstaller.java
+++
b/infrastructure/src/main/java/org/apache/geode/infrastructure/aws/MetadataInstaller.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.ConnectException;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collection;
import java.util.Collections;
@@ -33,30 +34,33 @@ import
net.schmizz.sshj.transport.verification.PromiscuousVerifier;
import net.schmizz.sshj.xfer.FilePermission;
import net.schmizz.sshj.xfer.FileSystemFile;
-public class KeyInstaller {
+
+public class MetadataInstaller {
public static final Config CONFIG = new DefaultConfig();
private static final int RETRIES = 30;
private final String user;
+ private final Path metadata;
private final Path privateKey;
- public KeyInstaller(String user, Path privateKey) {
- this.user = user;
- this.privateKey = privateKey;
+ public MetadataInstaller(String benchmarkTag) {
+ this.user = AwsBenchmarkMetadata.USER;
+ this.privateKey =
Paths.get(AwsBenchmarkMetadata.keyPairFileName(benchmarkTag));
+ this.metadata =
Paths.get(AwsBenchmarkMetadata.metadataFileName(benchmarkTag));
}
-
- public void installPrivateKey(Collection<String> hosts) {
- hosts.forEach(this::installKey);
+ public void installMetadata(Collection<String> hosts) {
+ hosts.forEach(this::installMetadata);
}
- private void installKey(String host) {
+ private void installMetadata(String host) {
try (SSHClient client = new SSHClient(CONFIG)) {
client.addHostKeyVerifier(new PromiscuousVerifier());
connect(host, client);
client.authPublickey(user, privateKey.toFile().getAbsolutePath());
SFTPClient sftpClient = client.newSFTPClient();
- String dest = "/home/" + user + "/.ssh/id_rsa";
- sftpClient.put(new FileSystemFile(privateKey.toFile()), dest);
+ String dest = "/home/" + user + "/geode-benchmarks-metadata.json";
+
+ sftpClient.put(new FileSystemFile(metadata.toFile()), dest);
sftpClient.setattr(dest, new FileAttributes.Builder()
.withPermissions(Collections.singleton(FilePermission.USR_R)).build());
} catch (IOException e) {
