This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b11c5f12f8 HDDS-5779. Create s3g bucket creation performance test
(#3629)
b11c5f12f8 is described below
commit b11c5f12f855a0620f764e3bd42ce7840db6791b
Author: Victor Zeddys <[email protected]>
AuthorDate: Sat Aug 20 10:06:32 2022 +0300
HDDS-5779. Create s3g bucket creation performance test (#3629)
---
.../dist/src/main/smoketest/s3/freon.robot | 44 +++++++++++++
.../java/org/apache/hadoop/ozone/freon/Freon.java | 1 +
.../hadoop/ozone/freon/S3BucketGenerator.java | 73 ++++++++++++++++++++++
.../hadoop/ozone/freon/S3EntityGenerator.java | 65 +++++++++++++++++++
.../apache/hadoop/ozone/freon/S3KeyGenerator.java | 47 ++++----------
5 files changed, 195 insertions(+), 35 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot
b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot
new file mode 100644
index 0000000000..f92ba87757
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/freon.robot
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 gateway test with aws cli
+Library OperatingSystem
+Library String
+Library BuiltIn
+Resource ./commonawslib.robot
+Suite Setup Setup v4 headers
+
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g:9878
+
+*** Keywords ***
+# Export access key and secret to the environment
+Setup aws credentials
+ ${accessKey} = Execute aws configure get aws_access_key_id
+ ${secret} = Execute aws configure get aws_secret_access_key
+ Set Environment Variable AWS_SECRET_ACCESS_KEY ${secret}
+ Set Environment Variable AWS_ACCESS_KEY_ID ${accessKey}
+
+Freon S3BG
+ [arguments] ${prefix}=s3bg ${n}=100 ${threads}=10
${args}=${EMPTY}
+ ${result} = Execute ozone freon s3bg -e ${ENDPOINT_URL} -t
${threads} -n ${n} -p ${prefix} ${args}
+ Should contain ${result} Successful executions:
${n}
+
+*** Test Cases ***
+Run Freon S3BG
+ [Setup] Setup aws credentials
+ Freon S3BG
\ No newline at end of file
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
index 4a1cd84169..b6bb6055ee 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
@@ -52,6 +52,7 @@ import picocli.CommandLine.Option;
HadoopFsValidator.class,
SameKeyReader.class,
S3KeyGenerator.class,
+ S3BucketGenerator.class,
DatanodeChunkGenerator.class,
DatanodeChunkValidator.class,
DatanodeBlockPutter.class,
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java
new file mode 100644
index 0000000000..ef7a85a412
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3BucketGenerator.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import com.codahale.metrics.Timer;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine.Command;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Generate buckets via the s3 interface.
+ *
+ * For a secure cluster,
+ * $> init user keytab
+ * $> kinit -kt /etc/security/keytabs/testuser.keytab testuser/[email protected]
+ * $> eval $(ozone s3 getsecret -e)
+ * for getting and exporting access_key_id and secret_access_key
+ * to freon shell test environment
+ * secret access key.
+ */
+@Command(name = "s3bg",
+ aliases = "s3-bucket-generator",
+ description = "Create buckets via the s3 interface.\n" +
+ "Command requires AWS authentication environment " +
+ "variables set, AWS_ACCESS_KEY_ID and " +
+ "AWS_SECRET_ACCESS_KEY",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true,
+ showDefaultValues = true)
+public class S3BucketGenerator extends S3EntityGenerator
+ implements Callable<Void> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(S3BucketGenerator.class);
+
+ private Timer timer;
+
+ @Override
+ public Void call() throws Exception {
+ s3ClientInit();
+
+ timer = getMetrics().timer("bucket-create");
+
+ runTests(this::createBucket);
+
+ return null;
+ }
+
+ private void createBucket(long bucketNum) throws Exception {
+ String bName = getPrefix() + bucketNum;
+ timer.time(() -> {
+ getS3().createBucket(bName);
+ return null;
+ });
+ }
+}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java
new file mode 100644
index 0000000000..1b9210dd72
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3EntityGenerator.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import picocli.CommandLine;
+
+
+/**
+ * Initiliazing common aspects of S3BucketGenerator and S3KeyGenerator.
+ */
+public class S3EntityGenerator extends BaseFreonGenerator {
+ @CommandLine.Option(names = {"-e", "--endpoint"},
+ description = "S3 HTTP endpoint",
+ defaultValue = "http://localhost:9878";)
+ private String endpoint;
+
+ private AmazonS3 s3;
+
+ protected void s3ClientInit() {
+ init();
+
+ AmazonS3ClientBuilder amazonS3ClientBuilder =
+ AmazonS3ClientBuilder.standard()
+ .withCredentials(new EnvironmentVariableCredentialsProvider());
+
+ if (endpoint.length() > 0) {
+ amazonS3ClientBuilder
+ .withPathStyleAccessEnabled(true)
+ .withEndpointConfiguration(
+ new EndpointConfiguration(endpoint, "us-east-1"));
+ } else {
+ amazonS3ClientBuilder.withRegion(Regions.DEFAULT_REGION);
+ }
+
+ s3 = amazonS3ClientBuilder.build();
+ }
+
+ public String getEndpoint() {
+ return endpoint;
+ }
+
+ public AmazonS3 getS3() {
+ return s3;
+ }
+}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java
index 708f4a14f8..3a3537d8e4 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/S3KeyGenerator.java
@@ -24,20 +24,16 @@ import java.util.concurrent.Callable;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
-import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import static
com.amazonaws.services.s3.internal.SkipMd5CheckStrategy.DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.UploadPartRequest;
+import static
com.amazonaws.services.s3.internal.SkipMd5CheckStrategy.DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY;
import com.amazonaws.services.s3.model.UploadPartResult;
import com.codahale.metrics.Timer;
import org.apache.commons.lang3.RandomStringUtils;
+
import static org.apache.hadoop.ozone.OzoneConsts.OM_MULTIPART_MIN_SIZE;
import org.slf4j.Logger;
@@ -50,12 +46,15 @@ import picocli.CommandLine.Option;
*/
@Command(name = "s3kg",
aliases = "s3-key-generator",
- description = "Create random keys via the s3 interface.",
+ description = "Create random keys via the s3 interface." +
+ "Command requires AWS authentication environment " +
+ "variables set, AWS_ACCESS_KEY_ID and " +
+ "AWS_SECRET_ACCESS_KEY",
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true,
showDefaultValues = true)
@SuppressWarnings("java:S2245") // no need for secure random
-public class S3KeyGenerator extends BaseFreonGenerator
+public class S3KeyGenerator extends S3EntityGenerator
implements Callable<Void> {
private static final Logger LOG =
@@ -73,11 +72,6 @@ public class S3KeyGenerator extends BaseFreonGenerator
defaultValue = "10240")
private int fileSize;
- @Option(names = {"-e", "--endpoint"},
- description = "S3 HTTP endpoint",
- defaultValue = "http://localhost:9878";)
- private String endpoint;
-
@Option(names = {"--multi-part-upload"},
description = "User multi part upload",
defaultValue = "false")
@@ -93,8 +87,6 @@ public class S3KeyGenerator extends BaseFreonGenerator
private String content;
- private AmazonS3 s3;
-
@Override
public Void call() throws Exception {
@@ -102,23 +94,8 @@ public class S3KeyGenerator extends BaseFreonGenerator
throw new IllegalArgumentException(
"Size of multipart upload parts should be at least 5MB (5242880)");
}
- init();
-
- AmazonS3ClientBuilder amazonS3ClientBuilder =
- AmazonS3ClientBuilder.standard()
- .withCredentials(new EnvironmentVariableCredentialsProvider());
-
- if (endpoint.length() > 0) {
- amazonS3ClientBuilder
- .withPathStyleAccessEnabled(true)
- .withEndpointConfiguration(
- new EndpointConfiguration(endpoint, "us-east-1"));
-
- } else {
- amazonS3ClientBuilder.withRegion(Regions.DEFAULT_REGION);
- }
- s3 = amazonS3ClientBuilder.build();
+ s3ClientInit();
content = RandomStringUtils.randomAscii(fileSize);
@@ -139,7 +116,7 @@ public class S3KeyGenerator extends BaseFreonGenerator
new InitiateMultipartUploadRequest(bucketName, keyName);
final InitiateMultipartUploadResult initiateMultipartUploadResult =
- s3.initiateMultipartUpload(initiateRequest);
+ getS3().initiateMultipartUpload(initiateRequest);
final String uploadId = initiateMultipartUploadResult.getUploadId();
List<PartETag> parts = new ArrayList<>();
@@ -156,16 +133,16 @@ public class S3KeyGenerator extends BaseFreonGenerator
StandardCharsets.UTF_8)));
final UploadPartResult uploadPartResult =
- s3.uploadPart(uploadPartRequest);
+ getS3().uploadPart(uploadPartRequest);
parts.add(uploadPartResult.getPartETag());
}
- s3.completeMultipartUpload(
+ getS3().completeMultipartUpload(
new CompleteMultipartUploadRequest(bucketName, keyName, uploadId,
parts));
} else {
- s3.putObject(bucketName, generateObjectName(counter),
+ getS3().putObject(bucketName, generateObjectName(counter),
content);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
