This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4001f9a230 HDDS-11081. Use thread-local instance of FileSystem in
Freon tests (#7091)
4001f9a230 is described below
commit 4001f9a2307f109430c979bd6bc7246cbee5bfdb
Author: Ivan Andika <[email protected]>
AuthorDate: Sat Aug 24 00:47:24 2024 +0800
HDDS-11081. Use thread-local instance of FileSystem in Freon tests (#7091)
---
.../ozone/freon/TestHadoopDirTreeGenerator.java | 6 +-
.../ozone/freon/HadoopBaseFreonGenerator.java | 85 ++++++++++++++++++++++
.../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 25 ++-----
.../hadoop/ozone/freon/HadoopFsGenerator.java | 59 ++-------------
.../hadoop/ozone/freon/HadoopFsValidator.java | 28 ++-----
.../ozone/freon/HadoopNestedDirGenerator.java | 24 ++----
.../apache/hadoop/ozone/freon/HsyncGenerator.java | 21 +-----
7 files changed, 117 insertions(+), 131 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
index 3140681d3d..9137eca6c0 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
@@ -152,10 +152,10 @@ public class TestHadoopDirTreeGenerator {
Path rootDir = new Path(rootPath.concat("/"));
// verify root path details
FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
+ // verify the num of peer directories, expected span count is 1
+ // as it has only one dir at root.
+ verifyActualSpan(1, fileStatuses);
for (FileStatus fileStatus : fileStatuses) {
- // verify the num of peer directories, expected span count is 1
- // as it has only one dir at root.
- verifyActualSpan(1, fileStatuses);
int actualDepth =
traverseToLeaf(fileSystem, fileStatus.getPath(), 1, depth, span,
fileCount, StorageSize.parse(perFileSize, StorageUnit.BYTES));
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopBaseFreonGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopBaseFreonGenerator.java
new file mode 100644
index 0000000000..eeb6514a72
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopBaseFreonGenerator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import picocli.CommandLine.Option;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.Optional;
+
+/**
+ * Base class for Freon generator tests that requires {@link FileSystem}
instance.
+ */
+public abstract class HadoopBaseFreonGenerator extends BaseFreonGenerator {
+
+ @Option(names = {"-r", "--rpath", "--path"},
+ description = "Hadoop FS file system path. Use full path.",
+ defaultValue = "o3fs://bucket1.vol1")
+ private String rootPath;
+
+ private final ThreadLocal<FileSystem> threadLocalFileSystem =
+ ThreadLocal.withInitial(this::createFS);
+
+ private OzoneConfiguration configuration;
+ private URI uri;
+
+ @Override
+ public void init() {
+ super.init();
+ configuration = createOzoneConfiguration();
+ uri = URI.create(rootPath);
+ String scheme = Optional.ofNullable(uri.getScheme())
+ .orElseGet(() -> FileSystem.getDefaultUri(configuration)
+ .getScheme());
+ String disableCacheName =
+ String.format("fs.%s.impl.disable.cache", scheme);
+ print("Disabling FS cache: " + disableCacheName);
+ configuration.setBoolean(disableCacheName, true);
+ }
+
+ @Override
+ protected void taskLoopCompleted() {
+ FileSystem fileSystem = threadLocalFileSystem.get();
+ try {
+ fileSystem.close();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ protected String getRootPath() {
+ return rootPath;
+ }
+
+ protected FileSystem getFileSystem() {
+ return threadLocalFileSystem.get();
+ }
+
+ private FileSystem createFS() {
+ try {
+ return FileSystem.get(uri, configuration);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java
index 91d78234ba..3eb879d5c0 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java
@@ -19,17 +19,14 @@ package org.apache.hadoop.ozone.freon;
import com.codahale.metrics.Timer;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageSize;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import java.net.URI;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicLong;
@@ -45,17 +42,12 @@ import java.util.concurrent.atomic.AtomicLong;
mixinStandardHelpOptions = true,
showDefaultValues = true)
@SuppressWarnings("java:S2245") // no need for secure random
-public class HadoopDirTreeGenerator extends BaseFreonGenerator
+public class HadoopDirTreeGenerator extends HadoopBaseFreonGenerator
implements Callable<Void> {
private static final Logger LOG =
LoggerFactory.getLogger(HadoopDirTreeGenerator.class);
- @Option(names = {"-r", "--rpath"},
- description = "Hadoop FS root path",
- defaultValue = "o3fs://bucket2.vol2")
- private String rootPath;
-
@Option(names = {"-d", "--depth"},
description = "Number of directories to be generated recursively",
defaultValue = "5")
@@ -99,8 +91,6 @@ public class HadoopDirTreeGenerator extends BaseFreonGenerator
private ContentGenerator contentGenerator;
- private FileSystem fileSystem;
-
@Override
public Void call() throws Exception {
String s;
@@ -111,10 +101,7 @@ public class HadoopDirTreeGenerator extends
BaseFreonGenerator
s = "Invalid span value, span value should be greater than zero!";
print(s);
} else {
- init();
- OzoneConfiguration configuration = createOzoneConfiguration();
- fileSystem = FileSystem.get(URI.create(rootPath), configuration);
-
+ super.init();
contentGenerator = new ContentGenerator(fileSize.toBytes(), bufferSize);
timer = getMetrics().timer("file-create");
@@ -152,7 +139,7 @@ public class HadoopDirTreeGenerator extends
BaseFreonGenerator
created.
*/
private void createDir(long counter) throws Exception {
- String dir = makeDirWithGivenNumberOfFiles(rootPath);
+ String dir = makeDirWithGivenNumberOfFiles(getRootPath());
if (depth > 1) {
createSubDirRecursively(dir, 1, 1);
}
@@ -196,8 +183,8 @@ public class HadoopDirTreeGenerator extends
BaseFreonGenerator
private String makeDirWithGivenNumberOfFiles(String parent)
throws Exception {
String dir = RandomStringUtils.randomAlphanumeric(length);
- dir = parent.toString().concat("/").concat(dir);
- fileSystem.mkdirs(new Path(dir));
+ dir = parent.concat("/").concat(dir);
+ getFileSystem().mkdirs(new Path(dir));
totalDirsCnt.incrementAndGet();
// Add given number of files into the created directory.
createFiles(dir);
@@ -212,7 +199,7 @@ public class HadoopDirTreeGenerator extends
BaseFreonGenerator
LOG.debug("FilePath:{}", file);
}
timer.time(() -> {
- try (FSDataOutputStream output = fileSystem.create(file)) {
+ try (FSDataOutputStream output = getFileSystem().create(file)) {
contentGenerator.write(output);
}
return null;
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
index dd386fb926..1f910c9398 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java
@@ -16,17 +16,12 @@
*/
package org.apache.hadoop.ozone.freon;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.net.URI;
-import java.util.Optional;
import java.util.concurrent.Callable;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import com.codahale.metrics.Timer;
import org.apache.hadoop.hdds.conf.StorageSize;
@@ -42,14 +37,9 @@ import picocli.CommandLine.Option;
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true,
showDefaultValues = true)
-public class HadoopFsGenerator extends BaseFreonGenerator
+public class HadoopFsGenerator extends HadoopBaseFreonGenerator
implements Callable<Void> {
- @Option(names = {"--path"},
- description = "Hadoop FS file system path. Use full path.",
- defaultValue = "o3fs://bucket1.vol1")
- private String rootPath;
-
@Option(names = {"-s", "--size"},
description = "Size of the generated files. " +
StorageSizeConverter.STORAGE_SIZE_DESCRIPTION,
@@ -77,29 +67,12 @@ public class HadoopFsGenerator extends BaseFreonGenerator
private Timer timer;
- private OzoneConfiguration configuration;
- private URI uri;
- private final ThreadLocal<FileSystem> threadLocalFileSystem =
- ThreadLocal.withInitial(this::createFS);
-
@Override
public Void call() throws Exception {
- init();
-
- configuration = createOzoneConfiguration();
- uri = URI.create(rootPath);
- String scheme = Optional.ofNullable(uri.getScheme())
- .orElseGet(() -> FileSystem.getDefaultUri(configuration)
- .getScheme());
- String disableCacheName =
- String.format("fs.%s.impl.disable.cache", scheme);
- print("Disabling FS cache: " + disableCacheName);
- configuration.setBoolean(disableCacheName, true);
-
- Path file = new Path(rootPath + "/" + generateObjectName(0));
- try (FileSystem fileSystem = threadLocalFileSystem.get()) {
- fileSystem.mkdirs(file.getParent());
- }
+ super.init();
+
+ Path file = new Path(getRootPath() + "/" + generateObjectName(0));
+ getFileSystem().mkdirs(file.getParent());
contentGenerator =
new ContentGenerator(fileSize.toBytes(), bufferSize, copyBufferSize,
@@ -113,8 +86,8 @@ public class HadoopFsGenerator extends BaseFreonGenerator
}
private void createFile(long counter) throws Exception {
- Path file = new Path(rootPath + "/" + generateObjectName(counter));
- FileSystem fileSystem = threadLocalFileSystem.get();
+ Path file = new Path(getRootPath() + "/" + generateObjectName(counter));
+ FileSystem fileSystem = getFileSystem();
timer.time(() -> {
try (FSDataOutputStream output = fileSystem.create(file)) {
@@ -123,22 +96,4 @@ public class HadoopFsGenerator extends BaseFreonGenerator
return null;
});
}
-
- private FileSystem createFS() {
- try {
- return FileSystem.get(uri, configuration);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- @Override
- protected void taskLoopCompleted() {
- FileSystem fileSystem = threadLocalFileSystem.get();
- try {
- fileSystem.close();
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsValidator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsValidator.java
index fe160efd73..1566eaed8a 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsValidator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsValidator.java
@@ -16,22 +16,18 @@
*/
package org.apache.hadoop.ozone.freon;
-import java.net.URI;
import java.security.MessageDigest;
import java.util.concurrent.Callable;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import com.codahale.metrics.Timer;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine.Command;
-import picocli.CommandLine.Option;
/**
* Data generator tool test om performance.
@@ -42,36 +38,24 @@ import picocli.CommandLine.Option;
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true,
showDefaultValues = true)
-public class HadoopFsValidator extends BaseFreonGenerator
+public class HadoopFsValidator extends HadoopBaseFreonGenerator
implements Callable<Void> {
private static final Logger LOG =
LoggerFactory.getLogger(HadoopFsValidator.class);
- @Option(names = {"--path"},
- description = "Hadoop FS file system path",
- defaultValue = "o3fs://bucket1.vol1")
- private String rootPath;
-
private ContentGenerator contentGenerator;
private Timer timer;
- private FileSystem fileSystem;
-
private byte[] referenceDigest;
@Override
public Void call() throws Exception {
+ super.init();
- init();
-
- OzoneConfiguration configuration = createOzoneConfiguration();
-
- fileSystem = FileSystem.get(URI.create(rootPath), configuration);
-
- Path file = new Path(rootPath + "/" + generateObjectName(0));
- try (FSDataInputStream stream = fileSystem.open(file)) {
+ Path file = new Path(getRootPath() + "/" + generateObjectName(0));
+ try (FSDataInputStream stream = getFileSystem().open(file)) {
referenceDigest = getDigest(stream);
}
@@ -83,10 +67,10 @@ public class HadoopFsValidator extends BaseFreonGenerator
}
private void validateFile(long counter) throws Exception {
- Path file = new Path(rootPath + "/" + generateObjectName(counter));
+ Path file = new Path(getRootPath() + "/" + generateObjectName(counter));
byte[] content = timer.time(() -> {
- try (FSDataInputStream input = fileSystem.open(file)) {
+ try (FSDataInputStream input = getFileSystem().open(file)) {
return IOUtils.toByteArray(input);
}
});
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java
index 8e631e5b6a..ff74a54fbc 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopNestedDirGenerator.java
@@ -16,13 +16,10 @@
*/
package org.apache.hadoop.ozone.freon;
-import java.net.URI;
import java.util.concurrent.Callable;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.commons.lang3.RandomStringUtils;
import org.slf4j.Logger;
@@ -41,17 +38,12 @@ import picocli.CommandLine.Option;
mixinStandardHelpOptions = true,
showDefaultValues = true)
@SuppressWarnings("java:S2245") // no need for secure random
-public class HadoopNestedDirGenerator extends BaseFreonGenerator
+public class HadoopNestedDirGenerator extends HadoopBaseFreonGenerator
implements Callable<Void> {
private static final Logger LOG =
LoggerFactory.getLogger(HadoopNestedDirGenerator.class);
- @Option(names = {"-r", "--rpath"},
- description = "Hadoop FS directory system path",
- defaultValue = "o3fs://bucket2.vol2")
- private String rootPath;
-
@Option(names = {"-d", "--depth"},
description = "Number of directories to be generated recursively",
defaultValue = "5")
@@ -70,8 +62,6 @@ public class HadoopNestedDirGenerator extends
BaseFreonGenerator
defaultValue = "10")
private int length;
- private FileSystem fileSystem;
-
@Override
public Void call() throws Exception {
String s;
@@ -82,9 +72,7 @@ public class HadoopNestedDirGenerator extends
BaseFreonGenerator
s = "Invalid span value, span value should be greater or equal to zero!";
print(s);
} else {
- init();
- OzoneConfiguration configuration = createOzoneConfiguration();
- fileSystem = FileSystem.get(URI.create(rootPath), configuration);
+ super.init();
runTests(this::createDir);
}
return null;
@@ -109,14 +97,14 @@ public class HadoopNestedDirGenerator extends
BaseFreonGenerator
dirString = dirString.concat("/").concat(RandomStringUtils.
randomAlphanumeric(length));
}
- Path file = new Path(rootPath.concat("/").concat(dirString));
- fileSystem.mkdirs(file.getParent());
+ Path file = new Path(getRootPath().concat("/").concat(dirString));
+ getFileSystem().mkdirs(file.getParent());
String leafDir = dirString.substring(0, dirString.length() - length);
String tmp = "/0";
for (int i = 1; i <= span; i++) {
String childDir = leafDir.concat(Integer.toString(i)).concat(tmp);
- Path dir = new Path(rootPath.concat("/").concat(childDir));
- fileSystem.mkdirs(dir.getParent());
+ Path dir = new Path(getRootPath().concat("/").concat(childDir));
+ getFileSystem().mkdirs(dir.getParent());
}
String message = "\nSuccessfully created directories. " +
"Total Directories with level = " + depth + " and span = " + span;
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java
index 962a744806..8de2ee032d 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java
@@ -16,7 +16,6 @@
*/
package org.apache.hadoop.ozone.freon;
-import java.net.URI;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
@@ -54,17 +53,12 @@ import picocli.CommandLine.Option;
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true,
showDefaultValues = true)
-public class HsyncGenerator extends BaseFreonGenerator implements
Callable<Void> {
+public class HsyncGenerator extends HadoopNestedDirGenerator implements
Callable<Void> {
private static final Logger LOG =
LoggerFactory.getLogger(HsyncGenerator.class);
@CommandLine.ParentCommand
private Freon freon;
- @Option(names = {"--path"},
- description = "Hadoop FS file system path. Use full path.",
- defaultValue = "o3fs://bucket1.vol1")
- private String rootPath;
-
@Option(names = {"--bytes-per-write"},
description = "Size of each write",
defaultValue = "1024")
@@ -78,7 +72,6 @@ public class HsyncGenerator extends BaseFreonGenerator
implements Callable<Void>
private Timer timer;
private OzoneConfiguration configuration;
- private FileSystem[] fileSystems;
private FSDataOutputStream[] outputStreams;
private Path[] files;
private AtomicInteger[] callsPerFile;
@@ -94,23 +87,20 @@ public class HsyncGenerator extends BaseFreonGenerator
implements Callable<Void>
@Override
public Void call() throws Exception {
- init();
+ super.init();
if (configuration == null) {
configuration = freon.createOzoneConfiguration();
}
- URI uri = URI.create(rootPath);
- fileSystems = new FileSystem[numberOfFiles];
outputStreams = new FSDataOutputStream[numberOfFiles];
files = new Path[numberOfFiles];
callsPerFile = new AtomicInteger[numberOfFiles];
+ FileSystem fileSystem = getFileSystem();
for (int i = 0; i < numberOfFiles; i++) {
- FileSystem fileSystem = FileSystem.get(uri, configuration);
- Path file = new Path(rootPath + "/" + generateObjectName(i));
+ Path file = new Path(getRootPath() + "/" + generateObjectName(i));
fileSystem.mkdirs(file.getParent());
outputStreams[i] = fileSystem.create(file);
- fileSystems[i] = fileSystem;
files[i] = file;
callsPerFile[i] = new AtomicInteger();
@@ -126,9 +116,6 @@ public class HsyncGenerator extends BaseFreonGenerator
implements Callable<Void>
for (FSDataOutputStream outputStream : outputStreams) {
outputStream.close();
}
- for (FileSystem fs : fileSystems) {
- fs.close();
- }
}
StringBuilder distributionReport = new StringBuilder();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
