adoroszlai commented on code in PR #7463:
URL: https://github.com/apache/ozone/pull/7463#discussion_r1853315792
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java:
##########
@@ -87,55 +97,75 @@ public HsyncGenerator() {
@Override
public Void call() throws Exception {
- super.init();
+ init();
if (configuration == null) {
configuration = freon.createOzoneConfiguration();
}
+ URI uri = URI.create(rootPath);
- outputStreams = new FSDataOutputStream[numberOfFiles];
- files = new Path[numberOfFiles];
- callsPerFile = new AtomicInteger[numberOfFiles];
- FileSystem fileSystem = getFileSystem();
- for (int i = 0; i < numberOfFiles; i++) {
- Path file = new Path(getRootPath() + "/" + generateObjectName(i));
- fileSystem.mkdirs(file.getParent());
- outputStreams[i] = fileSystem.create(file);
- files[i] = file;
- callsPerFile[i] = new AtomicInteger();
-
- LOG.info("Created file for testing: {}", file);
- }
+ FileSystem fileSystem = FileSystem.get(uri, configuration);
+ Path file = new Path(rootPath + "/" + generateObjectName(0));
+ fileSystem.mkdirs(file.getParent());
+ outputStream = fileSystem.create(file);
+
+ LOG.info("Created file for testing: {}", file);
timer = getMetrics().timer("hsync-generator");
data = PayloadUtils.generatePayload(writeSize);
+ startTransactionWriter();
+
try {
runTests(this::sendHsync);
} finally {
- for (FSDataOutputStream outputStream : outputStreams) {
outputStream.close();
- }
+ fileSystem.close();
}
- StringBuilder distributionReport = new StringBuilder();
- for (int i = 0; i < numberOfFiles; i++) {
- distributionReport.append("\t").append(files[i]).append(":
").append(callsPerFile[i]).append("\n");
- }
+ return null;
+ }
- LOG.info("Hsync generator finished, calls distribution: \n {}",
distributionReport);
+ private void startTransactionWriter() {
+ Thread transactionWriter = new Thread(this::generateTransactions);
+ transactionWriter.setDaemon(true);
+ transactionWriter.start();
+ }
- return null;
+ private void generateTransactions() {
+ int transaction = 0;
+ while (true) {
+ for (int i = 0; i < writesPerTransaction; i++) {
+ try {
+ if (writeSize > 1) {
+ outputStream.write(data);
+ } else {
+ outputStream.write(i);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ try {
+ writtenTransactions.put(transaction++);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
private void sendHsync(long counter) throws Exception {
timer.time(() -> {
- int i = ((int) counter) % numberOfFiles;
- FSDataOutputStream outputStream = outputStreams[i];
- outputStream.write(data);
- outputStream.hsync();
- callsPerFile[i].incrementAndGet();
- return null;
+ while (true) {
+ int transaction = writtenTransactions.take();
+ int lastSyncedTransaction = this.lastSyncedTransaction.get();
+ if (transaction > lastSyncedTransaction) {
+ outputStream.hsync();
+ this.lastSyncedTransaction.compareAndSet(lastSyncedTransaction,
transaction);
Review Comment:
```
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java
162: 'lastSyncedTransaction' hides a field.
```
```suggestion
int lastSynced = lastSyncedTransaction.get();
if (transaction > lastSynced) {
outputStream.hsync();
lastSyncedTransaction.compareAndSet(lastSynced, transaction);
```
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java:
##########
@@ -87,55 +97,75 @@ public HsyncGenerator() {
@Override
public Void call() throws Exception {
- super.init();
+ init();
if (configuration == null) {
configuration = freon.createOzoneConfiguration();
}
+ URI uri = URI.create(rootPath);
- outputStreams = new FSDataOutputStream[numberOfFiles];
- files = new Path[numberOfFiles];
- callsPerFile = new AtomicInteger[numberOfFiles];
- FileSystem fileSystem = getFileSystem();
- for (int i = 0; i < numberOfFiles; i++) {
- Path file = new Path(getRootPath() + "/" + generateObjectName(i));
- fileSystem.mkdirs(file.getParent());
- outputStreams[i] = fileSystem.create(file);
- files[i] = file;
- callsPerFile[i] = new AtomicInteger();
-
- LOG.info("Created file for testing: {}", file);
- }
+ FileSystem fileSystem = FileSystem.get(uri, configuration);
+ Path file = new Path(rootPath + "/" + generateObjectName(0));
+ fileSystem.mkdirs(file.getParent());
+ outputStream = fileSystem.create(file);
+
+ LOG.info("Created file for testing: {}", file);
timer = getMetrics().timer("hsync-generator");
data = PayloadUtils.generatePayload(writeSize);
+ startTransactionWriter();
+
try {
runTests(this::sendHsync);
} finally {
- for (FSDataOutputStream outputStream : outputStreams) {
outputStream.close();
Review Comment:
```
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HsyncGenerator.java
122: 'finally' child has incorrect indentation level 8, expected level
should be 6.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
