Gargi-jais11 commented on code in PR #9858:
URL: https://github.com/apache/ozone/pull/9858#discussion_r2910765370
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerPolicyPerformance.java:
##########
@@ -181,90 +177,165 @@ private void testPolicyPerformance(String policyName,
ContainerChoosingPolicy po
for (int i = 0; i < NUM_THREADS; i++) {
executor.submit(() -> {
try {
- long threadStart = System.nanoTime();
- int containerChosen = 0;
- int containerNotChosen = 0;
+ int containerCandidateChosen = 0;
+ int containerCandidateNotChosen = 0;
int failures = 0;
- // Choose a random volume
- HddsVolume srcVolume = volumes.get(rand.nextInt(NUM_VOLUMES));
- HddsVolume destVolume;
-
- do {
- destVolume = volumes.get(rand.nextInt(NUM_VOLUMES));
- } while (srcVolume.equals(destVolume));
for (int j = 0; j < NUM_ITERATIONS; j++) {
+ // Block all volumes except the highest-util (source) - no
destination has space
+ if (rand.nextDouble() < 0.05 && volumes.size() >= 2) {
+ List<HddsVolume> sorted = new ArrayList<>(volumes);
+ sorted.sort(Comparator.comparingDouble(v ->
+ v.getCurrentUsage().getUsedSpace() / (double) Math.max(1,
v.getCurrentUsage().getCapacity())));
+ for (int k = 0; k < sorted.size() - 1; k++) {
+ HddsVolume v = sorted.get(k);
+ long avail = v.getCurrentUsage().getAvailable();
+ if (avail > 0) {
+ v.incCommittedBytes(avail);
+ }
+ }
+ }
+
+ long threadStart = System.nanoTime();
try {
- ContainerData c = policy.chooseContainer(ozoneContainer,
srcVolume,
- destVolume, inProgressContainerIDs, THRESHOLD, volumeSet,
null);
- if (c == null) {
- containerNotChosen++;
+ ContainerCandidate candidate =
pol.chooseVolumesAndContainer(ozoneContainer,
+ volumeSet, deltaMap, inProgressContainerIDs, THRESHOLD);
+
+ if (candidate == null) {
+ containerCandidateNotChosen++;
} else {
- containerChosen++;
+ containerCandidateChosen++;
+ ContainerID chosenId =
ContainerID.valueOf(candidate.getContainerData().getContainerID());
if (inProgressContainerIDs.size() < MAX_IN_PROGRESS) {
-
inProgressContainerIDs.add(ContainerID.valueOf(c.getContainerID()));
+ inProgressContainerIDs.add(chosenId);
}
+ deltaMap.compute(candidate.getSourceVolume(), (k, v) -> (v ==
null ? 0L : v)
+ - candidate.getContainerData().getBytesUsed());
}
} catch (Exception e) {
failures++;
+ } finally {
+ totalTimeNanos.addAndGet(System.nanoTime() - threadStart);
}
}
- long threadEnd = System.nanoTime();
- totalTimeNanos.addAndGet(threadEnd - threadStart);
- containerChosenCount.addAndGet(containerChosen);
- containerNotChosenCount.addAndGet(containerNotChosen);
+ pairChosenCount.addAndGet(containerCandidateChosen);
+ pairNotChosenCount.addAndGet(containerCandidateNotChosen);
failureCount.addAndGet(failures);
} finally {
latch.countDown();
}
});
}
- // Wait max 5 minutes for test completion
assertTrue(latch.await(5, TimeUnit.MINUTES), "Test timed out");
long totalOperations = (long) NUM_THREADS * NUM_ITERATIONS;
- double avgTimePerOp = (double) totalTimeNanos.get() / totalOperations;
- double opsPerSec = totalOperations / (totalTimeNanos.get() /
1_000_000_000.0);
-
- System.out.println("Performance results for " + policyName);
+ int selections = pairChosenCount.get();
+ int noSelection = pairNotChosenCount.get();
+ System.out.println("Performance results for " + testName);
System.out.println("Total volumes: " + NUM_VOLUMES);
System.out.println("Total containers: " + NUM_CONTAINERS);
System.out.println("Total threads: " + NUM_THREADS);
System.out.println("Total operations: " + totalOperations);
- System.out.println("Container Chosen operations: " +
containerChosenCount.get());
- System.out.println("Container Not Chosen operations: " +
containerNotChosenCount.get());
- System.out.println("Failed operations: " + failureCount.get());
+ System.out.println("Container Candidate Chosen (src+dest+container): " +
selections);
Review Comment:
```
Created 20 volumes in 15 ms
Created 100000 containers in 1841 ms
Performance results for VolumeAndContainerSelection
Total volumes: 20
Total containers: 100000
Total threads: 10
Total operations: 100000
Container Candidate Chosen (src+dest+container): 99970
No Container Candidate Chosen: 30
Failed: 0
Total time (ms): 16476
Avg time per operation (ns): 164762.39932
Operations per second: 6069.345943778164
Created 20 volumes in 24 ms
Created 100000 containers in 1731 ms
Performance results for VolumeAndContainerSelection
Total volumes: 20
Total containers: 100000
Total threads: 10
Total operations: 100000
Container Candidate Chosen (src+dest+container): 99969
No Container Candidate Chosen: 31
Failed: 0
Total time (ms): 14662
Avg time per operation (ns): 146624.73389
Operations per second: 6820.131729959111
Created 20 volumes in 28 ms
Created 100000 containers in 1873 ms
Performance results for VolumeAndContainerSelection
Total volumes: 20
Total containers: 100000
Total threads: 10
Total operations: 100000
Container Candidate Chosen (src+dest+container): 99970
No Container Candidate Chosen: 30
Failed: 0
Total time (ms): 14194
Avg time per operation (ns): 141946.47331
Operations per second: 7044.909089189403
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
