Gargi-jais11 commented on PR #8512:
URL: https://github.com/apache/ozone/pull/8512#issuecomment-2911828413
> ```
> public class TestContainerChoosingPolicyPerformance {
>
> private static final int NUM_VOLUMES = 20;
> private static final int NUM_CONTAINERS = 1000000;
> private static final int NUM_THREADS = 100;
> private static final int NUM_ITERATIONS = 10000;
>
> private static final OzoneConfiguration CONF = new OzoneConfiguration();
>
> @TempDir
> private Path baseDir;
>
> private List<HddsVolume> volumes;
> private Collection<Container<?>> containers;
> private OzoneContainer ozoneContainer;
> private ContainerChoosingPolicy containerChoosingPolicy;
> private ExecutorService executor;
> private ContainerController containerController;
>
> // Simulate containers currently being balanced (in progress)
> private Set<Long> inProgressContainerIDs = ConcurrentHashMap.newKeySet();
>
> @BeforeEach
> public void setup() throws Exception {
> containers = new ArrayList<>();
> createVolumes(); //create 20 volumes
> createContainers(); //create 1 million containers with 1000 open rest
closed
> ozoneContainer = mock(OzoneContainer.class);
> containerController = mock(ContainerController.class);
> when(ozoneContainer.getController()).thenReturn(containerController);
>
> // Mock containerController to return containers for a given volume
> for (HddsVolume volume : volumes) {
> List<Container<?>> volumeContainers = new ArrayList<>();
> for (Container<?> container : containers) {
> if (container.getContainerData().getVolume().equals(volume)) {
> volumeContainers.add(container);
> }
> }
>
when(containerController.getContainers(volume)).thenReturn(volumeContainers.iterator());
> }
>
> containerChoosingPolicy = new DefaultContainerChoosingPolicy();
> executor = Executors.newFixedThreadPool(NUM_THREADS);
> }
>
> // @AfterEach
> public void cleanUp() {
> volumes.forEach(HddsVolume::shutdown);
>
> // Shutdown executor service
> if (executor != null && !executor.isShutdown()) {
> executor.shutdownNow();
> }
>
> // Clear containers and in-progress container IDs
> if (containers != null) {
> containers.clear();
> }
> inProgressContainerIDs.clear();
> }
>
> @Test
> public void testConcurrentVolumeChoosing() throws Exception {
> for (int i = 0; i < 3; i++) {
> setup();
> testPolicyPerformance("ContainerChoosingPolicy",
containerChoosingPolicy);
> cleanUp();
> }
> }
>
> /*
> * SuccessCount: Number of successful container choices from the policy.
> * FailureCount: Failures due to any exceptions thrown during container
choice.
> */
> private void testPolicyPerformance(String policyName,
ContainerChoosingPolicy policy) throws Exception {
> CountDownLatch latch = new CountDownLatch(NUM_THREADS);
> AtomicInteger containerChosenCount = new AtomicInteger(0);
> AtomicInteger containerNotChosenCount = new AtomicInteger(0);
> AtomicInteger failureCount = new AtomicInteger(0);
> AtomicLong totalTimeNanos = new AtomicLong(0);
>
> Random rand = new Random();
>
> for (int i = 0; i < NUM_THREADS; i++) {
> executor.submit(() -> {
> try {
> long threadStart = System.nanoTime();
> int containerChosen = 0;
> int containerNotChosen = 0;
> int failures = 0;
>
> for (int j = 0; j < NUM_ITERATIONS; j++) {
> try {
> // Choose a random volume
> HddsVolume volume = volumes.get(rand.nextInt(NUM_VOLUMES));
> ContainerData c = policy.chooseContainer(ozoneContainer,
volume, inProgressContainerIDs);
> if (c == null) {
> containerNotChosen++;
> inProgressContainerIDs.add(c.getContainerID());
> } else {
> containerChosen++;
> }
> } catch (Exception e) {
> failures++;
> }
> }
>
> long threadEnd = System.nanoTime();
> totalTimeNanos.addAndGet(threadEnd - threadStart);
> containerChosenCount.addAndGet(containerChosen);
> containerNotChosenCount.addAndGet(containerNotChosen);
> failureCount.addAndGet(failures);
> } finally {
> latch.countDown();
> }
> });
> }
>
> // Wait max 5 minutes for test completion
> assertTrue(latch.await(5, TimeUnit.MINUTES), "Test timed out");
>
> long totalOperations = (long) NUM_THREADS * NUM_ITERATIONS;
> double avgTimePerOp = (double) totalTimeNanos.get() / totalOperations;
> double opsPerSec = totalOperations / (totalTimeNanos.get() /
1_000_000_000.0);
>
> System.out.println("Performance results for " + policyName);
> System.out.println("Total operations: " + totalOperations);
> System.out.println("Container Chosen operations: " +
containerChosenCount.get());
> System.out.println("Container Not Chosen operations: " +
containerNotChosenCount.get());
> System.out.println("Failed operations: " + failureCount.get());
> System.out.println("Total time (ms): " + totalTimeNanos.get() /
1_000_000);
> System.out.println("Average time per operation (ns): " + avgTimePerOp);
> System.out.println("Operations per second: " + opsPerSec);
> }
>
> public void createVolumes() throws IOException {
> // Create volumes with mocked space usage
> volumes = new ArrayList<>();
> for (int i = 0; i < NUM_VOLUMES; i++) {
> String volumePath = baseDir.resolve("disk" + i).toString();
> SpaceUsageSource source = MockSpaceUsageSource.fixed(1000000000,
1000000000 - i * 50000);
> SpaceUsageCheckFactory factory = MockSpaceUsageCheckFactory.of(
> source, Duration.ZERO, SpaceUsagePersistence.None.INSTANCE);
> HddsVolume volume = new HddsVolume.Builder(volumePath)
> .conf(CONF)
> .usageCheckFactory(factory)
> .build();
> volumes.add(volume);
> }
> }
>
> public void createContainers() {
> List<Long> closedContainerIDs = new ArrayList<>();
> Random random = new Random();
>
> for (int i = 0; i < NUM_CONTAINERS; i++) {
> boolean isOpen = i < 1000; // First 1000 containers are open
> int volumeIndex = i % NUM_VOLUMES; // Distribute containers across
volumes
> HddsVolume volume = volumes.get(volumeIndex);
>
> KeyValueContainerData containerData = new KeyValueContainerData(
> i, ContainerLayoutVersion.FILE_PER_BLOCK ,
ContainerTestHelper.CONTAINER_MAX_SIZE,
> UUID.randomUUID().toString(), UUID.randomUUID().toString());
>
> containerData.setState(isOpen ? ContainerDataProto.State.OPEN :
ContainerDataProto.State.CLOSED);
> containerData.setVolume(volume);
>
> KeyValueContainer container = new KeyValueContainer(containerData,
CONF);
>
> containers.add(container);
>
> // Collect IDs of closed containers
> if (!isOpen) {
> closedContainerIDs.add((long) i);
> }
> }
>
> // Randomly select 1000 closed containers to be in-progress
> Collections.shuffle(closedContainerIDs, random);
> inProgressContainerIDs.addAll(closedContainerIDs.subList(0, 1000));
> }
> }
> ```
This is the Micro benchmark performance test for container choosing policy
in diskBalancer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org
For additional commands, e-mail: issues-h...@ozone.apache.org
