[
https://issues.apache.org/jira/browse/YARN-11745?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
chhinlinghean updated YARN-11745:
---------------------------------
Description:
The TimSort Transitivity rules got broken down when comparing both queues with
resources (0, 0), another queue with resources(some number, some number) and
with the same queues absolute capacity.
*Steps reproduce with Unit test:*
/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
{code:java}
@Test
public void testComparatorClassDoesNotViolateTimSortContract() {
String partition = "testPartition";
List<PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting>
queues = new ArrayList<>();
for (int i = 0; i < 300; i++) {
queues.add(createTestElement(partition, Resource.newInstance(0, 0))); //
Need to be (0, 0)
queues.add(createTestElement(partition, Resource.newInstance(8, 20))); //
Could be any number
queues.add(createTestElement(partition, Resource.newInstance(8, 8))); //
Could be any number
}
Collections.shuffle(queues);
Collections.sort(queues, new PriorityUtilizationQueueOrderingPolicy(true)
.new PriorityQueueComparator(partition));
}
private PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting
createTestElement(
String partition, Resource resource
) {
QueueCapacities mockQueueCapacities = mock(QueueCapacities.class);
when(mockQueueCapacities.getAbsoluteUsedCapacity(partition)).thenReturn(3.2f);
// Could be any number
when(mockQueueCapacities.getUsedCapacity(partition)).thenReturn(1.0f); //
Could be any number
when(mockQueueCapacities.getAbsoluteCapacity(partition)).thenReturn(4.2f); //
Could be any number
CSQueue mockQueue = mock(CSQueue.class);
when(mockQueue.getQueueCapacities()).thenReturn(mockQueueCapacities);
when(mockQueue.getPriority()).thenReturn(Priority.newInstance(5)); // Could
be any number
when(mockQueue.getAccessibleNodeLabels()).thenReturn(Collections.singleton("label1"));
QueueResourceQuotas randomQuotas = mock(QueueResourceQuotas.class);
when(randomQuotas.getConfiguredMinResource(partition)).thenReturn(resource);
when(mockQueue.getQueueResourceQuotas()).thenReturn(randomQuotas);
return new
PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting(
mockQueue, partition
);
}{code}
*How to fix it?*
Instead of checking with an AND condition when both queues resources are not
none to compare its resources, we should check with an OR condition instead.
Because in the case one queue's resource is none another one is not we should
still compare by its resources.
Previous code:
{code:java}
if (!minEffRes1.equals(Resources.none()) &&
!minEffRes2.equals(Resources.none())) {
return minEffRes2.compareTo(minEffRes1);
}
float abs1 = q1Sort.absoluteCapacity;
float abs2 = q2Sort.absoluteCapacity;
return Float.compare(abs2, abs1); {code}
Changed code to:
{code:java}
if (!minEffRes1.equals(Resources.none()) ||
!minEffRes2.equals(Resources.none())) {
return minEffRes2.compareTo(minEffRes1);
}
float abs1 = q1Sort.absoluteCapacity;
float abs2 = q2Sort.absoluteCapacity;
return Float.compare(abs2, abs1); {code}
was:
The TimSort Transitivity rules got broken down when comparing both queues with
resources (0, 0), another queue with resources(some number, some number) and
with the same queues absolute capacity.
*Steps reproduce with Unit test:*
{code:java}
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.junit.Before;
import org.junit.Test;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestThrowingIllegalArgumentException {
private List<Resource> resources;
private String partition;
private Random random;
private CSQueue mockQueue;
@Before
public void setUp() {
// Initialize reusable objects
partition = "testPartition";
resources = new ArrayList<>();
resources.add(Resource.newInstance(0, 0));
resources.add(Resource.newInstance(8, 0));
resources.add(Resource.newInstance(8, 8));
random = new Random();
mockQueue = mock(CSQueue.class);
}
@Test
public void testComparatorClass_WhenThrowingIllegalArgumentException() {
List<PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting>
collect =
IntStream.range(0, 1000)
.mapToObj(i -> crtTestElem())
.collect(Collectors.toList());
Collections.sort(collect, new
PriorityUtilizationQueueOrderingPolicy(true)
.new PriorityQueueComparator(partition));
}
private
PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting
crtTestElem() {
QueueCapacities mockQueueCapacities = mock(QueueCapacities.class);
when(mockQueueCapacities.getAbsoluteUsedCapacity(partition)).thenReturn(0.0f);
when(mockQueueCapacities.getUsedCapacity(partition)).thenReturn(1.3f);
when(mockQueueCapacities.getAbsoluteCapacity(partition)).thenReturn(4.0f);
when(mockQueue.getQueueCapacities()).thenReturn(mockQueueCapacities);
when(mockQueue.getPriority()).thenReturn(Priority.newInstance(5));
when(mockQueue.getAccessibleNodeLabels()).thenReturn(Collections.singleton("label1"));
QueueResourceQuotas randomQuotas = mock(QueueResourceQuotas.class);
Resource randomResource =
resources.get(random.nextInt(resources.size()));
when(randomQuotas.getConfiguredMinResource(partition)).thenReturn(randomResource);
when(mockQueue.getQueueResourceQuotas()).thenReturn(randomQuotas);
// Construct and return a test element
return new
PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting(
mockQueue, partition
);
}
} {code}
*How to fix it?*
Instead of checking with an AND condition when both queues resources are not
none to compare its resources, we should check with an OR condition instead.
Because in the case one queue's resource is none another one is not we should
still compare by its resources.
Previous code:
{code:java}
if (!minEffRes1.equals(Resources.none()) &&
!minEffRes2.equals(Resources.none())) {
return minEffRes2.compareTo(minEffRes1);
}
float abs1 = q1Sort.absoluteCapacity;
float abs2 = q2Sort.absoluteCapacity;
return Float.compare(abs2, abs1); {code}
Changed code to:
{code:java}
if (!minEffRes1.equals(Resources.none()) ||
!minEffRes2.equals(Resources.none())) {
return minEffRes2.compareTo(minEffRes1);
}
float abs1 = q1Sort.absoluteCapacity;
float abs2 = q2Sort.absoluteCapacity;
return Float.compare(abs2, abs1); {code}
> YARN ResourceManager throws java.lang.IllegalArgumentExceptio: Comparison
> method violates its general contract!
> ---------------------------------------------------------------------------------------------------------------
>
> Key: YARN-11745
> URL: https://issues.apache.org/jira/browse/YARN-11745
> Project: Hadoop YARN
> Issue Type: Bug
> Components: yarn
> Affects Versions: 3.4.0
> Reporter: chhinlinghean
> Assignee: chhinlinghean
> Priority: Major
> Attachments: ExamplesQueueResourceZeroproblem.pdf
>
>
> The TimSort Transitivity rules got broken down when comparing both queues
> with resources (0, 0), another queue with resources(some number, some number)
> and with the same queues absolute capacity.
> *Steps reproduce with Unit test:*
> /hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
> {code:java}
> @Test
> public void testComparatorClassDoesNotViolateTimSortContract() {
> String partition = "testPartition";
>
> List<PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting>
> queues = new ArrayList<>();
> for (int i = 0; i < 300; i++) {
> queues.add(createTestElement(partition, Resource.newInstance(0, 0))); //
> Need to be (0, 0)
> queues.add(createTestElement(partition, Resource.newInstance(8, 20))); //
> Could be any number
> queues.add(createTestElement(partition, Resource.newInstance(8, 8))); //
> Could be any number
> }
> Collections.shuffle(queues);
> Collections.sort(queues, new PriorityUtilizationQueueOrderingPolicy(true)
> .new PriorityQueueComparator(partition));
> }
> private
> PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting
> createTestElement(
> String partition, Resource resource
> ) {
> QueueCapacities mockQueueCapacities = mock(QueueCapacities.class);
>
> when(mockQueueCapacities.getAbsoluteUsedCapacity(partition)).thenReturn(3.2f);
> // Could be any number
> when(mockQueueCapacities.getUsedCapacity(partition)).thenReturn(1.0f); //
> Could be any number
> when(mockQueueCapacities.getAbsoluteCapacity(partition)).thenReturn(4.2f);
> // Could be any number
> CSQueue mockQueue = mock(CSQueue.class);
> when(mockQueue.getQueueCapacities()).thenReturn(mockQueueCapacities);
> when(mockQueue.getPriority()).thenReturn(Priority.newInstance(5)); // Could
> be any number
>
> when(mockQueue.getAccessibleNodeLabels()).thenReturn(Collections.singleton("label1"));
> QueueResourceQuotas randomQuotas = mock(QueueResourceQuotas.class);
> when(randomQuotas.getConfiguredMinResource(partition)).thenReturn(resource);
> when(mockQueue.getQueueResourceQuotas()).thenReturn(randomQuotas);
> return new
> PriorityUtilizationQueueOrderingPolicy.PriorityQueueResourcesForSorting(
> mockQueue, partition
> );
> }{code}
> *How to fix it?*
> Instead of checking with an AND condition when both queues resources are not
> none to compare its resources, we should check with an OR condition instead.
> Because in the case one queue's resource is none another one is not we should
> still compare by its resources.
> Previous code:
> {code:java}
> if (!minEffRes1.equals(Resources.none()) &&
> !minEffRes2.equals(Resources.none())) {
> return minEffRes2.compareTo(minEffRes1);
> }
> float abs1 = q1Sort.absoluteCapacity;
> float abs2 = q2Sort.absoluteCapacity;
> return Float.compare(abs2, abs1); {code}
> Changed code to:
> {code:java}
> if (!minEffRes1.equals(Resources.none()) ||
> !minEffRes2.equals(Resources.none())) {
> return minEffRes2.compareTo(minEffRes1);
> }
> float abs1 = q1Sort.absoluteCapacity;
> float abs2 = q2Sort.absoluteCapacity;
> return Float.compare(abs2, abs1); {code}
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: yarn-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: yarn-issues-h...@hadoop.apache.org
