brumi1024 commented on code in PR #8845: URL: https://github.com/apache/hadoop/pull/8845#discussion_r4388570395
########## hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueConfigResolverParity.java: ########## @@ -0,0 +1,1268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.ReservationACL; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.ConfigSnapshot; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.QueueConfigResolver; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.QueueProperty; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.QueueProperty.Kind; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.ResolutionInputs; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.ResolutionInputs.DynamicQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.Resolved; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.ResolvedQueueConfig; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.ResolvedQueueTree; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.TestQueueConfigResolver; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.ValueSource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy; +import org.junit.jupiter.api.Test; + +import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager.NO_LABEL; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.resolver.QueueProperties.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Parity between the resolver and trunk behavior. + * <ul> + * <li>Every {@code CapacitySchedulerConfiguration} per-queue getter returns + * the resolved value, or throws the resolved failure, for every queue of a + * set of configurations.</li> + * <li>Inherited and derived values equal the values of the queues a MockRM + * builds from the same configuration.</li> + * <li>For dynamic v1 and v2 queues created in a MockRM, the getters of the + * queue context configuration, which holds the template writes, return the + * values {@link QueueConfigResolver#resolveDynamicQueue} resolves.</li> + * </ul> + * For configurations trunk rejects at startup, the resolver must hold a + * failed value with the exception type and message that failed queue + * initialization. + */ +public class TestQueueConfigResolverParity { + /** + * Configurations that trunk rejects with a check (mixed capacity + * modes of siblings) rather than a parse failure, so the resolver has no + * failed value for them. + */ + private static final Set<String> CHECK_REJECTIONS = new HashSet<>( + Arrays.asList("absolute with weight units", "absolute before percentage")); + private static final String P = CapacitySchedulerConfiguration.PREFIX; + private static final int GB = 1024; + + /** Reads the value a getter returns for a queue and label. */ + private interface Getter { + Object get(CapacitySchedulerConfiguration conf, QueuePath queue, + String label, ResolutionInputs inputs); + } + + /** Properties a queue may inherit from its parent queue. */ + private static final Set<QueueProperty<?>> INHERITED = + new HashSet<>(Arrays.<QueueProperty<?>>asList(USER_LIMIT, + USER_LIMIT_FACTOR, USER_WEIGHTS, STATE, ACCESSIBLE_NODE_LABELS, + DEFAULT_NODE_LABEL_EXPRESSION, QUEUE_ORDERING_POLICY, + MAXIMUM_ALLOCATION, PREEMPTION_DISABLED, + INTRA_QUEUE_PREEMPTION_DISABLED, MAXIMUM_APPLICATION_LIFETIME, + DEFAULT_APPLICATION_LIFETIME)); + /** Properties templates do not write for dynamic queues. */ + private static final Set<QueueProperty<?>> NOT_TEMPLATED = + new HashSet<>(Arrays.<QueueProperty<?>>asList(QUEUES, USER_WEIGHTS, + ACL_SUBMIT_RESERVATIONS, ACL_LIST_RESERVATIONS, + ACL_ADMINISTER_RESERVATIONS, ALLOW_ZERO_CAPACITY_SUM, + AUTO_CREATE_CHILD_QUEUE_ENABLED, AUTO_CREATE_CHILD_QUEUE_MAX_QUEUES, + AUTO_CREATE_CHILD_QUEUE_FAIL_ON_EXCEEDING_CAPACITY, + AUTO_CREATED_QUEUE_MANAGEMENT_POLICY_NAME, LEAF_QUEUE_TEMPLATE, + AUTO_QUEUE_CREATION_V2_ENABLED, AUTO_QUEUE_CREATION_V2_MAX_DEPTH, + AQC_V2_TEMPLATE, AQC_V2_LEAF_TEMPLATE, AQC_V2_PARENT_TEMPLATE, + RESERVABLE, RESERVATION_WINDOW_MS, AVERAGE_CAPACITY_PERCENT, + INSTANTANEOUS_MAX_CAPACITY_PERCENT, RESERVATION_ADMISSION_POLICY_NAME, + RESERVATION_AGENT, RESERVATION_PLANNER, + RESERVATION_MOVE_ON_EXPIRY_ENABLED, RESERVATION_ENFORCEMENT_WINDOW_MS, + SHOW_RESERVATIONS_AS_QUEUES)); + + private static final Map<QueueProperty<?>, Getter> GETTERS = + new LinkedHashMap<>(); + + static { + GETTERS.put(QUEUES, (c, q, l, i) -> c.getQueues(q)); + GETTERS.put(CAPACITY, (c, q, l, i) -> c.getNonLabeledQueueCapacity(q)); + GETTERS.put(CAPACITY_WEIGHT, (c, q, l, i) -> l.equals(NO_LABEL) + ? c.getNonLabeledQueueWeight(q) : c.getLabeledQueueWeight(q, l)); + GETTERS.put(CAPACITY_IS_ABSOLUTE_RESOURCE, (c, q, l, i) -> + c.checkConfigTypeIsAbsoluteResource(l, q, QUEUE_RESOURCE_TYPES)); + GETTERS.put(MINIMUM_RESOURCE, (c, q, l, i) -> + c.getMinimumResourceRequirement(l, q, QUEUE_RESOURCE_TYPES)); + GETTERS.put(CAPACITY_VECTOR, (c, q, l, i) -> c + .parseConfiguredResourceVector(q, Collections.singleton(l)).get(l)); + GETTERS.put(MAXIMUM_CAPACITY, + (c, q, l, i) -> c.getNonLabeledQueueMaximumCapacity(q)); + GETTERS.put(MAXIMUM_RESOURCE, (c, q, l, i) -> + c.getMaximumResourceRequirement(l, q, QUEUE_RESOURCE_TYPES)); + GETTERS.put(MAXIMUM_CAPACITY_VECTOR, (c, q, l, i) -> c + .parseConfiguredMaximumCapacityVector(q, Collections.singleton(l), + QueueCapacityVector.newInstance()).get(l)); + GETTERS.put(LABELED_CAPACITY, + (c, q, l, i) -> c.getLabeledQueueCapacity(q, l)); + GETTERS.put(LABELED_MAXIMUM_CAPACITY, + (c, q, l, i) -> c.getLabeledQueueMaximumCapacity(q, l)); + GETTERS.put(MAXIMUM_AM_RESOURCE_PERCENT, (c, q, l, i) -> + c.getMaximumApplicationMasterResourcePerQueuePercent(q)); + GETTERS.put(LABELED_MAXIMUM_AM_RESOURCE_PERCENT, + (c, q, l, i) -> c.getMaximumAMResourcePercentPerPartition(q, l)); + GETTERS.put(MAXIMUM_APPLICATIONS, + (c, q, l, i) -> c.getMaximumApplicationsPerQueue(q)); + GETTERS.put(USER_LIMIT, (c, q, l, i) -> c.getUserLimit(q)); + GETTERS.put(USER_LIMIT_FACTOR, (c, q, l, i) -> c.getUserLimitFactor(q)); + GETTERS.put(STATE, (c, q, l, i) -> c.getConfiguredState(q)); + GETTERS.put(ACCESSIBLE_NODE_LABELS, + (c, q, l, i) -> c.getAccessibleNodeLabels(q)); + GETTERS.put(DEFAULT_NODE_LABEL_EXPRESSION, + (c, q, l, i) -> c.getDefaultNodeLabelExpression(q)); + GETTERS.put(ACL_SUBMIT_APPLICATIONS, + (c, q, l, i) -> c.getAcl(q, QueueACL.SUBMIT_APPLICATIONS)); + GETTERS.put(ACL_ADMINISTER_QUEUE, + (c, q, l, i) -> c.getAcl(q, QueueACL.ADMINISTER_QUEUE)); + GETTERS.put(ACL_APPLICATION_MAX_PRIORITY, (c, q, l, i) -> + c.getPriorityAcls(q, i.getClusterMaximumApplicationPriority())); + GETTERS.put(ACL_SUBMIT_RESERVATIONS, (c, q, l, i) -> + c.getReservationAcls(q).get(ReservationACL.SUBMIT_RESERVATIONS)); + GETTERS.put(ACL_LIST_RESERVATIONS, (c, q, l, i) -> + c.getReservationAcls(q).get(ReservationACL.LIST_RESERVATIONS)); + GETTERS.put(ACL_ADMINISTER_RESERVATIONS, (c, q, l, i) -> + c.getReservationAcls(q).get(ReservationACL.ADMINISTER_RESERVATIONS)); + GETTERS.put(QUEUE_ORDERING_POLICY, (c, q, l, i) -> + c.getQueueOrderingPolicy(q, null).getConfigName()); + GETTERS.put(PRIORITY, (c, q, l, i) -> c.getQueuePriority(q).getPriority()); + GETTERS.put(DEFAULT_APPLICATION_PRIORITY, + (c, q, l, i) -> c.getDefaultApplicationPriorityConfPerQueue(q)); + GETTERS.put(MAXIMUM_ALLOCATION_MB, + (c, q, l, i) -> c.getQueueMaximumAllocationMb(q)); + GETTERS.put(MAXIMUM_ALLOCATION_VCORES, + (c, q, l, i) -> c.getQueueMaximumAllocationVcores(q)); + GETTERS.put(PREEMPTION_DISABLED, + (c, q, l, i) -> c.getPreemptionDisabled(q, false)); + GETTERS.put(INTRA_QUEUE_PREEMPTION_DISABLED, + (c, q, l, i) -> c.getIntraQueuePreemptionDisabled(q, false)); + GETTERS.put(MAXIMUM_APPLICATION_LIFETIME, + (c, q, l, i) -> c.getMaximumLifetimePerQueue(q)); + GETTERS.put(DEFAULT_APPLICATION_LIFETIME, + (c, q, l, i) -> c.getDefaultLifetimePerQueue(q)); + GETTERS.put(MAX_PARALLEL_APPS, + (c, q, l, i) -> c.getMaxParallelAppsForQueue(q)); + GETTERS.put(MULTI_NODE_SORTING_POLICY, + (c, q, l, i) -> c.getMultiNodesSortingAlgorithmPolicy(q)); + GETTERS.put(ALLOW_ZERO_CAPACITY_SUM, + (c, q, l, i) -> c.getAllowZeroCapacitySum(q)); + GETTERS.put(AUTO_CREATE_CHILD_QUEUE_ENABLED, + (c, q, l, i) -> c.isAutoCreateChildQueueEnabled(q)); + GETTERS.put(AUTO_CREATE_CHILD_QUEUE_MAX_QUEUES, + (c, q, l, i) -> c.getAutoCreatedQueuesMaxChildQueuesLimit(q)); + GETTERS.put(AUTO_CREATE_CHILD_QUEUE_FAIL_ON_EXCEEDING_CAPACITY, + (c, q, l, i) -> + c.getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(q)); + GETTERS.put(AUTO_CREATED_QUEUE_MANAGEMENT_POLICY_NAME, + (c, q, l, i) -> c.getAutoCreatedQueueManagementPolicy(q)); + GETTERS.put(AUTO_QUEUE_CREATION_V2_ENABLED, + (c, q, l, i) -> c.isAutoQueueCreationV2Enabled(q)); + GETTERS.put(AUTO_QUEUE_CREATION_V2_MAX_QUEUES, + (c, q, l, i) -> c.getAutoCreatedQueuesV2MaxChildQueuesLimit(q)); + GETTERS.put(AUTO_QUEUE_CREATION_V2_MAX_DEPTH, + (c, q, l, i) -> c.getMaximumAutoCreatedQueueDepth(q)); + GETTERS.put(AUTO_QUEUE_AUTO_REMOVAL_ENABLED, + (c, q, l, i) -> c.isAutoExpiredDeletionEnabled(q)); + GETTERS.put(AQC_V2_TEMPLATE, (c, q, l, i) -> + new AutoCreatedQueueTemplate(c, q).getTemplateProperties()); + GETTERS.put(AQC_V2_LEAF_TEMPLATE, (c, q, l, i) -> + new AutoCreatedQueueTemplate(c, q).getLeafOnlyProperties()); + GETTERS.put(AQC_V2_PARENT_TEMPLATE, (c, q, l, i) -> + new AutoCreatedQueueTemplate(c, q).getParentOnlyProperties()); + GETTERS.put(RESERVABLE, (c, q, l, i) -> c.isReservable(q)); + GETTERS.put(RESERVATION_WINDOW_MS, + (c, q, l, i) -> c.getReservationWindow(q)); + GETTERS.put(AVERAGE_CAPACITY_PERCENT, + (c, q, l, i) -> c.getAverageCapacity(q)); + GETTERS.put(INSTANTANEOUS_MAX_CAPACITY_PERCENT, + (c, q, l, i) -> c.getInstantaneousMaxCapacity(q)); + GETTERS.put(RESERVATION_ADMISSION_POLICY_NAME, + (c, q, l, i) -> c.getReservationAdmissionPolicy(q)); + GETTERS.put(RESERVATION_AGENT, (c, q, l, i) -> c.getReservationAgent(q)); + GETTERS.put(RESERVATION_PLANNER, (c, q, l, i) -> c.getReplanner(q)); + GETTERS.put(RESERVATION_MOVE_ON_EXPIRY_ENABLED, + (c, q, l, i) -> c.getMoveOnExpiry(q)); + GETTERS.put(RESERVATION_ENFORCEMENT_WINDOW_MS, + (c, q, l, i) -> c.getEnforcementWindow(q)); + GETTERS.put(SHOW_RESERVATIONS_AS_QUEUES, + (c, q, l, i) -> c.getShowReservationAsQueues(q)); + } + + /** + * Properties without a plain getter, and why: user weights, the maximum + * allocation, the ordering policy parameters and the v1 template are + * compared against live queues or their template writes instead; the + * application ordering policy getter builds the policy object. + */ + private static final Set<QueueProperty<?>> WITHOUT_GETTER = + new HashSet<>(Arrays.<QueueProperty<?>>asList(USER_WEIGHTS, + MAXIMUM_ALLOCATION, ORDERING_POLICY_PARAMETERS, LEAF_QUEUE_TEMPLATE, + APP_ORDERING_POLICY)); + + @Test + public void testEveryPropertyIsCompared() { + for (QueueProperty<?> property : ALL_PROPERTIES) { + assertTrue(GETTERS.containsKey(property) + || WITHOUT_GETTER.contains(property), property.toString()); + } + } + + @Test + public void testGettersMatchResolvedValues() throws Exception { + int compared = 0; + for (Map.Entry<String, Configuration> entry : corpus().entrySet()) { + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(entry.getValue(), false); + ResolutionInputs inputs = TestQueueConfigResolver.inputs(conf, + Collections.<DynamicQueue>emptyList()); + ResolvedQueueTree tree = + QueueConfigResolver.resolve(conf.getConfigSnapshot(), inputs); + ConfiguredNodeLabels labels = new ConfiguredNodeLabels(conf); + for (ResolvedQueueConfig queue : tree.getQueues()) { + if (queue.isDynamic()) { + // Without a scheduler nothing writes the template entries + continue; + } + compared += assertGetterParity(entry.getKey(), conf, inputs, queue); + Set<String> expectedLabels = queue.getQueuePath().isRoot() + ? labels.getAllConfiguredLabels() + : labels.getLabelsByQueue(queue.getQueuePath().getFullPath()); + assertEquals(expectedLabels, + queue.get(CONFIGURED_NODE_LABELS).getValue(), entry.getKey()); + } + } + assertTrue(compared > 1000, "compared " + compared + " values"); + } + + @Test + public void testRejectedConfigurationsHoldFailedValues() throws Exception { + for (Map.Entry<String, Configuration> entry : rejections().entrySet()) { + String name = entry.getKey(); + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(entry.getValue(), false); + ResolutionInputs inputs = TestQueueConfigResolver.inputs(conf, + Collections.<DynamicQueue>emptyList()); + ResolvedQueueTree tree = + QueueConfigResolver.resolve(conf.getConfigSnapshot(), inputs); + for (ResolvedQueueConfig queue : tree.getQueues()) { + assertGetterParity(name, conf, inputs, queue); + } + Throwable rootCause = null; + MockRM rm = null; + try { + // Queues are parsed when the RM is initialized + rm = startRM(entry.getValue()); + } catch (Exception e) { + rootCause = e; + while (rootCause.getCause() != null) { + rootCause = rootCause.getCause(); + } + } finally { + if (rm != null) { + rm.stop(); + } + } + assertNotNull(rootCause, name + " was accepted"); + assertRejectionResolved(name, tree, rootCause); + } + } + + /** The existing dynamic queues of a live hierarchy. */ + private static List<DynamicQueue> dynamicQueues(CSQueue queue, + List<DynamicQueue> dynamic) { + if (queue instanceof AutoCreatedLeafQueue) { + dynamic.add(new DynamicQueue(queue.getQueuePathObject(), true, true)); + } else if (!(queue instanceof ReservationQueue) + && ((AbstractCSQueue) queue).isDynamicQueue()) { + dynamic.add(new DynamicQueue(queue.getQueuePathObject(), false, + queue instanceof AbstractLeafQueue)); + } + if (queue.getChildQueues() != null) { + for (CSQueue child : queue.getChildQueues()) { + dynamicQueues(child, dynamic); + } + } + return dynamic; + } + + /** + * The exception that failed queue initialization must be a resolved + * failure, with the same type and message. + */ + private static void assertRejectionResolved(String name, + ResolvedQueueTree tree, Throwable rootCause) { + List<String> failures = new ArrayList<>(); + for (ResolvedQueueConfig queue : tree.getQueues()) { + for (Resolved<?> value : queue.explain().values()) { + if (value.isFailed()) { + try { + value.getValue(); + } catch (RuntimeException e) { + failures.add(e.getClass().getName() + ": " + e.getMessage()); + } + } + } + } + String expected = rootCause.getClass().getName() + ": " + + rootCause.getMessage(); + if (CHECK_REJECTIONS.contains(name)) { + assertTrue(!failures.contains(expected), name); Review Comment: Done. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
