zentol commented on a change in pull request #11443: [FLINK-14791][coordination] ResourceManager tracks ClusterPartitions URL: https://github.com/apache/flink/pull/11443#discussion_r400727826
########## File path: flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResourceManagerPartitionTrackerImplTest.java ########## @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.io.network.partition; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.jobgraph.IntermediateDataSetID; +import org.apache.flink.runtime.taskexecutor.partition.ClusterPartitionReport; +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.junit.Assert.assertEquals; + +/** + * Test for the {@link ResourceManagerPartitionTrackerImpl}. + */ +public class ResourceManagerPartitionTrackerImplTest extends TestLogger { + + private static final ClusterPartitionReport EMPTY_PARTITION_REPORT = new ClusterPartitionReport(Collections.emptySet()); + + private static final ResourceID TASK_EXECUTOR_ID_1 = ResourceID.generate(); + private static final ResourceID TASK_EXECUTOR_ID_2 = ResourceID.generate(); + private static final IntermediateDataSetID DATA_SET_ID = new IntermediateDataSetID(); + private static final ResultPartitionID PARTITION_ID_1 = new ResultPartitionID(); + private static final ResultPartitionID PARTITION_ID_2 = new ResultPartitionID(); + + @Test + public void testProcessEmptyClusterPartitionReport() { + TestClusterPartitionReleaser partitionReleaser = new TestClusterPartitionReleaser(); + final ResourceManagerPartitionTracker tracker = new ResourceManagerPartitionTrackerImpl(partitionReleaser); + + tracker.processTaskExecutorClusterPartitionReport(TASK_EXECUTOR_ID_1, EMPTY_PARTITION_REPORT); + assertThat(partitionReleaser.releaseCalls, empty()); + } + + /** + * Verifies that a task executor hosting multiple partitions of a data set receives a release call if a subset of + * its partitions is lost. + */ + @Test + public void testReportProcessingWithPartitionLossOnSameTaskExecutor() { + TestClusterPartitionReleaser partitionReleaser = new TestClusterPartitionReleaser(); + final ResourceManagerPartitionTracker tracker = new ResourceManagerPartitionTrackerImpl(partitionReleaser); + + tracker.processTaskExecutorClusterPartitionReport( + TASK_EXECUTOR_ID_1, + createClusterPartitionReport(DATA_SET_ID, 2, PARTITION_ID_1, PARTITION_ID_2)); + + tracker.processTaskExecutorClusterPartitionReport( + TASK_EXECUTOR_ID_1, + createClusterPartitionReport(DATA_SET_ID, 2, PARTITION_ID_2)); + + assertThat(partitionReleaser.releaseCalls, contains(Tuple2.of(TASK_EXECUTOR_ID_1, Collections.singleton(DATA_SET_ID)))); + } + + /** + * Verifies that a task executor hosting partitions of a data set receives a release call if a partition of the + * data set is lost on another task executor. + */ + @Test + public void testReportProcessingWithPartitionLossOnOtherTaskExecutor() { + TestClusterPartitionReleaser partitionReleaser = new TestClusterPartitionReleaser(); + final ResourceManagerPartitionTracker tracker = new ResourceManagerPartitionTrackerImpl(partitionReleaser); + + tracker.processTaskExecutorClusterPartitionReport( + TASK_EXECUTOR_ID_1, + createClusterPartitionReport(DATA_SET_ID, 2, PARTITION_ID_1)); + + tracker.processTaskExecutorClusterPartitionReport( + TASK_EXECUTOR_ID_2, + createClusterPartitionReport(DATA_SET_ID, 2, PARTITION_ID_2)); + + tracker.processTaskExecutorClusterPartitionReport(TASK_EXECUTOR_ID_1, EMPTY_PARTITION_REPORT); + + assertThat(partitionReleaser.releaseCalls, contains(Tuple2.of(TASK_EXECUTOR_ID_2, Collections.singleton(DATA_SET_ID)))); + } + + @Test + public void testListDataSetsBasics() { + TestClusterPartitionReleaser partitionReleaser = new TestClusterPartitionReleaser(); + final ResourceManagerPartitionTracker tracker = new ResourceManagerPartitionTrackerImpl(partitionReleaser); + + assertEquals(0, tracker.listDataSets().size()); + + tracker.processTaskExecutorClusterPartitionReport( + TASK_EXECUTOR_ID_1, + createClusterPartitionReport(DATA_SET_ID, 1, PARTITION_ID_1)); + + final Map<IntermediateDataSetID, DataSetMetaInfo> listing = tracker.listDataSets(); + assertThat(listing, hasKey(DATA_SET_ID)); + DataSetMetaInfo metaInfo = listing.get(DATA_SET_ID); + assertEquals(1, metaInfo.getNumTotalPartitions()); + + tracker.processTaskExecutorClusterPartitionReport( + TASK_EXECUTOR_ID_1, + EMPTY_PARTITION_REPORT); + assertEquals(0, tracker.listDataSets().size()); + } + + @Test + public void testListDataSetsMultiplePartitionsOnSingleTaskExecutor() { Review comment: I've addressed your second comment (I think). As for the other one, your state transitions aren't quite correct, or rather overly simplified. It goes from `unknown -> partially-complete -> complete -> partially-complete`. Trying to split these up just means we duplicate code we already have. You can't go `partially-complete -> complete` without first having to go `unknown -> partially-complete`, so if you were to write separate tests you're running the same code `unknown -> partially-complete` twice. And this is true for all state transitions. Assertions are usable enough for us to differentiate between failures at different stages, so I don't buy in to the whole "test exactly one thing" mentality. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
