zhuzhurk commented on a change in pull request #12375: URL: https://github.com/apache/flink/pull/12375#discussion_r438668891
########## File path: flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/PhysicalSlotRequestBulkTrackerTest.java ########## @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster.slotpool; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; +import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; +import org.apache.flink.runtime.jobmanager.scheduler.Locality; +import org.apache.flink.runtime.jobmanager.slots.TestingSlotOwner; +import org.apache.flink.runtime.jobmaster.SlotInfo; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.clock.ManualClock; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * Tests for {@link PhysicalSlotRequestBulkTracker}. + */ +public class PhysicalSlotRequestBulkTrackerTest extends TestLogger { + + private static final Time TIMEOUT = Time.milliseconds(5000L); + + private ManualClock clock = new ManualClock(); + + private PhysicalSlotRequestBulkTracker bulkTracker; + + private Set<PhysicalSlot> slots; + + private Supplier<Set<SlotInfo>> slotsRetriever; + + @Before + public void setup() throws Exception { + slots = new HashSet<>(); + slotsRetriever = () -> slots.stream().collect(Collectors.toSet()); Review comment: done. ########## File path: flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/PhysicalSlotRequestBulkTrackerTest.java ########## @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster.slotpool; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; +import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; +import org.apache.flink.runtime.jobmanager.scheduler.Locality; +import org.apache.flink.runtime.jobmanager.slots.TestingSlotOwner; +import org.apache.flink.runtime.jobmaster.SlotInfo; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.clock.ManualClock; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * Tests for {@link PhysicalSlotRequestBulkTracker}. + */ +public class PhysicalSlotRequestBulkTrackerTest extends TestLogger { + + private static final Time TIMEOUT = Time.milliseconds(5000L); + + private ManualClock clock = new ManualClock(); Review comment: done. ########## File path: flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/PhysicalSlotRequestBulkTrackerTest.java ########## @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster.slotpool; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; +import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; +import org.apache.flink.runtime.jobmanager.scheduler.Locality; +import org.apache.flink.runtime.jobmanager.slots.TestingSlotOwner; +import org.apache.flink.runtime.jobmaster.SlotInfo; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.runtime.taskmanager.LocalTaskManagerLocation; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.clock.ManualClock; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * Tests for {@link PhysicalSlotRequestBulkTracker}. + */ +public class PhysicalSlotRequestBulkTrackerTest extends TestLogger { + + private static final Time TIMEOUT = Time.milliseconds(5000L); + + private ManualClock clock = new ManualClock(); + + private PhysicalSlotRequestBulkTracker bulkTracker; + + private Set<PhysicalSlot> slots; + + private Supplier<Set<SlotInfo>> slotsRetriever; + + @Before + public void setup() throws Exception { + slots = new HashSet<>(); + slotsRetriever = () -> slots.stream().collect(Collectors.toSet()); + bulkTracker = new PhysicalSlotRequestBulkTracker(slotsRetriever, clock); + } + + @Test + public void testTrackBulk() { + final PhysicalSlotRequestBulk bulk = new PhysicalSlotRequestBulk(Collections.emptyList()); + + assertThat(bulkTracker.isTracked(bulk), is(false)); + + bulkTracker.track(bulk); + + assertThat(bulkTracker.isTracked(bulk), is(true)); + assertThat(bulk.getUnfulfillableSince(), is(clock.relativeTimeMillis())); + } + + @Test + public void testUntrackBulk() { + final PhysicalSlotRequestBulk bulk = new PhysicalSlotRequestBulk(Collections.emptyList()); + + bulkTracker.track(bulk); + bulkTracker.untrack(bulk); + + assertThat(bulkTracker.isTracked(bulk), is(false)); + } + + @Test + public void testBulkFulfilledOnCheck() { + final PhysicalSlotRequest request = createPhysicalSlotRequest(); + final PhysicalSlotRequestBulk bulk = new PhysicalSlotRequestBulk(Arrays.asList(request)); Review comment: done. ########## File path: flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/BulkSlotProviderImplTest.java ########## @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster.slotpool; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; +import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor; +import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter; +import org.apache.flink.runtime.jobmanager.slots.TestingSlotOwner; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FlinkException; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.clock.ManualClock; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Tests for {@link BulkSlotProviderImpl}. + */ +public class BulkSlotProviderImplTest extends TestLogger { + + private static final Time TIMEOUT = Time.milliseconds(1000L); + + private static ScheduledExecutorService singleThreadScheduledExecutorService; + + private static ComponentMainThreadExecutor mainThreadExecutor; + + private TestingSlotPoolImpl slotPool; + + private BulkSlotProviderImpl bulkSlotProvider; + + private TestingSlotOwner slotOwner; + + private ManualClock clock; + + @BeforeClass + public static void setupClass() { + singleThreadScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forSingleThreadExecutor(singleThreadScheduledExecutorService); + } + + @AfterClass + public static void teardownClass() { + if (singleThreadScheduledExecutorService != null) { + singleThreadScheduledExecutorService.shutdownNow(); + } + } + + @Before + public void setup() throws Exception { + clock = new ManualClock(); + + slotPool = new SlotPoolBuilder(mainThreadExecutor).build(); + + bulkSlotProvider = new BulkSlotProviderImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool); + bulkSlotProvider.start(mainThreadExecutor); + + slotOwner = new TestingSlotOwner(); + slotOwner.setReturnAllocatedSlotConsumer( + logicalSlot -> + slotPool.releaseSlot( + logicalSlot.getSlotRequestId(), + new FlinkException("Slot is being returned to the SlotPool."))); + } + + @After + public void teardown() { + CompletableFuture.runAsync(() -> slotPool.close(), mainThreadExecutor).join(); + } + + @Test + public void testBulkSlotAllocationFulfilledWithAvailableSlots() throws Exception { + final PhysicalSlotRequest request1 = createPhysicalSlotRequest(); + final PhysicalSlotRequest request2 = createPhysicalSlotRequest(); + final List<PhysicalSlotRequest> requests = Arrays.asList(request1, request2); + + addSlotToSlotPool(); + addSlotToSlotPool(); + + final CompletableFuture<Collection<PhysicalSlotRequest.Result>> slotFutures = allocateSlots(requests); + + final Collection<PhysicalSlotRequest.Result> results = slotFutures.get(TIMEOUT.getSize(), TIMEOUT.getUnit()); + final Collection<SlotRequestId> resultRequestIds = results.stream() + .map(PhysicalSlotRequest.Result::getSlotRequestId) + .collect(Collectors.toList()); + + assertThat(resultRequestIds, containsInAnyOrder(request1.getSlotRequestId(), request2.getSlotRequestId())); + } + + @Test + public void testBulkSlotAllocationFulfilledWithNewSlots() { + final List<PhysicalSlotRequest> requests = Arrays.asList( + createPhysicalSlotRequest(), + createPhysicalSlotRequest()); + final CompletableFuture<Collection<PhysicalSlotRequest.Result>> slotFutures = allocateSlots(requests); + + addSlotToSlotPool(); + + assertThat(slotFutures.isDone(), is(false)); + + addSlotToSlotPool(); + + assertThat(slotFutures.isDone(), is(true)); + assertThat(slotFutures.isCompletedExceptionally(), is(false)); + } + + @Test + public void testBulkSlotAllocationTimeoutsIfUnfulfillable() { + final List<PhysicalSlotRequest> requests = Arrays.asList( + createPhysicalSlotRequest(), + createPhysicalSlotRequest()); + final CompletableFuture<Collection<PhysicalSlotRequest.Result>> slotFutures = allocateSlots(requests); + + addSlotToSlotPool(); + + assertThat(slotPool.getAllocatedSlots().listSlotInfo(), hasSize(1)); + + clock.advanceTime(TIMEOUT.toMilliseconds() + 1L, TimeUnit.MILLISECONDS); + + try { + // wait util the requests timed out + slotFutures.get(); + fail("Expected that the slot futures time out."); + } catch (Exception e) { + final Optional<Throwable> cause = ExceptionUtils.findThrowableWithMessage( + e, + "Slot request bulk is not fulfillable!"); + assertThat(cause.isPresent(), is(true)); + assertThat(cause.get(), instanceOf(TimeoutException.class)); + } + } + + @Test + public void testFailedBulkSlotAllocationReleasesAllocatedSlot() { + final List<PhysicalSlotRequest> requests = Arrays.asList( + createPhysicalSlotRequest(), + createPhysicalSlotRequest()); + final CompletableFuture<Collection<PhysicalSlotRequest.Result>> slotFutures = allocateSlots(requests); + + addSlotToSlotPool(); + + assertThat(slotPool.getAllocatedSlots().listSlotInfo(), hasSize(1)); + + clock.advanceTime(TIMEOUT.toMilliseconds() + 1L, TimeUnit.MILLISECONDS); + + try { + // wait util the requests timed out + slotFutures.get(); + fail("Expected that the slot futures time out."); + } catch (Exception e) { + // expected + } Review comment: done. ########## File path: flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/slotpool/BulkSlotProviderImplTest.java ########## @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster.slotpool; + +import org.apache.flink.api.common.time.Time; +import org.apache.flink.runtime.clusterframework.types.ResourceProfile; +import org.apache.flink.runtime.clusterframework.types.SlotProfile; +import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor; +import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter; +import org.apache.flink.runtime.jobmanager.slots.TestingSlotOwner; +import org.apache.flink.runtime.jobmaster.SlotRequestId; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FlinkException; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.clock.ManualClock; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Tests for {@link BulkSlotProviderImpl}. + */ +public class BulkSlotProviderImplTest extends TestLogger { + + private static final Time TIMEOUT = Time.milliseconds(1000L); + + private static ScheduledExecutorService singleThreadScheduledExecutorService; + + private static ComponentMainThreadExecutor mainThreadExecutor; + + private TestingSlotPoolImpl slotPool; + + private BulkSlotProviderImpl bulkSlotProvider; + + private TestingSlotOwner slotOwner; + + private ManualClock clock; + + @BeforeClass + public static void setupClass() { + singleThreadScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forSingleThreadExecutor(singleThreadScheduledExecutorService); + } + + @AfterClass + public static void teardownClass() { + if (singleThreadScheduledExecutorService != null) { + singleThreadScheduledExecutorService.shutdownNow(); + } + } + + @Before + public void setup() throws Exception { + clock = new ManualClock(); + + slotPool = new SlotPoolBuilder(mainThreadExecutor).build(); + + bulkSlotProvider = new BulkSlotProviderImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool); + bulkSlotProvider.start(mainThreadExecutor); + + slotOwner = new TestingSlotOwner(); Review comment: done. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected]
