wilfred-s commented on a change in pull request #100: [YUNIKORN-84] Always
waiting in recovery phase as long as rejected node exists
URL:
https://github.com/apache/incubator-yunikorn-k8shim/pull/100#discussion_r406302434
##########
File path: pkg/cache/context_recovery_test.go
##########
@@ -106,83 +109,91 @@ func TestNodesRecovery(t *testing.T) {
dispatcher.Start()
defer dispatcher.Stop()
- var node1 = v1.Node{
- ObjectMeta: apis.ObjectMeta{
- Name: "host0001",
- Namespace: "default",
- UID: "uid_0001",
- },
- Status: v1.NodeStatus{
- Capacity: utils.NewK8sResourceList(
- utils.K8sResource{
- ResourceName: v1.ResourceMemory,
- Value: 1024,
- }, utils.K8sResource{
- ResourceName: v1.ResourceCPU,
- Value: 10,
- }),
- },
- }
-
- var node2 = v1.Node{
- ObjectMeta: apis.ObjectMeta{
- Name: "host0002",
- Namespace: "default",
- UID: "uid_0002",
- },
- Status: v1.NodeStatus{
- Capacity: utils.NewK8sResourceList(
- utils.K8sResource{
- ResourceName: v1.ResourceMemory,
- Value: 1024,
- }, utils.K8sResource{
- ResourceName: v1.ResourceCPU,
- Value: 10,
- }),
- },
+ numNodes := 3
+ nodes := make([]*v1.Node, numNodes)
+ expectedStates := make([]string, numNodes)
+ for i := 0; i < numNodes; i++ {
+ nodes[i] = &v1.Node{
+ ObjectMeta: apis.ObjectMeta{
+ Name: "host000%d" + strconv.Itoa(i),
+ Namespace: "default",
+ UID: types.UID("uid_000" +
strconv.Itoa(i)),
+ },
+ Status: v1.NodeStatus{
+ Capacity: utils.NewK8sResourceList(
+ utils.K8sResource{
+ ResourceName: v1.ResourceMemory,
+ Value: 1024,
+ }, utils.K8sResource{
+ ResourceName: v1.ResourceCPU,
+ Value: 10,
+ }),
+ },
+ }
+ expectedStates[i] = events.States().Node.Recovering
}
nodeLister := test.NewNodeListerMock()
- nodeLister.AddNode(&node1)
- nodeLister.AddNode(&node2)
+ for _, node := range nodes {
+ nodeLister.AddNode(node)
+ }
apiProvide4test.SetNodeLister(nodeLister)
mockedAppRecover := test.NewMockedRecoverableAppManager()
- if err := context.recover([]interfaces.Recoverable{mockedAppRecover},
3*time.Second); err == nil {
+ if err := context.recover([]interfaces.Recoverable{mockedAppRecover},
1*time.Second); err == nil {
t.Fatalf("expecting timeout here!")
} else {
t.Logf("context stays waiting for recovery, error: %v", err)
}
- sn1 := context.nodes.getNode("host0001")
- sn2 := context.nodes.getNode("host0002")
-
- assert.Assert(t, sn1 != nil)
- assert.Assert(t, sn2 != nil)
+ // verify all nodes were added into context
+ schedulerNodes := make([]*SchedulerNode, len(nodes))
+ for i, node := range nodes {
+ schedulerNodes[i] = context.nodes.getNode(node.Name)
+ assert.Assert(t, schedulerNodes[i] != nil)
+ }
- // node1 recovery is done
+ // dispatch NodeAccepted event for the first node, its expected state
should be Healthy
dispatcher.Dispatch(CachedSchedulerNodeEvent{
- NodeID: "host0001",
+ NodeID: schedulerNodes[0].name,
Event: events.NodeAccepted,
})
+ expectedStates[0] = events.States().Node.Healthy
+ if err := utils.WaitForCondition(func() bool {
+ return reflect.DeepEqual(getNodeStates(schedulerNodes),
expectedStates)
+ }, 100*time.Millisecond, 3*time.Second); err != nil {
+ t.Fatal("unexpected node states")
+ }
+ // dispatch NodeRejected event for the second node, its expected state
should be Rejected
+ dispatcher.Dispatch(CachedSchedulerNodeEvent{
+ NodeID: schedulerNodes[1].name,
+ Event: events.NodeRejected,
+ })
+ expectedStates[1] = events.States().Node.Rejected
if err := utils.WaitForCondition(func() bool {
- return sn1.getNodeState() ==
string(events.States().Node.Healthy) &&
- sn2.getNodeState() ==
string(events.States().Node.Recovering)
- }, time.Second, 5*time.Second); err != nil {
+ return reflect.DeepEqual(getNodeStates(schedulerNodes),
expectedStates)
+ }, 100*time.Millisecond, 3*time.Second); err != nil {
t.Fatal("unexpected node states")
Review comment:
same as above log the error to help find test failures
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
