This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new b6809ea8 [YUNIKORN-2565] Remove Node reference from nodeEvents (#867)
b6809ea8 is described below
commit b6809ea80d4459705ea740aa574ec4ef48aea829
Author: Peter Bacsko <[email protected]>
AuthorDate: Thu May 23 12:24:03 2024 +0200
[YUNIKORN-2565] Remove Node reference from nodeEvents (#867)
Closes: #867
Signed-off-by: Peter Bacsko <[email protected]>
---
pkg/scheduler/objects/node.go | 22 +++---
pkg/scheduler/objects/node_events.go | 46 ++++++-------
pkg/scheduler/objects/node_events_test.go | 111 +++++++++++-------------------
pkg/scheduler/objects/node_test.go | 2 +-
pkg/scheduler/objects/utilities_test.go | 2 +-
5 files changed, 77 insertions(+), 106 deletions(-)
diff --git a/pkg/scheduler/objects/node.go b/pkg/scheduler/objects/node.go
index 6ea167ef..09efa2d3 100644
--- a/pkg/scheduler/objects/node.go
+++ b/pkg/scheduler/objects/node.go
@@ -77,7 +77,7 @@ func NewNode(proto *si.NodeInfo) *Node {
schedulable: true,
listeners: make([]NodeListener, 0),
}
- sn.nodeEvents = newNodeEvents(sn, events.GetEventSystem())
+ sn.nodeEvents = newNodeEvents(events.GetEventSystem())
// initialise available resources
var err error
sn.availableResource, err =
resources.SubErrorNegative(sn.totalResource, sn.occupiedResource)
@@ -165,7 +165,7 @@ func (sn *Node) SetCapacity(newCapacity
*resources.Resource) *resources.Resource
delta := resources.Sub(newCapacity, sn.totalResource)
sn.totalResource = newCapacity
sn.refreshAvailableResource()
- sn.nodeEvents.sendNodeCapacityChangedEvent()
+ sn.nodeEvents.sendNodeCapacityChangedEvent(sn.NodeID,
sn.totalResource.Clone())
return delta
}
@@ -184,7 +184,7 @@ func (sn *Node) SetOccupiedResource(occupiedResource
*resources.Resource) {
return
}
sn.occupiedResource = occupiedResource
- sn.nodeEvents.sendNodeOccupiedResourceChangedEvent()
+ sn.nodeEvents.sendNodeOccupiedResourceChangedEvent(sn.NodeID,
sn.occupiedResource.Clone())
sn.refreshAvailableResource()
}
@@ -234,7 +234,7 @@ func (sn *Node) SetSchedulable(schedulable bool) {
sn.Lock()
defer sn.Unlock()
sn.schedulable = schedulable
- sn.nodeEvents.sendNodeSchedulableChangedEvent(sn.schedulable)
+ sn.nodeEvents.sendNodeSchedulableChangedEvent(sn.NodeID, sn.schedulable)
}
// Can this node be used in scheduling.
@@ -304,7 +304,7 @@ func (sn *Node) RemoveAllocation(allocationKey string)
*Allocation {
delete(sn.allocations, allocationKey)
sn.allocatedResource.SubFrom(alloc.GetAllocatedResource())
sn.availableResource.AddTo(alloc.GetAllocatedResource())
- sn.nodeEvents.sendAllocationRemovedEvent(alloc.allocationKey,
alloc.allocatedResource)
+ sn.nodeEvents.sendAllocationRemovedEvent(sn.NodeID,
alloc.allocationKey, alloc.allocatedResource)
return alloc
}
@@ -327,7 +327,7 @@ func (sn *Node) AddAllocation(alloc *Allocation) bool {
sn.allocations[alloc.GetAllocationKey()] = alloc
sn.allocatedResource.AddTo(res)
sn.availableResource.SubFrom(res)
- sn.nodeEvents.sendAllocationAddedEvent(alloc.allocationKey, res)
+ sn.nodeEvents.sendAllocationAddedEvent(sn.NodeID,
alloc.allocationKey, res)
return true
}
return false
@@ -490,7 +490,7 @@ func (sn *Node) Reserve(app *Application, ask
*AllocationAsk) error {
return fmt.Errorf("reservation does not fit on node %s, appID
%s, ask %s", sn.NodeID, app.ApplicationID, ask.GetAllocatedResource().String())
}
sn.reservations[appReservation.getKey()] = appReservation
- sn.nodeEvents.sendReservedEvent(ask.GetAllocatedResource(),
ask.GetAllocationKey())
+ sn.nodeEvents.sendReservedEvent(sn.NodeID, ask.GetAllocatedResource(),
ask.GetAllocationKey())
// reservation added successfully
return nil
}
@@ -512,7 +512,7 @@ func (sn *Node) unReserve(app *Application, ask
*AllocationAsk) (int, error) {
}
if _, ok := sn.reservations[resKey]; ok {
delete(sn.reservations, resKey)
- sn.nodeEvents.sendUnreservedEvent(ask.GetAllocatedResource(),
ask.GetAllocationKey())
+ sn.nodeEvents.sendUnreservedEvent(sn.NodeID,
ask.GetAllocatedResource(), ask.GetAllocationKey())
return 1, nil
}
// reservation was not found
@@ -587,9 +587,11 @@ func (sn *Node) getListeners() []NodeListener {
}
func (sn *Node) SendNodeAddedEvent() {
- sn.nodeEvents.sendNodeAddedEvent()
+ sn.RLock()
+ defer sn.RUnlock()
+ sn.nodeEvents.sendNodeAddedEvent(sn.NodeID, sn.totalResource.Clone())
}
func (sn *Node) SendNodeRemovedEvent() {
- sn.nodeEvents.sendNodeRemovedEvent()
+ sn.nodeEvents.sendNodeRemovedEvent(sn.NodeID)
}
diff --git a/pkg/scheduler/objects/node_events.go
b/pkg/scheduler/objects/node_events.go
index 3a60dba4..a01ceafd 100644
--- a/pkg/scheduler/objects/node_events.go
+++ b/pkg/scheduler/objects/node_events.go
@@ -27,46 +27,45 @@ import (
type nodeEvents struct {
eventSystem events.EventSystem
- node *Node
}
-func (n *nodeEvents) sendNodeAddedEvent() {
+func (n *nodeEvents) sendNodeAddedEvent(nodeID string, capacity
*resources.Resource) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, "Node added to the
scheduler", common.Empty, si.EventRecord_ADD,
- si.EventRecord_DETAILS_NONE, n.node.GetCapacity())
+ event := events.CreateNodeEventRecord(nodeID, "Node added to the
scheduler", common.Empty, si.EventRecord_ADD,
+ si.EventRecord_DETAILS_NONE, capacity)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendNodeRemovedEvent() {
+func (n *nodeEvents) sendNodeRemovedEvent(nodeID string) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, "Node removed from
the scheduler", common.Empty, si.EventRecord_REMOVE,
+ event := events.CreateNodeEventRecord(nodeID, "Node removed from the
scheduler", common.Empty, si.EventRecord_REMOVE,
si.EventRecord_NODE_DECOMISSION, nil)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendAllocationAddedEvent(allocKey string, res
*resources.Resource) {
+func (n *nodeEvents) sendAllocationAddedEvent(nodeID, allocKey string, res
*resources.Resource) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, common.Empty,
allocKey, si.EventRecord_ADD,
+ event := events.CreateNodeEventRecord(nodeID, common.Empty, allocKey,
si.EventRecord_ADD,
si.EventRecord_NODE_ALLOC, res)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendAllocationRemovedEvent(allocKey string, res
*resources.Resource) {
+func (n *nodeEvents) sendAllocationRemovedEvent(nodeID, allocKey string, res
*resources.Resource) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, common.Empty,
allocKey, si.EventRecord_REMOVE,
+ event := events.CreateNodeEventRecord(nodeID, common.Empty, allocKey,
si.EventRecord_REMOVE,
si.EventRecord_NODE_ALLOC, res)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendNodeSchedulableChangedEvent(ready bool) {
+func (n *nodeEvents) sendNodeSchedulableChangedEvent(nodeID string, ready
bool) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
@@ -76,50 +75,49 @@ func (n *nodeEvents) sendNodeSchedulableChangedEvent(ready
bool) {
} else {
reason = "schedulable: false"
}
- event := events.CreateNodeEventRecord(n.node.NodeID, reason,
common.Empty, si.EventRecord_SET,
+ event := events.CreateNodeEventRecord(nodeID, reason, common.Empty,
si.EventRecord_SET,
si.EventRecord_NODE_SCHEDULABLE, nil)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendNodeCapacityChangedEvent() {
+func (n *nodeEvents) sendNodeCapacityChangedEvent(nodeID string, total
*resources.Resource) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, common.Empty,
common.Empty, si.EventRecord_SET,
- si.EventRecord_NODE_CAPACITY, n.node.totalResource)
+ event := events.CreateNodeEventRecord(nodeID, common.Empty,
common.Empty, si.EventRecord_SET,
+ si.EventRecord_NODE_CAPACITY, total)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendNodeOccupiedResourceChangedEvent() {
+func (n *nodeEvents) sendNodeOccupiedResourceChangedEvent(nodeID string,
occupied *resources.Resource) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, common.Empty,
common.Empty, si.EventRecord_SET,
- si.EventRecord_NODE_OCCUPIED, n.node.occupiedResource)
+ event := events.CreateNodeEventRecord(nodeID, common.Empty,
common.Empty, si.EventRecord_SET,
+ si.EventRecord_NODE_OCCUPIED, occupied)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendReservedEvent(res *resources.Resource, askID string) {
+func (n *nodeEvents) sendReservedEvent(nodeID string, res *resources.Resource,
askID string) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, common.Empty,
askID, si.EventRecord_ADD,
+ event := events.CreateNodeEventRecord(nodeID, common.Empty, askID,
si.EventRecord_ADD,
si.EventRecord_NODE_RESERVATION, res)
n.eventSystem.AddEvent(event)
}
-func (n *nodeEvents) sendUnreservedEvent(res *resources.Resource, askID
string) {
+func (n *nodeEvents) sendUnreservedEvent(nodeID string, res
*resources.Resource, askID string) {
if !n.eventSystem.IsEventTrackingEnabled() {
return
}
- event := events.CreateNodeEventRecord(n.node.NodeID, common.Empty,
askID, si.EventRecord_REMOVE,
+ event := events.CreateNodeEventRecord(nodeID, common.Empty, askID,
si.EventRecord_REMOVE,
si.EventRecord_NODE_RESERVATION, res)
n.eventSystem.AddEvent(event)
}
-func newNodeEvents(node *Node, evt events.EventSystem) *nodeEvents {
+func newNodeEvents(evt events.EventSystem) *nodeEvents {
return &nodeEvents{
eventSystem: evt,
- node: node,
}
}
diff --git a/pkg/scheduler/objects/node_events_test.go
b/pkg/scheduler/objects/node_events_test.go
index 07f1d6a3..bafc4f93 100644
--- a/pkg/scheduler/objects/node_events_test.go
+++ b/pkg/scheduler/objects/node_events_test.go
@@ -30,17 +30,15 @@ import (
)
func TestSendNodeAddedEvent(t *testing.T) {
- node := &Node{
- NodeID: nodeID1,
- }
+ resource :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 1})
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendNodeAddedEvent()
+ ne := newNodeEvents(eventSystem)
+ ne.sendNodeAddedEvent(nodeID1, resource)
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendNodeAddedEvent()
+ ne = newNodeEvents(eventSystem)
+ ne.sendNodeAddedEvent(nodeID1, resource)
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -48,21 +46,20 @@ func TestSendNodeAddedEvent(t *testing.T) {
assert.Equal(t, "Node added to the scheduler", event.Message)
assert.Equal(t, si.EventRecord_ADD, event.EventChangeType)
assert.Equal(t, si.EventRecord_DETAILS_NONE, event.EventChangeDetail)
- assert.Equal(t, 0, len(event.Resource.Resources))
+ assert.Equal(t, 1, len(event.Resource.Resources))
+ protoRes := resources.NewResourceFromProto(event.Resource)
+ assert.DeepEqual(t, protoRes, resource)
}
func TestSendNodeRemovedEvent(t *testing.T) {
- node := &Node{
- NodeID: nodeID1,
- }
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendNodeRemovedEvent()
+ ne := newNodeEvents(eventSystem)
+ ne.sendNodeRemovedEvent(nodeID1)
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendNodeRemovedEvent()
+ ne = newNodeEvents(eventSystem)
+ ne.sendNodeRemovedEvent(nodeID1)
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -74,19 +71,16 @@ func TestSendNodeRemovedEvent(t *testing.T) {
}
func TestSendAllocationAddedEvent(t *testing.T) {
- node := &Node{
- NodeID: nodeID1,
- }
resource :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 1})
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendAllocationAddedEvent("alloc-0", resource)
+ ne := newNodeEvents(eventSystem)
+ ne.sendAllocationAddedEvent(nodeID1, "alloc-0", resource)
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendAllocationAddedEvent("alloc-0", resource)
+ ne = newNodeEvents(eventSystem)
+ ne.sendAllocationAddedEvent(nodeID1, "alloc-0", resource)
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -100,19 +94,16 @@ func TestSendAllocationAddedEvent(t *testing.T) {
}
func TestSendAllocationRemovedEvent(t *testing.T) {
- node := &Node{
- NodeID: nodeID1,
- }
resource :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 1})
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendAllocationRemovedEvent("alloc-0", resource)
+ ne := newNodeEvents(eventSystem)
+ ne.sendAllocationRemovedEvent(nodeID1, "alloc-0", resource)
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendAllocationRemovedEvent("alloc-0", resource)
+ ne = newNodeEvents(eventSystem)
+ ne.sendAllocationRemovedEvent(nodeID1, "alloc-0", resource)
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
assert.Equal(t, "alloc-0", event.ReferenceID)
@@ -126,18 +117,14 @@ func TestSendAllocationRemovedEvent(t *testing.T) {
func TestSendOccupiedResourceChangedEvent(t *testing.T) {
resource :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 1})
- node := &Node{
- NodeID: nodeID1,
- occupiedResource: resource,
- }
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendNodeOccupiedResourceChangedEvent()
+ ne := newNodeEvents(eventSystem)
+ ne.sendNodeOccupiedResourceChangedEvent(nodeID1, resource)
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendNodeOccupiedResourceChangedEvent()
+ ne = newNodeEvents(eventSystem)
+ ne.sendNodeOccupiedResourceChangedEvent(nodeID1, resource)
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -152,18 +139,14 @@ func TestSendOccupiedResourceChangedEvent(t *testing.T) {
func TestSendCapacityChangedEvent(t *testing.T) {
resource :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 1})
- node := &Node{
- NodeID: nodeID1,
- totalResource: resource,
- }
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendNodeCapacityChangedEvent()
+ ne := newNodeEvents(eventSystem)
+ ne.sendNodeCapacityChangedEvent(nodeID1, resource)
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendNodeCapacityChangedEvent()
+ ne = newNodeEvents(eventSystem)
+ ne.sendNodeCapacityChangedEvent(nodeID1, resource)
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -177,18 +160,14 @@ func TestSendCapacityChangedEvent(t *testing.T) {
}
func TestNodeSchedulableChangedEvent(t *testing.T) {
- node := &Node{
- NodeID: nodeID1,
- schedulable: true,
- }
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendNodeSchedulableChangedEvent(false)
+ ne := newNodeEvents(eventSystem)
+ ne.sendNodeSchedulableChangedEvent(nodeID1, false)
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendNodeSchedulableChangedEvent(false)
+ ne = newNodeEvents(eventSystem)
+ ne.sendNodeSchedulableChangedEvent(nodeID1, false)
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -199,7 +178,7 @@ func TestNodeSchedulableChangedEvent(t *testing.T) {
assert.Equal(t, 0, len(event.Resource.Resources))
eventSystem.Reset()
- ne.sendNodeSchedulableChangedEvent(true)
+ ne.sendNodeSchedulableChangedEvent(nodeID1, true)
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event = eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -212,18 +191,14 @@ func TestNodeSchedulableChangedEvent(t *testing.T) {
func TestNodeReservationEvent(t *testing.T) {
resource :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 1})
- node := &Node{
- NodeID: nodeID1,
- schedulable: true,
- }
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendReservedEvent(resource, "alloc-0")
+ ne := newNodeEvents(eventSystem)
+ ne.sendReservedEvent(nodeID1, resource, "alloc-0")
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendReservedEvent(resource, "alloc-0")
+ ne = newNodeEvents(eventSystem)
+ ne.sendReservedEvent(nodeID1, resource, "alloc-0")
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
@@ -238,18 +213,14 @@ func TestNodeReservationEvent(t *testing.T) {
func TestNodeUnreservationEvent(t *testing.T) {
resource :=
resources.NewResourceFromMap(map[string]resources.Quantity{"first": 1})
- node := &Node{
- NodeID: nodeID1,
- schedulable: true,
- }
eventSystem := mock.NewEventSystemDisabled()
- ne := newNodeEvents(node, eventSystem)
- ne.sendUnreservedEvent(resource, "alloc-0")
+ ne := newNodeEvents(eventSystem)
+ ne.sendUnreservedEvent(nodeID1, resource, "alloc-0")
assert.Equal(t, 0, len(eventSystem.Events), "unexpected event")
eventSystem = mock.NewEventSystem()
- ne = newNodeEvents(node, eventSystem)
- ne.sendUnreservedEvent(resource, "alloc-0")
+ ne = newNodeEvents(eventSystem)
+ ne.sendUnreservedEvent(nodeID1, resource, "alloc-0")
assert.Equal(t, 1, len(eventSystem.Events), "event was not generated")
event := eventSystem.Events[0]
assert.Equal(t, nodeID1, event.ObjectID)
diff --git a/pkg/scheduler/objects/node_test.go
b/pkg/scheduler/objects/node_test.go
index 25b32a4a..dccb2790 100644
--- a/pkg/scheduler/objects/node_test.go
+++ b/pkg/scheduler/objects/node_test.go
@@ -664,7 +664,7 @@ func TestNodeEvents(t *testing.T) {
"ready": "true",
})
node := NewNode(proto)
- node.nodeEvents = newNodeEvents(node, mockEvents)
+ node.nodeEvents = newNodeEvents(mockEvents)
node.SendNodeAddedEvent()
assert.Equal(t, 1, len(mockEvents.Events))
diff --git a/pkg/scheduler/objects/utilities_test.go
b/pkg/scheduler/objects/utilities_test.go
index 4cab3457..bc751553 100644
--- a/pkg/scheduler/objects/utilities_test.go
+++ b/pkg/scheduler/objects/utilities_test.go
@@ -176,8 +176,8 @@ func newNodeInternal(nodeID string, total, occupied
*resources.Resource) *Node {
allocations: make(map[string]*Allocation),
schedulable: true,
reservations: make(map[string]*reservation),
+ nodeEvents: newNodeEvents(events.GetEventSystem()),
}
- sn.nodeEvents = newNodeEvents(sn, events.GetEventSystem())
return sn
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
