Mark Bergsma has uploaded a new change for review.
https://gerrit.wikimedia.org/r/187346
Change subject: Add no-gravity configuration option
......................................................................
Add no-gravity configuration option
If no-gravity = True on an LVS service, instead of removing a server
from IPVS for depooling, its weight is set to 0 instead.
This adds a lot of complications, as previously held invariants
(e.g. (server.pooled <==> server listed in IPVS)) no longer hold.
* server.pooled has been renamed to server.listed to clarify the
distinction between "being used" and "being configured in IPVS"
* server.listWeight has been added to distinguish between desired server
weight when pooled vs weight configured in IPVS
The existing code is also inconsistent with (boolean) server state
variables that indicate _intent_ and _current state_. This is (at
least partially) resolved by creating new variables:
* server.list vs server.listed
* server.pool vs server.pooled
These are now maintained as separate concerns in the code. Tests still
need to be adapted.
Bug: T86650
Change-Id: I1f8aec57401481520eec80c7f9bfd0ca5271f7c2
---
M pybal/ipvs.py
M pybal/pybal.py
2 files changed, 98 insertions(+), 26 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/debs/pybal
refs/changes/46/187346/1
diff --git a/pybal/ipvs.py b/pybal/ipvs.py
index 6d4aa76..25cba80 100644
--- a/pybal/ipvs.py
+++ b/pybal/ipvs.py
@@ -141,8 +141,8 @@
cmd = " ".join(['-a', cls.subCommandService(service),
cls.subCommandServer(server)])
# Include weight if specified
- if server.weight:
- cmd += ' -w %d' % server.weight
+ if server.listWeight:
+ cmd += ' -w %d' % server.listWeight
return cmd
@@ -159,8 +159,8 @@
cmd = " ".join(['-e', cls.subCommandService(service),
cls.subCommandServer(server)])
# Include weight if specified
- if server.weight:
- cmd += ' -w %d' % server.weight
+ if server.listWeight:
+ cmd += ' -w %d' % server.listWeight
return cmd
@@ -192,6 +192,7 @@
self.configuration = configuration
self.ipvsManager.DryRun = configuration.getboolean('dryrun', False)
+ self.noGravity = configuration.getboolean('no-gravity', False)
if self.configuration.getboolean('bgp', False):
from pybal import BGPFailover
@@ -228,8 +229,15 @@
[self.ipvsManager.commandEditServer(self.service(), server)
for server in newServers & self.servers] + \
[self.ipvsManager.commandRemoveServer(self.service(), server)
for server in self.servers - newServers]
- self.servers = newServers
self.ipvsManager.modifyState(cmdList)
+
+ # Update server.listed & server.pooled with new status
+ for server in self.servers + newServers:
+ server.listed = server in newServers
+ server.pooled = (server.listed and server.listWeight > 0)
+
+ self.servers = newServers
+
def addServer(self, server):
"""Adds (pools) a single Server to the LVS state"""
@@ -243,7 +251,8 @@
self.servers.add(server)
self.ipvsManager.modifyState(cmdList)
- server.pooled = True
+ server.listed = True
+ server.pooled = (server.weight != 0)
def removeServer(self, server):
"""Removes (depools) a single Server from the LVS state"""
@@ -252,8 +261,25 @@
self.servers.remove(server) # May raise KeyError
- server.pooled = False
self.ipvsManager.modifyState(cmdList)
+ server.listed = False
+ server.pooled = False
+
+
+ def editServer(self, server):
+ """
+ Changes the IPVS attributes of an existing server.
+
+ Handles weight 0 as special (depooled) case
+ Does NOT modify server.weight
+ """
+
+ assert server.listed and server in self.servers
+
+ cmdList = [self.ipvsManager.commandEditServer(self.service(), server)]
+ self.ipvsManager.modifyState(cmdList)
+ server.pooled = (server.weight != 0)
+
def initServer(self, server):
"""Initializes a server instance with LVS service specific
configuration."""
diff --git a/pybal/pybal.py b/pybal/pybal.py
index 6dae0d3..4a4249e 100644
--- a/pybal/pybal.py
+++ b/pybal/pybal.py
@@ -49,15 +49,19 @@
self.monitors = set()
# A few invariants that SHOULD be maintained (but currently may not
be):
- # P0: pooled => enabled /\ ready
- # P1: up => pooled \/ !enabled \/ !ready
+ # P0: list => enabled /\ ready
+ # P1: up => pool \/ !enabled \/ !ready
# P2: pooled => up \/ !canDepool
- self.weight = self.DEF_WEIGHT
- self.up = False
- self.pooled = False
- self.enabled = True
- self.ready = False
+ self.weight = self.DEF_WEIGHT # Desired weight of the server when
pooled
+ self.listWeight = None # Weight the server will actually be
listed with in IPVS
+ self.up = False # True if there is consensus among
monitors that this server is up
+ self.list = None # Whether this server will be listed
in IPVS (regardless of state)
+ self.listed = False # True IFF added as realserver to IPVS
(regardless of weight)
+ self.pool = False # Whether this server will be pooled
(listed with weight > 0)
+ self.pooled = False # True IFF listed with weight > 0
+ self.enabled = True # Whether this server will be
considered for pooling at all
+ self.ready = False # Set to True when this server has
initialized and can be listed
self.modified = None
def __eq__(self, other):
@@ -156,7 +160,9 @@
self.ready = True
self.up = self.DEF_STATE
- self.pooled = self.DEF_STATE
+ self.pool = self.DEF_STATE and self.weight > 0
+ self.maintainState()
+ self.setListWeight()
self.maintainState()
self.createMonitoringInstances(coordinator)
@@ -218,14 +224,24 @@
self.up and "up" or (self.calcPartialStatus() and
"partially up" or "down"),
self.pooled and "pooled" or "not pooled")
+ def setListState(self):
+ """Sets the list attributes as appropriate for no-gravity"""
+
+ self.list = self.ready and self.enabled and (self.lvsservice.noGravity
or self.pool)
+
+ if self.lvsservice.noGravity and not self.pool:
+ self.listWeight = 0
+ else:
+ self.listWeight = self.weight
+
def maintainState(self):
"""Maintains a few invariants on configuration changes"""
# P0
if not self.enabled or not self.ready:
- self.pooled = False
+ self.list = False
# P1
- if not self.pooled and self.enabled:
+ if not self.pool and self.enabled:
self.up = False
def merge(self, configuration):
@@ -289,7 +305,7 @@
# Hand over enabled servers to LVSService
self.lvsservice.assignServers(
- set([server for server in self.servers.itervalues() if
server.pooled]))
+ set([server for server in self.servers.itervalues() if
server.list]))
def refreshModifiedServers(self):
"""
@@ -300,7 +316,10 @@
if not server.modified: continue
server.up = server.calcStatus()
- server.pooled = server.enabled and server.up
+ server.pool = server.enabled and server.up
+ server.setListState()
+ server.maintainState()
+
def resultDown(self, monitor, reason=None):
"""
@@ -327,7 +346,7 @@
if not server.up and server.calcStatus():
print self, "Server %s (%s) is up" % (server.host,
server.textStatus())
server.up = True
- if server.enabled and server.ready: self.repool(server)
+ if server.enabled and server.ready: self.repool(server)
def depool(self, server):
"""Depools a single Server, if possible"""
@@ -335,7 +354,16 @@
assert server.pooled
if self.canDepool():
- self.lvsservice.removeServer(server)
+ server.pool = False
+ # Set to weight 0 or remove
+ if self.lvsservice.noGravity:
+ assert server.listed and server.listWeight > 0
+ server.listWeight = 0
+ self.lvsservice.editServer(server)
+ else:
+ assert server.listed
+ self.list = False
+ self.lvsservice.removeServer(server)
self.pooledDownServers.discard(server)
else:
self.pooledDownServers.add(server)
@@ -350,10 +378,18 @@
assert server.enabled and server.ready
if not server.pooled:
- self.lvsservice.addServer(server)
+ server.pool = True
+ # Set original weight or re-add
+ if self.lvsservice.noGravity:
+ assert server.listed and server.listWeight == 0
+ self.lvsservice.editServer(server)
+ else:
+ assert not server.listed
+ server.list = True
+ self.lvsservice.addServer(server)
else:
print self, "Leaving previously pooled but down server",
server.host, "pooled"
-
+
# If it had been pooled in down state before, remove it from the list
self.pooledDownServers.discard(server)
@@ -361,15 +397,25 @@
while len(self.pooledDownServers) > 0 and self.canDepool():
self.depool(self.pooledDownServers.pop())
+ def pooledServers(self):
+ """
+ Returns a set of pooled servers.
+
+ This is a subset of self.servers as servers with weight 0
+ are not considered 'pooled'
+ """
+
+ return set([server for server in self.servers.itervalues() if
server.pooled])
+
def canDepool(self):
"""Returns a boolean denoting whether another server can be depooled"""
# Construct a list of servers that have status 'down'
downServers = [server for server in self.servers.itervalues() if not
server.up]
-
+
# The total amount of pooled servers may never drop below a configured
threshold
- return len(self.servers) - len(downServers) >= len(self.servers) *
self.lvsservice.getDepoolThreshold()
-
+ return len(self.pooledServers) - len(downServers) >=
len(self.pooledServers) * self.lvsservice.getDepoolThreshold()
+
def loadServers(self, configURL=None):
"""Periodic task to load a new server list/configuration file from a
specified URL."""
--
To view, visit https://gerrit.wikimedia.org/r/187346
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1f8aec57401481520eec80c7f9bfd0ca5271f7c2
Gerrit-PatchSet: 1
Gerrit-Project: operations/debs/pybal
Gerrit-Branch: master
Gerrit-Owner: Mark Bergsma <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
