Giuseppe Lavagetto has submitted this change and it was merged.
Change subject: Allow the etcd driver to handle deleted or inactive nodes
......................................................................
Allow the etcd driver to handle deleted or inactive nodes
As it was designed now, it was practically impossible to remove a node
from a running pybal instance.
Bug: T125397
Change-Id: I909efea01a33b842e4e0299264c240423b59d952
---
M pybal/etcd.py
M pybal/test/test_config.py
A pybal/test/test_etcd.py
3 files changed, 154 insertions(+), 6 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pybal/etcd.py b/pybal/etcd.py
index c0f3f20..7bb4268 100644
--- a/pybal/etcd.py
+++ b/pybal/etcd.py
@@ -25,8 +25,13 @@
def decode_node(node):
"""Decode an individual node from an etcd response."""
key = node['key'].rsplit('/', 1)[-1]
+ # handle deletions by returning the key and a value of None
+ if 'value' not in node:
+ return key, None
value = json.loads(node['value'])
pooled = value.pop('pooled', None)
+ if pooled == 'inactive':
+ return key, None
if pooled is not None:
value['enabled'] = pooled == 'yes'
return key, value
@@ -141,7 +146,18 @@
else:
self.waitIndex = etcdIdx + 1
config = copy.deepcopy(self.lastConfig)
- config.update(decode_etcd_data(update))
+
+ # Read new data
+ new_data = decode_etcd_data(update)
+ config.update(new_data)
+
+ # Remove deleted/inactive nodes
+ to_remove = [k for k, v in new_data.items() if v is None]
+ for k in to_remove:
+ if k in config:
+ del config[k]
+
+ # Now update pybal config
if config != self.lastConfig:
self.coordinator.onConfigUpdate(copy.deepcopy(config))
self.lastConfig = config
diff --git a/pybal/test/test_config.py b/pybal/test/test_config.py
index e32e5aa..c604ff3 100644
--- a/pybal/test/test_config.py
+++ b/pybal/test/test_config.py
@@ -6,15 +6,12 @@
This module contains tests for `pybal.config`.
"""
-import json
-import os
-import tempfile
-
-from twisted.python.failure import Failure
import mock
+import json
import pybal
import pybal.config
+import pybal.etcd
from .fixtures import PyBalTestCase, MockClientGetPage
diff --git a/pybal/test/test_etcd.py b/pybal/test/test_etcd.py
new file mode 100644
index 0000000..09418d1
--- /dev/null
+++ b/pybal/test/test_etcd.py
@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+"""
+ PyBal unit tests
+ ~~~~~~~~~~~~~~~~
+
+ This module contains tests for `pybal.etcd`.
+
+"""
+
+import copy
+import mock
+import urlparse
+
+import pybal
+import pybal.config
+import pybal.etcd
+from .fixtures import PyBalTestCase
+
+
+
+class EtcdConfigurationObserverTestCase(PyBalTestCase):
+ def setUp(self):
+ super(EtcdConfigurationObserverTestCase, self).setUp()
+ self.observer = self.getObserver()
+
+ def getObserver(self, url='etcd://example.com/config/text'):
+ return pybal.etcd.EtcdConfigurationObserver(
+ self.coordinator, url)
+
+ def testParseConfigUrl(self):
+ """Test initialization"""
+ obs = self.getObserver()
+ self.assertEquals(obs.host, 'example.com')
+ self.assertEquals(obs.port, 2379)
+ self.assertEquals(obs.key, '/config/text')
+ self.assertEquals(obs.waitIndex, None)
+
+ def testGetPath(self):
+ self.assertEquals(self.observer.getPath(),
+ '/v2/keys/config/text?recursive=true')
+ self.observer.waitIndex = 4
+ url = urlparse.urlparse(self.observer.getPath())
+ self.assertEquals(url.path, '/v2/keys/config/text')
+ self.assertDictEqual(dict(urlparse.parse_qsl(url.query)),
+ {'wait': 'true', 'waitIndex': '4',
+ 'recursive': 'true' })
+
+ def testGetMaxModifiedIndex(self):
+ nodes = {
+ 'modifiedIndex': 40,
+ 'nodes': [
+ {'modifiedIndex': 4},
+ {'modifiedIndex': 3},
+ {'modifiedIndex': 15},
+ ]
+ }
+ self.assertEquals(40, self.observer.getMaxModifiedIndex(nodes))
+
+ def testOnUpdate(self):
+ create = {
+ "action":"set",
+ "node":
+ {
+ "key": "/testdir/1",
+ "value": "{\"pooled\": \"yes\", \"weight\": 10}",
+ "modifiedIndex": 11,
+ "createdIndex": 11,
+ }
+ }
+ create_another = copy.deepcopy(create)
+ create_another['node']['key'] = "/testdir/2"
+ depool = {
+ "action": "set",
+ "node": {
+ "key": "/testdir/1",
+ "value": "{\"pooled\": \"no\", \"weight\": 10}",
+ "modifiedIndex": 12,
+ "createdIndex": 12
+ },
+ "prevNode": {
+ "key": "/testdir/1",
+ "value": "{\"pooled\": \"yes\", \"weight\": 10}",
+ "modifiedIndex": 11,
+ "createdIndex": 11
+ }
+ }
+ inactive = copy.deepcopy(depool)
+ inactive['node']['value'] = "{\"pooled\": \"inactive\", \"weight\":
10}"
+ delete = {
+ "action": "delete",
+ "node": {
+ "key": "/testdir/1",
+ "modifiedIndex": 13,
+ "createdIndex": 12
+ },
+ "prevNode": {
+ "key": "/testdir/1",
+ "value": "{\"pooled\": \"no\", \"weight\": 10}",
+ "modifiedIndex": 12,
+ "createdIndex": 12
+ }
+ }
+ self.observer.coordinator.onConfigUpdate = mock.MagicMock()
+ self.observer.lastConfig = {}
+ # Add a node
+ self.observer.onUpdate(create, 0)
+ self.observer.coordinator.onConfigUpdate.assert_called_with({'1':
{'enabled': True, u'weight': 10}})
+ # Add another one
+ self.observer.onUpdate(create_another, 11)
+ self.observer.coordinator.onConfigUpdate.assert_called_with(
+ {'1': {'enabled': True, u'weight': 10},
+ '2': {'enabled': True, u'weight': 10}}
+ )
+
+ # Depool a server
+ self.observer.onUpdate(depool, 12)
+ self.observer.coordinator.onConfigUpdate.assert_called_with(
+ {'1': {'enabled': False, u'weight': 10},
+ '2': {'enabled': True, u'weight': 10}}
+ )
+
+ # Set it to inactive
+ self.observer.onUpdate(inactive, 12)
+ self.observer.coordinator.onConfigUpdate.assert_called_with(
+ {'2': {'enabled': True, u'weight': 10}}
+ )
+
+ # repool it
+ self.observer.onUpdate(create, 11)
+
+ # Delete a server
+ self.observer.onUpdate(delete, 13)
+ self.observer.coordinator.onConfigUpdate.assert_called_with(
+ {'2': {'enabled': True, u'weight': 10}}
+ )
--
To view, visit https://gerrit.wikimedia.org/r/268361
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I909efea01a33b842e4e0299264c240423b59d952
Gerrit-PatchSet: 3
Gerrit-Project: operations/debs/pybal
Gerrit-Branch: master
Gerrit-Owner: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
