Hashar has uploaded a new change for review.
https://gerrit.wikimedia.org/r/235690
Change subject: Debug dying task managers
......................................................................
Debug dying task managers
Change-Id: I81684026071b5abd6162e9f117e7fad08da61ea2
---
M nodepool/task_manager.py
1 file changed, 29 insertions(+), 25 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/debs/nodepool
refs/changes/90/235690/1
diff --git a/nodepool/task_manager.py b/nodepool/task_manager.py
index e48de1a..6e9cc95 100644
--- a/nodepool/task_manager.py
+++ b/nodepool/task_manager.py
@@ -79,33 +79,37 @@
def run(self):
last_ts = 0
- while True:
- task = self.queue.get()
- if not task:
- if not self._running:
- break
- continue
+ try:
while True:
- delta = time.time() - last_ts
- if delta >= self.rate:
- break
- time.sleep(self.rate - delta)
- self.log.debug("Manager %s running task %s (queue: %s)" %
- (self.name, task, self.queue.qsize()))
- start = time.time()
- self.runTask(task)
- last_ts = time.time()
- dt = last_ts - start
- self.log.debug("Manager %s ran task %s in %ss" %
- (self.name, task, dt))
- if statsd:
- #nodepool.task.PROVIDER.subkey
- subkey = type(task).__name__
- key = 'nodepool.task.%s.%s' % (self.name, subkey)
- statsd.timing(key, dt)
- statsd.incr(key)
+ task = self.queue.get()
+ if not task:
+ if not self._running:
+ break
+ continue
+ while True:
+ delta = time.time() - last_ts
+ if delta >= self.rate:
+ break
+ time.sleep(self.rate - delta)
+ self.log.debug("Manager %s running task %s (queue: %s)" %
+ (self.name, task, self.queue.qsize()))
+ start = time.time()
+ self.runTask(task)
+ last_ts = time.time()
+ dt = last_ts - start
+ self.log.debug("Manager %s ran task %s in %ss" %
+ (self.name, task, dt))
+ if statsd:
+ #nodepool.task.PROVIDER.subkey
+ subkey = type(task).__name__
+ key = 'nodepool.task.%s.%s' % (self.name, subkey)
+ statsd.timing(key, dt)
+ statsd.incr(key)
- self.queue.task_done()
+ self.queue.task_done()
+ except Exception:
+ self.log.exception("Task manager died.")
+ raise
def submitTask(self, task):
if not self._running:
--
To view, visit https://gerrit.wikimedia.org/r/235690
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I81684026071b5abd6162e9f117e7fad08da61ea2
Gerrit-PatchSet: 1
Gerrit-Project: operations/debs/nodepool
Gerrit-Branch: patch-queue/debian
Gerrit-Owner: Hashar <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
