Commit f5225970f1977ffab944ec047c8f990695e7c8e2:
rough in a monitoring system
Branch: refs/heads/master
Author: Sam Ruby <[email protected]>
Committer: Sam Ruby <[email protected]>
Pusher: rubys <[email protected]>
------------------------------------------------------------
www/status/css/status.css | ++ --
www/status/index.cgi | ++++++++++ ---
www/status/index.html | ------------
www/status/js/status.js | ++++++++++++ ---
www/status/monitor.rb | +++++++++
www/status/monitors/svn.rb | ++++++++++
------------------------------------------------------------
314 changes: 270 additions, 44 deletions.
------------------------------------------------------------
diff --git a/www/status/css/status.css b/www/status/css/status.css
index 10555ee..e26b596 100644
--- a/www/status/css/status.css
+++ b/www/status/css/status.css
@@ -1,5 +1,5 @@
-.just-padding {
- padding: 15px;
+body {
+ padding: 0 15px;
}
.list-group.list-group-root {
diff --git a/www/status/index.cgi b/www/status/index.cgi
index 26aec36..127fd81 100755
--- a/www/status/index.cgi
+++ b/www/status/index.cgi
@@ -1,16 +1,39 @@
#!/usr/bin/ruby
-require 'wunderbar'
+require 'json'
+require 'time'
-# the following is what infrastructure team sees:
-print "Status: 200 OK\r\n\r\n"
+json = File.expand_path('../status.json', __FILE__)
+status = JSON.parse(File.read(json)) rescue {}
-# For human consumption:
+# Get new status every minute
+if not status['mtime'] or Time.now - Time.parse(status['mtime']) > 60
+ begin
+ require_relative './monitor'
+ status = Monitor.new.status || {}
+ rescue Exception => e
+ print "Status: 500 Internal Server Error\r\n"
+ print "Context-Type: text/plain\r\n\r\n"
+ puts e.to_s
+ puts "\nbacktrace:"
+ e.backtrace.each {|line| puts " #{line}"}
+ exit
+ end
+end
+
+# The following is what infrastructure team sees:
+if %w(success info).include? status['level']
+ print "Status: 200 OK\r\n\r\n"
+else
+ print "Status: 400 #{status['title'] || 'failure'}\r\n\r\n"
+end
+
+# What the browser sees:
print <<-EOF
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8"/>
- <title>Whimsy status</title>
+ <title>Whimsy-Test Status</title>
<link rel="stylesheet" type="text/css" href="css/bootstrap.min.css"/>
<link rel="stylesheet" type="text/css" href="css/status.css"/>
@@ -21,11 +44,16 @@ print <<-EOF
</head>
<body>
- <div class="just-padding">
- <div class="list-group list-group-root well">
- Loading...
- </div>
+ <h1>Whimsy-Test Status</h1>
+
+ <div class="list-group list-group-root well">
+ Loading...
</div>
+
+ <p>
+ This status is monitored by:
+ <a href="https://www.pingmybox.com/dashboard?location=470";>Ping My
Box</a>
+ </p>
</body>
</html>
EOF
diff --git a/www/status/index.html b/www/status/index.html
deleted file mode 100644
index dfe3ab6..0000000
--- a/www/status/index.html
+++ /dev/null
@@ -1,24 +0,0 @@
-<!DOCTYPE html>
-<html>
- <head>
- <meta charset="UTF-8"/>
- <title>Whimsy status</title>
-
- <link rel="stylesheet" type="text/css" href="css/bootstrap.min.css"/>
- <link rel="stylesheet" type="text/css" href="css/status.css"/>
-
- <script type="text/javascript" src="js/jquery.min.js"></script>
- <script type="text/javascript" src="js/bootstrap.min.js"></script>
- <script type="text/javascript" src="js/status.js"></script>
- </head>
-
- <body>
- <div class="just-padding">
- <div class="list-group list-group-root well">
- Loading...
- </div>
- </div>
- </body>
-</html>
-
-
diff --git a/www/status/js/status.js b/www/status/js/status.js
index 900a875..5c8c23f 100644
--- a/www/status/js/status.js
+++ b/www/status/js/status.js
@@ -1,15 +1,21 @@
$(function() {
- function list(status, prefix, container) {
+ // convert status into .list-group-item and .list-group elements, and
+ // insert into the container. Use prefix when generating ids.
+ function listGroup(status, prefix, container) {
var items = Object.keys(status);
+ // sort items in a case insensitive manner
items.sort(function (a, b) {
return a.toLowerCase().localeCompare(b.toLowerCase());
});
items.forEach(function(item) {
var value = status[item];
- var anchor = $('<a>').addClass('list-group-item').text(item).
+
+ // build an anchor line
+ var anchor = $('<a>').addClass('list-group-item').
+ text(value.text || item).
addClass('alert-' + value.level).
attr('href', '#' + prefix + item).
attr('data-toggle', 'collapse');
@@ -19,35 +25,68 @@ $(function() {
var div = $('<div>').addClass('list-group').addClass('collapse').
attr('id', prefix + item);
+ // build nested content (recursively if value.data is an object)
if (!value.data) {
+ div.append($('<a>').addClass('list-group-item').
+ append($('<em>empty</em>')));
} else if (Array.isArray(value.data)) {
value.data.forEach(function(subitem) {
div.append($('<a>').addClass('list-group-item').
text(subitem.toString()));
});
} else if (typeof value.data == 'object') {
- list(value.data, prefix + item, div);
+ listGroup(value.data, prefix + item + '-', div);
} else {
div.append($('<a>').addClass('list-group-item').
text(value.data.toString()));
}
-
+
+ // append each to the container
container.append(anchor);
container.append(div);
});
}
-
+ // fetch status from the server
$.get('status.json', function(status) {
+ // remove 'loading...' line
$('.well').text('');
- list(status.data, '', $('.well'));
+ // replace with status
+ listGroup(status.data, '', $('.well'));
+
+ // make toggles active
$('.list-group-item').on('click', function() {
- $('.glyphicon', this)
- .toggleClass('glyphicon-chevron-right')
- .toggleClass('glyphicon-chevron-down');
+ var glyphicon = $('.glyphicon', this);
+
+ // update location hash in the url
+ if (glyphicon.hasClass('glyphicon-chevron-right')) {
+ location.hash = $(this).attr('href');
+ }
+
+ // toggle the content
+ glyphicon.
+ toggleClass('glyphicon-chevron-right').
+ toggleClass('glyphicon-chevron-down');
});
+ // if hash is present in location, show that item
+ if (location.hash) {
+ // find element
+ var element = $('a[href="' + location.hash + '"]');
+
+ // expand all parents
+ element.parents('.list-group').each(function() {
+ $('a[href="#' + this.getAttribute('id') + '"]').click();
+ });
+
+ // expand this item
+ element.click()
+
+ // scroll to this item
+ $('html, body').animate({scrollTop: element.offset().top}, 1000);
+ }
+
});
});
diff --git a/www/status/monitor.rb b/www/status/monitor.rb
new file mode 100644
index 0000000..d3c92ef
--- /dev/null
+++ b/www/status/monitor.rb
@@ -0,0 +1,143 @@
+#
+# Overall monitor class is responsible for loading and running each
+# monitor in the `monitors` directory, collecting and normalizing the
+# results and outputting it as JSON.
+#
+
+require 'json'
+require 'time'
+
+class Monitor
+ # match http://getbootstrap.com/components/#alerts
+ LEVELS = %w(success info warning danger)
+
+ attr_reader :status
+
+ def initialize
+ status_file = File.expand_path('../status.json', __FILE__)
+ File.open(status_file, File::RDWR|File::CREAT, 0644) do |file|
+ # lock the file
+ mtime = File.exist?(status_file) ? File.mtime(status_file) : Time.at(0)
+ file.flock(File::LOCK_EX)
+
+ # fetch previous status
+ baseline = JSON.parse(file.read) rescue {}
+ baseline['data'] = {} unless baseline['data'].instance_of? Hash
+
+ # If status was updated while waiting for the lock, use the new status
+ if not File.exist?(status_file) or mtime != File.mtime(status_file)
+ @status = baseline
+ return
+ end
+
+ # invoke each monitor, collecting status from each
+ newstatus = {}
+ self.class.singleton_methods.sort.each do |method|
+ # invoke method to determine current status
+ begin
+ previous = baseline[method] || {mtime: Time.at(0).gmtime.iso8601}
+ status = Monitor.send(method, previous) || previous
+ rescue Exception => e
+ status = {
+ level: 'danger',
+ data: {
+ exception: {
+ level: 'danger',
+ text: e.inspect,
+ data: e.backtrace
+ }
+ }
+ }
+ end
+
+ # default mtime to now
+ status['mtime'] ||= Time.now.gmtime.iso8601 if status.instance_of? Hash
+
+ # update baseline
+ newstatus[method] = status
+ end
+
+ # normalize status
+ @status = normalize(data: newstatus)
+
+ # update results
+ file.rewind
+ file.write JSON.pretty_generate(@status)
+ file.flush
+ file.truncate(file.pos)
+ end
+ end
+
+ ISSUE_TYPE = {
+ 'success' => 'successes',
+ 'info' => 'updates',
+ 'warning' => 'warnings',
+ 'danger' => 'issues'
+ }
+
+ ISSUE_TYPE.default = 'problems'
+
+ # default fields, and propagate status 'upwards'
+ def normalize(status)
+ # convert symbols to strings
+ status.keys.each do |key|
+ status[key.to_s] = status.delete(key) if key.instance_of? Symbol
+ end
+
+ # normalize data
+ if status['data'].instance_of? Hash
+ # recursively normalize the data structure
+ status['data'].values.each {|value| normalize(value)}
+ elsif not status['data']
+ # default data
+ status['data'] = 'missing'
+ status['level'] ||= 'danger'
+ end
+
+ # normalize level (filling in title when this occurs)
+ if status['level']
+ if not LEVELS.include? status['level']
+ status['title'] ||= "invalid status: #{status['level'].inspect}"
+ status['level'] = 'danger'
+ end
+ else
+ if status['data'].instance_of? Hash
+ # find the values with the highest status level
+ highest = status['data'].
+ group_by {|key, value| LEVELS.index(value['level']) || 9}.max
+
+ # adopt that level
+ status['level'] = LEVELS[highest.first] || 'danger'
+
+ group = highest.last
+ if group.length > 1
+ # indicate the number of item with that status
+ status['title'] = "#{group.length} #{ISSUE_TYPE[status['level']]}"
+ else
+ # indicate the item with the problem
+ key, value = group.first
+ if value['title']
+ status['title'] ||= "#{key} #{value['title']}"
+ else
+ status['title'] ||= "#{key} #{value['data'].inspect}"
+ end
+ end
+ else
+ # default level
+ status['level'] ||= 'success'
+ end
+ end
+
+ status
+ end
+end
+
+# load the monitors
+Dir[File.expand_path('../monitors/*.rb', __FILE__)].each do |monitor|
+ require monitor
+end
+
+# for debugging purposes
+if __FILE__ == $0
+ puts JSON.pretty_generate(Monitor.new.status)
+end
diff --git a/www/status/monitors/svn.rb b/www/status/monitors/svn.rb
new file mode 100644
index 0000000..9fcf16b
--- /dev/null
+++ b/www/status/monitors/svn.rb
@@ -0,0 +1,40 @@
+#
+# Monitor status of svn updates
+#
+
+def Monitor.svn(previous_status)
+ # read cron log
+ log = File.expand_path('../../www/logs/svn-update')
+ updates = File.read(log).split("\n/srv/svn/")
+ updates.shift
+
+ status = {}
+
+ # extract status for each repository
+ updates.each do |update|
+ level = 'success'
+ data = update[/^At revision \d+\.$/]
+
+ lines = update.split("\n")
+ repository = lines.shift
+
+ lines.reject! {|line| line == "Updating '.':"}
+ lines.reject! {|line| line =~ /^At revision \d+\.$/}
+
+ unless lines.empty?
+ level = 'info'
+ data = lines.dup
+ end
+
+ lines.reject! {|line| line =~ /^[ADU] /}
+
+ unless lines.empty?
+ level = 'danger'
+ data = lines.dup
+ end
+
+ status[repository] = {level: level, data: data}
+ end
+
+ {data: status}
+end
