Ayounsi has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/363296 )
Change subject: Add diffscan module.
......................................................................
Add diffscan module.
Bug: T169624
Change-Id: I19cf84c6e1a1a33af201ddef61eeba47c04cb949
---
M hieradata/labs.yaml
A modules/diffscan/files/diffscanpy
A modules/diffscan/manifests/init.pp
A modules/diffscan/templates/targets.txt.erb
A modules/profile/manifests/diffscan.pp
5 files changed, 570 insertions(+), 0 deletions(-)
Approvals:
Muehlenhoff: Looks good to me, but someone else must approve
jenkins-bot: Verified
Ayounsi: Looks good to me, approved
diff --git a/hieradata/labs.yaml b/hieradata/labs.yaml
index 377d824..a30b000 100644
--- a/hieradata/labs.yaml
+++ b/hieradata/labs.yaml
@@ -116,3 +116,11 @@
spice_hostname: 'labspice.wikimedia.org'
scheduler_pool:
- labvirt1001
+
+profile::diffscan::ipranges:
+ - 185.15.56.0/22
+ - 91.198.174.0/24
+ - 198.35.26.0/23
+ - 208.80.152.0/22
+profile::diffscan::emailto: [email protected]
+profile::diffscan::groupname: Labs-to-public-v4
diff --git a/modules/diffscan/files/diffscanpy
b/modules/diffscan/files/diffscanpy
new file mode 100644
index 0000000..a35ae71
--- /dev/null
+++ b/modules/diffscan/files/diffscanpy
@@ -0,0 +1,479 @@
+#!/usr/bin/python2
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+# Copyright (c) 2015 Mozilla Corporation
+# Author: [email protected]
+
+import sys
+import os
+import fcntl
+from string import Template
+import getopt
+import re
+import time
+import subprocess
+import tempfile
+import cPickle
+import errno
+import tempfile
+import shutil
+import calendar
+
+# Edit the nmap_scanoptions variable below to configure generic options
+# that are passed by nmap to the script. This script generates email using
+# the sendmail command, so also ensure that command is in your path when
+# it is run.
+
+# Change this to suit your environment
+#
+# Be sure to include -vv so hosts that are down are reported in the
+# output for correct tracking.
+nmap_scanoptions = '-vv -sS -PE -PS22,25,80,443,3306,8443,9100 -T4 ' + \
+ '--privileged'
+
+nmap_topports = Template('--top-ports $topports')
+nmap_logoptions = Template('-oG $tmppath')
+nmap_inoptions = Template('-iL $inpath')
+nmap_portspec = Template('-p $portspec')
+
+append_path = ':/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:' + \
+ '/usr/local/sbin:/usr/lib'
+
+class ScanData(object):
+ def __init__(self):
+ self.scantime = time.gmtime()
+ self.hosts = {}
+ self.dnsmap = {}
+ self.uphosts = []
+ self.downhosts = []
+
+ def get_hosts(self):
+ return self.hosts.keys()
+
+ def get_host_ports(self, h):
+ return self.hosts[h]
+
+ def open_exists(self, addr, port, proto):
+ if addr not in self.hosts:
+ return False
+ cand = [port, proto]
+ if cand not in self.hosts[addr]:
+ return False
+ return True
+
+ def total_services(self):
+ ret = 0
+ for i in self.hosts:
+ ret += len(self.hosts[i])
+ return ret
+
+ def add_open(self, addr, port, proto, hn):
+ if proto != 'tcp' and proto != 'udp':
+ raise Exception('unknown protocol %s' % proto)
+ if addr not in self.hosts:
+ self.hosts[addr] = []
+ self.dnsmap[addr] = hn
+ self.hosts[addr].append([int(port), proto])
+
+class Alert(object):
+ def __init__(self, host, port, proto, dns, open_prev, closed_prev,
+ statstr):
+ self.host = host
+ self.port = port
+ self.proto = proto
+ self.dns = dns
+ self.open_prev = open_prev
+ self.closed_prev = closed_prev
+ self.statstr = statstr
+
+ @staticmethod
+ def alert_header():
+ return 'STATUS HOST PORT PROTO OPREV CPREV DNS'
+
+ def __str__(self):
+ return '%s %s %s %s %s %s %s' % (self.statstr, self.host,
+ str(self.port), self.proto,
+ str(self.open_prev), str(self.closed_prev),
+ self.dns)
+
+class ScanState(object):
+ KEEP_SCANS = 7
+
+ def __init__(self):
+ self._lastscan = None
+ self._scanlist = []
+ self._alerts_open = []
+ self._alerts_closed = []
+ self._outfile = None
+
+ def up_trend(self):
+ ret = ''
+ for i in self._scanlist:
+ if len(ret) == 0:
+ ret = '%d' % len(i.uphosts)
+ else:
+ ret += ',%d' % len(i.uphosts)
+ return ret
+
+ def down_trend(self):
+ ret = ''
+ for i in self._scanlist:
+ if len(ret) == 0:
+ ret = '%d' % len(i.downhosts)
+ else:
+ ret += ',%d' % len(i.downhosts)
+ return ret
+
+ def register_outfile(self, o):
+ self._outfile = o
+
+ def clear_outfile(self):
+ self._outfile = None
+
+ def clear_alerts(self):
+ self._alerts_open = []
+ self._alerts_closed = []
+
+ def last_scan_total_services(self):
+ return self._lastscan.total_services()
+
+ def previous_scan_total_services(self):
+ if len(self._scanlist) > 1:
+ return self._scanlist[1].total_services()
+ return 0
+
+ def set_last(self, last):
+ self._lastscan = last
+ if len(self._scanlist) == self.KEEP_SCANS:
+ self._scanlist.pop()
+ self._scanlist.insert(0, last)
+ self.clear_alerts()
+
+ def calculate(self):
+ self.calculate_new_open()
+ self.calculate_new_closed()
+
+ def prev_service_status(self, addr, port, proto):
+ openprev = 0
+ closedprev = 0
+ if len(self._scanlist) <= 1:
+ return (0, 0)
+ for s in self._scanlist[1:]:
+ if s.open_exists(addr, port, proto):
+ openprev += 1
+ else:
+ closedprev += 1
+ return (openprev, closedprev)
+
+ def calculate_new_open(self):
+ if len(self._scanlist) <= 1:
+ return
+ for i in self._lastscan.get_hosts():
+ for p in self._lastscan.get_host_ports(i):
+ prevscan = self._scanlist[1]
+ if not prevscan.open_exists(i, p[0], p[1]):
+ statstr = 'OPEN'
+ dns = self._lastscan.dnsmap[i]
+ # If this host isn't in the previous up or down list,
+ # note it as a new host
+ if (i not in prevscan.uphosts) and \
+ (i not in prevscan.downhosts):
+ statstr = 'OPENNEWHOST'
+ openprev, closedprev = \
+ self.prev_service_status(i, p[0], p[1])
+ self._alerts_open.append(Alert(i, p[0], p[1], dns,
+ openprev, closedprev, statstr))
+
+ def calculate_new_closed(self):
+ if len(self._scanlist) <= 1:
+ return
+ prevscan = self._scanlist[1]
+ for i in prevscan.get_hosts():
+ for p in prevscan.get_host_ports(i):
+ if not self._lastscan.open_exists(i, p[0], p[1]):
+ statstr = 'CLOSED'
+ # See if the host existed in the current scan, if it did
+ # use that hostname, otherwise grab previous
+ if i in self._lastscan.dnsmap:
+ dns = self._lastscan.dnsmap[i]
+ else:
+ # If we didn't have a dns map entry for it, that means
+ # the host wasn't even up, note this in the status
+ statstr = 'CLOSEDDOWN'
+ dns = prevscan.dnsmap[i]
+ openprev, closedprev = \
+ self.prev_service_status(i, p[0], p[1])
+ self._alerts_closed.append(Alert(i, p[0], p[1], dns,
+ openprev, closedprev, statstr))
+
+ def print_open_alerts(self):
+ self._outfile.write('%s\n' % Alert.alert_header())
+ for i in self._alerts_open:
+ self._outfile.write('%s\n' % str(i))
+
+ def print_closed_alerts(self):
+ self._outfile.write('%s\n' % Alert.alert_header())
+ for i in self._alerts_closed:
+ self._outfile.write('%s\n' % str(i))
+
+lockfile = None
+state = None
+tmpfile = None
+debugging = False
+myhost = None
+recip = None
+groupname = None
+topports = 2000
+portspec = None
+nosmtp = False
+
+statefile = './diffscan.state'
+outdir = './diffscan_out'
+
+def outdir_setup():
+ if not os.path.isdir(outdir):
+ os.mkdir(outdir, 0755)
+ if not os.access(outdir, os.W_OK):
+ sys.stderr.write('%s not writable\n' % outdir)
+ sys.exit(1)
+
+def copy_nmap_out(p):
+ tval = int(calendar.timegm(time.gmtime()))
+ pidval = os.getpid()
+ fname = os.path.join(outdir, 'nmap-%d-%d.out' % (tval, pidval))
+ shutil.copyfile(p, fname)
+
+def load_scanstate():
+ try:
+ f = open(statefile, 'r')
+ except IOError as e:
+ if e.errno == errno.ENOENT:
+ return ScanState()
+ else:
+ raise
+ ret = cPickle.load(f)
+ f.close()
+ return ret
+
+def write_scanstate():
+ f = open(statefile, 'w')
+ cPickle.dump(state, f)
+ f.close()
+
+def parse_output(path):
+ new = ScanData()
+
+ f = open(path, 'r')
+ while True:
+ buf = f.readline()
+ if buf == None:
+ break
+ if buf == '':
+ break
+ buf = buf.strip()
+ m = re.search('Host: (\S+) \(([^)]*)\).*Status: Up', buf)
+ if m != None:
+ addr = m.group(1)
+ new.uphosts.append(addr)
+ m = re.search('Host: (\S+) \(([^)]*)\).*Status: Down', buf)
+ if m != None:
+ addr = m.group(1)
+ new.downhosts.append(addr)
+ m = re.search('Host: (\S+) \(([^)]*)\).*Ports: (.*)$', buf)
+ if m != None:
+ addr = m.group(1)
+ hn = m.group(2)
+ if len(hn) == 0:
+ hn = 'unknown'
+ p = [x.split('/') for x in m.group(3).split(',')]
+ for i in p:
+ if i[1] != 'open':
+ continue
+ new.add_open(addr.strip(), i[0].strip(), i[2].strip(), hn)
+ f.close()
+
+ state.set_last(new)
+
+def diffscan_fail_notify(errmsg):
+ if nosmtp:
+ return
+ buf = 'Subject: diffscan2 %s %s\n' % (groupname, myhost)
+ buf += 'From: diffscan2 <noreply@%s>\n' % myhost
+ buf += 'To: %s\n' % ','.join(recip)
+ buf += '\n'
+ buf += 'diffscan execution failed\n\n'
+ buf += '%s\n' % errmsg
+ sp = subprocess.Popen(['sendmail', '-t'], stdin=subprocess.PIPE)
+ sp.communicate(buf)
+
+def run_nmap(targets):
+ nmap_args = []
+ nmap_args += nmap_scanoptions.split()
+
+ tf = tempfile.mkstemp()
+ os.close(tf[0])
+ if portspec != None:
+ nmap_args += nmap_portspec.substitute(portspec=portspec).split()
+ else:
+ nmap_args += nmap_topports.substitute(topports=topports).split()
+ nmap_args += nmap_logoptions.substitute(tmppath=tf[1]).split()
+ nmap_args += nmap_inoptions.substitute(inpath=targets).split()
+
+ nfd = open('/dev/null', 'w')
+ try:
+ ret = subprocess.call(['nmap',] + nmap_args, stdout=nfd)
+ except Exception as e:
+ os.remove(tf[1])
+ diffscan_fail_notify('executing of nmap failed, %s' % str(e))
+ return False
+ nfd.close()
+
+ if ret != 0:
+ os.remove(tf[1])
+ diffscan_fail_notify('nmap failed with return code %d, exiting' % ret)
+ return False
+
+ parse_output(tf[1])
+
+ copy_nmap_out(tf[1])
+ os.remove(tf[1])
+ return True
+
+def usage():
+ sys.stdout.write('usage: diffscan.py [options] targets_file' \
+ ' recipients groupname\n\n' \
+ 'options:\n\n' \
+ '\t-h\t\tusage information\n' \
+ '\t-m num\t\ttop ports to scan (2000, see nmap --top-ports)\n' \
+ '\t-n\t\tno smtp, write output to stdout (recipient ignored)\n' \
+ '\t-o path\t\tdirectory to save nmap output (./diffscan_out)\n' \
+ '\t-p spec\t\tinstead of top ports use port spec (see nmap -p)\n' \
+ '\t-s path\t\tpath to state file (./diffscan.state)\n\n')
+ sys.exit(0)
+
+def create_lock():
+ global lockfile
+
+ lfname = statefile + '.lock'
+ lockfile = open(lfname, 'w')
+ fcntl.lockf(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ lockfile.write(str(os.getpid()))
+ lockfile.flush()
+
+def release_lock():
+ lfname = statefile + '.lock'
+
+ lockfile.close()
+ os.remove(lfname)
+
+def domain():
+ global statefile
+ global state
+ global outdir
+ global tmpfile
+ global debugging
+ global myhost
+ global recip
+ global groupname
+ global topports
+ global portspec
+ global nosmtp
+
+ os.environ['PATH'] = os.environ['PATH'] + append_path
+
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], 'dhm:no:p:s:')
+ except getopt.GetoptError:
+ usage()
+ for o, a in opts:
+ if o == '-h':
+ usage()
+ elif o == '-o':
+ outdir = a
+ elif o == '-p':
+ portspec = a
+ elif o == '-d':
+ debugging = True
+ elif o == '-m':
+ topports = a
+ elif o == '-n':
+ nosmtp = True
+ elif o == '-s':
+ statefile = a
+ if len(args) < 3:
+ usage()
+ targetfile = args[0]
+ recip = args[1].split(',')
+ groupname = args[2]
+
+ outdir_setup()
+
+ create_lock()
+
+ state = load_scanstate()
+
+ if not nosmtp:
+ tmpout = tempfile.mkstemp()
+ tmpfile = os.fdopen(tmpout[0], 'w')
+ else:
+ tmpfile = sys.stdout
+ state.register_outfile(tmpfile)
+
+ myhost = os.uname()[1]
+ tmpfile.write('Subject: diffscan2 %s %s\n' % (groupname, myhost))
+ tmpfile.write('From: diffscan2 <noreply@%s>\n' % myhost)
+ tmpfile.write('To: %s\n' % ','.join(recip))
+ tmpfile.write('\n')
+
+ tmpfile.write('diffscan2 results output\n\n')
+
+ if not run_nmap(targetfile):
+ if not nosmtp:
+ tmpfile.close()
+ os.remove(tmpout[1])
+ sys.exit(1)
+ state.calculate()
+ tmpfile.write('New Open Service List\n')
+ tmpfile.write('---------------------\n')
+ state.print_open_alerts()
+ tmpfile.write('\n')
+ tmpfile.write('New Closed Service List\n')
+ tmpfile.write('---------------------\n')
+ state.print_closed_alerts()
+
+ tmpfile.write('\n')
+ tmpfile.write('OPREV: number of times service was open in previous ' \
+ 'scans\n')
+ tmpfile.write('CPREV: number of times service was closed in ' \
+ 'previous scans\n')
+ tmpfile.write('maximum previous scans stored: %d\n' % state.KEEP_SCANS)
+ tmpfile.write('current total services: %d\n' % \
+ state.last_scan_total_services())
+ tmpfile.write('previous total services: %d\n' % \
+ state.previous_scan_total_services())
+ tmpfile.write('up trend: %s\n' % state.up_trend())
+ tmpfile.write('down trend: %s\n' % state.down_trend())
+
+ state.clear_outfile()
+ write_scanstate()
+
+ if not nosmtp:
+ tmpfile.close()
+
+ f = open(tmpout[1], 'r')
+ buf = f.read()
+ f.close()
+ if debugging:
+ sys.stdout.write(buf)
+ sp = subprocess.Popen(['sendmail', '-t'], stdin=subprocess.PIPE)
+ sp.communicate(buf)
+ os.remove(tmpout[1])
+
+ release_lock()
+
+if __name__ == '__main__':
+ domain()
+
+sys.exit(0)
diff --git a/modules/diffscan/manifests/init.pp
b/modules/diffscan/manifests/init.pp
new file mode 100644
index 0000000..fa5aece
--- /dev/null
+++ b/modules/diffscan/manifests/init.pp
@@ -0,0 +1,51 @@
+# == Class: diffscan
+#
+# This class installs & manages diffscan,
+# an nmap wrapper for differential port scans.
+# See https://github.com/ameihm0912/diffscan2
+#
+# == Parameters
+#
+# [*ipranges*]
+# The list of IP/masks to scan. See nmap doc for accepted formats.
+#
+# [*emailto*]
+# Diff emails recipient. Defaults to "root".
+#
+# [*groupname*]
+# An identifier to distinguish between several instances.
+# Defaults to "diffscan".
+#
+class diffscan(
+ $ipranges={},
+ $emailto='[email protected]',
+ $groupname='diffscan'
+) {
+ file { '/srv/diffscan':
+ ensure => 'directory',
+ owner => 'root',
+ group => 'root',
+ mode => '0775',
+ }
+ file { "/srv/diffscan/targets-${groupname}.txt":
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ content => template('diffscan/targets.txt.erb'),
+ }
+ file { '/srv/diffscan/diffscan.py':
+ ensure => present,
+ owner => 'root',
+ group => 'root',
+ mode => '0554',
+ source => 'puppet:///modules/diffscan/diffscanpy',
+ }
+ cron { "diffscan-${groupname}":
+ ensure => present,
+ user => 'root', # nmap needs root privileges
+ command => "/srv/diffscan/diffscan.py targets-${groupname}.txt
${emailto} ${groupname}",
+ hour => '0',
+ }
+
+}
diff --git a/modules/diffscan/templates/targets.txt.erb
b/modules/diffscan/templates/targets.txt.erb
new file mode 100644
index 0000000..cec63fa
--- /dev/null
+++ b/modules/diffscan/templates/targets.txt.erb
@@ -0,0 +1,3 @@
+<% @ipranges.each do |iprange| %>
+<%= iprange %>
+<% end %>
diff --git a/modules/profile/manifests/diffscan.pp
b/modules/profile/manifests/diffscan.pp
new file mode 100644
index 0000000..e9fb6fa
--- /dev/null
+++ b/modules/profile/manifests/diffscan.pp
@@ -0,0 +1,29 @@
+# == Class: diffscan
+#
+# This class installs & manages diffscan,
+# an nmap wrapper for differential port scans.
+# See https://github.com/ameihm0912/diffscan2
+#
+# == Parameters
+#
+# [*ipranges*]
+# The list of IP/masks to scan. See nmap doc for accepted formats.
+#
+# [*emailto*]
+# Diff emails recipient. Defaults to "root".
+#
+# [*groupname*]
+# An identifier to distinguish between several instances.
+# Defaults to "diffscan".
+#
+class profile::diffscan(
+ $ipranges=hiera('profile::diffscan::ipranges'),
+ $emailto=hiera('profile::diffscan::emailto'),
+ $groupname=hiera('profile::diffscan::groupname'),
+) {
+ class { '::diffscan':
+ ipranges => $ipranges,
+ emailto => $emailto,
+ groupname => $groupname,
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/363296
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I19cf84c6e1a1a33af201ddef61eeba47c04cb949
Gerrit-PatchSet: 6
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ayounsi <[email protected]>
Gerrit-Reviewer: Ayounsi <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Muehlenhoff <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
