ArielGlenn has submitted this change and it was merged.
Change subject: allow for more than one mysql conf file, audit all of /a
......................................................................
allow for more than one mysql conf file, audit all of /a
Change-Id: Id4636e613a80eceeb3450f3fe21d3c83c909cb33
---
M dataretention/audit_files.py
1 file changed, 69 insertions(+), 51 deletions(-)
Approvals:
ArielGlenn: Verified; Looks good to me, approved
diff --git a/dataretention/audit_files.py b/dataretention/audit_files.py
index 7343ca8..91f4b1f 100644
--- a/dataretention/audit_files.py
+++ b/dataretention/audit_files.py
@@ -3047,14 +3047,14 @@
cf = {
'root_locations': ["/root"],
- 'logs_locations': ["/var/log", "/a/search", "/a/sqldata",
+ 'logs_locations': ["/var/log", "/a",
"/var/store"],
'homes_locations': ["/home", "/data/db20/home"],
'rotate_basedir': "/etc/logrotate.d",
'rotate_mainconf': "/etc/logrotate.conf",
- 'mysqlconf': "/etc/mysql/my.cnf",
+ 'mysqlconf': ["/etc/mysql/my.cnf", "/etc/my.cnf"],
# ignore these
'ignored_dirs': {
@@ -3103,7 +3103,13 @@
"request-tracker4/mason_data",
"salt", "samba", "smokeping/images", "svnusers", "yum"],
+ '/a':
+ ['mediawiki-config'],
+
'/a/sqldata':
+ ["*"],
+
+ '/a/sqldata-cache':
["*"],
'/srv/sqldata':
@@ -3146,6 +3152,7 @@
".rediscli_history", ".rnd", ".screenrc",
".selected_editor", ".sh_history",
"swift.conf", ".tcshrc",
+ "__tokudb_lock_dont_delete_me_*",
".toprc", ".tramp_history",
"twemproxy.conf", ".variables",
".vcl", ".viminfo", ".viminfo.tmp",
@@ -4535,59 +4542,70 @@
check how long mysql logs are kept around
'''
# note that I also see my.cnf.s3 and we don't check those (yet)
- try:
- contents = open(Config.cf['mysqlconf']).read()
- except:
- # file or directory probably doesn't exist
- return ''
- lines = contents.split('\n')
- for line in lines:
- line = line.strip()
- if not line:
+ output = ''
+ for filename in Config.cf['mysqlconf']:
+ found = False
+ try:
+ contents = open(filename).read()
+ except:
+ # file or directory probably doesn't exist
continue
- if line.startswith('datadir'):
- fields = line.split('=',1)
- fields = [field.strip() for field in fields]
- if fields[0] != 'datadir':
+ lines = contents.split('\n')
+ for line in lines:
+ line = line.strip()
+ if not line:
continue
- if not fields[1].startswith('/'):
- continue
- datadir = fields[1]
- # strip trailing slash if needed
- if len(datadir) > 1 and datadir.endswith('/'):
- datadir = datadir[:-1]
- # we can skip all bin logs, relay logs, and pid files in this
- # directory. anything else should get looked at.
- if '.' in self.hostname:
- hostname = self.hostname.split('.')[0]
- else:
- hostname = self.hostname
- ignore_these = [hostname + '-bin', hostname + '-relay-bin',
- hostname + '.pid', hostname + '-bin.index',
- hostname + '-relay-bin.index']
+ if line.startswith('datadir'):
+ fields = line.split('=',1)
+ fields = [field.strip() for field in fields]
+ if fields[0] != 'datadir':
+ continue
+ if not fields[1].startswith('/'):
+ continue
+ datadir = fields[1]
+ # strip trailing slash if needed
+ if len(datadir) > 1 and datadir.endswith('/'):
+ datadir = datadir[:-1]
+ # we can skip all bin logs, relay logs, and pid files in
this
+ # directory. anything else should get looked at.
+ if '.' in self.hostname:
+ hostname = self.hostname.split('.')[0]
+ else:
+ hostname = self.hostname
+ ignore_these = [hostname + '-bin', hostname + '-relay-bin',
+ hostname + '.pid', hostname + '-bin.index',
+ hostname + '-relay-bin.index']
- # add these files to ignore list; a one line report on
- # mysql log expiry configuration is sufficient
- if datadir not in self.ignored['files']:
- self.ignored['files'][datadir] = ignore_these
- else:
- self.ignored['files'][datadir].extend(ignore_these)
+ # add these files to ignore list; a one line report on
+ # mysql log expiry configuration is sufficient
+ if datadir not in self.ignored['files']:
+ self.ignored['files'][datadir] = ignore_these
+ else:
+ self.ignored['files'][datadir].extend(ignore_these)
+ # skip the subdirectories in here, they will be full of
mysql dbs
+ if datadir not in self.ignored['dirs']:
+ self.ignored['files'][datadir] = ['*']
+ else:
+ self.ignored['files'][datadir].append('*')
- if line.startswith('expire_logs_days'):
- fields = line.split('=',1)
- fields = [field.strip() for field in fields]
- if fields[0] != 'expire_logs_days':
- continue
- if not fields[1].isdigit():
- continue
- if int(fields[1]) > Config.cf['cutoff']/86400:
- return ('WARNING: mysql logs expired after %s days'
- % fields[1])
- else:
- return ''
- # if we don't find the entry, the default value is '0'
- # which means logs are never expired
- return 'WARNING: mysql logs never expired'
+ if line.startswith('expire_logs_days'):
+ fields = line.split('=',1)
+ fields = [field.strip() for field in fields]
+ if fields[0] != 'expire_logs_days':
+ continue
+ if not fields[1].isdigit():
+ continue
+ found = True
+ if int(fields[1]) > Config.cf['cutoff']/86400:
+ if output:
+ output = output + '\n'
+ output = output + ('WARNING: some mysql logs expired
after %s days in %s'
+ % (fields[1], filename))
+ if not found:
+ if output:
+ output = output + '\n'
+ output = output + 'WARNING: some mysql logs never expired in '
+ filename
+ return(output)
def do_local_audit(self):
'''
--
To view, visit https://gerrit.wikimedia.org/r/233447
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id4636e613a80eceeb3450f3fe21d3c83c909cb33
Gerrit-PatchSet: 2
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
