ArielGlenn has submitted this change and it was merged.
Change subject: last 'executor' code removed, all salt module now
......................................................................
last 'executor' code removed, all salt module now
Change-Id: I5700057c71b372f9d4afd39376dac385b38ac3d6
---
M dataretention/data_auditor.py
M dataretention/retention/cli.py
M dataretention/retention/retentionaudit.py
A dataretention/retention/userconfretriever.py
4 files changed, 109 insertions(+), 68 deletions(-)
Approvals:
ArielGlenn: Verified; Looks good to me, approved
diff --git a/dataretention/data_auditor.py b/dataretention/data_auditor.py
index 07d23a5..9d7c343 100644
--- a/dataretention/data_auditor.py
+++ b/dataretention/data_auditor.py
@@ -8,6 +8,7 @@
from retention.remotelogauditor import RemoteLogsAuditor
from retention.remotehomeauditor import RemoteHomesAuditor
from retention.examiner import RemoteFileExaminer, RemoteDirExaminer
+from retention.userconfretriever import RemoteUserCfRetriever
def usage(message=None):
if message:
@@ -89,6 +90,10 @@
linecount (-l) -- number of lines of file content to be displayed
(efault: 1)
+ userconf (-u) -- instead of an audit, display the per user config
+ files on the given hosts; this may not be specified
+ with 'audit'.
+
For 'logs' audit type:
system (-S) -- show system logs (e.g. syslog, messages) along with
@@ -111,6 +116,7 @@
verbose = False
ignore_also = None
dir_info = None
+ getuserconfs = False
batchno = 1
file_info = None
linecount = 1
@@ -125,7 +131,7 @@
try:
(options, remainder) = getopt.gnu_getopt(
- sys.argv[1:], "a:b:d:Df:F:l:i:Ie:m:oprsSt:T:vh",
+ sys.argv[1:], "a:b:d:Df:F:l:i:Ie:m:oprsSt:T:uvh",
["audit=", "files=",
"filecontents=", "linecount=",
"ignore=",
@@ -134,7 +140,8 @@
"oldest", "prettyprint", "report",
"dirsizes", "examine", "batchno",
"sample", "system",
- "target=", "timeout=", "verbose", "help"])
+ "target=", "timeout=",
+ "userconf", "verbose", "help"])
except getopt.GetoptError as err:
usage("Unknown option specified: " + str(err))
@@ -186,6 +193,8 @@
if not val.isdigit():
usage("timeout must be a number")
timeout = int(val)
+ elif opt in ["-u", "--userconf"]:
+ getuserconfs = True
elif opt in ["-h", "--help"]:
usage()
elif opt in ["-v", "--verbose"]:
@@ -199,12 +208,12 @@
if hosts_expr is None:
usage("Mandatory target argument not specified")
- count = len(filter(None, [audit_type, dir_info, file_info]))
+ count = len(filter(None, [audit_type, dir_info, file_info, getuserconfs]))
if count == 0:
- usage("One of 'audit', 'examine' "
+ usage("One of 'audit', 'examine', 'userconf' "
"or 'filecontents' must be specified")
elif count > 1:
- usage("Only one of 'audit', 'examine' "
+ usage("Only one of 'audit', 'examine' 'userconf' "
"or 'filecontents' may be specified")
if dir_info is not None:
@@ -216,6 +225,10 @@
fileexam = RemoteFileExaminer(file_info, hosts_expr, linecount,
timeout)
fileexam.run()
sys.exit(0)
+ elif getuserconfs:
+ getconfs = RemoteUserCfRetriever(hosts_expr, timeout, 'homes')
+ getconfs.run()
+ sys.exit(0)
if audit_type not in ['root', 'logs', 'homes']:
usage("audit type must be one of 'root', 'logs', 'homes'")
diff --git a/dataretention/retention/cli.py b/dataretention/retention/cli.py
index 617b370..5fb1848 100644
--- a/dataretention/retention/cli.py
+++ b/dataretention/retention/cli.py
@@ -6,12 +6,11 @@
import readline
import atexit
import traceback
-import salt.client
sys.path.append('/srv/audits/retention/scripts/')
from retention.status import Status
-from retention.rule import Rule, RuleStore
+from retention.rule import RuleStore
import retention.remotefileauditor
from retention.localhomeaudit import LocalHomesAuditor
from retention.locallogaudit import LocalLogsAuditor
@@ -22,66 +21,7 @@
from retention.examiner import RemoteDirExaminer, RemoteFileExaminer
import retention.fileutils
import retention.ruleutils
-
-class LocalIgnores(object):
- '''
- retrieval and display dirs / files listed as to
- be ignored in per-user lists on local host
- '''
- def __init__(self, host, timeout, audit_type):
- self.host = host
- self.timeout = timeout
- self.audit_type = audit_type
- self.locations = audit_type + "_locations"
-
- def run(self, quiet=False):
- '''
- do all the work
-
- note that 'quiet' applies only to remotely
- run, and the same is true for returning the contents.
- maybe we want to fix that
- '''
-
- local_ignores = {}
-
- if retention.utils.running_locally(self.host):
- local_ignores = LocalHomesAuditor.get_local_ignores(self.locations)
- output = json.dumps(local_ignores)
- print output
- else:
- client = salt.client.LocalClient()
- code = "# -*- coding: utf-8 -*-\n"
- code += self.generate_executor()
- with open(__file__, 'r') as fp_:
- code += fp_.read()
- result = client.cmd([self.host], "cmd.exec_code",
- ["python2", code],
- expr_form='list',
- timeout=self.timeout)
- if self.host in result:
- input = result[self.host]
- try:
- local_ignores = json.loads(
- input, object_hook=JsonHelper.decode_dict)
- except:
- print "WARNING: failed to get local ignores on host",
- print self.host,
- print "got this:", input
- local_ignores = {}
-
- if not quiet:
- print local_ignores
-
- return local_ignores
-
- def generate_executor(self):
- code = """
-def executor():
- de = LocalIgnores('localhost', %d, '%s')
- de.run()
-""" % (self.timeout, self.audit_type)
- return code
+from retention.userconfretriever import RemoteUserCfRetriever
class CommandLine(object):
@@ -420,7 +360,7 @@
print "exiting at user request"
break
else:
- local_ign = LocalIgnores(host_todo, self.timeout,
self.audit_type)
+ local_ign = RemoteUserCfRetriever(host_todo, self.timeout,
self.audit_type)
self.local_ignores = local_ign.run(True)
local_ignored_dirs, local_ignored_files =
LocalHomesAuditor.process_local_ignores(
self.local_ignores, self.ignored)
diff --git a/dataretention/retention/retentionaudit.py
b/dataretention/retention/retentionaudit.py
index c3a5ae3..8a30f05 100644
--- a/dataretention/retention/retentionaudit.py
+++ b/dataretention/retention/retentionaudit.py
@@ -8,6 +8,7 @@
from retention.locallogaudit import LocalLogsAuditor
from retention.localhomeaudit import LocalHomesAuditor
from retention.examiner import LocalFileExaminer, LocalDirExaminer
+from retention.userconfretriever import LocalUserCfRetriever
log = logging.getLogger(__name__)
@@ -58,3 +59,7 @@
result = dexaminer.run()
return result
+def retrieve_usercfs(timeout, audit_type):
+ ucfsretriever = LocalUserCfRetriever(timeout, audit_type)
+ result = ucfsretriever.run()
+ return result
diff --git a/dataretention/retention/userconfretriever.py
b/dataretention/retention/userconfretriever.py
new file mode 100644
index 0000000..545f934
--- /dev/null
+++ b/dataretention/retention/userconfretriever.py
@@ -0,0 +1,83 @@
+import sys
+import json
+import salt.client
+
+sys.path.append('/srv/audits/retention/scripts/')
+
+import retention.remotefileauditor
+from retention.localhomeaudit import LocalHomesAuditor
+import retention.utils
+from retention.utils import JsonHelper
+import retention.fileutils
+import retention.ruleutils
+
+class RemoteUserCfRetriever(object):
+ '''
+ retrieval and display dirs / files listed as to
+ be ignored in per-user lists on remote host
+ '''
+ def __init__(self, host, timeout, audit_type):
+ self.host = host
+ self.timeout = timeout
+ self.audit_type = audit_type
+ self.locations = audit_type + "_locations"
+
+ def run(self, quiet=False):
+ '''
+ do all the work
+
+ note that 'quiet' applies only to remotely
+ run, and the same is true for returning the contents.
+ maybe we want to fix that
+ '''
+
+ local_ignores = {}
+
+ client = salt.client.LocalClient()
+ module_args = [self.timeout, self.audit_type]
+
+ result = client.cmd([self.host], "retentionaudit.retrieve_usercfs",
+ module_args, expr_form='list',
+ timeout=self.timeout)
+
+ if self.host in result:
+ input = result[self.host]
+ try:
+ local_ignores = json.loads(
+ input, object_hook=JsonHelper.decode_dict)
+ except:
+ print "WARNING: failed to get local ignores on host",
+ print self.host,
+ print "got this:", input
+ local_ignores = {}
+
+ if not quiet:
+ print local_ignores
+
+ return local_ignores
+
+class LocalUserCfRetriever(object):
+ '''
+ retrieval and display dirs / files listed as to
+ be ignored in per-user lists on local host
+ '''
+ def __init__(self, timeout, audit_type='homes'):
+ self.timeout = timeout
+ self.audit_type = audit_type
+ self.locations = audit_type + "_locations"
+
+ def run(self, quiet=False):
+ '''
+ do all the work
+
+ note that 'quiet' applies only to remotely
+ run, and the same is true for returning the contents.
+ maybe we want to fix that
+ '''
+
+ local_ignores = {}
+
+ local_ignores = LocalHomesAuditor.get_local_ignores(self.locations)
+ output = json.dumps(local_ignores)
+ print output
+ return output
--
To view, visit https://gerrit.wikimedia.org/r/233461
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5700057c71b372f9d4afd39376dac385b38ac3d6
Gerrit-PatchSet: 2
Gerrit-Project: operations/software
Gerrit-Branch: master
Gerrit-Owner: ArielGlenn <[email protected]>
Gerrit-Reviewer: ArielGlenn <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
