5 new commits in pytest: https://bitbucket.org/hpk42/pytest/commits/edc9d6f6e76a/ Changeset: edc9d6f6e76a Branch: refactor_LsofFdLeakChecker User: Marc Abramowitz Date: 2014-04-01 19:15:27 Summary: testing/conftest.py: Refactor lsof fd leak checking
Isolate the logic into one class to make easier to understand, more maintainable. This may aid in later plugging in an alternative implementation, such as one that uses psutil (https://bitbucket.org/hpk42/pytest/pull-request/137/use-psutil-to-detect-open-files-in-tests/diff) Affected #: 1 file diff -r d4b093bc36df977c017b43286953286542c788cf -r edc9d6f6e76ac45d9d1adadb710a239c030af28e testing/conftest.py --- a/testing/conftest.py +++ b/testing/conftest.py @@ -4,7 +4,23 @@ pytest_plugins = "pytester", import os, py -pid = os.getpid() + +class LsofFdLeakChecker(object): + def get_open_files(self): + out = self._exec_lsof() + open_files = self._parse_lsof_output(out) + return open_files + + def _exec_lsof(self): + pid = os.getpid() + return py.process.cmdexec("lsof -p %d" % pid) + + def _parse_lsof_output(self, out): + def isopen(line): + return ("REG" in line or "CHR" in line) and ( + "deleted" not in line and 'mem' not in line and "txt" not in line) + return [x for x in out.split("\n") if isopen(x)] + def pytest_addoption(parser): parser.addoption('--lsof', @@ -15,11 +31,12 @@ config._basedir = py.path.local() if config.getvalue("lsof"): try: - out = py.process.cmdexec("lsof -p %d" % pid) + config._fd_leak_checker = LsofFdLeakChecker() + config._openfiles = config._fd_leak_checker.get_open_files() except py.process.cmdexec.Error: pass else: - config._numfiles = len(getopenfiles(out)) + config._numfiles = len(config._openfiles) #def pytest_report_header(): # return "pid: %s" % os.getpid() @@ -31,8 +48,7 @@ return [x for x in out.split("\n") if isopen(x)] def check_open_files(config): - out2 = py.process.cmdexec("lsof -p %d" % pid) - lines2 = getopenfiles(out2) + lines2 = config._fd_leak_checker.get_open_files() if len(lines2) > config._numfiles + 3: error = [] error.append("***** %s FD leackage detected" % https://bitbucket.org/hpk42/pytest/commits/7a00688fef66/ Changeset: 7a00688fef66 Branch: refactor_LsofFdLeakChecker User: Marc Abramowitz Date: 2014-04-01 22:41:35 Summary: Improve LsofFdLeakChecker; more reliable and useful leak checking * Make it invoke lsof with options for machine-readable output * Parse out file descriptor and filename from lsof output * Draw attention to file descriptors now open that weren't open before Affected #: 1 file diff -r edc9d6f6e76ac45d9d1adadb710a239c030af28e -r 7a00688fef6649283f9316d1c0d0872c0af7bd71 testing/conftest.py --- a/testing/conftest.py +++ b/testing/conftest.py @@ -13,13 +13,24 @@ def _exec_lsof(self): pid = os.getpid() - return py.process.cmdexec("lsof -p %d" % pid) + return py.process.cmdexec("lsof -Ffn0 -p %d" % pid) def _parse_lsof_output(self, out): def isopen(line): - return ("REG" in line or "CHR" in line) and ( - "deleted" not in line and 'mem' not in line and "txt" not in line) - return [x for x in out.split("\n") if isopen(x)] + return line.startswith('f') and ( + "deleted" not in line and 'mem' not in line and "txt" not in line and 'cwd' not in line) + + open_files = [] + + for line in out.split("\n"): + if isopen(line): + fields = line.split('\0') + fd = int(fields[0][1:]) + filename = fields[1][1:] + if filename.startswith('/'): + open_files.append((fd, filename)) + + return open_files def pytest_addoption(parser): @@ -49,11 +60,16 @@ def check_open_files(config): lines2 = config._fd_leak_checker.get_open_files() - if len(lines2) > config._numfiles + 3: + new_fds = sorted(set([t[0] for t in lines2]) - set([t[0] for t in config._openfiles])) + open_files = [t for t in lines2 if t[0] in new_fds] + if open_files: error = [] - error.append("***** %s FD leackage detected" % - (len(lines2)-config._numfiles)) - error.extend(lines2) + error.append("***** %s FD leackage detected" % len(open_files)) + error.extend([str(f) for f in open_files]) + error.append("*** Before:") + error.extend([str(f) for f in config._openfiles]) + error.append("*** After:") + error.extend([str(f) for f in lines2]) error.append(error[0]) # update numfile so that the overall test run continuess config._numfiles = len(lines2) https://bitbucket.org/hpk42/pytest/commits/6751cb39a91f/ Changeset: 6751cb39a91f Branch: refactor_LsofFdLeakChecker User: Marc Abramowitz Date: 2014-04-01 23:13:11 Summary: testing/conftest.py: Reintialize config._openfiles for each test And no longer need getopenfiles or config._numfiles Affected #: 1 file diff -r 7a00688fef6649283f9316d1c0d0872c0af7bd71 -r 6751cb39a91f3fc7ed0c023db7fb9d4571ff1b6a testing/conftest.py --- a/testing/conftest.py +++ b/testing/conftest.py @@ -38,7 +38,8 @@ action="store_true", dest="lsof", default=False, help=("run FD checks if lsof is available")) -def pytest_configure(config): +def pytest_runtest_setup(item): + config = item.config config._basedir = py.path.local() if config.getvalue("lsof"): try: @@ -46,18 +47,10 @@ config._openfiles = config._fd_leak_checker.get_open_files() except py.process.cmdexec.Error: pass - else: - config._numfiles = len(config._openfiles) #def pytest_report_header(): # return "pid: %s" % os.getpid() -def getopenfiles(out): - def isopen(line): - return ("REG" in line or "CHR" in line) and ( - "deleted" not in line and 'mem' not in line and "txt" not in line) - return [x for x in out.split("\n") if isopen(x)] - def check_open_files(config): lines2 = config._fd_leak_checker.get_open_files() new_fds = sorted(set([t[0] for t in lines2]) - set([t[0] for t in config._openfiles])) @@ -71,13 +64,11 @@ error.append("*** After:") error.extend([str(f) for f in lines2]) error.append(error[0]) - # update numfile so that the overall test run continuess - config._numfiles = len(lines2) raise AssertionError("\n".join(error)) def pytest_runtest_teardown(item, __multicall__): item.config._basedir.chdir() - if hasattr(item.config, '_numfiles'): + if hasattr(item.config, '_openfiles'): x = __multicall__.execute() check_open_files(item.config) return x https://bitbucket.org/hpk42/pytest/commits/373fcb6b914d/ Changeset: 373fcb6b914d Branch: refactor_LsofFdLeakChecker User: Marc Abramowitz Date: 2014-04-02 00:36:54 Summary: Remove cast of fd to int and sorting Casting of fd can break for non-numeric fd (e.g.: "rtd" on Linux) and isn't necessary since we don't need to sort. Affected #: 1 file diff -r 6751cb39a91f3fc7ed0c023db7fb9d4571ff1b6a -r 373fcb6b914d6295c7b34ce33d464ab2aae27a9e testing/conftest.py --- a/testing/conftest.py +++ b/testing/conftest.py @@ -25,7 +25,7 @@ for line in out.split("\n"): if isopen(line): fields = line.split('\0') - fd = int(fields[0][1:]) + fd = fields[0][1:] filename = fields[1][1:] if filename.startswith('/'): open_files.append((fd, filename)) @@ -53,7 +53,7 @@ def check_open_files(config): lines2 = config._fd_leak_checker.get_open_files() - new_fds = sorted(set([t[0] for t in lines2]) - set([t[0] for t in config._openfiles])) + new_fds = set([t[0] for t in lines2]) - set([t[0] for t in config._openfiles]) open_files = [t for t in lines2 if t[0] in new_fds] if open_files: error = [] https://bitbucket.org/hpk42/pytest/commits/1cf062005b1c/ Changeset: 1cf062005b1c User: hpk42 Date: 2014-04-02 09:24:16 Summary: Merged in msabramo/pytest/refactor_LsofFdLeakChecker (pull request #138) testing/conftest.py: Refactor lsof fd leak checking Affected #: 1 file diff -r d966b6d87ac51aa38c68d73291fdfeb467dec4b9 -r 1cf062005b1c2743006cde11ee746b8dd53ea40f testing/conftest.py --- a/testing/conftest.py +++ b/testing/conftest.py @@ -4,48 +4,71 @@ pytest_plugins = "pytester", import os, py -pid = os.getpid() + +class LsofFdLeakChecker(object): + def get_open_files(self): + out = self._exec_lsof() + open_files = self._parse_lsof_output(out) + return open_files + + def _exec_lsof(self): + pid = os.getpid() + return py.process.cmdexec("lsof -Ffn0 -p %d" % pid) + + def _parse_lsof_output(self, out): + def isopen(line): + return line.startswith('f') and ( + "deleted" not in line and 'mem' not in line and "txt" not in line and 'cwd' not in line) + + open_files = [] + + for line in out.split("\n"): + if isopen(line): + fields = line.split('\0') + fd = fields[0][1:] + filename = fields[1][1:] + if filename.startswith('/'): + open_files.append((fd, filename)) + + return open_files + def pytest_addoption(parser): parser.addoption('--lsof', action="store_true", dest="lsof", default=False, help=("run FD checks if lsof is available")) -def pytest_configure(config): +def pytest_runtest_setup(item): + config = item.config config._basedir = py.path.local() if config.getvalue("lsof"): try: - out = py.process.cmdexec("lsof -p %d" % pid) + config._fd_leak_checker = LsofFdLeakChecker() + config._openfiles = config._fd_leak_checker.get_open_files() except py.process.cmdexec.Error: pass - else: - config._numfiles = len(getopenfiles(out)) #def pytest_report_header(): # return "pid: %s" % os.getpid() -def getopenfiles(out): - def isopen(line): - return ("REG" in line or "CHR" in line) and ( - "deleted" not in line and 'mem' not in line and "txt" not in line) - return [x for x in out.split("\n") if isopen(x)] - def check_open_files(config): - out2 = py.process.cmdexec("lsof -p %d" % pid) - lines2 = getopenfiles(out2) - if len(lines2) > config._numfiles + 3: + lines2 = config._fd_leak_checker.get_open_files() + new_fds = set([t[0] for t in lines2]) - set([t[0] for t in config._openfiles]) + open_files = [t for t in lines2 if t[0] in new_fds] + if open_files: error = [] - error.append("***** %s FD leackage detected" % - (len(lines2)-config._numfiles)) - error.extend(lines2) + error.append("***** %s FD leackage detected" % len(open_files)) + error.extend([str(f) for f in open_files]) + error.append("*** Before:") + error.extend([str(f) for f in config._openfiles]) + error.append("*** After:") + error.extend([str(f) for f in lines2]) error.append(error[0]) - # update numfile so that the overall test run continuess - config._numfiles = len(lines2) raise AssertionError("\n".join(error)) def pytest_runtest_teardown(item, __multicall__): item.config._basedir.chdir() - if hasattr(item.config, '_numfiles'): + if hasattr(item.config, '_openfiles'): x = __multicall__.execute() check_open_files(item.config) return x Repository URL: https://bitbucket.org/hpk42/pytest/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email. _______________________________________________ pytest-commit mailing list pytest-commit@python.org https://mail.python.org/mailman/listinfo/pytest-commit
