This is an automated email from the git hooks/post-receive script. tille pushed a commit to branch master in repository python-qcli.
commit 04d1d1dcab5fc7928dd790bdc59c7dd3ca7de0c6 Author: Andreas Tille <[email protected]> Date: Wed Oct 18 15:35:16 2017 +0200 New upstream version 0.1.1 --- PKG-INFO | 10 ++ debian/changelog | 15 -- debian/compat | 1 - debian/control | 23 ---- debian/copyright | 45 ------ debian/rules | 33 ----- debian/source/format | 1 - debian/watch | 2 - qcli/__init__.py | 23 ++++ qcli/option_parsing.py | 350 +++++++++++++++++++++++++++++++++++++++++++++++ qcli/test.py | 178 ++++++++++++++++++++++++ qcli/util.py | 57 ++++++++ scripts/qcli_make_rst | 243 ++++++++++++++++++++++++++++++++ scripts/qcli_make_script | 129 +++++++++++++++++ setup.py | 18 +++ 15 files changed, 1008 insertions(+), 120 deletions(-) diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..536a5f4 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: qcli +Version: 0.1.1 +Summary: UNKNOWN +Home-page: UNKNOWN +Author: UNKNOWN +Author-email: UNKNOWN +License: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index eb5232f..0000000 --- a/debian/changelog +++ /dev/null @@ -1,15 +0,0 @@ -python-qcli (0.1.1-1) unstable; urgency=medium - - * New upstream version - * Fixed watch file - * cme fix dpkg-control - * debhelper 10 - * use --buildsystem=pybuild - - -- Andreas Tille <[email protected]> Wed, 07 Dec 2016 12:23:23 +0100 - -python-qcli (0.1.0-1) unstable; urgency=low - - * Initial release (Closes: #733135) - - -- Andreas Tille <[email protected]> Thu, 26 Dec 2013 07:55:53 +0100 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index f599e28..0000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/debian/control b/debian/control deleted file mode 100644 index 685ae00..0000000 --- a/debian/control +++ /dev/null @@ -1,23 +0,0 @@ -Source: python-qcli -Maintainer: Debian Med Packaging Team <[email protected]> -Uploaders: Andreas Tille <[email protected]> -Section: python -Priority: optional -Build-Depends: debhelper (>= 10), - dh-python, - python-all-dev, - python-nose -Standards-Version: 3.9.8 -Vcs-Browser: https://anonscm.debian.org/viewvc/debian-med/trunk/packages/python-qcli/trunk/ -Vcs-Svn: svn://anonscm.debian.org/debian-med/trunk/packages/python-qcli/trunk/ -Homepage: https://pypi.python.org/pypi/qcli/0.1.0 -X-Python-Version: >= 2.6 - -Package: python-qcli -Architecture: any -Depends: ${shlibs:Depends}, - ${python:Depends}, - ${misc:Depends} -Description: separated module of pyqi needed for QIIME package - The qiime package needs this as new dependency which is not part of the - main pyqi package. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 1714ff6..0000000 --- a/debian/copyright +++ /dev/null @@ -1,45 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Contact: Greg Caporaso <[email protected]> -Source: https://pypi.python.org/pypi/qcli - -Files: * -Copyright: 2013-2014 Greg Caporaso <[email protected]> -License: BSDlike -Comment: Upstream has confirmed the license online - Since there is no explicit copy of the licensing conditions upstream - was asked for explicite permission to distribute. This was given on - the Debian Med mailing list here: - . - https://lists.alioth.debian.org/pipermail/debian-med-packaging/2013-December/024166.html - . - The package is only temporary needed for QIIME 1.8 and in the next - major upstream release of QIIME it will be replaced by pyqi which - has an explicite licensing statement. - -Files: debian/* -Copyright: 2013-2016 Andreas Tille <[email protected]> -License: BSDlike - -License: BSDlike - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name BiPy nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - . - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE BIPY DEVELOPMENT TEAM BE LIABLE FOR ANY - DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/debian/rules b/debian/rules deleted file mode 100755 index ae40d9d..0000000 --- a/debian/rules +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/make -f - -DH_VERBOSE := 1 - -pkg := $(shell dpkg-parsechangelog | sed -n 's/^Source: //p') -bindir := $(CURDIR)/debian/$(pkg)/usr/bin -examples := $(CURDIR)/debian/$(pkg)/usr/share/doc/$(pkg)/examples - -pybuilddir := $(shell dpkg-architecture -qDEB_BUILD_ARCH_OS)-$(shell dpkg-architecture -qDEB_BUILD_GNU_CPU) - -export PYBUILD_NAME=qcli - - -%: - dh $@ --with python2 --buildsystem=pybuild - -# we only need this package as qiime dependency - there is no point in bloating /usr/bin with these scripts -override_dh_installexamples: - mv $(bindir) $(examples) - -override_dh_auto_test: -ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) - set -e -x;\ - for pyv in `pyversions -dv` ; do \ - cd build/; \ - ln -s ../python-code/tests; \ - env PYTHONPATH=lib.$(pybuilddir)-$${pyv} nosetests ; \ - done -endif - -get-orig-source: - mkdir -p ../tarballs - uscan --verbose --force-download --destdir=../tarballs diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 163aaf8..0000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (quilt) diff --git a/debian/watch b/debian/watch deleted file mode 100644 index b3a10e8..0000000 --- a/debian/watch +++ /dev/null @@ -1,2 +0,0 @@ -version=4 -http://pypi.debian.net/qcli/qcli-(.+)\.(?:tar(?:\.gz|\.bz2)?|tgz) diff --git a/qcli/__init__.py b/qcli/__init__.py new file mode 100644 index 0000000..57a605d --- /dev/null +++ b/qcli/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +__author__ = "The BiPy Development Team" +__copyright__ = "Copyright 2013, The BiPy Project" +__credits__ = ["Rob Knight", + "Greg Caporaso", + "Gavin Huttley", + "Daniel McDonald", + "Jai Ram Rideout"] +__license__ = "GPL" +__version__ = "0.1.1" +__maintainer__ = "Greg Caporaso" +__email__ = "[email protected]" + +# import most commonly used objects and functions so they can +# be imported directly from qlci (e.g., from qcli import make_option) +from qcli.option_parsing import ( + make_option, + parse_command_line_parameters) +from qcli.test import (run_script_usage_tests) +from qcli.util import (qcli_system_call) + +__all__ = ['option_parsing','test','util'] diff --git a/qcli/option_parsing.py b/qcli/option_parsing.py new file mode 100644 index 0000000..5eb537b --- /dev/null +++ b/qcli/option_parsing.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python +""" Utilities for parsing command line options and arguments + +This code was derived from PyCogent (www.pycogent.org) and QIIME +(www.qiime.org), where it was initally developed. It has been ported +to qcli to support accessing this functionality without those +dependencies. + +""" + +from copy import copy +import types +import sys +from optparse import (OptionParser, OptionGroup, Option, + OptionValueError, OptionError) +from os import popen, remove, makedirs, getenv +from os.path import join, abspath, exists, isdir, isfile, split +from glob import glob + + +__author__ = "Greg Caporaso, Gavin Huttley, Rob Knight, Daniel McDonald" +__copyright__ = "Copyright 2013, The BiPy Project" +__credits__ = ["Greg Caporaso", + "Daniel McDonald", + "Gavin Huttley", + "Rob Knight", + "Jose Antonio Navas Molina"] +__license__ = "GPL" +__version__ = "0.1.1" +__maintainer__ = "Greg Caporaso" +__email__ = "[email protected]" + + +class QOptionParser(OptionParser): + """QCLI's OptionParser subclass""" + def __init__(self, error_suffix='', **kwargs): + OptionParser.__init__(self, **kwargs) + # error suffix specifies a message that will be appended to every + # error shown with the error method + self.error_suffix = error_suffix + + def error(self, msg): + # based on the built-in optparse.py error method + self.exit(2, "Error in %s: %s\n%s" % (self.get_prog_name(), msg, + self.error_suffix)) + +## Definition of CogentOption option type, a subclass of Option that +## contains specific types for filepaths and directory paths. This +## will be particularly useful for graphical interfaces that make +## use of the script_info dictionary as they can then distinguish +## paths from ordinary strings +def check_existing_filepath(option, opt, value): + if not exists(value): + raise OptionValueError( + "option %s: file does not exist: %r" % (opt, value)) + elif not isfile(value): + raise OptionValueError( + "option %s: not a regular file (can't be a directory!): %r" % (opt, value)) + else: + return value + +def check_existing_filepaths(option, opt, value): + paths = [] + for v in value.split(','): + fps = glob(v) + if len(fps) == 0: + raise OptionValueError( + "No filepaths match pattern/name '%s'. " + "All patterns must be matched at least once." % v) + else: + paths.extend(fps) + values = [] + for v in paths: + check_existing_filepath(option,opt,v) + values.append(v) + return values + +def check_existing_dirpath(option, opt, value): + if not exists(value): + raise OptionValueError( + "option %s: directory does not exist: %r" % (opt, value)) + elif not isdir(value): + raise OptionValueError( + "option %s: not a directory (can't be a file!): %r" % (opt, value)) + else: + return value + +def check_new_filepath(option, opt, value): + return value + +def check_new_dirpath(option, opt, value): + return value + +def check_existing_path(option, opt, value): + if not exists(value): + raise OptionValueError( + "option %s: path does not exist: %r" % (opt, value)) + return value + +def check_new_path(option, opt, value): + return value + +def check_multiple_choice(option, opt, value): + #split_char = ';' if ';' in value else ',' + values = value.split(option.split_char) + for v in values: + if v not in option.mchoices: + choices = ",".join(map(repr, option.mchoices)) + raise OptionValueError( + "option %s: invalid choice: %r (choose from %s)" + % (opt, v, choices)) + return values + +def check_blast_db(option, opt, value): + db_dir, db_name = split(abspath(value)) + if not exists(db_dir): + raise OptionValueError( + "option %s: path does not exists: %r" % (opt, db_dir)) + elif not isdir(db_dir): + raise OptionValueError( + "option %s: not a directory: %r" % (opt, db_dir)) + return value + +class QcliOption(Option): + ATTRS = Option.ATTRS + ['mchoices','split_char'] + + TYPES = Option.TYPES + ("existing_path", + "new_path", + "existing_filepath", + "existing_filepaths", + "new_filepath", + "existing_dirpath", + "new_dirpath", + "multiple_choice", + "blast_db") + TYPE_CHECKER = copy(Option.TYPE_CHECKER) + # for cases where the user specifies an existing file or directory + # as input, but it can be either a dir or a file + TYPE_CHECKER["existing_path"] = check_existing_path + # for cases where the user specifies a new file or directory + # as output, but it can be either a dir or a file + TYPE_CHECKER["new_path"] = check_new_path + # for cases where the user passes a single existing file + TYPE_CHECKER["existing_filepath"] = check_existing_filepath + # for cases where the user passes one or more existing files + # as a comma-separated list - paths are returned as a list + TYPE_CHECKER["existing_filepaths"] = check_existing_filepaths + # for cases where the user is passing a new path to be + # create (e.g., an output file) + TYPE_CHECKER["new_filepath"] = check_new_filepath + # for cases where the user is passing an existing directory + # (e.g., containing a set of input files) + TYPE_CHECKER["existing_dirpath"] = check_existing_dirpath + # for cases where the user is passing a new directory to be + # create (e.g., an output dir which will contain many result files) + TYPE_CHECKER["new_dirpath"] = check_new_dirpath + # for cases where the user is passing one or more values + # as comma- or semicolon-separated list + # choices are returned as a list + TYPE_CHECKER["multiple_choice"] = check_multiple_choice + # for cases where the user is passing a blast database option + # blast_db is returned as a string + TYPE_CHECKER["blast_db"] = check_blast_db + + def _check_multiple_choice(self): + if self.type == "multiple_choice": + if self.mchoices is None: + raise OptionError( + "must supply a list of mchoices for type '%s'" % self.type, self) + elif type(self.mchoices) not in (types.TupleType, types.ListType): + raise OptionError( + "choices must be a list of strings ('%s' supplied)" + % str(type(self.mchoices)).split("'")[1], self) + if self.split_char is None: + self.split_char = ',' + elif self.mchoices is not None: + raise OptionError( + "must not supply mchoices for type %r" % self.type, self) + + CHECK_METHODS = Option.CHECK_METHODS + [_check_multiple_choice] + +# When this code was in PyCogent, the option object was called +# CogentOption, so leaving that name in place for backward compatibility. +make_option = CogentOption = QcliOption + +## End definition of new option type + +def build_usage_lines(required_options, + script_description, + script_usage, + optional_input_line, + required_input_line): + """ Build the usage string from components + """ + line1 = 'usage: %prog [options] ' + '{%s}' %\ + ' '.join(['%s %s' % (str(ro),ro.dest.upper())\ + for ro in required_options]) + usage_examples = [] + for title, description, command in script_usage: + title = title.strip(':').strip() + description = description.strip(':').strip() + command = command.strip() + if title: + usage_examples.append('%s: %s\n %s' %\ + (title,description,command)) + else: + usage_examples.append('%s\n %s' % (description,command)) + usage_examples = '\n\n'.join(usage_examples) + lines = (line1, + '', # Blank line + optional_input_line, + required_input_line, + '', # Blank line + script_description, + '', # Blank line + 'Example usage: ',\ + 'Print help message and exit', + ' %prog -h\n', + usage_examples) + return '\n'.join(lines) + +def set_parameter(key,kwargs,default=None): + try: + return kwargs[key] + except KeyError: + return default + +def set_required_parameter(key,kwargs): + try: + return kwargs[key] + except KeyError: + raise KeyError,\ + "parse_command_line_parameters requires value for %s" % key + +def parse_command_line_parameters(**kwargs): + """ Constructs the OptionParser object and parses command line arguments + + parse_command_line_parameters takes a dict of objects via kwargs which + it uses to build command line interfaces according to standards + developed in the Knight Lab, and enforced in QIIME. The currently + supported options are listed below with their default values. If no + default is provided, the option is required. + + script_description + script_usage = [("","","")] + version + required_options=None + optional_options=None + suppress_verbose=False + disallow_positional_arguments=True + help_on_no_arguments=True + optional_input_line = '[] indicates optional input (order unimportant)' + required_input_line = '{} indicates required input (order unimportant)' + + These values can either be passed directly, as: + parse_command_line_parameters(script_description="My script",\ + script_usage=[('Print help','%prog -h','')],\ + version=1.0) + + or they can be passed via a pre-constructed dict, as: + d = {'script_description':"My script",\ + 'script_usage':[('Print help','%prog -h','')],\ + 'version':1.0} + parse_command_line_parameters(**d) + + """ + # Get the options, or defaults if none were provided. + script_description = set_required_parameter('script_description',kwargs) + version = set_required_parameter('version',kwargs) + script_usage = set_parameter('script_usage',kwargs,[("","","")]) + required_options = set_parameter('required_options',kwargs,[]) + optional_options = set_parameter('optional_options',kwargs,[]) + suppress_verbose = set_parameter('suppress_verbose',kwargs,False) + disallow_positional_arguments =\ + set_parameter('disallow_positional_arguments',kwargs,True) + help_on_no_arguments = set_parameter('help_on_no_arguments',kwargs,True) + optional_input_line = set_parameter('optional_input_line',kwargs,\ + '[] indicates optional input (order unimportant)') + required_input_line = set_parameter('required_input_line',kwargs,\ + '{} indicates required input (order unimportant)') + # command_line_text will usually be nothing, but can be passed for + # testing purposes + command_line_args = set_parameter('command_line_args',kwargs,None) + + # Build the usage and version strings + usage = build_usage_lines(required_options,script_description,script_usage,\ + optional_input_line,required_input_line) + version = 'Version: %prog ' + version + + # Instantiate the command line parser object + parser = QOptionParser(error_suffix=kwargs.pop('error_suffix', ''), + usage=usage, version=version) + parser.exit = set_parameter('exit_func',kwargs,parser.exit) + + # If no arguments were provided, print the help string (unless the + # caller specified not to) + if help_on_no_arguments and (not command_line_args) and len(sys.argv) == 1: + parser.print_usage() + return parser.exit(-1) + + + # Process the required options + if required_options: + # Define an option group so all required options are + # grouped together, and under a common header + required = OptionGroup(parser, "REQUIRED options", + "The following options must be provided under all circumstances.") + for ro in required_options: + # if the option doesn't already end with [REQUIRED], + # add it. + if not ro.help.strip().endswith('[REQUIRED]'): + ro.help += ' [REQUIRED]' + required.add_option(ro) + parser.add_option_group(required) + + # Add a verbose parameter (if the caller didn't specify not to) + if not suppress_verbose: + parser.add_option('-v','--verbose',action='store_true',\ + dest='verbose',help='Print information during execution '+\ + '-- useful for debugging [default: %default]',default=False) + + # Add the optional options + map(parser.add_option,optional_options) + + # Parse the command line + # command_line_text will None except in test cases, in which + # case sys.argv[1:] will be parsed + opts,args = parser.parse_args(command_line_args) + + # If positional arguments are not allowed, and any were provided, + # raise an error. + if disallow_positional_arguments and len(args) != 0: + parser.error("Positional argument detected: %s\n" % str(args[0]) +\ + " Be sure all parameters are identified by their option name.\n" +\ + " (e.g.: include the '-i' in '-i INPUT_DIR')") + + # Test that all required options were provided. + if required_options: + required_option_ids = [o.dest for o in required.option_list] + for required_option_id in required_option_ids: + if getattr(opts,required_option_id) == None: + return parser.error('Required option --%s omitted.' \ + % required_option_id) + + # Return the parser, the options, and the arguments. The parser is returned + # so users have access to any additional functionality they may want at + # this stage -- most commonly, it will be used for doing custom tests of + # parameter values. + return parser, opts, args + diff --git a/qcli/test.py b/qcli/test.py new file mode 100644 index 0000000..36675b4 --- /dev/null +++ b/qcli/test.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +""" Utilities for parsing command line options and arguments + +This code was derived from QIIME (www.qiime.org), where it was initally +developed. It has been ported to qcli to support accessing this functionality +without those dependencies. + +""" + +import signal +from os.path import isdir, split, join, abspath, exists +from os import chdir, getcwd +from shutil import copytree, rmtree +from glob import glob +from site import addsitedir +from qcli.util import (qcli_system_call, + remove_files) + + +__author__ = "Greg Caporaso" +__copyright__ = "Copyright 2013, The BiPy Project" +__credits__ = ["Greg Caporaso"] +__license__ = "GPL" +__version__ = "0.1.1" +__maintainer__ = "Greg Caporaso" +__email__ = "[email protected]" + +### Code for timing out tests that exceed a time limit +## The test case timing code included in this file is adapted from +## recipes provided at: +## http://code.activestate.com/recipes/534115-function-timeout/ +## http://stackoverflow.com/questions/492519/timeout-on-a-python-function-call + +# to use this, call initiate_timeout(allowed_seconds_per_test) in +# TestCase.setUp() and then disable_timeout() in TestCase.tearDown() + +class TimeExceededError(Exception): + pass + +def initiate_timeout(seconds=60): + + def timeout(signum, frame): + raise TimeExceededError,\ + "Test failed to run in allowed time (%d seconds)." % seconds + + signal.signal(signal.SIGALRM, timeout) + # set the 'alarm' to go off in seconds seconds + signal.alarm(seconds) + +def disable_timeout(): + # turn off the alarm + signal.alarm(0) + +### End code for timing out tests that exceed a time limit + +def run_script_usage_tests(test_data_dir, + scripts_dir, + working_dir, + verbose=False, + tests=None, + failure_log_fp=None, + force_overwrite=False, + timeout=60): + """ Test script_usage examples when test data is present in test_data_dir + + Returns a result summary string and the number of script usage + examples (i.e. commands) that failed. + """ + # process input filepaths and directories + test_data_dir = abspath(test_data_dir) + working_dir = join(working_dir,'script_usage_tests') + if force_overwrite and exists(working_dir): + rmtree(working_dir) + if failure_log_fp != None: + failure_log_fp = abspath(failure_log_fp) + + if tests == None: + tests = [split(d)[1] for d in sorted(glob('%s/*' % test_data_dir)) if isdir(d)] + + if verbose: + print 'Tests to run:\n %s' % ' '.join(tests) + + addsitedir(scripts_dir) + + failed_tests = [] + warnings = [] + total_tests = 0 + for test in tests: + + # import the usage examples - this is possible because we added + # scripts_dir to the PYTHONPATH above + script_fn = '%s/%s.py' % (scripts_dir,test) + script = __import__(test) + usage_examples = script.script_info['script_usage'] + + if verbose: + print 'Testing %d usage examples from: %s' % (len(usage_examples),script_fn) + + # init the test environment + test_input_dir = '%s/%s' % (test_data_dir,test) + test_working_dir = '%s/%s' % (working_dir,test) + copytree(test_input_dir,test_working_dir) + chdir(test_working_dir) + + # remove pre-exisitng output files if any + try: + script_usage_output_to_remove = script.script_info['script_usage_output_to_remove'] + except KeyError: + script_usage_output_to_remove = [] + for e in script_usage_output_to_remove: + rmtree(e.replace('$PWD',getcwd()),ignore_errors=True) + remove_files([e.replace('$PWD',getcwd())],error_on_missing=False) + + if verbose: + print ' Running tests in: %s' % getcwd() + print ' Tests:' + + for usage_example in usage_examples: + if '%prog' not in usage_example[2]: + warnings.append('%s usage examples do not all use %%prog to represent the command name. You may not be running the version of the command that you think you are!' % test) + cmd = usage_example[2].replace('%prog',script_fn) + if verbose: + print ' %s' % cmd, + + timed_out = False + initiate_timeout(timeout) + try: + stdout, stderr, return_value = qcli_system_call(cmd) + except TimeExceededError: + timed_out = True + else: + disable_timeout() + + total_tests += 1 + if timed_out: + # Add a string instead of return_value - if fail_tests ever ends + # up being returned from this function we'll want to code this as + # an int for consistency in the return value type. + failed_tests.append((cmd, "", "", "None, time exceeded")) + if verbose: print ": Timed out" + elif return_value != 0: + failed_tests.append((cmd, stdout, stderr, return_value)) + if verbose: print ": Failed" + else: + pass + if verbose: print ": Pass" + + if verbose: + print '' + + if failure_log_fp: + failure_log_f = open(failure_log_fp,'w') + if len(failed_tests) == 0: + failure_log_f.write('All script interface tests passed.\n') + else: + i = 1 + for cmd, stdout, stderr, return_value in failed_tests: + failure_log_f.write('**Failed test %d:\n%s\n\nReturn value: %s\n\nStdout:\n%s\n\nStderr:\n%s\n\n' % (i,cmd,str(return_value), stdout, stderr)) + i += 1 + failure_log_f.close() + + + if warnings: + print 'Warnings:' + for warning in warnings: + print ' ' + warning + print '' + + result_summary = 'Ran %d commands to test %d scripts. %d of these commands failed.' % (total_tests,len(tests),len(failed_tests)) + if len(failed_tests) > 0: + failed_scripts = set([split(e[0].split()[0])[1] for e in failed_tests]) + result_summary += '\nFailed scripts were: %s' % " ".join(failed_scripts) + if failure_log_fp: + result_summary += "\nFailures are summarized in %s" % failure_log_fp + + rmtree(working_dir) + + return result_summary, len(failed_tests) diff --git a/qcli/util.py b/qcli/util.py new file mode 100644 index 0000000..24dd3d9 --- /dev/null +++ b/qcli/util.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +""" Utilities for parsing command line options and arguments + +This code was derived from QIIME (www.qiime.org), where it was initally +developed. It has been ported to qcli to support accessing this functionality +without those dependencies. + +""" + +__author__ = "Greg Caporaso" +__copyright__ = "Copyright 2013, The BiPy Project" +__credits__ = ["Greg Caporaso"] +__license__ = "GPL" +__version__ = "0.1.1" +__maintainer__ = "Greg Caporaso" +__email__ = "[email protected]" + +from os import remove +from subprocess import Popen, PIPE, STDOUT + +def qcli_system_call(cmd, shell=True): + """Call cmd and return (stdout, stderr, return_value). + + cmd can be either a string containing the command to be run, or a sequence + of strings that are the tokens of the command. + + Please see Python's subprocess. Popen for a description of the shell + parameter and how cmd is interpreted differently based on its value. + + This function is ported from QIIME (previously qiime_system_call). + """ + proc = Popen(cmd, + shell=shell, + universal_newlines=True, + stdout=PIPE, + stderr=PIPE) + # communicate pulls all stdout/stderr from the PIPEs to + # avoid blocking -- don't remove this line! + stdout, stderr = proc.communicate() + return_value = proc.returncode + return stdout, stderr, return_value + +def remove_files(list_of_filepaths, error_on_missing=True): + """Remove list of filepaths, optionally raising an error if any are missing + + This function is ported from PyCogent. + """ + missing = [] + for fp in list_of_filepaths: + try: + remove(fp) + except OSError: + missing.append(fp) + + if error_on_missing and missing: + raise OSError, "Some filepaths were not accessible: %s" % '\t'.join(missing) + \ No newline at end of file diff --git a/scripts/qcli_make_rst b/scripts/qcli_make_rst new file mode 100755 index 0000000..25d53e8 --- /dev/null +++ b/scripts/qcli_make_rst @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# File created on 15 Feb 2010 +from __future__ import division + +__author__ = "Jesse Stombaugh" +__copyright__ = "Copyright 2013, The BiPy Project" +__credits__ = ["Jesse Stombaugh", + "Greg Caporaso"] +__license__ = "GPL" +__version__ = "0.1.0" +__maintainer__ = "Jesse Stombaugh" +__email__ = "[email protected]" + +import os +from os import makedirs +from os.path import exists, abspath, split, splitext +from string import replace +import types +import re +from sys import exit, stderr +from site import addsitedir +from qcli import (parse_command_line_parameters, + make_option) + +rst_text= \ +'''\ +.. _%s: + +.. index:: %s.py + +*%s.py* -- %s +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Description:** + +%s + + +**Usage:** :file:`%s.py [options]` + +**Input Arguments:** + +.. note:: + +%s + +**Output:** + +%s + +%s + +''' + +script_info={} +script_info['brief_description']="""Make Sphinx RST file for one or more qcli-based scripts""" +script_info['script_description'] = """This script will take a qcli script and convert the usage strings and options to generate a documentation .rst file.""" +script_info['script_usage']=[] +script_info['script_usage'].append(("""Create RST for many files""", + """Create rst files for all files ending with .py in the scripts/ directory. Write the rst files to the rst directory. Note that if the value you pass for -i contains a wildcard character (e.g., "*"), the value must be wrapped in quotes.""", + """%prog -i "scripts/*py" -o rst""")) +script_info['output_description']="""This script will output one or more Sphinx rst-formatted files.""" + +script_info['required_options'] = [ + make_option('-i','--input_fps',type='existing_filepaths', + help='the input file(s) to generate rst files for'), + make_option('-o','--output_dir',type='new_filepath', + help='the directory where the resulting rst file(s) should be written'), +] + +script_info['version'] = __version__ + + +def convert_py_file_to_link(input_str): + m=re.compile('[\w]+\.py') + python_script_names=set(m.findall(input_str)) + + if python_script_names: + script_w_link=input_str + for i in python_script_names: + individual_script_name=os.path.splitext(i) + script_w_link=script_w_link.replace(i, '`'+ i + ' <./' + \ + individual_script_name[0] + '.html>`_') + + return script_w_link + else: + return input_str + +def main(): + option_parser, opts, args = parse_command_line_parameters(**script_info) + + #Create a list of the scripts to create rst files for. + file_names = opts.input_fps + for fn in file_names: + addsitedir(abspath(split(fn)[0])) + + #Identify the directory where results should be written. + output_dir = opts.output_dir + if output_dir and not output_dir.endswith('/'): + output_dir = output_dir + '/' + if not exists(output_dir): + makedirs(output_dir) + + script={} + #Iterate through list of filenames + for filename in file_names: + #Get only the name of the script and remove other path information. + filename = splitext(split(filename)[1])[0] + + #Import the script file to get the dictionary values + try: + script=__import__(filename) + except ImportError: + raise ImportError, "Can't import %s" % filename + + #Define output file path + outf=os.path.join(output_dir,'%s.rst' % filename) + + #This try block attempts to parse the dictionary and if the dictionary + #is not present, then it will write that information to stdout + try: + + imported_brief_description=script.script_info['brief_description'] + imported_script_description=script.script_info['script_description'] + + new_script_description = \ + convert_py_file_to_link(imported_script_description) + #print new_script_description + inputs='' + if script.script_info.has_key('required_options') and \ + script.script_info['required_options']<>[]: + inputs= '\t\n\t**[REQUIRED]**\n\t\t\n' + for i in script.script_info['required_options']: + # when no default is provided in the call to make_option, + # the value of i.default is a tuple -- this try/except + # handles the diff types that i.default can be + try: + if i.default<>'': + if i.default[0] == 'NO': + # i.default is a tuple, so defualt hasn't been + # set by the user, and it should therefore be + # None + defaults = None + else: + # i.default is a string + defaults = i.default + else: + defaults=None + except TypeError: + # i.default is not a string or a tuple (e.g., it's an + # int or None) + defaults = i.default + + p=re.compile('\%default') + help_str=p.sub(str(defaults),i.help) + new_help_str=convert_py_file_to_link(help_str) + new_help_str=new_help_str[0].upper() + new_help_str[1:] + + cmd_arg=str(i).replace('--','`-`-').replace('/',', ') + inputs=inputs+'\t'+str(cmd_arg)+'\n\t\t'+ new_help_str+'\n' + + + if script.script_info.has_key('optional_options') and \ + script.script_info['optional_options']<>[]: + inputs=inputs + '\t\n\t**[OPTIONAL]**\n\t\t\n' + for i in script.script_info['optional_options']: + # when no default is provided in the call to make_option, + # the value of i.default is a tuple -- this try/except + # handles the diff types that i.default can be + try: + if i.default<>'': + if i.default[0] == 'NO': + # i.default is a tuple, so defualt hasn't been + # set by the user, and it should therefore be + # None + defaults = None + else: + # i.default is a string + defaults = i.default + else: + defaults=i.default + except TypeError: + # i.default is not a string or a tuple (e.g., it's an + # int or None) + defaults = i.default + + p=re.compile('\%default') + help_str=p.sub(str(defaults),i.help) + new_help_str=convert_py_file_to_link(help_str) + new_help_str=new_help_str[0].upper() + new_help_str[1:] + + cmd_arg=str(i).replace('--','`-`-').replace('/',', ') + inputs=inputs+'\t'+str(cmd_arg)+'\n\t\t'+ new_help_str+'\n' + + if (not script.script_info.has_key('required_options') and not script.script_info.has_key('optional_options')) or \ + (script.script_info['required_options']==[] and script.script_info['optional_options']==[]): + inputs='\t\n\tNone' + + script_examples='' + for ex in script.script_info['script_usage']: + example_title=ex[0].strip() + if example_title <> '': + if example_title.endswith(':'): + script_examples += '\n**' + ex[0] + '**\n' + else: + script_examples += '\n**' + ex[0] + ':**\n' + if ex[1] <> '': + script_ex=convert_py_file_to_link(ex[1]) + script_examples += '\n' + script_ex + '\n' + if ex[2] <>'': + new_cmd=ex[2].replace('%prog',filename+'.py') + script_examples += '\n::\n\n\t' + new_cmd + '\n' + + script_out = \ + script.script_info['output_description'].replace('--','`-`-') + new_script_out=convert_py_file_to_link(script_out) + + output_text = rst_text % (filename, + filename, + filename, + imported_brief_description, + new_script_description, + filename, + inputs, + new_script_out, + script_examples) + + ###Write rst file + f = open(outf, 'w') + f.write((output_text.replace('%prog',filename+'.py'))) + f.close() + + #script.close() + except AttributeError: + print "%s: This file does not contain the appropriate dictionary" \ + % (filename) + except KeyError: + print "%s: This file does not contain necessary fields" \ + % (filename) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/qcli_make_script b/scripts/qcli_make_script new file mode 100755 index 0000000..1c68c03 --- /dev/null +++ b/scripts/qcli_make_script @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# make_qiime_py_file.py + +""" +This is a script which will add headers and footers to new qcli scripts +and make them executable. +""" + +__author__ = "Greg Caporaso" +__copyright__ = "Copyright 2013, The BiPy Project" +__credits__ = ["Greg Caporaso"] +__license__ = "GPL" +__version__ = "0.1.0" +__maintainer__ = "Greg Caporaso" +__email__ = "[email protected]" + +from sys import exit +from os import popen +from os.path import exists +from time import strftime +from optparse import OptionParser +from qcli import (parse_command_line_parameters, + make_option) + +script_info={} +script_info['brief_description']="""Create a template qcli script.""" +script_info['script_description']="""This script will create a template qcli script and make it executable.""" +script_info['script_usage']=[] +script_info['script_usage'].append(("""Example usage""","""Create a new script""","""%prog -a "Greg Caporaso" -e [email protected] -o my_script.py""")) +script_info['script_usage_output_to_remove'] = ['my_script.py'] +script_info['output_description']="""The result of this script is a qcli template script.""" + +script_info['required_options']=[\ + make_option('-o','--output_fp',help="The output filepath.") +] + +script_info['optional_options']=[ + make_option('-a','--author_name', + help="The script author's (probably you) name to be included in"+\ + " the header variables. This will typically need to be enclosed "+\ + " in quotes to handle spaces. [default:%default]",default='AUTHOR_NAME'), + make_option('-e','--author_email', + help="The script author's (probably you) e-mail address to be included in"+\ + " the header variables. [default:%default]",default='AUTHOR_EMAIL'), + make_option('-c','--copyright', + help="The copyright information to be included in"+\ + " the header variables. [default:%default]",default='Copyright 2013, The BiPy project') +] + +script_info['version'] = __version__ + +def main(): + option_parser, opts, args = parse_command_line_parameters(**script_info) + + + header_block = """#!/usr/bin/env python +# File created on %s +from __future__ import division + +__author__ = "AUTHOR_NAME" +__copyright__ = "COPYRIGHT" +__credits__ = ["AUTHOR_NAME"] +__license__ = "GPL" +__version__ = "0.1.0" +__maintainer__ = "AUTHOR_NAME" +__email__ = "AUTHOR_EMAIL" +""" % strftime('%d %b %Y') + + script_block = """ +from qcli import (parse_command_line_parameters, + make_option) + +script_info = {} +script_info['brief_description'] = "" +script_info['script_description'] = "" +script_info['script_usage'] = [] +script_info['script_usage'].append(("","","")) +script_info['output_description']= "" +script_info['required_options'] = [ + # Example required option + #make_option('-i','--input_fp',type="existing_filepath",help='the input filepath'), +] +script_info['optional_options'] = [ + # Example optional option + #make_option('-o','--output_dir',type="new_dirpath",help='the output directory [default: %default]'), +] +script_info['version'] = __version__""" + + output_fp = opts.output_fp + + # Check to see if the file which was requested to be created + # already exists -- if it does, print a message and exit + if exists(output_fp): + print '\n'.join(["The file name you requested already exists.",\ + " Delete extant file and rerun script if it should be overwritten.",\ + " Otherwise change the file name (-o).",\ + "Creating no files and exiting..."]) + exit(1) + + # Create the header data + header_block = header_block.replace('AUTHOR_NAME',opts.author_name) + header_block = header_block.replace('AUTHOR_EMAIL',opts.author_email) + header_block = header_block.replace('COPYRIGHT',opts.copyright) + lines = [header_block] + + lines.append(script_block) + lines += ['','','','def main():',\ + ' option_parser, opts, args =\\',\ + ' parse_command_line_parameters(**script_info)',\ + '','',\ + 'if __name__ == "__main__":',\ + ' main()'] + + + # Open the new file for writing and write it. + f = open(output_fp,'w') + f.write('\n'.join(lines)) + f.close() + + # change mode to 755 + chmod_string = ' '.join(['chmod 755',output_fp]) + popen(chmod_string) + + + + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2f0fd4e --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python + +__author__ = "The BiPy Development Team" +__copyright__ = "Copyright 2013, The BiPy Project" +__credits__ = ["Rob Knight", "Greg Caporaso", ] +__license__ = "GPL" +__version__ = "0.1.1" +__maintainer__ = "Greg Caporaso" +__email__ = "[email protected]" + +from distutils.core import setup +from glob import glob + +setup(name='qcli', + version='0.1.1', + packages=['qcli'], + scripts=glob('scripts/qcli*') + ) \ No newline at end of file -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/python-qcli.git _______________________________________________ debian-med-commit mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/debian-med-commit
