This is an automated email from the git hooks/post-receive script. osallou pushed a commit to branch master in repository biomaj3-core.
commit 38666e248d0cf8eff8d7a2fa3d4aefc3a82f05c4 Author: Olivier Sallou <[email protected]> Date: Thu Aug 17 07:20:40 2017 +0000 New upstream version 3.0.9 --- .gitignore | 72 +++++ .travis.yml | 31 ++ CHANGES.txt | 22 ++ LICENSE | 662 +++++++++++++++++++++++++++++++++++++++++ MANIFEST.in | 2 + README.md | 10 + biomaj_core/__init__.py | 0 biomaj_core/bmajindex.py | 213 +++++++++++++ biomaj_core/config.py | 482 ++++++++++++++++++++++++++++++ biomaj_core/mimes-bio.txt | 18 ++ biomaj_core/utils.py | 315 ++++++++++++++++++++ requirements.txt | 5 + setup.cfg | 2 + setup.py | 53 ++++ tests/alu.properties | 43 +++ tests/bank/process/test.sh | 11 + tests/bank/test.fasta.gz | Bin 0 -> 45 bytes tests/bank/test2.fasta | 2 + tests/bank/test_100.txt | 1 + tests/biomaj_tests.py | 297 ++++++++++++++++++ tests/computed.properties | 44 +++ tests/computed2.properties | 45 +++ tests/computederror.properties | 43 +++ tests/directhttp.properties | 41 +++ tests/error.properties | 43 +++ tests/global.properties | 123 ++++++++ tests/local.properties | 41 +++ tests/locallist.properties | 44 +++ tests/localprocess.properties | 100 +++++++ tests/multi.properties | 60 ++++ tests/noargs.properties | 48 +++ tests/noexe.properties | 48 +++ tests/noname.properties | 49 +++ tests/prenoargs.properties | 45 +++ tests/prenoexe.properties | 45 +++ tests/prenoname.properties | 45 +++ tests/rmnoargs.properties | 45 +++ tests/rmnoexe.properties | 45 +++ tests/rmnoname.properties | 45 +++ tests/sub1.properties | 43 +++ tests/sub2.properties | 41 +++ tests/testhttp.properties | 43 +++ 42 files changed, 3367 insertions(+) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..754c2a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,72 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# Coveralls +.coveralls.yml + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# PyCharm +.idea + +# Vim +.viminfo +# Less history +.lesshst + +.dbshell +.emacs* +.ipython +.mongo* +#*.properties + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5f2c44e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,31 @@ +language: python +sudo: false +python: + - "2.7" + - "3.4" + - "3.5" +# Apply only on main branches +branches: + except: + - /^feature.*$/ +# command to install dependencies +#before_install: +# - "sudo apt-get update -qq" +# - "sudo apt-get install -qq libldap2-dev libsasl2-dev" +install: + - "pip install flake8" + - "pip install -r requirements.txt" + - "pip install coverage" + - "pip install python-coveralls" + - "python setup.py -q install" +# - "echo data_file=$TRAVIS_BUILD_DIR/.coverage >> .coveragerc" +# command to run tests +#before_script: +# - sleep 10 +#script: nosetests --with-coverage --cover-package=biomaj -a '!network' +#script: nosetests --with-coverage --cover-package=biomaj +script: + - python setup.py test + - flake8 --ignore E501,E123 biomaj_core/*.py +# - coveralls + diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..6964b4f --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,22 @@ +3.0.9: + #1 Add checks on bank properties +3.0.8: + Add DOCKER_URL, MONGO_URL, MONGO_DB env variable in override + Do not fail on elasticsearch index creation, just log + Add rsync check +3.0.7: + set logging only if necessary +3.0.6: + Fix configparser import for python 2 vs 3 +3.0.5: + Add config override with env variables +3.0.4: + Code cleanup +3.0.3: + Put index management in core +3.0.2: + Add missing deps +3.0.1: + Add README etc.. +3.0.0: + separation of biomaj and biomaj_core diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cebe035 --- /dev/null +++ b/LICENSE @@ -0,0 +1,662 @@ +GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +<http://www.gnu.org/licenses/>. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..5dcc15a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include *.txt *.md +recursive-include biomaj_core *.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..e6d7650 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +# Biomaj + +Biomaj core library + + +Properties can be overriden by environment variables with pattern BIOMAJ_X_Y_Z for property x.y.z + +Example: + +use BIOMAJ_LDAP_HOST for ldap.host diff --git a/biomaj_core/__init__.py b/biomaj_core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/biomaj_core/bmajindex.py b/biomaj_core/bmajindex.py new file mode 100644 index 0000000..a6c68bb --- /dev/null +++ b/biomaj_core/bmajindex.py @@ -0,0 +1,213 @@ +from builtins import str +from builtins import object +import logging +import copy +from elasticsearch import Elasticsearch + + +class BmajIndex(object): + """ + ElasticSearch indexation and search + """ + + """ + ElasticSearch server + """ + es = None + + """ + Index name + """ + index = 'biomaj' + + """ + Do indexing + """ + do_index = False + + """ + Skip if failure (tests) + """ + skip_if_failure = False + + @staticmethod + def load(hosts=None, index='biomaj', do_index=True): + """ + Initialize index + + :param hosts: List of elastic search nodes to connect to + :type hosts: list + :param do_index: index data or not + :type do_index: bool + """ + if hosts is None: + hosts = ['localhost'] + if not do_index: + return + BmajIndex.index = index + BmajIndex.do_index = do_index + if BmajIndex.es is None: + BmajIndex.es = Elasticsearch(hosts) + + mapping = { + "mappings": { + "production": { + "date_detection": False + }, + "releasestats": { + "date_detection": False, + } + } + } + try: + if not BmajIndex.es.indices.exists(index=BmajIndex.index): + BmajIndex.es.indices.create(index=BmajIndex.index, body=mapping) + except Exception as e: + logging.error('ElasticSearch connection error, check server is running and configuration: ' + str(e)) + + @staticmethod + def _bulk_delete(query, flush=True): + try: + page = BmajIndex.es.search( + index=BmajIndex.index, + doc_type='production', + search_type="query_then_fetch", + size=1000, + body={ + 'query': { + 'match': { + 'bank': query['bank'] + } + } + }) + + if page is None: + return + bulk_delete = '' + for del_hit in page['hits']['hits']: + if ('release' in query and query['release'] == del_hit['_source']['release']) or 'release' not in query: + bulk_delete += "{ \"delete\" : {\"_index\":\"" + BmajIndex.index + "\",\"_type\":\"production\", \"_id\" : \"" + del_hit['_id'] + "\" } }\n" + if bulk_delete: + BmajIndex.es.bulk(body=bulk_delete) + if flush: + BmajIndex.es.indices.flush(index=BmajIndex.index, force=True) + except Exception as e: + if BmajIndex.skip_if_failure: + BmajIndex.do_index = False + else: + logging.error('Failed to delete bank index: ' + str(e)) + + @staticmethod + def delete_all_bank(bank_name): + """ + Delete complete index for a bank + """ + if not BmajIndex.do_index: + return + BmajIndex._bulk_delete({"bank": bank_name}, True) + + @staticmethod + def remove(bank_name, release): + """ + Remove a production release + + :param bank_name: Name of the bank + :type bank_name: str + :param release: production release + :type release: str + """ + if not BmajIndex.do_index: + return + BmajIndex._bulk_delete({"release": release, "bank": bank_name}) + """ + try: + query = { + "query" : { + "term" : {"release" : release, "bank": bank_name} + } + } + BmajIndex.es.delete_by_query(index=BmajIndex.index, body=query) + except Exception as e: + logging.error('Index:Remove:'+bank_name+'_'+str(release)+':Exception:'+str(e)) + if BmajIndex.skip_if_failure: + BmajIndex.do_index = False + """ + + @staticmethod + def search(query): + if not BmajIndex.do_index: + return None + res = BmajIndex.es.search(index=BmajIndex.index, + doc_type='production', + search_type="query_then_fetch", + body=query) + return res['hits']['hits'] + + @staticmethod + def searchq(query, size=1000): + """ + Lucene syntax search + + :param query: Lucene search string + :type query: str + :param size: number of results + :type size: int + :return: list of matches + """ + if not BmajIndex.do_index: + return None + res = BmajIndex.es.search(index=BmajIndex.index, doc_type='production', q=query, size=size) + return res['hits']['hits'] + + @staticmethod + def add_stat(stat_id, stat): + """ + Add some statistics, must contain release and bank properties. + """ + if not BmajIndex.do_index: + return + if stat['release'] is None or stat['bank'] is None: + return False + try: + BmajIndex.es.index(index=BmajIndex.index, doc_type='releasestats', id=stat_id, body=stat) + except Exception: + if BmajIndex.skip_if_failure: + BmajIndex.do_index = False + else: + return False + return True + + @staticmethod + def add(bank_name, prod, flush=False): + """ + Index a production release + + :param bank_name: Name of the bank + :type bank_name: str + :param prod: session release object + :type prod: dict + :param flush: Force flushing + :type flush: bool + """ + if not BmajIndex.do_index: + return + obj = copy.deepcopy(prod) + if obj['release'] is None: + return + obj['bank'] = bank_name + formats = obj['formats'] + try: + for fkey, fvalue in formats.items(): + for elt in fvalue: + elt['format'] = fkey + elt['bank'] = bank_name + elt['release'] = obj['release'] + if 'status' in obj: + elt['status'] = obj['status'] + BmajIndex.es.index(index=BmajIndex.index, doc_type='production', body=elt) + if flush: + BmajIndex.es.indices.flush(index=BmajIndex.index, force=True) + except Exception as e: + logging.error('Index:Add:' + bank_name + '_' + str(obj['release']) + ':Exception:' + str(e)) + if BmajIndex.skip_if_failure: + BmajIndex.do_index = False diff --git a/biomaj_core/config.py b/biomaj_core/config.py new file mode 100644 index 0000000..9f60c5d --- /dev/null +++ b/biomaj_core/config.py @@ -0,0 +1,482 @@ +from __future__ import print_function +from future import standard_library +from builtins import str +from builtins import object +import logging +import logging.config +import os +import time +import sys + +from biomaj_core.bmajindex import BmajIndex + +if sys.version < '3': + import ConfigParser as configparser +else: + import configparser + +standard_library.install_aliases() + + +class BiomajConfig(object): + """ + Manage Biomaj configuration + """ + + DEFAULTS = { + 'http.parse.dir.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})', + 'http.parse.file.line': r'<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1})', + 'http.group.dir.name': 1, + 'http.group.dir.date': 2, + 'http.group.file.name': 1, + 'http.group.file.date': 2, + 'http.group.file.size': 3, + 'visibility.default': 'public', + 'historic.logfile.level': 'INFO', + 'bank.num.threads': 2, + 'files.num.threads': 4, + 'use_elastic': 0, + 'use_drmaa': 0, + 'db.type': '', + 'db.formats': '', + 'keep.old.version': 1, + 'docker.sudo': '1', + 'auto_publish': 0 + } + + # Old biomaj level compatibility + LOGLEVEL = { + 'DEBUG': logging.DEBUG, + 'VERBOSE': logging.INFO, + 'INFO': logging.INFO, + 'WARN': logging.WARNING, + 'ERR': logging.ERROR + } + + """ + Global configuration file + """ + global_config = None + + """ + Per use global configuration file, overriding global_config + """ + user_config = None + allow_user_config = False + + @staticmethod + def load_config(config_file=None, allow_user_config=True): + """ + Loads general config + + :param config_file: global.properties file path + :type config_file: str + :param allow_user_config: use ~/.biomaj.cfg if present + :type allow_user_config: bool + """ + BiomajConfig.allow_user_config = allow_user_config + + if config_file is None: + env_file = os.environ.get('BIOMAJ_CONF') + if env_file is not None and os.path.exists(env_file): + config_file = env_file + else: + env_file = 'global.properties' + if os.path.exists(env_file): + config_file = env_file + + if config_file is None or not os.path.exists(config_file): + raise Exception('Missing global configuration file') + + BiomajConfig.config_file = os.path.abspath(config_file) + + BiomajConfig.global_config = configparser.ConfigParser() + + # if allow_user_config and os.path.exists(os.path.expanduser('~/.biomaj.cfg')): + # BiomajConfig.user_config_file = os.path.expanduser('~/.biomaj.cfg') + # BiomajConfig.user_config = configparser.ConfigParser() + # BiomajConfig.user_config.read([os.path.expanduser('~/.biomaj.cfg')]) + # else: + # BiomajConfig.user_config_file = None + + BiomajConfig.global_config.read([config_file]) + + # ElasticSearch indexation support + do_index = False + if BiomajConfig.global_config.get('GENERAL', 'use_elastic') and \ + BiomajConfig.global_config.get('GENERAL', 'use_elastic') == "1": + do_index = True + if do_index: + if BiomajConfig.global_config.get('GENERAL', 'elastic_nodes'): + elastic_hosts = BiomajConfig.global_config.get('GENERAL', + 'elastic_nodes')\ + .split(',') + else: + elastic_hosts = ['localhost'] + elastic_index = BiomajConfig.global_config.get('GENERAL', 'elastic_index') + if elastic_index is None: + elastic_index = 'biomaj' + + if BiomajConfig.global_config.has_option('GENERAL', 'test') and \ + BiomajConfig.global_config.get('GENERAL', 'test') == "1": + # Test connection to elasticsearch, if not available skip indexing for tests + BmajIndex.skip_if_failure = True + + BmajIndex.load(index=elastic_index, + hosts=elastic_hosts, + do_index=do_index) + + def __init__(self, bank, options=None): + """ + Loads bank configuration + + :param bank: bank name + :type bank: str + :param options: bank options + :type options: argparse + """ + self.log_level = logging.INFO + self.name = bank + if BiomajConfig.global_config is None: + BiomajConfig.load_config() + self.config_bank = configparser.ConfigParser() + conf_dir = BiomajConfig.global_config.get('GENERAL', 'conf.dir') + if not os.path.exists(os.path.join(conf_dir, bank + '.properties')): + logging.error('Bank configuration file does not exists') + raise Exception('Configuration file ' + bank + + '.properties does not exists') + try: + config_files = [BiomajConfig.config_file] + self.user_config_file = None + if self.allow_user_config and hasattr(options, 'user')\ + and options.user\ + and os.path.exists(os.path.expanduser('~' + options.user + '/.biomaj.cfg')): + self.user_config_file = os.path.expanduser('~/' + options.user + + '.biomaj.cfg') + self.user_config = configparser.ConfigParser() + self.user_config.read([os.path.expanduser('~/' + options.user + + '.biomaj.cfg')]) + + if self.user_config_file is not None: + config_files.append(self.user_config_file) + config_files.append(os.path.join(conf_dir, bank + '.properties')) + self.config_bank.read(config_files) + except Exception as e: + logging.error("Configuration file error " + str(e)) + raise Exception("Configuration file error " + str(e)) + + self.last_modified = int(os.stat(os.path.join(conf_dir, bank + + '.properties')).st_mtime) + + do_log = False + if options is None: + do_log = True + elif hasattr(options, 'no_log') and not options.no_log: + do_log = True + elif type(options) is dict and 'no_log' in options and not options['no_log']: + do_log = True + + if do_log: + if os.path.exists(os.path.expanduser('~/.biomaj.cfg')): + logging.config.fileConfig(os.path.expanduser('~/.biomaj.cfg')) + else: + logging.config.fileConfig(BiomajConfig.config_file) + + logger = logging.getLogger() + bank_log_dir = os.path.join(self.get('log.dir'), bank, str(time.time())) + if not os.path.exists(bank_log_dir): + os.makedirs(bank_log_dir) + hdlr = logging.FileHandler(os.path.join(bank_log_dir, bank + '.log')) + self.log_file = os.path.join(bank_log_dir, bank + '.log') + if options is not None and options.get_option('log') is not None: + hdlr.setLevel(BiomajConfig.LOGLEVEL[options.get_option('log')]) + self.log_level = BiomajConfig.LOGLEVEL[options.get_option('log')] + else: + hdlr.setLevel(BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')]) + self.log_level = BiomajConfig.LOGLEVEL[self.get('historic.logfile.level')] + formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + ''' + # For monolithic, update default logger/handlers? + existing_logger_names = list(logging.getLogger().manager.loggerDict.keys()) + for existing_logger in existing_logger_names: + if existing_logger.startswith('biomaj'): + tmp_logger = logging.getLogger(existing_logger) + tmp_logger.setLevel(self.log_level) + for handler in tmp_logger.handlers: + handler.setLevel(self.log_level) + ''' + else: + self.log_file = 'none' + + cache_dir = self.get('cache.dir') + if cache_dir is None: + print("Configuration file error: cache.dir empty") + logging.error("cache.dir is not defined") + sys.exit(1) + + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + process_dir = self.get('process.dir') + if process_dir is None: + print("Configuration file error: process.dir empty") + logging.error("process.dir is not defined") + sys.exit(1) + + if not os.path.exists(process_dir): + os.makedirs(process_dir) + + data_dir = self.get('data.dir') + if data_dir is None: + print("Configuration file error: data.dir empty") + logging.error("data.dir is not defined") + sys.exit(1) + + if not os.path.exists(data_dir): + os.makedirs(data_dir) + + lock_dir = self.get('lock.dir') + if lock_dir is None: + print("Configuration file error: lock.dir empty") + logging.error("lock.dir is not defined") + sys.exit(1) + + if not os.path.exists(lock_dir): + os.makedirs(lock_dir) + + def reset_logger(self): + ''' + Reconfigure logging + ''' + if self.log_file == 'none': + return + if os.path.exists(os.path.expanduser('~/.biomaj.cfg')): + logging.config.fileConfig(os.path.expanduser('~/.biomaj.cfg')) + else: + logging.config.fileConfig(BiomajConfig.config_file) + + logger = logging.getLogger() + hdlr = logging.FileHandler(self.log_file) + hdlr.setLevel(self.log_level) + formatter = logging.Formatter('%(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + + def set(self, prop, value, section='GENERAL'): + self.config_bank.set(section, prop, value) + + def _in_env(self, prop): + ''' + Search for env variable BIOMAJ_X_Y_Z for property X.Y.Z + ''' + env_prop = 'BIOMAJ_' + prop.upper().replace('.', '_') + if env_prop in os.environ: + return os.environ[env_prop] + else: + return None + + def get_bool(self, prop, section='GENERAL', escape=True, default=None): + """ + Get a boolean property from bank or general configration. Optionally in section. + """ + value = None + + if self._in_env(prop): + value = self._in_env(prop) + else: + value = self.get(prop, section, escape, default) + + if value is None: + return False + if value is True or value == 'true' or value == '1': + return True + else: + return False + + def get(self, prop, section='GENERAL', escape=True, default=None): + """ + Get a property from bank or general configration. Optionally in section. + """ + if self._in_env(prop): + return self._in_env(prop) + # Compatibility fields + if prop == 'depends': + depend = self.get('db.source', section, escape, None) + if depend: + return depend + + if self.config_bank.has_option(section, prop): + val = self.config_bank.get(section, prop) + if prop == 'remote.dir' and not val.endswith('/'): + val = val + '/' + # If regexp, escape backslashes + if escape and (prop == 'local.files' or prop == 'remote.files' or + prop == 'http.parse.dir.line' or + prop == 'http.parse.file.line'): + val = val.replace('\\\\', '\\') + return val + + if self.user_config is not None: + if self.user_config.has_option(section, prop): + return self.user_config.get(section, prop) + + if BiomajConfig.global_config.has_option(section, prop): + return BiomajConfig.global_config.get(section, prop) + + if prop in BiomajConfig.DEFAULTS: + return BiomajConfig.DEFAULTS[prop] + + return default + + def get_time(self): + """ + Return last modification time of config files + """ + return self.last_modified + + def check(self): + """ + Check configuration + """ + self.set('localrelease', '') + self.set('remoterelease', '') + + status = True + if not self.get('data.dir'): + logging.error('data.dir is not set') + status = False + if not self.get('conf.dir'): + logging.error('conf.dir is not set') + status = False + if not self.get('log.dir'): + logging.error('log.dir is not set') + status = False + if not self.get('process.dir'): + logging.error('process.dir is not set') + status = False + if not self.get('lock.dir'): + logging.error('lock.dir is not set') + status = False + if not self.get('cache.dir'): + logging.error('cache.dir is not set') + status = False + if not self.get('db.fullname'): + logging.warn('db.fullname is not set') + if not self.get('db.formats'): + logging.warn('db.formats is not set') + if self.get('use_ldap'): + if not self.get('ldap.host') or not self.get('ldap.port') or \ + not self.get('ldap.dn'): + logging.error('use_ldap set to 1 but missing configuration') + status = False + if self.get('use_elastic'): + if not self.get('elastic_nodes') or not self.get('elastic_index'): + logging.error('use_elastic set to 1 but missing configuration') + status = False + + # if not self.get('celery.queue') or not self.get('celery.broker'): + # logging.warn('celery config is not set, that\'s fine if you do not use Celery for background tasks') + + if not self.get('mail.smtp.host'): + logging.warn('SMTP mail config not set, you will not be able to send emails') + if self.get('mail.smtp.host') and not self.get('mail.from'): + logging.error('Mail origin mail.from not set') + status = False + + if not self.get('offline.dir.name'): + logging.error('offline.dir.name is not set') + status = False + elif self.get('offline.dir.name').startswith('/'): + logging.error('offline dir must be relative to data.dir and should not start with a /') + status = False + if not self.get('dir.version'): + logging.error('dir.version is not set') + status = False + elif self.get('dir.version').startswith('/'): + logging.error('dir.version must be relative to data.dir and should not start with a /') + status = False + if not self.get('protocol'): + logging.error('protocol is not set') + status = False + else: + protocol = self.get('protocol') + allowed_protocols = ['none', 'multi', 'local', 'ftp', 'sftp', 'http', + 'https', 'directftp', 'directhttp', 'directhttps', 'rsync'] + if protocol not in allowed_protocols: + logging.error('Protocol not supported: ' + protocol) + status = False + if protocol not in ['multi', 'none']: + if protocol != 'local' and not self.get('server'): + logging.error('server not set') + status = False + if not self.get('remote.dir'): + logging.error('remote.dir not set') + status = False + elif not self.get('remote.dir').endswith('/'): + logging.error('remote.dir must end with a /') + return False + if protocol not in ['direcftp', 'directhttp', 'directhttps'] and\ + not self.get('remote.files') and\ + not self.get('remote.list'): + logging.error('remote.files not set') + status = False + if not self.get('local.files'): + logging.error('local.files is not set') + status = False + # Remove processes + processes = ['db.remove.process', 'db.pre.process'] + for process in processes: + if self.get(process): + metas = self.get(process).split(',') + for meta in metas: + if not self.get(meta): + logging.error('Metaprocess ' + meta + ' not defined') + status = False + else: + procs = self.get(meta).split(',') + for proc in procs: + if not self.get(proc + '.name'): + logging.error('Process ' + proc + ' not defined') + status = False + else: + if not self.get(proc + '.exe'): + logging.error('Process exe for ' + proc + + ' not defined') + status = False + if not self.get(proc + '.args'): + logging.error( + 'Process args for ' + proc + + ' not defined') + status = False + # Check blocks + if self.get('BLOCKS'): + blocks = self.get('BLOCKS').split(',') + for block in blocks: + if not self.get(block + '.db.post.process'): + logging.error('Block ' + block + ' not defined') + status = False + else: + metas = self.get(block + '.db.post.process').split(',') + for meta in metas: + if not self.get(meta): + logging.error('Metaprocess ' + meta + ' not defined') + status = False + else: + procs = self.get(meta).split(',') + for proc in procs: + if not self.get(proc + '.name'): + logging.error('Process ' + proc + + ' not defined') + status = False + else: + if not self.get(proc + '.exe'): + logging.error('Process exe for ' + + proc + ' not defined') + status = False + if not self.get(proc + '.args'): + logging.error('Process args for ' + + proc + ' not defined') + status = False + return status diff --git a/biomaj_core/mimes-bio.txt b/biomaj_core/mimes-bio.txt new file mode 100644 index 0000000..c794bf6 --- /dev/null +++ b/biomaj_core/mimes-bio.txt @@ -0,0 +1,18 @@ +# Biological file mime types +application/fasta fasta fa fsa +application/bam bam bai +application/gff gff gff3 +application/bed bed +application/fastq fastq +application/gtf gtf +application/octet-stream ab1 scf +application/axt axt +application/csFasta csfasta +application/FasttqSolexa fastqsolexa +application/Interval interval +application/Laj laj +application/Lav lav +application/Maf maf +application/QualityScore qual +application/BlastXml blastxml +application/Wiggle wig diff --git a/biomaj_core/utils.py b/biomaj_core/utils.py new file mode 100644 index 0000000..56f8eea --- /dev/null +++ b/biomaj_core/utils.py @@ -0,0 +1,315 @@ +import os +import re +import logging +import shutil +import datetime +import time +import subprocess +from subprocess import CalledProcessError +import socket + +from mimetypes import MimeTypes + + +class Utils(object): + """ + Utility classes + """ + + mime = None + + @staticmethod + def service_config_override(config): + if 'RABBITMQ_HOST' in os.environ and os.environ['RABBITMQ_HOST']: + config['rabbitmq']['host'] = os.environ['RABBITMQ_HOST'] + if 'RABBITMQ_PORT' in os.environ and os.environ['RABBITMQ_PORT']: + config['rabbitmq']['port'] = int(os.environ['RABBITMQ_PORT']) + if 'RABBITMQ_USER' in os.environ and os.environ['RABBITMQ_USER']: + config['rabbitmq']['user'] = os.environ['RABBITMQ_USER'] + if 'RABBITMQ_PASSWORD' in os.environ and os.environ['RABBITMQ_PASSWORD']: + config['rabbitmq']['password'] = os.environ['RABBITMQ_PASSWORD'] + if 'RABBITMQ_VHOST' in os.environ and os.environ['RABBITMQ_VHOST']: + config['rabbitmq']['virtual_host'] = os.environ['RABBITMQ_VHOST'] + if 'consul' not in config: + config['consul'] = {'host': None, 'id': None} + if not config['consul']['id']: + if 'HOSTNAME' in os.environ and os.environ['HOSTNAME']: + config['consul']['id'] = os.environ['HOSTNAME'] + else: + config['consul']['id'] = socket.gethostname() + + if 'web' not in config: + config['web'] = {} + if 'BIOMAJ_HOSTNAME' in os.environ and os.environ['BIOMAJ_HOSTNAME']: + config['web']['hostname'] = os.environ['BIOMAJ_HOSTNAME'] + else: + if 'HOSTNAME' in os.environ and os.environ['HOSTNAME']: + config['web']['hostname'] = os.environ['HOSTNAME'] + else: + config['web']['hostname'] = socket.gethostname() + + if 'docker' not in config: + config['docker'] = {} + if 'DOCKER_URL' in os.environ and os.environ['DOCKER_URL']: + config['docker']['url'] = os.environ['DOCKER_URL'] + else: + config['docker']['url'] = None + + if 'REDIS_HOST' in os.environ and os.environ['REDIS_HOST']: + config['redis']['host'] = os.environ['REDIS_HOST'] + if 'REDIS_PORT' in os.environ and os.environ['REDIS_PORT']: + config['redis']['port'] = int(os.environ['REDIS_PORT']) + if 'REDIS_PREFIX' in os.environ and os.environ['REDIS_PREFIX']: + config['redis']['prefix'] = os.environ['REDIS_PREFIX'] + if 'CONSUL_HOST' in os.environ and os.environ['CONSUL_HOST']: + config['consul']['host'] = os.environ['CONSUL_HOST'] + if 'CONSUL_ID' in os.environ and os.environ['CONSUL_ID']: + config['consul']['id'] = os.environ['CONSUL_ID'] + if 'WEB_PORT' in os.environ and os.environ['WEB_PORT']: + config['web']['port'] = int(os.environ['WEB_PORT']) + if 'WEB_LOCAL_ENDPOINT' in os.environ and os.environ['WEB_LOCAL_ENDPOINT']: + config['web']['local_endpoint'] = os.environ['WEB_LOCAL_ENDPOINT'] + if 'mongo' not in config: + config['mongo'] = {'url': None, 'db': 'biomaj'} + if 'MONGO_URL' in os.environ and os.environ['MONGO_URL']: + config['mongo']['url'] = os.environ['MONGO_URL'] + if 'MONGO_DB' in os.environ and os.environ['MONGO_DB']: + config['mongo']['db'] = os.environ['MONGO_DB'] + + @staticmethod + def get_folder_size(folder): + """ + Get directory path full size + + :param folder: directory path + :type folder: str + """ + if not os.path.exists(folder): + return -1 + folder_size = 0 + for (path, dirs, files) in os.walk(folder): + for ffile in files: + filename = os.path.join(path, ffile) + folder_size += os.path.getsize(filename) + return folder_size + + @staticmethod + def detect_format(filename): + """ + try to detect file format by extension + """ + if Utils.mime is None: + Utils.mime = MimeTypes() + mimesfile = os.path.join(os.path.dirname(__file__), 'mimes-bio.txt') + Utils.mime.read(mimesfile, True) + return Utils.mime.guess_type(filename, True) + + @staticmethod + def get_more_recent_file(files): + """ + Return the date of the most recent file in list. + + Each file is a dict like with (at least) parameters: year, month, day + """ + release = None + for rfile in files: + if release is None: + release = {'year': rfile['year'], 'month': rfile['month'], 'day': rfile['day']} + else: + rel_date = datetime.date(int(release['year']), int(release['month']), int(release['day'])) + file_date = datetime.date(int(rfile['year']), int(rfile['month']), int(rfile['day'])) + if file_date > rel_date: + release['year'] = rfile['year'] + release['month'] = rfile['month'] + release['day'] = rfile['day'] + return release + + @staticmethod + def month_to_num(date): + return { + 'Jan': 1, + 'Feb': 2, + 'Mar': 3, + 'Apr': 4, + 'May': 5, + 'Jun': 6, + 'Jul': 7, + 'Aug': 8, + 'Sep': 9, + 'Oct': 10, + 'Nov': 11, + 'Dec': 12, + '01': 1, + '02': 2, + '03': 3, + '04': 4, + '05': 5, + '06': 6, + '07': 7, + '08': 8, + '09': 9, + '10': 10, + '11': 11, + '12': 12 + }[date] + + @staticmethod + def copy_files(files_to_copy, to_dir, move=False, lock=None): + """ + Copy or move files to to_dir, keeping directory structure. + + Copy keeps the original file stats. + Files should have attributes name and root: + - root: root directory + - name: relative path of file in root directory + + /root/file/file1 will be copied in to_dir/file/file1 + + :param files_to_copy: list of files to copy + :type files_to_copy: list + :param to_dir: destination directory + :type to_dir: str + :param move: move instead of copy + :type move: bool + :param lock: thread lock object for multi-threads + :type lock: Lock + """ + logger = logging.getLogger('biomaj') + nb_files = len(files_to_copy) + cur_files = 1 + for file_to_copy in files_to_copy: + logger.debug(str(cur_files) + '/' + str(nb_files) + ' copy file ' + file_to_copy['name']) + cur_files += 1 + from_file = file_to_copy['root'] + '/' + file_to_copy['name'] + to_file = to_dir + '/' + file_to_copy['name'] + if lock is not None: + lock.acquire() + try: + if not os.path.exists(os.path.dirname(to_file)): + os.makedirs(os.path.dirname(to_file)) + except Exception as e: + logger.error(e) + finally: + lock.release() + + else: + if not os.path.exists(os.path.dirname(to_file)): + try: + os.makedirs(os.path.dirname(to_file)) + except Exception as e: + logger.error(e) + if move: + shutil.move(from_file, to_file) + else: + start_time = datetime.datetime.now() + start_time = time.mktime(start_time.timetuple()) + shutil.copyfile(from_file, to_file) + end_time = datetime.datetime.now() + end_time = time.mktime(end_time.timetuple()) + file_to_copy['download_time'] = end_time - start_time + shutil.copystat(from_file, to_file) + + @staticmethod + def copy_files_with_regexp(from_dir, to_dir, regexps, move=False, lock=None): + """ + Copy or move files from from_dir to to_dir matching regexps. + Copy keeps the original file stats. + + :param from_dir: origin directory + :type from_dir: str + :param to_dir: destination directory + :type to_dir: str + :param regexps: list of regular expressions that files in from_dir should match to be copied + :type regexps: list + :param move: move instead of copy + :type move: bool + :param lock: thread lock object for multi-threads + :type lock: Lock + :return: list of copied files with their size + """ + logger = logging.getLogger('biomaj') + files_to_copy = [] + for root, dirs, files in os.walk(from_dir, topdown=True): + for name in files: + for reg in regexps: + file_relative_path = os.path.join(root, name).replace(from_dir, '') + if file_relative_path.startswith('/'): + file_relative_path = file_relative_path.replace('/', '', 1) + if reg == "**/*": + files_to_copy.append({'name': file_relative_path}) + continue + if re.match(reg, file_relative_path): + files_to_copy.append({'name': file_relative_path}) + continue + + for file_to_copy in files_to_copy: + from_file = from_dir + '/' + file_to_copy['name'] + to_file = to_dir + '/' + file_to_copy['name'] + + if lock is not None: + lock.acquire() + try: + if not os.path.exists(os.path.dirname(to_file)): + os.makedirs(os.path.dirname(to_file)) + except Exception as e: + logger.error(e) + finally: + lock.release() + else: + if not os.path.exists(os.path.dirname(to_file)): + os.makedirs(os.path.dirname(to_file)) + if move: + shutil.move(from_file, to_file) + else: + shutil.copyfile(from_file, to_file) + shutil.copystat(from_file, to_file) + file_to_copy['size'] = os.path.getsize(to_file) + f_stat = datetime.datetime.fromtimestamp(os.path.getmtime(to_file)) + file_to_copy['year'] = str(f_stat.year) + file_to_copy['month'] = str(f_stat.month) + file_to_copy['day'] = str(f_stat.day) + (file_format, encoding) = Utils.detect_format(to_file) + file_to_copy['format'] = file_format + return files_to_copy + + @staticmethod + def uncompress(archivefile, remove=True): + """ + Test if file is an archive, and uncompress it + Remove archive file if specified + + :param file: full path to file to check and uncompress + :type file: str + :param remove: remove archive if present + :type remove: bool + :return: True if ok, False if an error occured + """ + is_archive = False + logger = logging.getLogger('biomaj') + try: + if archivefile.endswith('.tar.gz'): + subprocess.check_call("tar xfz " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True) + is_archive = True + elif archivefile.endswith('.tar'): + subprocess.check_call("tar xf " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True) + is_archive = True + elif archivefile.endswith('.bz2'): + subprocess.check_call("tar xjf " + archivefile + " --overwrite -C " + os.path.dirname(archivefile), shell=True) + is_archive = True + elif archivefile.endswith('.gz'): + subprocess.check_call("gunzip -f " + archivefile, shell=True) + is_archive = True + elif archivefile.endswith('.zip'): + subprocess.check_call("unzip -o " + archivefile + " -d " + os.path.dirname(archivefile), shell=True) + is_archive = True + except CalledProcessError as uncompresserror: + logger.error("Uncompress error of %s: %s" % (archivefile, str(uncompresserror))) + return False + + if is_archive: + logger.debug('Uncompress:uncompress:' + archivefile) + + if is_archive and remove and os.path.exists(archivefile): + os.remove(archivefile) + + return True diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..89deaa9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +nose +mock +future +elasticsearch + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c6e79c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6adbbc1 --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +try: + from setuptools import setup, find_packages +except ImportError: + from distutils.core import setup + +from distutils.command.install import install +import os + + +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, 'README.md')) as f: + README = f.read() +with open(os.path.join(here, 'CHANGES.txt')) as f: + CHANGES = f.read() + + +config = { + 'description': 'BioMAJ core library', + 'long_description': README + '\n\n' + CHANGES, + 'author': 'Olivier Sallou', + 'url': 'http://biomaj.genouest.org', + 'download_url': 'http://biomaj.genouest.org', + 'author_email': '[email protected]', + 'version': '3.0.9', + 'classifiers': [ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + # Indicate who your project is intended for + 'Intended Audience :: Science/Research', + 'Topic :: Scientific/Engineering :: Bio-Informatics', + # Pick your license as you wish (should match "license" above) + 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4' + ], + 'install_requires': [ + 'future', + 'elasticsearch' + ], + 'tests_require': ['nose', 'mock'], + 'test_suite': 'nose.collector', + 'packages': find_packages(), + 'include_package_data': True, + 'name': 'biomaj_core' +} + +setup(**config) diff --git a/tests/alu.properties b/tests/alu.properties new file mode 100644 index 0000000..0e729e0 --- /dev/null +++ b/tests/alu.properties @@ -0,0 +1,43 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="alu.n : alu repeat element. alu.a : translation of alu.n repeats" +db.name=alu +db.type=nucleic_protein + +offline.dir.name=offline/ncbi/blast/alu_tmp +dir.version=ncbi/blast/alu + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=ftp.ncbi.nih.gov +remote.dir=/blast/db/FASTA/ + +release.file= +release.regexp= +release.file.compressed= + +remote.files=^alu.*\.gz$ + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^alu\.(a|n).* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/bank/process/test.sh b/tests/bank/process/test.sh new file mode 100755 index 0000000..2d510e2 --- /dev/null +++ b/tests/bank/process/test.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "Testing a process" + +echo "test meta data" +echo "##BIOMAJ#blast#nucleic#organism:hg19,chr:chr1#blast/chr1/chr1db" +echo "##BIOMAJ#blast#nucleic#organism:hg19,chr:chr2#blast/chr2/chr2db" + +echo "test meta data 2" + +echo "##BIOMAJ#fasta#nucleic#organism:hg19#fasta/chr1.fa,fasta/chr2.fa" diff --git a/tests/bank/test.fasta.gz b/tests/bank/test.fasta.gz new file mode 100644 index 0000000..666d6f2 Binary files /dev/null and b/tests/bank/test.fasta.gz differ diff --git a/tests/bank/test2.fasta b/tests/bank/test2.fasta new file mode 100644 index 0000000..410ca0f --- /dev/null +++ b/tests/bank/test2.fasta @@ -0,0 +1,2 @@ +>test2 +gcgcgcgcgcgcgcgccgcgcgcgcgcgcgcggc diff --git a/tests/bank/test_100.txt b/tests/bank/test_100.txt new file mode 100644 index 0000000..c7f7c3b --- /dev/null +++ b/tests/bank/test_100.txt @@ -0,0 +1 @@ +This is a sample file to extract Release 103 from a text file diff --git a/tests/biomaj_tests.py b/tests/biomaj_tests.py new file mode 100644 index 0000000..a8a245d --- /dev/null +++ b/tests/biomaj_tests.py @@ -0,0 +1,297 @@ +from nose.tools import * +from nose.plugins.attrib import attr + +import shutil +import os +import tempfile +import stat + +from biomaj_core.config import BiomajConfig +from biomaj_core.utils import Utils +from biomaj_core.bmajindex import BmajIndex + +import unittest + + +class UtilsForTest: + """ + Copy properties files to a temp directory and update properties to + use a temp directory + """ + + def __init__(self): + """ + Setup the temp dirs and files. + """ + self.global_properties = None + self.bank_properties = None + + self.test_dir = tempfile.mkdtemp('biomaj') + + self.conf_dir = os.path.join(self.test_dir, 'conf') + if not os.path.exists(self.conf_dir): + os.makedirs(self.conf_dir) + self.data_dir = os.path.join(self.test_dir, 'data') + if not os.path.exists(self.data_dir): + os.makedirs(self.data_dir) + self.log_dir = os.path.join(self.test_dir, 'log') + if not os.path.exists(self.log_dir): + os.makedirs(self.log_dir) + self.process_dir = os.path.join(self.test_dir, 'process') + if not os.path.exists(self.process_dir): + os.makedirs(self.process_dir) + self.lock_dir = os.path.join(self.test_dir, 'lock') + if not os.path.exists(self.lock_dir): + os.makedirs(self.lock_dir) + self.cache_dir = os.path.join(self.test_dir, 'cache') + if not os.path.exists(self.cache_dir): + os.makedirs(self.cache_dir) + + if self.global_properties is None: + self.__copy_global_properties() + + if self.bank_properties is None: + self.__copy_test_bank_properties() + + def clean(self): + """ + Deletes temp directory + """ + shutil.rmtree(self.test_dir) + + def __copy_test_bank_properties(self): + if self.bank_properties is not None: + return + self.bank_properties = ['alu', 'local', 'testhttp', 'directhttp', + 'noname', 'noexe', 'noargs', 'prenoname', + 'prenoexe', 'prenoargs', 'rmnoname', 'rmnoexe', + 'rmnoargs'] + curdir = os.path.dirname(os.path.realpath(__file__)) + for b in self.bank_properties: + from_file = os.path.join(curdir, b+'.properties') + to_file = os.path.join(self.conf_dir, b+'.properties') + shutil.copyfile(from_file, to_file) + + self.bank_process = ['test.sh'] + curdir = os.path.dirname(os.path.realpath(__file__)) + procdir = os.path.join(curdir, 'bank/process') + for proc in self.bank_process: + from_file = os.path.join(procdir, proc) + to_file = os.path.join(self.process_dir, proc) + shutil.copyfile(from_file, to_file) + os.chmod(to_file, stat.S_IRWXU) + + # Manage local bank test, use bank test subdir as remote + properties = ['multi.properties', 'computederror.properties', + 'error.properties', 'local.properties', + 'localprocess.properties', 'testhttp.properties', + 'computed.properties', 'computed2.properties', + 'sub1.properties', 'sub2.properties'] + for prop in properties: + from_file = os.path.join(curdir, prop) + to_file = os.path.join(self.conf_dir, prop) + fout = open(to_file,'w') + with open(from_file,'r') as fin: + for line in fin: + if line.startswith('remote.dir'): + fout.write("remote.dir="+os.path.join(curdir, + 'bank') + "\n") + elif line.startswith('remote.files'): + fout.write(line.replace('/tmp', os.path.join(curdir, + 'bank'))) + else: + fout.write(line) + fout.close() + + def __copy_global_properties(self): + if self.global_properties is not None: + return + self.global_properties = os.path.join(self.conf_dir, 'global.properties') + curdir = os.path.dirname(os.path.realpath(__file__)) + global_template = os.path.join(curdir, 'global.properties') + fout = open(self.global_properties, 'w') + with open(global_template,'r') as fin: + for line in fin: + if line.startswith('conf.dir'): + fout.write("conf.dir="+self.conf_dir+"\n") + elif line.startswith('log.dir'): + fout.write("log.dir="+self.log_dir+"\n") + elif line.startswith('data.dir'): + fout.write("data.dir="+self.data_dir+"\n") + elif line.startswith('process.dir'): + fout.write("process.dir="+self.process_dir+"\n") + elif line.startswith('lock.dir'): + fout.write("lock.dir="+self.lock_dir+"\n") + else: + fout.write(line) + fout.close() + + +class TestBiomajUtils(unittest.TestCase): + + def setUp(self): + self.utils = UtilsForTest() + + def tearDown(self): + self.utils.clean() + + def test_properties_override(self): + BiomajConfig.load_config(self.utils.global_properties, + allow_user_config=False) + config = BiomajConfig('local') + ldap_host = config.get('ldap.host') + self.assertTrue(ldap_host == 'localhost') + os.environ['BIOMAJ_LDAP_HOST'] = 'someserver' + ldap_host = config.get('ldap.host') + self.assertTrue(ldap_host == 'someserver') + + def test_service_config_override(self): + config = { + 'rabbitmq': { 'host': '1.2.3.4'} + } + Utils.service_config_override(config) + self.assertTrue(config['rabbitmq']['host'] == '1.2.3.4') + os.environ['RABBITMQ_HOST'] = '4.3.2.1' + Utils.service_config_override(config) + self.assertTrue(config['rabbitmq']['host'] == '4.3.2.1') + + def test_mimes(self): + fasta_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'bank/test2.fasta') + (mime, encoding) = Utils.detect_format(fasta_file) + self.assertTrue('application/fasta' == mime) + + @attr('compress') + def test_uncompress(self): + from_file = { 'root': os.path.dirname(os.path.realpath(__file__)), + 'name': 'bank/test.fasta.gz' + } + + to_dir = self.utils.data_dir + Utils.copy_files([from_file], to_dir) + Utils.uncompress(os.path.join(to_dir, from_file['name'])) + self.assertTrue(os.path.exists(to_dir+'/bank/test.fasta')) + + def test_copy_with_regexp(self): + from_dir = os.path.dirname(os.path.realpath(__file__)) + to_dir = self.utils.data_dir + Utils.copy_files_with_regexp(from_dir, to_dir, ['.*\.py']) + self.assertTrue(os.path.exists(to_dir+'/biomaj_tests.py')) + + def test_copy(self): + from_dir = os.path.dirname(os.path.realpath(__file__)) + local_file = 'biomaj_tests.py' + files_to_copy = [ {'root': from_dir, 'name': local_file}] + to_dir = self.utils.data_dir + Utils.copy_files(files_to_copy, to_dir) + self.assertTrue(os.path.exists(to_dir+'/biomaj_tests.py')) + + @attr('check') + def test_check_method(self): + """Check .name, .exe and .args are well check during bank configuration + checking""" + BiomajConfig.load_config(self.utils.global_properties) + for conf in ['noname', 'noexe', 'noargs', 'prenoname', 'prenoexe', + 'prenoargs', 'rmnoname', 'rmnoexe', 'rmnoargs']: + config = BiomajConfig(conf) + self.assertFalse(config.check()) + + +@attr('elastic') +class TestElastic(unittest.TestCase): + """ + test indexing and search + """ + + def setUp(self): + BmajIndex.es = None + self.utils = UtilsForTest() + curdir = os.path.dirname(os.path.realpath(__file__)) + BiomajConfig.load_config(self.utils.global_properties, + allow_user_config=False) + if BmajIndex.do_index == False: + self.skipTest("Skipping indexing tests due to elasticsearch not available") + # Delete all banks + b = Bank('local') + b.banks.remove({}) + BmajIndex.delete_all_bank('local') + + self.config = BiomajConfig('local') + data_dir = self.config.get('data.dir') + lock_file = os.path.join(data_dir,'local.lock') + if os.path.exists(lock_file): + os.remove(lock_file) + + def tearDown(self): + data_dir = self.config.get('data.dir') + lock_file = os.path.join(data_dir,'local.lock') + if os.path.exists(lock_file): + os.remove(lock_file) + self.utils.clean() + BmajIndex.delete_all_bank('test') + + def test_index(self): + BmajIndex.do_index = True + prod = { + "data_dir" : "/tmp/test/data", + "formats" : { + "fasta" : [ + { + "files" : [ + "fasta/chr1.fa", + "fasta/chr2.fa" + ], + "types" : [ + "nucleic" + ], + "tags" : { + "organism" : "hg19" + } + } + ], + "blast": [ + { + "files" : [ + "blast/chr1/chr1db" + ], + "types" : [ + "nucleic" + ], + "tags" : { + "chr" : "chr1", + "organism" : "hg19" + } + } + ] + + }, + "freeze" : False, + "session" : 1416229253.930908, + "prod_dir" : "alu-2003-11-26", + "release" : "2003-11-26", + "types" : [ + "nucleic" + ] + } + + BmajIndex.add('test',prod, True) + + query = { + 'query' : { + 'match' : {'bank': 'test'} + } + } + res = BmajIndex.search(query) + self.assertTrue(len(res)==2) + + + def test_remove_all(self): + self.test_index() + query = { + 'query' : { + 'match' : {'bank': 'test'} + } + } + BmajIndex.delete_all_bank('test') + res = BmajIndex.search(query) + self.assertTrue(len(res)==0) diff --git a/tests/computed.properties b/tests/computed.properties new file mode 100644 index 0000000..214baf4 --- /dev/null +++ b/tests/computed.properties @@ -0,0 +1,44 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="computed local system bank test" +db.name=local0 +db.type=nucleic_protein + +offline.dir.name=offline/test/local0_tmp +dir.version=test/local0 + +depends=sub1 +sub1.files.move=flat/test_.* + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/tmp/ +remote.files=^test.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/computed2.properties b/tests/computed2.properties new file mode 100644 index 0000000..2768000 --- /dev/null +++ b/tests/computed2.properties @@ -0,0 +1,45 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="computed local system bank test" +db.name=local0 +db.type=nucleic_protein + +offline.dir.name=offline/test/local0_tmp +dir.version=test/local0 + +depends=sub1 + +ref.release=sub1 + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/tmp/ +remote.files=^test.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/computederror.properties b/tests/computederror.properties new file mode 100644 index 0000000..ce4bae1 --- /dev/null +++ b/tests/computederror.properties @@ -0,0 +1,43 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="computed error local system bank test" +db.name=computederror +db.type=nucleic_protein + +offline.dir.name=offline/test/computederror_tmp +dir.version=test/computederror + +depends=sub2,error + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/tmp/ +remote.files=^test.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/directhttp.properties b/tests/directhttp.properties new file mode 100644 index 0000000..30f673d --- /dev/null +++ b/tests/directhttp.properties @@ -0,0 +1,41 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="directhttp system bank test" +db.name=directhttp +db.type=nucleic_protein + +offline.dir.name=offline/test/local_tmp +dir.version=test/directhttp + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=directhttp +server=ftp2.fr.debian.org + +release.protocol=directhttp +release.server=ftp2.fr.debian.org +release.remote.dir=/debian/README +release.file=README +release.regexp=([0-9.]+), +release.file.compressed= + +#remote.dir=common/downloads/Current_Release/Pfalciparum3D7/fasta/data/PlasmoDB-25_Pfalciparum3D7_Genome.fasta +#plasmo/communityDownload.do?fname=Atg3_alignment.txt +remote.dir=/debian/README.html +remote.files= + +local.files=debian/README.html + +## Post Process ## The files should be located in the projectfiles/process +BLOCKS= + +### Deployment ### + +keep.old.version=1 diff --git a/tests/error.properties b/tests/error.properties new file mode 100644 index 0000000..2e50f00 --- /dev/null +++ b/tests/error.properties @@ -0,0 +1,43 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="error local system bank test" +db.name=error +db.type=nucleic_protein + +offline.dir.name=offline/test/error_tmp +dir.version=test/error + +depends=sub2 + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/error/ +remote.files=^error.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^error.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/global.properties b/tests/global.properties new file mode 100644 index 0000000..75cb02c --- /dev/null +++ b/tests/global.properties @@ -0,0 +1,123 @@ +[GENERAL] +test=1 +conf.dir=/tmp/biomaj/config +log.dir=/tmp/biomaj/log +process.dir=/tmp/biomaj/process +#The root directory where all databases are stored. +#If your data is not stored under one directory hirearchy +#you can override this value in the database properties file. +data.dir=/tmp/biomaj/ +lock.dir=/tmp/biomaj/lock +cache.dir=/tmp/biomaj/cache + +db.url=mongodb://localhost:27017 +db.name=biomaj_test + +use_ldap=1 +ldap.host=localhost +ldap.port=389 +ldap.dn=nodomain + +# Use ElasticSearch for index/search capabilities +use_elastic=0 +#Comma separated list of elasticsearch nodes host1,host2:port2 +elastic_nodes=localhost +elastic_index=biomaj_test + +celery.queue=biomaj +celery.broker=mongodb://localhost:27017/biomaj_celery + +# Get directory stats (can be time consuming depending on number of files etc...) +data.stats=1 + +# List of user admin (linux user id, comma separated) +admin= + +# Auto publish on updates (do not need publish flag, can be ovveriden in bank property file) +auto_publish=0 + +######################## +# Global properties file + + +#To override these settings for a specific database go to its +#properties file and uncomment or add the specific line you want +#to override. + +#---------------- +# Mail Configuration +#--------------- +#Uncomment thes lines if you want receive mail when the workflow is finished + +mail.smtp.host= +mail.admin= +mail.from= + +#--------------------- +#Proxy authentification +#--------------------- +#proxyHost= +#proxyPort= +#proxyUser= +#proxyPassword= + +#Number of thread for processes +bank.num.threads=2 + +#Number of threads to use for downloading +files.num.threads=4 + +#to keep more than one release increase this value +keep.old.version=0 + +#---------------------- +# Release configuration +#---------------------- +release.separator=_ + +#The historic log file is generated in log/ +#define level information for output : DEBUG,INFO,WARN,ERR +historic.logfile.level=DEBUG + +#http.parse.dir.line=<a[\s]+href="([\S]+)/".*alt="\[DIR\]">.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2}) +http.parse.dir.line=<img[\s]+src="[\S]+"[\s]+alt="\[DIR\]"[\s]*/?>[\s]*<a[\s]+href="([\S]+)/"[\s]*>.*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2}) +http.parse.file.line=<img[\s]+src="[\S]+"[\s]+alt="\[[\s]+\]"[\s]*/?>[\s]<a[\s]+href="([\S]+)".*([\d]{2}-[\w\d]{2,5}-[\d]{4}\s[\d]{2}:[\d]{2})[\s]+([\d\.]+[MKG]{0,1}) + +http.group.dir.name=1 +http.group.dir.date=2 +http.group.file.name=1 +http.group.file.date=2 +http.group.file.size=3 + + +# Bank default access +visibility.default=public + + +[loggers] +keys = root, biomaj + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = console + +[logger_biomaj] +level = DEBUG +handlers = console +qualname = biomaj +propagate=0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s diff --git a/tests/local.properties b/tests/local.properties new file mode 100644 index 0000000..7f6f5fd --- /dev/null +++ b/tests/local.properties @@ -0,0 +1,41 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=local +db.type=nucleic_protein + +offline.dir.name=offline/test/local_tmp +dir.version=test/local + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/tmp/ +remote.files=^test.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/locallist.properties b/tests/locallist.properties new file mode 100644 index 0000000..a901b2c --- /dev/null +++ b/tests/locallist.properties @@ -0,0 +1,44 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=locallist +db.type=nucleic_protein + +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files= +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/localprocess.properties b/tests/localprocess.properties new file mode 100644 index 0000000..7166186 --- /dev/null +++ b/tests/localprocess.properties @@ -0,0 +1,100 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=local +db.type=nucleic_protein + +offline.dir.name=offline/test/local_tmp +dir.version=test/local + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/tmp/ +remote.files=^test.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Pre process +db.pre.process=META0 + +## Remove process +db.remove.process=META0 + +## Post Process ## The files should be located in the projectfiles/process directory +BLOCKS=BLOCK1,BLOCK2 +BLOCK1.db.post.process=META0 +BLOCK2.db.post.process=META1,META2,META3 +META0=PROC0 +META1=PROC1,PROC2 +META2=PROC3 +META3=PROC4,PROC5 + + +PROC0.name=test0 +PROC0.desc=sample test +PROC0.cluster=false +PROC0.type=test +PROC0.exe=echo +PROC0.args=test $datadir + +PROC1.name=test1 +PROC1.desc=sample test +PROC1.cluster=false +PROC1.type=test +PROC1.exe=touch +PROC1.args=$datadir/$dirversion/$localrelease/proc1.txt + +PROC2.name=test2 +PROC2.desc=sample test +PROC2.cluster=false +PROC2.type=test +PROC2.exe=touch +PROC2.args=$datadir/$dirversion/$localrelease/proc2.txt + +PROC3.name=test3 +PROC3.desc=sample test +PROC3.cluster=false +PROC3.type=test +PROC3.exe=echo +PROC3.args=test 3 + +PROC4.name=test4 +PROC4.desc=sample test +PROC4.cluster=false +PROC4.type=test +PROC4.exe=echo +PROC4.args=test 4 + +PROC5.name=test5 +PROC5.desc=sample test +PROC5.cluster=false +PROC5.type=testmetadata +PROC5.exe=test.sh +PROC5.args= +PROC5.format=test +PROC5.types=any +PROC5.tags=chr:chr1,organism:hg19 +# If files is set, then the post-process does not have to print generated files on STDOUT (but can) +# in this case, the list of files will be extracted from this list with above format/types/tags +PROC5.files=dir1/file1,dir1/file2,dir1/file3 + +### Deployment ### + +keep.old.version=1 diff --git a/tests/multi.properties b/tests/multi.properties new file mode 100644 index 0000000..82e08f9 --- /dev/null +++ b/tests/multi.properties @@ -0,0 +1,60 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname=test for multi protocol +db.name=multi +db.type=test + +offline.dir.name=offline/multi_tmp +dir.version=multi + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=multi +server= +remote.dir= + +remote.file.0.protocol = directhttp +remote.file.0.server = httpbin.org +remote.file.0.path = /get +remote.file.0.params.keys = key1,key2 +remote.file.0.params.key1 = value1 +remote.file.0.params.key2 = value2 +remote.file.0.name = test1.json + +remote.file.1.protocol = directhttp +remote.file.1.method = POST +remote.file.1.server = httpbin.org +remote.file.1.path = /post +remote.file.1.params.keys = key1,key2 +remote.file.1.params.key1 = value1 +remote.file.1.params.key2 = value2 +remote.file.1.name = test2.json + +release.file= +release.regexp= +release.file.compressed= + +remote.files=^stable/Release$ + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/noargs.properties b/tests/noargs.properties new file mode 100644 index 0000000..36710c8 --- /dev/null +++ b/tests/noargs.properties @@ -0,0 +1,48 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=noargs +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + +BLOCKS=PROC1 +PROC1.db.post.process=JOB1 +JOB1=job + +job.name=job_name +job.exe=job_exe.sh +job.desc=job desc \ No newline at end of file diff --git a/tests/noexe.properties b/tests/noexe.properties new file mode 100644 index 0000000..ade982e --- /dev/null +++ b/tests/noexe.properties @@ -0,0 +1,48 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=noexe +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + +BLOCKS=PROC1 +PROC1.db.post.process=JOB1 +JOB1=job + +job.name=job_name +job.args=args1 arg2 +job.desc=job desc \ No newline at end of file diff --git a/tests/noname.properties b/tests/noname.properties new file mode 100644 index 0000000..518a0bf --- /dev/null +++ b/tests/noname.properties @@ -0,0 +1,49 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=noname +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + +BLOCKS=PROC1 +PROC1.db.post.process=JOB1 +JOB1=job + + +job.exe=job_exe.sh +job.args=args1 arg2 +job.desc=job desc \ No newline at end of file diff --git a/tests/prenoargs.properties b/tests/prenoargs.properties new file mode 100644 index 0000000..e94c69f --- /dev/null +++ b/tests/prenoargs.properties @@ -0,0 +1,45 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=prenoargs +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.pre.process=PREJOB1 +PREJOB1=prejob + +prejob.name=job_name +prejob.exe=job_exe.sh +prejob.desc=job desc \ No newline at end of file diff --git a/tests/prenoexe.properties b/tests/prenoexe.properties new file mode 100644 index 0000000..8d0d89c --- /dev/null +++ b/tests/prenoexe.properties @@ -0,0 +1,45 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=prenoexe +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.pre.process=PREJOB1 +PREJOB1=prejob + +prejob.name=job_name +prejob.args=args1 arg2 +prejob.desc=job desc \ No newline at end of file diff --git a/tests/prenoname.properties b/tests/prenoname.properties new file mode 100644 index 0000000..97dd1a5 --- /dev/null +++ b/tests/prenoname.properties @@ -0,0 +1,45 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=prenoname +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.pre.process=PREJOB1 +PREJOB1=prejob + +prejob.exe=job_exe.sh +prejob.args=args1 arg2 +prejob.desc=job desc \ No newline at end of file diff --git a/tests/rmnoargs.properties b/tests/rmnoargs.properties new file mode 100644 index 0000000..11b2d45 --- /dev/null +++ b/tests/rmnoargs.properties @@ -0,0 +1,45 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=rmnoargs +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.remove.process=RMJOB1 +RMJOB1=rmjob + +rmjob.name=job_name +rmjob.exe=job_exe.sh +rmjob.desc=job desc \ No newline at end of file diff --git a/tests/rmnoexe.properties b/tests/rmnoexe.properties new file mode 100644 index 0000000..f5dcbaf --- /dev/null +++ b/tests/rmnoexe.properties @@ -0,0 +1,45 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=rmnoexe +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.remove.process=RMJOB1 +RMJOB1=rmjob + +rmjob.name=job_name +rmjob.args=args1 arg2 +rmjob.desc=job desc \ No newline at end of file diff --git a/tests/rmnoname.properties b/tests/rmnoname.properties new file mode 100644 index 0000000..54cc993 --- /dev/null +++ b/tests/rmnoname.properties @@ -0,0 +1,45 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="local system bank test" +db.name=rmnoname +db.type=nucleic_protein +db.formats=fasta +offline.dir.name=offline/test/locallist_tmp +dir.version=test/locallist +mail.smtp.host=nowhere +mail.from=here +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=ftp +server=nowhere + +release.files= +release.regexp= +release.file.compressed= + +remote.dir= +remote.files=empty +remote.files.list=true +remote.files.1.path=/tmp/test.fasta.gz +remote.files.2.path=/tmp/test2.fasta + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.remove.process=RMJOB1 +RMJOB1=rmjob + +rmjob.exe=job_exe.sh +rmjob.args=args1 arg2 +rmjob.desc=job desc \ No newline at end of file diff --git a/tests/sub1.properties b/tests/sub1.properties new file mode 100644 index 0000000..8e0c69b --- /dev/null +++ b/tests/sub1.properties @@ -0,0 +1,43 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="sub local system bank test" +db.name=local1 +db.type=nucleic_protein + +offline.dir.name=offline/test/local1_tmp +dir.version=test/local1 + +depends=sub2 + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/tmp/ +remote.files=^test.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/sub2.properties b/tests/sub2.properties new file mode 100644 index 0000000..b9d3142 --- /dev/null +++ b/tests/sub2.properties @@ -0,0 +1,41 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname="sub local system bank test" +db.name=local2 +db.type=nucleic_protein + +offline.dir.name=offline/test/local2_tmp +dir.version=test/local2 + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=local +server= + +release.file= +release.regexp= +release.file.compressed= + +remote.dir=/tmp/ +remote.files=^test.* + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^test.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + +### Deployment ### + +keep.old.version=1 diff --git a/tests/testhttp.properties b/tests/testhttp.properties new file mode 100644 index 0000000..454753e --- /dev/null +++ b/tests/testhttp.properties @@ -0,0 +1,43 @@ +[GENERAL] +###################### +### Initialization ### + +db.fullname=test for http protocol +db.name=testhttp +db.type=package + +offline.dir.name=offline/testhttp_tmp +dir.version=testhttp + +frequency.update=0 + +### Synchronization ### + +files.num.threads=1 + +# NCBI (download fasta) +protocol=http +server=ftp2.fr.debian.org +remote.dir=/debian/dists/ + +release.file= +release.regexp= +release.file.compressed= + +remote.files=^stable/Release$ + +#Uncomment if you don't want to extract the data files. +#no.extract=true + +local.files=^.* + +## Post Process ## The files should be located in the projectfiles/process directory + +db.post.process= + + + + +### Deployment ### + +keep.old.version=1 -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/biomaj3-core.git _______________________________________________ debian-med-commit mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/debian-med-commit
