This is an automated email from the ASF dual-hosted git repository. wohali pushed a commit to branch upload-failed-logfiles in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 023bca12693ff29f781ac516db4bfc9ba5356ded Author: Joan Touzet <[email protected]> AuthorDate: Wed May 10 18:45:19 2017 -0400 Add auto log uploader for CI workflow This commit is intended to improve failure analysis in Travis and Jenkins runs by uploading couch log files after failed test runs. A new script has been added that, upon invocation from the top-level directory, tars up all EUnit logfiles and the dev/logs/node1.log file. It then constructs a document based on the running environment (Travis, Jenkins, manual). The document is posted to a CouchDB instance running on couchdb-vm.apache.org with credentials passed in the COUCHAUTH environment variable. The .travis.yml file has been updated to install the script's prerequisites (python-requests) and with the secret credentials, and to instruct travis to run the log uploader after any test failure. Similar steps will be taken to reconfigure Jenkins after these changes are merged to master. The test/javascript/run harness had to be modified to terminate after the first failure. This is because each new test wipes the logfile of the previous run. All tests can still be run, regardless of failure, by specifing the -a/--all flag, however it is expected that the reduction in test suite execution time when a test fails will be appreciated by many. --- .gitignore | 2 + .travis.yml | 21 ++++++-- build-aux/logfile-uploader.py | 113 ++++++++++++++++++++++++++++++++++++++++++ test/javascript/run | 6 ++- 4 files changed, 137 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 480904f..3e22192 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ log apache-couchdb-*/ bin/ config.erl +*.tar.gz +*.tar.bz2 dev/boot_node.beam dev/lib/ dev/logs/ diff --git a/.travis.yml b/.travis.yml index 580ac6c..156856c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,19 +17,29 @@ addons: - libicu-dev - libmozjs185-dev - pkg-config + - python-requests + - python-sphinx - help2man - shunit2 +python: + - "2.7" + git: depth: 10 +# logfile uploader uses requests +cache: + - pip + +# logfile uploader credentials +env: + global: + - secure: "UdA/gKIlyuXaW+hUgRx40t1TYjLCGxMqHvM5Uw7UbUH2dqEkgJiLfhZGchS1JVzl8M01VKZUUzS7v2nvRLiHZN1kvaw5kfq31VRoafUah8jfmvqNWZVdLovHl3aw5UX/HRt0RkbWbhdbdknTfh6+YinSZ+Nb54jCErMg9nabXtM=" + # Enable this block if you want to build docs & fauxton too -#cache: -# - pip #node_js: # - 6 -#install: -# - pip install sphinx #before_script: # - ./configure -c @@ -40,6 +50,9 @@ before_script: script: - make check +after_failure: + - build-aux/logfile-uploader.py + # Re-enable once test suite is reliable #notifications: # email: false diff --git a/build-aux/logfile-uploader.py b/build-aux/logfile-uploader.py new file mode 100755 index 0000000..7218b58 --- /dev/null +++ b/build-aux/logfile-uploader.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +from __future__ import print_function + +import datetime +import glob +import json +import os +import tarfile +import time + +import requests + +COUCH_URL = "https://couchdb-vm2.apache.org/ci_errorlogs"; +TARFILE = "couchlog.tar.gz" + +def _tojson(req): + """Support requests v0.x as well as 1.x+""" + if requests.__version__[0] == '0': + return json.loads(req.content) + return req.json() + +def collect_logfiles(): + """ Find and tarball all logfiles """ + tb = tarfile.open(name=TARFILE, mode='w:gz') + # EUnit + for log in glob.glob('src/*/.eunit/couch.log'): + tb.add(log) + # JS harness + for log in glob.glob('dev/logs/node1.log'): + tb.add(log) + tb.close() + +def build_ci_doc(): + """ Build a metadata document with relevant detail from CI env """ + doc = {} + if 'TRAVIS' in os.environ: + doc['builder'] = 'travis' + doc['build_id'] = os.environ['TRAVIS_JOB_ID'] + doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE'] + doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \ + os.environ['TRAVIS_JOB_ID'] + doc['branch'] = os.environ['TRAVIS_BRANCH'] + doc['commit'] = os.environ['TRAVIS_COMMIT'] + doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG'] + elif 'JENKINS_URL' in os.environ: + doc['builder'] = 'jenkins' + doc['build_id'] = os.environ['BUILD_NUMBER'] + doc['url'] = os.environ['BUILD_URL'] + doc['branch'] = os.environ['GIT_BRANCH'] + doc['commit'] = os.environ['GIT_COMMIT'] + doc['repo'] = os.environ['GIT_URL'] + else: + doc['builder'] = 'manual' + # TODO: shell out to get correct repo, commit, branch info? + doc['repo'] = 'https://github.com/apache/couchdb' + doc['build_id'] = str(time.time()) + + # shorten doc id + repo = doc['repo'].split('/')[-1] + repo = repo.replace('.git', '') + + doc['_id'] = doc['builder'] + '-' + repo + '-' + \ + doc['build_id'] + \ + '-' + datetime.datetime.utcnow().isoformat() + + return doc + +def upload_logs(): + lp = os.environ['COUCHAUTH'].split(':') + creds = (lp[0], lp[1]) + doc = build_ci_doc() + req = requests.post(COUCH_URL, + data=json.dumps(doc), + auth=creds, + headers={'Content-type': 'application/json'}) + req.raise_for_status() + req = _tojson(req) + with open(TARFILE, 'rb') as f: + # ancient versions of requests break if data is iterable + fdata = f.read() + req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE, + headers={'Content-type': 'application/x-gtar'}, + auth=creds, + params={'rev': req['rev']}, + data=fdata) + req2.raise_for_status() + return req2 + + +def main(): + """ Find latest logfile and upload to Couch logfile db. """ + print ("Uploading logfiles...") + collect_logfiles() + req = upload_logs() + print (req.url) + print (req.content) + print ("Upload complete!") + +if __name__ == '__main__': + main() diff --git a/test/javascript/run b/test/javascript/run index 10d00d8..7f366eb 100755 --- a/test/javascript/run +++ b/test/javascript/run @@ -99,7 +99,9 @@ def options(): return [ op.make_option("-s", "--start", metavar="FILENAME", default=None, help="Start from the given filename if multiple files " - "are passed") + "are passed"), + op.make_option("-a", "--all", action="store_true", dest="all", + help="Run all tests, even if one or more fail") ] @@ -141,6 +143,8 @@ def main(): passed += 1 else: failed += 1 + if not opts.all: + break sys.stderr.write("=======================================================" + os.linesep) -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
