Hi, I did a quick test, works. I committed that patch with minor modifications:
- changed arg line back to arg value for test args. The problem in the python script is the " ".join(testArgs), whoich breaks when test args are escaped. So it is better to pass the whole arg line as one parameter, escaped completely. The joining then is a no-op, because there is only one parameter. - removed useless svnversion. I hope this also works on Jenkins. I'll trigger a build. Uwe ----- Uwe Schindler H.-H.-Meier-Allee 63, D-28213 Bremen http://www.thetaphi.de eMail: [email protected] > -----Original Message----- > From: Uwe Schindler [mailto:[email protected]] > Sent: Thursday, August 28, 2014 11:01 AM > To: [email protected] > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: > buildAndPushRelease.py smokeTestRelease.py > > I think the attached patch should be correct: > > Index: build.xml > ========================================================== > ========= > --- build.xml (revision 1621092) > +++ build.xml (working copy) > @@ -407,13 +407,15 @@ > <!-- Tell Python not to write any bytecode cache into the filesystem: > --> > <arg value="-B"/> > <arg file="dev-tools/scripts/smokeTestRelease.py"/> > - <arg value="${fakeRelease.uri}"/> > + <arg value="--revision"/> > <arg value="skip"/> > + <arg value="--version"/> > <arg value="${fakeReleaseVersion}"/> > + <arg value="--tmp-dir"/> > <arg file="${fakeReleaseTmp}"/> > - <arg value="false"/> > - <arg value="-testArgs"/> > - <arg value="${smokeTestRelease.testArgs}"/> > + <arg value="--not-signed"/> > + <arg value="${fakeRelease.uri}"/> > + <arg line="${smokeTestRelease.testArgs}"/> > <env key="JAVA7_HOME" file="${JAVA7_HOME}"/> > </exec> > <delete dir="${fakeRelease}"/> > > ----- > Uwe Schindler > H.-H.-Meier-Allee 63, D-28213 Bremen > http://www.thetaphi.de > eMail: [email protected] > > > > -----Original Message----- > > From: Uwe Schindler [mailto:[email protected]] > > Sent: Thursday, August 28, 2014 10:39 AM > > To: [email protected] > > Subject: RE: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: > > buildAndPushRelease.py smokeTestRelease.py > > > > Hi, > > > > did the parameter names change by that? If yes, you need to fix the > > build.xml in the root folder to pass right params in "nigtly-smoke". > > > > Uwe > > > > ----- > > Uwe Schindler > > H.-H.-Meier-Allee 63, D-28213 Bremen > > http://www.thetaphi.de > > eMail: [email protected] > > > > > -----Original Message----- > > > From: [email protected] [mailto:[email protected]] > > > Sent: Thursday, August 28, 2014 10:02 AM > > > To: [email protected] > > > Subject: svn commit: r1621085 - in /lucene/dev/trunk/dev-tools/scripts: > > > buildAndPushRelease.py smokeTestRelease.py > > > > > > Author: rjernst > > > Date: Thu Aug 28 08:01:50 2014 > > > New Revision: 1621085 > > > > > > URL: http://svn.apache.org/r1621085 > > > Log: > > > Change buildAndPushRelease and smokeTestRelease scripts to use > > > argparse > > > > > > Modified: > > > lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py > > > lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py > > > > > > Modified: lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py > > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev- > > > > > > tools/scripts/buildAndPushRelease.py?rev=1621085&r1=1621084&r2=162108 > > > 5&view=diff > > > > > > ========================================================== > > > ==================== > > > --- lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py > > > (original) > > > +++ lucene/dev/trunk/dev-tools/scripts/buildAndPushRelease.py Thu > > > +++ Aug > > > 28 > > > +++ 08:01:50 2014 > > > @@ -13,6 +13,7 @@ > > > # See the License for the specific language governing permissions > > > and # limitations under the License. > > > > > > +import argparse > > > import datetime > > > import re > > > import time > > > @@ -20,16 +21,7 @@ import shutil > > > import os > > > import sys > > > import subprocess > > > - > > > -# Usage: python3.2 -u buildAndPushRelease.py [-sign gpgKey(eg: > > > 6E68DA61)] [-prepare] [-push userName] [-pushLocal dirName] [-smoke > > > tmpDir] /path/to/checkout version(eg: 3.4.0) rcNum(eg: 0) -# -# EG: > > > python3.2 -u buildAndPushRelease.py -prepare -push mikemccand -sign > > > 6E68DA61 /path/to/lucene_solr_4_7 4.7.0 0 > > > - > > > -# NOTE: if you specify -sign, you have to type in your gpg password > > > at -# some point while this runs; it's VERY confusing because the > > > output -# is directed to /tmp/release.log, so, you have to tail that > > > and when -# GPG wants your password, type it! Also sometimes you > > > have to type -# it twice in a row! > > > +import textwrap > > > > > > LOG = '/tmp/release.log' > > > > > > @@ -226,94 +218,89 @@ def pushLocal(version, root, rev, rcNum, > > > > > > print(' done!') > > > return 'file://%s/%s' % (os.path.abspath(localDir), dir) > > > - > > > -def main(): > > > - doPrepare = '-prepare' in sys.argv > > > - if doPrepare: > > > - sys.argv.remove('-prepare') > > > > > > - try: > > > - idx = sys.argv.index('-push') > > > - except ValueError: > > > - doPushRemote = False > > > - else: > > > - doPushRemote = True > > > - username = sys.argv[idx+1] > > > - del sys.argv[idx:idx+2] > > > +def read_version(path): > > > + version_props_file = os.path.join(path, 'lucene', > > > +'version.properties') > > > + return re.search(r'version\.base=(.*)', > > > +open(version_props_file).read()).group(1) > > > + > > > +def parse_config(): > > > + epilogue = textwrap.dedent(''' > > > + Example usage for a Release Manager: > > > + python3.2 -u buildAndPushRelease.py --push-remote mikemccand > > > +--sign > > > +6E68DA61 --rc-num 1 --version 4.7.0 /path/to/lucene_solr_4_7 > > > + ''') > > > + description = 'Utility to build, push, and test a release.' > > > + parser = argparse.ArgumentParser(description=description, > > > epilog=epilogue, > > > + > > > +formatter_class=argparse.RawDescriptionHelpFormatter) > > > + parser.add_argument('--no-prepare', dest='prepare', default=True, > > > action='store_false', > > > + help='Use the already built release in the > > > +provided checkout') > > > + parser.add_argument('--push-remote', metavar='USERNAME', > > > + help='Push the release to people.apache.org > > > +for the given user') > > > + parser.add_argument('--push-local', metavar='PATH', > > > + help='Push the release to the local path') > > > + parser.add_argument('--sign', metavar='KEYID', > > > + help='Sign the release with the given gpg > > > +key') > > > + parser.add_argument('--rc-num', metavar='NUM', type=int, default=1, > > > + help='Release Candidate number, required') > > > + parser.add_argument('--smoke-test', metavar='PATH', > > > + help='Run the smoker tester on the release in > > > +the given directory') > > > + parser.add_argument('root', metavar='checkout_path', > > > + help='Root of SVN checkout for lucene-solr') > > > + config = parser.parse_args() > > > + > > > + if config.push_remote is not None and config.push_local is not None: > > > + parser.error('Cannot specify --push-remote and --push-local > > > + together') if not config.prepare and config.sign: > > > + parser.error('Cannot sign already built release') if > > > + config.push_local is not None and os.path.exists(config.push_local): > > > + parser.error('Cannot push to local path that already exists') > > > + if config.rc_num <= 0: > > > + parser.error('Release Candidate number must be a positive > > > + integer') if not os.path.isdir(config.root): > > > + # TODO: add additional svn check to ensure dir is a real > > > + lucene-solr > > > checkout > > > + parser.error('Root path is not a valid lucene-solr checkout') > > > + if config.smoke_test is not None and > os.path.exists(config.smoke_test): > > > + parser.error('Smoke test path already exists') > > > > > > - try: > > > - idx = sys.argv.index('-smoke') > > > - except ValueError: > > > - smokeTmpDir = None > > > - else: > > > - smokeTmpDir = sys.argv[idx+1] > > > - del sys.argv[idx:idx+2] > > > - if os.path.exists(smokeTmpDir): > > > - print() > > > - print('ERROR: smoke tmpDir "%s" exists; please remove first' % > > > smokeTmpDir) > > > - print() > > > - sys.exit(1) > > > - > > > - try: > > > - idx = sys.argv.index('-pushLocal') > > > - except ValueError: > > > - doPushLocal = False > > > - else: > > > - doPushLocal = True > > > - localStagingDir = sys.argv[idx+1] > > > - del sys.argv[idx:idx+2] > > > - if os.path.exists(localStagingDir): > > > - print() > > > - print('ERROR: pushLocal dir "%s" exists; please remove first' % > > > localStagingDir) > > > - print() > > > - sys.exit(1) > > > - > > > - if doPushRemote and doPushLocal: > > > - print() > > > - print('ERROR: specify at most one of -push or -pushLocal (got both)') > > > - print() > > > - sys.exit(1) > > > - > > > - try: > > > - idx = sys.argv.index('-sign') > > > - except ValueError: > > > - gpgKeyID = None > > > - else: > > > - gpgKeyID = sys.argv[idx+1] > > > - del sys.argv[idx:idx+2] > > > + config.version = read_version(config.root) print('Building version: > > > + %s' % config.version) > > > > > > + if config.sign: > > > sys.stdout.flush() > > > import getpass > > > - gpgPassword = getpass.getpass('Enter GPG keystore password: ') > > > + config.key_id = config.sign > > > + config.key_password = getpass.getpass('Enter GPG keystore > password: > > > + ') > > > + else: > > > + config.gpg_password = None > > > > > > - root = os.path.abspath(sys.argv[1]) > > > - version = sys.argv[2] > > > - rcNum = int(sys.argv[3]) > > > + return config > > > + > > > +def main(): > > > + c = parse_config() > > > > > > - if doPrepare: > > > - rev = prepare(root, version, gpgKeyID, gpgPassword, smokeTmpDir is > > > None) > > > + if c.prepare: > > > + rev = prepare(c.root, c.version, c.key_id, c.key_password, > > > + key.smoke_test is None) > > > else: > > > os.chdir(root) > > > rev = open('rev.txt', encoding='UTF-8').read() > > > > > > - if doPushRemote: > > > + if c.push_remote: > > > url = push(version, root, rev, rcNum, username) > > > - elif doPushLocal: > > > - url = pushLocal(version, root, rev, rcNum, localStagingDir) > > > + elif c.push_local: > > > + url = pushLocal(version, root, rev, c.rc_num, c.push_local) > > > else: > > > url = None > > > > > > if url is not None: > > > print(' URL: %s' % url) > > > > > > - if smokeTmpDir is not None: > > > + if c.smoke_test is not None: > > > import smokeTestRelease > > > smokeTestRelease.DEBUG = False > > > - smokeTestRelease.smokeTest(url, rev, version, smokeTmpDir, > > gpgKeyID > > > is not None, '') > > > + smokeTestRelease.smokeTest(url, rev, c.version, c.smoke_test, > > > + c.sign is not None, '') > > > > > > if __name__ == '__main__': > > > try: > > > main() > > > - except: > > > - import traceback > > > - traceback.print_exc() > > > + except KeyboardInterrupt: > > > + print('Keyboard interrupt...exiting') > > > + > > > > > > Modified: lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py > > > URL: http://svn.apache.org/viewvc/lucene/dev/trunk/dev- > > > > > > tools/scripts/smokeTestRelease.py?rev=1621085&r1=1621084&r2=1621085& > > > view=diff > > > > > > ========================================================== > > > ==================== > > > --- lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py > > > (original) > > > +++ lucene/dev/trunk/dev-tools/scripts/smokeTestRelease.py Thu Aug > > > +++ 28 > > > +++ 08:01:50 2014 > > > @@ -13,6 +13,7 @@ > > > # See the License for the specific language governing permissions > > > and # limitations under the License. > > > > > > +import argparse > > > import os > > > import zipfile > > > import codecs > > > @@ -40,6 +41,7 @@ import checkJavaDocs import checkJavadocLinks > > > import io import codecs > > > +import textwrap > > > > > > # This tool expects to find /lucene and /solr off the base URL. > > > You # must have a working gpg, tar, unzip in your path. This has > > > been @@ -1248,55 > > > +1250,60 @@ def crawl(downloadedFiles, urlString, ta > > > downloadedFiles.append(path) > > > sys.stdout.write('.') > > > > > > -reAllowedVersion = re.compile(r'^\d+\.\d+\.\d+(-ALPHA|-BETA)?$') > > > - > > > -def main(): > > > - > > > - if len(sys.argv) < 5: > > > - print() > > > - print('Usage python -u %s BaseURL SvnRevision version tmpDir [ > > > isSigned(True|False) ] [ -testArgs "-Dwhat=ever [ ... ]" ]' > > > - % sys.argv[0]) > > > - print() > > > - print(' example: python3.2 -u dev-tools/scripts/smokeTestRelease.py > > > http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC1 > > > - > > > rev1469340 1469340 4.3.0 /path/to/a/tmp/dir') > > > - print() > > > - sys.exit(1) > > > - > > > - baseURL = sys.argv[1] > > > - svnRevision = sys.argv[2] > > > - version = sys.argv[3] > > > +version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)') > > > +revision_re = re.compile(r'rev(\d+)') def parse_config(): > > > + epilogue = textwrap.dedent(''' > > > + Example usage: > > > + python3.2 -u dev-tools/scripts/smokeTestRelease.py > > > +http://people.apache.org/~whoever/staging_area/lucene-solr-4.3.0-RC > > > +1- > > > re > > > +v1469340') > > > + ''') > > > + description = 'Utility to test a release.' > > > + parser = argparse.ArgumentParser(description=description, > > > epilog=epilogue, > > > + > > > +formatter_class=argparse.RawDescriptionHelpFormatter) > > > + parser.add_argument('--tmp-dir', metavar='PATH', > > > + help='Temporary directory to test inside, > > > +defaults to /tmp/smoke_lucene_$version_$revision') > > > + parser.add_argument('--not-signed', dest='is_signed', > > > action='store_false', default=True, > > > + help='Indicates the release is not signed') > > > + parser.add_argument('--revision', > > > + help='SVN revision number that release was > > > +built with, defaults to that in URL') > > > + parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?', > > > + help='Version of the release, defaults to > > > +that in > > > +URL') > > > + parser.add_argument('url', help='Url pointing to release to > > > +test') > > > + parser.add_argument('test_args', nargs=argparse.REMAINDER, > > > metavar='ARGS', > > > + help='Arguments to pass to ant for testing, e.g. > > > +-Dwhat=ever') > > > + c = parser.parse_args() > > > + > > > + if c.version is not None: > > > + if not version_re.match(c.version): > > > + parser.error('version "%s" does not match format > > > + X.Y.Z[-ALPHA|-BETA]' % c.version) > > > + else: > > > + version_match = version_re.search(c.url) > > > + if version_match is None: > > > + parser.error('Could not find version in URL') > > > + c.version = version_match.group(1) > > > + > > > + if c.revision is None: > > > + revision_match = revision_re.search(c.url) > > > + if revision_match is None: > > > + parser.error('Could not find revision in URL') > > > + c.revision = revision_match.group(1) > > > > > > - if not reAllowedVersion.match(version): > > > - raise RuntimeError('version "%s" does not match format X.Y.Z[- > ALPHA|- > > > BETA]' % version) > > > - > > > - tmpDirArgNum = 4 > > > - tmpDir = os.path.abspath(sys.argv[tmpDirArgNum]) > > > + if c.tmp_dir: > > > + c.tmp_dir = os.path.abspath(c.tmp_dir) > > > + else: > > > + tmp = '/tmp/smoke_lucene_%s_%s' % (c.version, c.revision) > > > + c.tmp_dir = tmp > > > + i = 1 > > > + while os.path.exists(c.tmp_dir): > > > + c.tmp_dir = tmp + '_%d' % i > > > + i += 1 > > > > > > - # TODO: yuck: positional-only args with more than one optional > > > arg totally sucks > > > - # TODO: consider naming all args > > > - isSigned = True > > > - testArgs = '' > > > - lastArgNum = len(sys.argv) - 1 > > > - if lastArgNum > tmpDirArgNum: > > > - nextArgNum = tmpDirArgNum + 1 > > > - if sys.argv[nextArgNum] == '-testArgs': > > > - if nextArgNum == lastArgNum: > > > - raise RuntimeError('missing expected argument to -testArgs') > > > - else: > > > - # TODO: should there be arg validation here? E.g. starts with > > > '-D'? > > > - testArgs = sys.argv[nextArgNum + 1] > > > - nextArgNum += 2 > > > - if nextArgNum <= lastArgNum: > > > - isSigned = (sys.argv[nextArgNum].lower() == "true") > > > - nextArgNum += 1 > > > - if nextArgNum <= lastArgNum and testArgs == '': > > > - if sys.argv[nextArgNum] == '-testArgs': > > > - if nextArgNum == lastArgNum: > > > - raise RuntimeError('missing expected argument to -testArgs') > > > - else: > > > - # TODO: should there be arg validation here? E.g. starts with > > > '-D'? > > > - testArgs = sys.argv[nextArgNum + 1] > > > + return c > > > > > > - smokeTest(baseURL, svnRevision, version, tmpDir, isSigned, > > > testArgs) > > > +def main(): > > > + c = parse_config() > > > + print('NOTE: output encoding is %s' % sys.stdout.encoding) > > > + smokeTest(c.url, c.revision, c.version, c.tmp_dir, c.is_signed, ' > > > +'.join(c.test_args)) > > > > > > def smokeTest(baseURL, svnRevision, version, tmpDir, isSigned, > testArgs): > > > > > > @@ -1350,10 +1357,8 @@ def smokeTest(baseURL, svnRevision, vers > > > print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - > > > startTime)) > > > > > > if __name__ == '__main__': > > > - print('NOTE: output encoding is %s' % sys.stdout.encoding) > > > try: > > > main() > > > - except: > > > - traceback.print_exc() > > > - sys.exit(1) > > > - sys.exit(0) > > > + except KeyboardInterrupt: > > > + print('Keyboard interrupt...exiting') > > > + > > > > > > > > --------------------------------------------------------------------- > > To unsubscribe, e-mail: [email protected] For > > additional commands, e-mail: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
