Modified: subversion/branches/verify-keep-going/tools/server-side/svnauthz.c URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnauthz.c?rev=1546002&r1=1546001&r2=1546002&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnauthz.c (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnauthz.c Wed Nov 27 11:52:35 2013 @@ -59,7 +59,9 @@ static const apr_getopt_option_t options {"repository", svnauthz__repos, 1, ("repository authz name")}, {"transaction", 't', 1, ("transaction id")}, {"is", svnauthz__is, 1, - ("instead of outputing, tests if the access is ARG\n" + ("instead of outputting, test if the access is\n" + " " + "exactly ARG\n" " " "ARG can be one of the following values:\n" " " @@ -67,10 +69,12 @@ static const apr_getopt_option_t options " " " r read-only access\n" " " - " no no access\n") + " no no access") }, - {"groups-file", svnauthz__groups_file, 1, ("path to the global groups file")}, - {"recursive", 'R', 0, ("recursive access to path")}, + {"groups-file", svnauthz__groups_file, 1, + ("use the groups from file ARG")}, + {"recursive", 'R', 0, + ("determine recursive access to PATH")}, {0, 0, 0, 0} }; @@ -129,27 +133,32 @@ static const svn_opt_subcommand_desc2_t {'t'} }, {"accessof", subcommand_accessof, {0} /* no aliases */, ("Print or test the permissions set by an authz file.\n" - "usage: 1. svnauthz accessof [--username USER] [--groups-file GROUPS_FILE] TARGET\n" - " 2. svnauthz accessof [--username USER] [--groups-file GROUPS_FILE] \\\n" - " -t TXN REPOS_PATH FILE_PATH\n\n" - " 1. Prints the access of USER based on TARGET.\n" + "usage: 1. svnauthz accessof TARGET\n" + " 2. svnauthz accessof -t TXN REPOS_PATH FILE_PATH\n" + "\n" + " 1. Prints the access of USER to PATH based on authorization file at TARGET.\n" " TARGET can be a path to a file or an absolute file:// URL to an authz\n" - " file in a repository, but cannot be a repository relative URL (^/).\n\n" - " 2. Prints the access of USER based on authz file at FILE_PATH in the\n" - " transaction TXN in the repository at REPOS_PATH.\n\n" - " If the --username argument is omitted then access of an anonymous user\n" - " will be printed. If --path argument is omitted prints if any access\n" - " to the repo is allowed. If --groups-file is specified, the groups from\n" - " GROUPS_FILE will be used.\n\n" + " file in a repository, but cannot be a repository relative URL (^/).\n" + "\n" + " 2. Prints the access of USER to PATH based on authz file at FILE_PATH in the\n" + " transaction TXN in the repository at REPOS_PATH.\n" + "\n" + " USER is the argument to the --username option; if that option is not\n" + " provided, then access of an anonymous user will be printed or tested.\n" + "\n" + " PATH is the argument to the --path option; if that option is not provided,\n" + " the maximal access to any path in the repository will be considered.\n" + "\n" "Outputs one of the following:\n" " rw write access (which also implies read)\n" " r read access\n" - " no no access\n\n" + " no no access\n" + "\n" "Returns:\n" - " 0 when syntax is OK and --is argument (if any) matches.\n" + " 0 when syntax is OK and '--is' argument (if any) matches.\n" " 1 when syntax is invalid.\n" " 2 operational error\n" - " 3 when --is argument doesn't match\n" + " 3 when '--is' argument doesn't match\n" ), {'t', svnauthz__username, svnauthz__path, svnauthz__repos, svnauthz__is, svnauthz__groups_file, 'R'} }, @@ -373,42 +382,6 @@ subcommand_accessof(apr_getopt_t *os, vo #undef EXIT_FAILURE #define EXIT_FAILURE 2 -/* Similar to svn_cmdline_handle_exit_error but with an exit_code argument - so we can comply with our contract and exit with 2 for internal failures. - Also is missing the pool argument since we don't need it given - main/sub_main. */ -static int -handle_exit_error(svn_error_t *err, const char *prefix, int exit_code) -{ - /* Issue #3014: - * Don't print anything on broken pipes. The pipe was likely - * closed by the process at the other end. We expect that - * process to perform error reporting as necessary. - * - * ### This assumes that there is only one error in a chain for - * ### SVN_ERR_IO_PIPE_WRITE_ERROR. See svn_cmdline_fputs(). */ - if (err->apr_err != SVN_ERR_IO_PIPE_WRITE_ERROR) - svn_handle_error2(err, stderr, FALSE, prefix); - svn_error_clear(err); - return exit_code; -} - -/* Report and clear the error ERR, and return EXIT_FAILURE. */ -#define EXIT_ERROR(err, exit_code) \ - handle_exit_error(err, "svnauthz: ", exit_code) - -/* A redefinition of the public SVN_INT_ERR macro, that suppresses the - * error message if it is SVN_ERR_IO_PIPE_WRITE_ERROR, amd with the - * program name 'svnauthz' instead of 'svn'. */ -#undef SVN_INT_ERR -#define SVN_INT_ERR(expr) \ - do { \ - svn_error_t *svn_err__temp = (expr); \ - if (svn_err__temp) \ - return EXIT_ERROR(svn_err__temp, EXIT_FAILURE); \ - } while (0) - - /* Return TRUE if the UI of 'svnauthz-validate' (svn 1.7 and earlier) should be emulated, given argv[0]. */ static svn_boolean_t @@ -476,8 +449,13 @@ canonicalize_access_file(const char **ca return SVN_NO_ERROR; } -static int -sub_main(int argc, const char *argv[], apr_pool_t *pool) +/* + * On success, leave *EXIT_CODE untouched and return SVN_NO_ERROR. On error, + * either return an error to be displayed, or set *EXIT_CODE to non-zero and + * return SVN_NO_ERROR. + */ +static svn_error_t * +sub_main(int *exit_code, int argc, const char *argv[], apr_pool_t *pool) { svn_error_t *err; @@ -488,7 +466,7 @@ sub_main(int argc, const char *argv[], a int i; /* Initialize the FS library. */ - SVN_INT_ERR(svn_fs_initialize(pool)); + SVN_ERR(svn_fs_initialize(pool)); received_opts = apr_array_make(pool, SVN_OPT_MAX_OPTIONS, sizeof(int)); @@ -497,7 +475,7 @@ sub_main(int argc, const char *argv[], a opt_state.txn = opt_state.repos_path = opt_state.groups_file = NULL; /* Parse options. */ - SVN_INT_ERR(svn_cmdline__getopt_init(&os, argc, argv, pool)); + SVN_ERR(svn_cmdline__getopt_init(&os, argc, argv, pool)); os->interleave = 1; if (!use_compat_mode(argv[0], pool)) @@ -512,8 +490,9 @@ sub_main(int argc, const char *argv[], a break; if (status != APR_SUCCESS) { - SVN_INT_ERR(subcommand_help(NULL, NULL, pool)); - return EXIT_FAILURE; + SVN_ERR(subcommand_help(NULL, NULL, pool)); + *exit_code = EXIT_FAILURE; + return SVN_NO_ERROR; } /* Stash the option code in an array before parsing it. */ @@ -526,7 +505,7 @@ sub_main(int argc, const char *argv[], a opt_state.help = TRUE; break; case 't': - SVN_INT_ERR(svn_utf_cstring_to_utf8(&opt_state.txn, arg, pool)); + SVN_ERR(svn_utf_cstring_to_utf8(&opt_state.txn, arg, pool)); break; case 'R': opt_state.recursive = TRUE; @@ -535,28 +514,29 @@ sub_main(int argc, const char *argv[], a opt_state.version = TRUE; break; case svnauthz__username: - SVN_INT_ERR(svn_utf_cstring_to_utf8(&opt_state.username, arg, pool)); + SVN_ERR(svn_utf_cstring_to_utf8(&opt_state.username, arg, pool)); break; case svnauthz__path: - SVN_INT_ERR(svn_utf_cstring_to_utf8(&opt_state.fspath, arg, pool)); + SVN_ERR(svn_utf_cstring_to_utf8(&opt_state.fspath, arg, pool)); opt_state.fspath = svn_fspath__canonicalize(opt_state.fspath, pool); break; case svnauthz__repos: - SVN_INT_ERR(svn_utf_cstring_to_utf8(&opt_state.repos_name, arg, pool)); + SVN_ERR(svn_utf_cstring_to_utf8(&opt_state.repos_name, arg, pool)); break; case svnauthz__is: - SVN_INT_ERR(svn_utf_cstring_to_utf8(&opt_state.is, arg, pool)); + SVN_ERR(svn_utf_cstring_to_utf8(&opt_state.is, arg, pool)); break; case svnauthz__groups_file: - SVN_INT_ERR( + SVN_ERR( svn_utf_cstring_to_utf8(&opt_state.groups_file, arg, pool)); break; default: { - SVN_INT_ERR(subcommand_help(NULL, NULL, pool)); - return EXIT_FAILURE; + SVN_ERR(subcommand_help(NULL, NULL, pool)); + *exit_code = EXIT_FAILURE; + return SVN_NO_ERROR; } } } @@ -594,8 +574,9 @@ sub_main(int argc, const char *argv[], a { svn_error_clear(svn_cmdline_fprintf(stderr, pool, ("subcommand argument required\n"))); - SVN_INT_ERR(subcommand_help(NULL, NULL, pool)); - return EXIT_FAILURE; + SVN_ERR(subcommand_help(NULL, NULL, pool)); + *exit_code = EXIT_FAILURE; + return SVN_NO_ERROR; } } else @@ -607,14 +588,15 @@ sub_main(int argc, const char *argv[], a const char *first_arg_utf8; os->ind++; - SVN_INT_ERR(svn_utf_cstring_to_utf8(&first_arg_utf8, + SVN_ERR(svn_utf_cstring_to_utf8(&first_arg_utf8, first_arg, pool)); svn_error_clear( svn_cmdline_fprintf(stderr, pool, ("Unknown subcommand: '%s'\n"), first_arg_utf8)); - SVN_INT_ERR(subcommand_help(NULL, NULL, pool)); - return EXIT_FAILURE; + SVN_ERR(subcommand_help(NULL, NULL, pool)); + *exit_code = EXIT_FAILURE; + return SVN_NO_ERROR; } } } @@ -628,13 +610,12 @@ sub_main(int argc, const char *argv[], a { if (os->ind +2 != argc) { - err = svn_error_create(SVN_ERR_CL_ARG_PARSING_ERROR, NULL, - ("Repository and authz file arguments " - "required")); - return EXIT_ERROR(err, EXIT_FAILURE); + return svn_error_create(SVN_ERR_CL_ARG_PARSING_ERROR, NULL, + ("Repository and authz file arguments " + "required")); } - SVN_INT_ERR(svn_utf_cstring_to_utf8(&opt_state.repos_path, os->argv[os->ind], + SVN_ERR(svn_utf_cstring_to_utf8(&opt_state.repos_path, os->argv[os->ind], pool)); os->ind++; @@ -644,24 +625,23 @@ sub_main(int argc, const char *argv[], a /* Exactly 1 non-option argument */ if (os->ind + 1 != argc) { - err = svn_error_create(SVN_ERR_CL_ARG_PARSING_ERROR, NULL, - ("Authz file argument required")); - return EXIT_ERROR(err, EXIT_FAILURE); + return svn_error_create(SVN_ERR_CL_ARG_PARSING_ERROR, NULL, + ("Authz file argument required")); } /* Grab AUTHZ_FILE from argv. */ - SVN_INT_ERR(svn_utf_cstring_to_utf8(&opt_state.authz_file, os->argv[os->ind], + SVN_ERR(svn_utf_cstring_to_utf8(&opt_state.authz_file, os->argv[os->ind], pool)); /* Canonicalize opt_state.authz_file appropriately. */ - SVN_INT_ERR(canonicalize_access_file(&opt_state.authz_file, + SVN_ERR(canonicalize_access_file(&opt_state.authz_file, opt_state.authz_file, opt_state.txn != NULL, pool)); /* Same for opt_state.groups_file if it is present. */ if (opt_state.groups_file) { - SVN_INT_ERR(canonicalize_access_file(&opt_state.groups_file, + SVN_ERR(canonicalize_access_file(&opt_state.groups_file, opt_state.groups_file, opt_state.txn != NULL, pool)); } @@ -687,13 +667,14 @@ sub_main(int argc, const char *argv[], a pool); svn_opt_format_option(&optstr, badopt, FALSE, pool); if (subcommand->name[0] == '-') - SVN_INT_ERR(subcommand_help(NULL, NULL, pool)); + SVN_ERR(subcommand_help(NULL, NULL, pool)); else svn_error_clear(svn_cmdline_fprintf(stderr, pool, ("Subcommand '%s' doesn't accept option '%s'\n" "Type 'svnauthz help %s' for usage.\n"), subcommand->name, optstr, subcommand->name)); - return EXIT_FAILURE; + *exit_code = EXIT_FAILURE; + return SVN_NO_ERROR; } } @@ -715,7 +696,8 @@ sub_main(int argc, const char *argv[], a { /* Follow our contract that says we exit with 1 if the file does not validate. */ - return EXIT_ERROR(err, 1); + *exit_code = 1; + return err; } else if (err->apr_err == SVN_ERR_AUTHZ_UNREADABLE || err->apr_err == SVN_ERR_AUTHZ_UNWRITABLE @@ -723,31 +705,22 @@ sub_main(int argc, const char *argv[], a { /* Follow our contract that says we exit with 3 if --is does not * match. */ - return EXIT_ERROR(err, 3); + *exit_code = 3; + return err; } - - return EXIT_ERROR(err, EXIT_FAILURE); - } - else - { - /* Ensure that everything is written to stdout, so the user will - see any print errors. */ - err = svn_cmdline_fflush(stdout); - if (err) - { - return EXIT_ERROR(err, EXIT_FAILURE); - } - return EXIT_SUCCESS; + return err; } + return SVN_NO_ERROR; } int main(int argc, const char *argv[]) { apr_pool_t *pool; - int exit_code; + int exit_code = EXIT_SUCCESS; + svn_error_t *err; /* Initialize the app. Send all error messages to 'stderr'. */ if (svn_cmdline_init(argv[0], stderr) != EXIT_SUCCESS) @@ -755,7 +728,18 @@ main(int argc, const char *argv[]) pool = svn_pool_create(NULL); - exit_code = sub_main(argc, argv, pool); + err = sub_main(&exit_code, argc, argv, pool); + + /* Flush stdout and report if it fails. It would be flushed on exit anyway + but this makes sure that output is not silently lost if it fails. */ + err = svn_error_compose_create(err, svn_cmdline_fflush(stdout)); + + if (err) + { + if (exit_code == 0) + exit_code = EXIT_FAILURE; + svn_cmdline_handle_exit_error(err, NULL, "svnauthz: "); + } svn_pool_destroy(pool); return exit_code;
Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/daemonize.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/daemonize.py?rev=1546002&r1=1546001&r2=1546002&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/daemonize.py (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/daemonize.py Wed Nov 27 11:52:35 2013 @@ -24,6 +24,7 @@ import os import signal import sys import time +import multiprocessing # requires Python 2.6 # possible return values from Daemon.daemonize() @@ -50,11 +51,11 @@ class Daemon(object): def daemonize_exit(self): try: result = self.daemonize() - except (ChildFailed, DaemonFailed) as e: + except (ChildFailed, DaemonFailed), e: # duplicate the exit code sys.exit(e.code) except (ChildTerminatedAbnormally, ChildForkFailed, - DaemonTerminatedAbnormally, DaemonForkFailed) as e: + DaemonTerminatedAbnormally, DaemonForkFailed), e: sys.stderr.write('ERROR: %s\n' % e) sys.exit(1) except ChildResumedIncorrectly: @@ -71,29 +72,41 @@ class Daemon(object): # in original process. daemon is up and running. we're done. def daemonize(self): - # fork off a child that can detach itself from this process. - try: - pid = os.fork() - except OSError as e: - raise ChildForkFailed(e.errno, e.strerror) - - if pid > 0: - # we're in the parent. let's wait for the child to finish setting - # things up -- on our exit, we want to ensure the child is accepting - # connections. - cpid, status = os.waitpid(pid, 0) - assert pid == cpid - if os.WIFEXITED(status): - code = os.WEXITSTATUS(status) - if code: - raise ChildFailed(code) - return DAEMON_RUNNING - - # the child did not exit cleanly. - raise ChildTerminatedAbnormally(status) + ### review error situations. map to backwards compat. ?? + ### be mindful of daemonize_exit(). + ### we should try and raise ChildFailed / ChildTerminatedAbnormally. + ### ref: older revisions. OR: remove exceptions. + + child_is_ready = multiprocessing.Event() + child_completed = multiprocessing.Event() + + p = multiprocessing.Process(target=self._first_child, + args=(child_is_ready, child_completed)) + p.start() + + # Wait for the child to finish setting things up (in case we need + # to communicate with it). It will only exit when ready. + ### use a timeout here! (parameterized, of course) + p.join() + + ### need to propagate errors, to adjust the return codes + if child_completed.is_set(): + ### what was the exit status? + return DAEMON_COMPLETE + if child_is_ready.is_set(): + return DAEMON_RUNNING + + ### how did we get here?! the immediate child should not exit without + ### signalling ready/complete. some kind of error. + return DAEMON_STARTED + def _first_child(self, child_is_ready, child_completed): # we're in the child. + ### NOTE: the original design was a bit bunk. Exceptions raised from + ### this point are within the child processes. We need to signal the + ### errors to the parent in other ways. + # decouple from the parent process os.chdir('/') os.umask(0) @@ -102,56 +115,86 @@ class Daemon(object): # remember this pid so the second child can signal it. thispid = os.getpid() - # register a signal handler so the SIGUSR1 doesn't stop the process. - # this object will also record whether if got signalled. - daemon_accepting = SignalCatcher(signal.SIGUSR1) - - # if the daemon process exits before sending SIGUSR1, then we need to see - # the problem. trap SIGCHLD with a SignalCatcher. + # if the daemon process exits before signalling readiness, then we + # need to see the problem. trap SIGCHLD with a SignalCatcher. daemon_exit = SignalCatcher(signal.SIGCHLD) # perform the second fork try: pid = os.fork() - except OSError as e: + except OSError, e: + ### this won't make it to the parent process raise DaemonForkFailed(e.errno, e.strerror) if pid > 0: # in the parent. - # we want to wait for the daemon to signal that it has created and - # bound the socket, and is (thus) ready for connections. if the - # daemon improperly exits before serving, we'll see SIGCHLD and the - # .pause will return. - ### we should add a timeout to this. allow an optional parameter to - ### specify the timeout, in case it takes a long time to start up. - signal.pause() + + # Wait for the child to be ready for operation. + while True: + # The readiness event will invariably be signalled early/first. + # If it *doesn't* get signalled because the child has prematurely + # exited, then we will pause 10ms before noticing the exit. The + # pause is acceptable since that is aberrant/unexpected behavior. + ### is there a way to break this wait() on a signal such as SIGCHLD? + ### parameterize this wait, in case the app knows children may + ### fail quickly? + if child_is_ready.wait(timeout=0.010): + # The child signalled readiness. Yay! + break + if daemon_exit.signalled: + # Whoops. The child exited without signalling :-( + break + # Python 2.6 compat: .wait() may exit when set, but return None + if child_is_ready.is_set(): + break + # A simple timeout. The child is taking a while to prepare. Go + # back and wait for readiness. if daemon_exit.signalled: + # Tell the parent that the child has exited. + ### we need to communicate the exit status, if possible. + child_completed.set() + # reap the daemon process, getting its exit code. bubble it up. cpid, status = os.waitpid(pid, 0) assert pid == cpid if os.WIFEXITED(status): code = os.WEXITSTATUS(status) if code: + ### this won't make it to the parent process raise DaemonFailed(code) + ### this return value is ignored return DAEMON_NOT_RUNNING # the daemon did not exit cleanly. + ### this won't make it to the parent process raise DaemonTerminatedAbnormally(status) - if daemon_accepting.signalled: - # the daemon is up and running, so save the pid and return success. - if self.pidfile: - open(self.pidfile, 'w').write('%d\n' % pid) - return DAEMON_STARTED + # child_is_ready got asserted. the daemon is up and running, so + # save the pid and return success. + if self.pidfile: + # Be wary of symlink attacks + try: + os.remove(self.pidfile) + except OSError: + pass + fd = os.open(self.pidfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0444) + os.write(fd, '%d\n' % pid) + os.close(fd) + ### this return value is ignored + return DAEMON_STARTED + + ### old code. what to do with this? throw ChildResumedIncorrectly + ### or just toss this and the exception. # some other signal popped us out of the pause. the daemon might not # be running. + ### this won't make it to the parent process raise ChildResumedIncorrectly() - # we're a deamon now. get rid of the final remnants of the parent. - # start by restoring default signal handlers + # we're a daemon now. get rid of the final remnants of the parent: + # restore the signal handlers and switch std* to the proper files. signal.signal(signal.SIGUSR1, signal.SIG_DFL) signal.signal(signal.SIGCHLD, signal.SIG_DFL) sys.stdout.flush() @@ -169,30 +212,31 @@ class Daemon(object): so.close() se.close() - # TEST: don't release the parent immediately. the whole parent stack - # should pause along with this sleep. + ### TEST: don't release the parent immediately. the whole parent stack + ### should pause along with this sleep. #time.sleep(10) # everything is set up. call the initialization function. self.setup() - # sleep for one second before signalling. we want to make sure the - # parent has called signal.pause() - ### we should think of a better wait around the race condition. - time.sleep(1) + ### TEST: exit before signalling. + #sys.exit(0) + #sys.exit(1) - # okay. the daemon is ready. signal the parent to tell it we're set. - os.kill(thispid, signal.SIGUSR1) + # the child is now ready for parent/anyone to communicate with it. + child_is_ready.set() # start the daemon now. self.run() # The daemon is shutting down, so toss the pidfile. - try: - os.remove(self.pidfile) - except OSError: - pass + if self.pidfile: + try: + os.remove(self.pidfile) + except OSError: + pass + ### this return value is ignored return DAEMON_COMPLETE def setup(self): @@ -202,6 +246,34 @@ class Daemon(object): raise NotImplementedError +class _Detacher(Daemon): + def __init__(self, target, logfile='/dev/null', pidfile=None, + args=(), kwargs={}): + Daemon.__init__(self, logfile, pidfile) + self.target = target + self.args = args + self.kwargs = kwargs + + def setup(self): + pass + + def run(self): + self.target(*self.args, **self.kwargs) + + +def run_detached(target, *args, **kwargs): + """Simple function to run TARGET as a detached daemon. + + The additional arguments/keywords will be passed along. This function + does not return -- sys.exit() will be called as appropriate. + + (capture SystemExit if logging/reporting is necessary) + ### if needed, a variant of this func could be written to not exit + """ + d = _Detacher(target, args=args, kwargs=kwargs) + d.daemonize_exit() + + class SignalCatcher(object): def __init__(self, signum): self.signalled = False Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py?rev=1546002&r1=1546001&r2=1546002&view=diff ============================================================================== --- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py (original) +++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py Wed Nov 27 11:52:35 2013 @@ -476,7 +476,15 @@ def handle_options(options): # Otherwise, we should write this (foreground) PID into the file. if options.pidfile and not options.daemon: pid = os.getpid() - open(options.pidfile, 'w').write('%s\n' % pid) + # Be wary of symlink attacks + try: + os.remove(options.pidfile) + except OSError: + pass + fd = os.open(options.pidfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0444) + os.write(fd, '%d\n' % pid) + os.close(fd) logging.info('pid %d written to %s', pid, options.pidfile) if options.gid: @@ -536,7 +544,8 @@ def main(args): # We manage the logfile ourselves (along with possible rotation). The # daemon process can just drop stdout/stderr into /dev/null. - d = Daemon('/dev/null', options.pidfile, options.umask, bdec) + d = Daemon('/dev/null', os.path.abspath(options.pidfile), + options.umask, bdec) if options.daemon: # Daemonize the process and call sys.exit() with appropriate code d.daemonize_exit() Modified: subversion/branches/verify-keep-going/win-tests.py URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/win-tests.py?rev=1546002&r1=1546001&r2=1546002&view=diff ============================================================================== --- subversion/branches/verify-keep-going/win-tests.py (original) +++ subversion/branches/verify-keep-going/win-tests.py Wed Nov 27 11:52:35 2013 @@ -84,6 +84,8 @@ def _usage_exit(): print(" --disable-bulk-updates : Disable bulk updates on HTTP server") print(" --ssl-cert : Path to SSL server certificate to trust.") print(" --javahl : Run the javahl tests instead of the normal tests") + print(" --swig=language : Run the swig perl/python/ruby tests instead of") + print(" the normal tests") print(" --list : print test doc strings only") print(" --milestone-filter=RE : RE is a regular expression pattern that (when") print(" used with --list) limits the tests listed to") @@ -108,29 +110,24 @@ CMDLINE_TEST_SCRIPT_NATIVE_PATH = CMDLIN sys.path.insert(0, os.path.join('build', 'generator')) sys.path.insert(1, 'build') -import gen_win +import gen_win_dependencies +import gen_base version_header = os.path.join('subversion', 'include', 'svn_version.h') cp = configparser.ConfigParser() cp.read('gen-make.opts') -gen_obj = gen_win.GeneratorBase('build.conf', version_header, - cp.items('options')) +gen_obj = gen_win_dependencies.GenDependenciesBase('build.conf', version_header, + cp.items('options')) all_tests = gen_obj.test_progs + gen_obj.bdb_test_progs \ + gen_obj.scripts + gen_obj.bdb_scripts client_tests = [x for x in all_tests if x.startswith(CMDLINE_TEST_SCRIPT_PATH)] -svn_dlls = [] -for section in gen_obj.sections.values(): - if section.options.get("msvc-export"): - dll_basename = section.name + "-" + str(gen_obj.version) + ".dll" - svn_dlls.append(os.path.join("subversion", section.name, dll_basename)) - opts, args = my_getopt(sys.argv[1:], 'hrdvqct:pu:f:', ['release', 'debug', 'verbose', 'quiet', 'cleanup', 'test=', 'url=', 'svnserve-args=', 'fs-type=', 'asp.net-hack', 'httpd-dir=', 'httpd-port=', 'httpd-daemon', 'httpd-server', 'http-short-circuit', 'httpd-no-log', 'disable-http-v2', 'disable-bulk-updates', 'help', - 'fsfs-packing', 'fsfs-sharding=', 'javahl', + 'fsfs-packing', 'fsfs-sharding=', 'javahl', 'swig=', 'list', 'enable-sasl', 'bin=', 'parallel', 'config-file=', 'server-minor-version=', 'log-level=', 'log-to-stdout', 'mode-filter=', 'milestone-filter=', @@ -156,6 +153,7 @@ http_bulk_updates = True list_tests = None milestone_filter = None test_javahl = None +test_swig = None enable_sasl = None svn_bin = None parallel = None @@ -216,6 +214,11 @@ for opt, val in opts: fsfs_packing = 1 elif opt == '--javahl': test_javahl = 1 + elif opt == '--swig': + if val not in ['perl', 'python', 'ruby']: + sys.stderr.write('Running \'%s\' swig tests not supported (yet).\n' + % (val,)) + test_swig = val elif opt == '--list': list_tests = 1 elif opt == '--milestone-filter': @@ -289,12 +292,18 @@ def create_target_dir(dirname): print("mkdir: %s" % tgt_dir) os.makedirs(tgt_dir) -def copy_changed_file(src, tgt): +def copy_changed_file(src, tgt=None, to_dir=None, cleanup=True): if not os.path.isfile(src): print('Could not find ' + src) sys.exit(1) - if os.path.isdir(tgt): - tgt = os.path.join(tgt, os.path.basename(src)) + + if to_dir and not tgt: + tgt = os.path.join(to_dir, os.path.basename(src)) + elif not tgt or (tgt and to_dir): + raise RuntimeError("Using 'tgt' *or* 'to_dir' is required" % (tgt,)) + elif tgt and os.path.isdir(tgt): + raise RuntimeError("'%s' is a directory. Use to_dir=" % (tgt,)) + if os.path.exists(tgt): assert os.path.isfile(tgt) if filecmp.cmp(src, tgt): @@ -306,57 +315,39 @@ def copy_changed_file(src, tgt): print("copy: %s" % src) print(" to: %s" % tgt) shutil.copy(src, tgt) - return 1 -def copy_execs(baton, dirname, names): - copied_execs = baton - for name in names: - if not name.endswith('.exe'): - continue - src = os.path.join(dirname, name) - tgt = os.path.join(abs_builddir, dirname, name) - create_target_dir(dirname) - if copy_changed_file(src, tgt): - copied_execs.append(tgt) + if cleanup: + copied_execs.append(tgt) def locate_libs(): "Move DLLs to a known location and set env vars" - dlls = [] - - # look for APR 1.x dll's and use those if found - apr_test_path = os.path.join(gen_obj.apr_path, objdir, 'libapr-1.dll') - if os.path.exists(apr_test_path): - suffix = "-1" - else: - suffix = "" - - if cp.has_option('options', '--with-static-apr'): - dlls.append(os.path.join(gen_obj.apr_path, objdir, - 'libapr%s.dll' % (suffix))) - dlls.append(os.path.join(gen_obj.apr_util_path, objdir, - 'libaprutil%s.dll' % (suffix))) - - if gen_obj.libintl_path is not None: - dlls.append(os.path.join(gen_obj.libintl_path, 'bin', 'intl3_svn.dll')) - - if gen_obj.bdb_lib is not None: - partial_path = os.path.join(gen_obj.bdb_path, 'bin', gen_obj.bdb_lib) - if objdir == 'Debug': - dlls.append(partial_path + 'd.dll') - else: - dlls.append(partial_path + '.dll') - - if gen_obj.sasl_path is not None: - dlls.append(os.path.join(gen_obj.sasl_path, 'lib', 'libsasl.dll')) + debug = (objdir == 'Debug') + + for lib in gen_obj._libraries.values(): + + if debug: + name, dir = lib.debug_dll_name, lib.debug_dll_dir + else: + name, dir = lib.dll_name, lib.dll_dir + + if name and dir: + src = os.path.join(dir, name) + if os.path.exists(src): + copy_changed_file(src, to_dir=abs_builddir, cleanup=False) + + for name in lib.extra_bin: + src = os.path.join(dir, name) + copy_changed_file(src, to_dir=abs_builddir) - for dll in dlls: - copy_changed_file(dll, abs_objdir) # Copy the Subversion library DLLs - if not cp.has_option('options', '--disable-shared'): - for svn_dll in svn_dlls: - copy_changed_file(os.path.join(abs_objdir, svn_dll), abs_objdir) + for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL): + if isinstance(i, gen_base.TargetLib) and i.msvc_export: + src = os.path.join(abs_objdir, i.filename) + if os.path.isfile(src): + copy_changed_file(src, to_dir=abs_builddir, + cleanup=False) # Copy the Apache modules if run_httpd and cp.has_option('options', '--with-httpd'): @@ -367,11 +358,11 @@ def locate_libs(): mod_dontdothat_path = os.path.join(abs_objdir, 'tools', 'server-side', 'mod_dontdothat', 'mod_dontdothat.so') - copy_changed_file(mod_dav_svn_path, abs_objdir) - copy_changed_file(mod_authz_svn_path, abs_objdir) - copy_changed_file(mod_dontdothat_path, abs_objdir) + copy_changed_file(mod_dav_svn_path, to_dir=abs_builddir, cleanup=False) + copy_changed_file(mod_authz_svn_path, to_dir=abs_builddir, cleanup=False) + copy_changed_file(mod_dontdothat_path, to_dir=abs_builddir, cleanup=False) - os.environ['PATH'] = abs_objdir + os.pathsep + os.environ['PATH'] + os.environ['PATH'] = abs_builddir + os.pathsep + os.environ['PATH'] def fix_case(path): path = os.path.normpath(path) @@ -489,15 +480,9 @@ class Httpd: self._create_mime_types_file() self._create_dontdothat_file() - # Determine version. - if os.path.exists(os.path.join(self.httpd_dir, - 'modules', 'mod_access_compat.so')): - self.httpd_ver = 2.3 - elif os.path.exists(os.path.join(self.httpd_dir, - 'modules', 'mod_auth_basic.so')): - self.httpd_ver = 2.2 - else: - self.httpd_ver = 2.0 + # Obtain version. + version_vals = gen_obj._libraries['httpd'].version.split('.') + self.httpd_ver = float('%s.%s' % (version_vals[0], version_vals[1])) # Create httpd config file fp = open(self.httpd_config, 'w') @@ -608,7 +593,7 @@ class Httpd: return 'LoadModule ' + name + " " + self._quote(full_path) + '\n' def _svn_module(self, name, path): - full_path = os.path.join(self.abs_objdir, path) + full_path = os.path.join(self.abs_builddir, path) return 'LoadModule ' + name + ' ' + self._quote(full_path) + '\n' def _svn_repo(self, name): @@ -688,21 +673,17 @@ class Httpd: print('Httpd.stop_daemon not implemented') # Move the binaries to the test directory +create_target_dir(abs_builddir) locate_libs() if create_dirs: - old_cwd = os.getcwd() - try: - os.chdir(abs_objdir) - baton = copied_execs - for dirpath, dirs, files in os.walk('subversion'): - copy_execs(baton, dirpath, files) - for dirpath, dirs, files in os.walk('tools/server-side'): - copy_execs(baton, dirpath, files) - except: - os.chdir(old_cwd) - raise - else: - os.chdir(old_cwd) + for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL): + if isinstance(i, gen_base.TargetExe): + src = os.path.join(abs_objdir, i.filename) + + if os.path.isfile(src): + dst = os.path.join(abs_builddir, i.filename) + create_target_dir(os.path.dirname(dst)) + copy_changed_file(src, dst) # Create the base directory for Python tests create_target_dir(CMDLINE_TEST_SCRIPT_NATIVE_PATH) @@ -760,7 +741,7 @@ else: print('Testing %s configuration on %s' % (objdir, repo_loc)) sys.path.insert(0, os.path.join(abs_srcdir, 'build')) -if not test_javahl: +if not test_javahl and not test_swig: import run_tests if log_to_stdout: log_file = None @@ -788,44 +769,203 @@ if not test_javahl: raise else: os.chdir(old_cwd) -else: +elif test_javahl: failed = False - args = ( - 'java.exe', - '-Dtest.rootdir=' + os.path.join(abs_builddir, 'javahl'), - '-Dtest.srcdir=' + os.path.join(abs_srcdir, - 'subversion/bindings/javahl'), - '-Dtest.rooturl=', - '-Dtest.fstype=' + fs_type , - '-Dtest.tests=', - - '-Djava.library.path=' - + os.path.join(abs_objdir, - 'subversion/bindings/javahl/native'), - '-classpath', - os.path.join(abs_srcdir, 'subversion/bindings/javahl/classes') +';' + - gen_obj.junit_path - ) - - sys.stderr.flush() - print('Running org.apache.subversion tests:') - sys.stdout.flush() - - r = subprocess.call(args + tuple(['org.apache.subversion.javahl.RunTests'])) - sys.stdout.flush() - sys.stderr.flush() - if (r != 0): - print('[Test runner reported failure]') - failed = True - print('Running org.tigris.subversion tests:') - sys.stdout.flush() - r = subprocess.call(args + tuple(['org.tigris.subversion.javahl.RunTests'])) - sys.stdout.flush() - sys.stderr.flush() + java_exe = None + + for path in os.environ["PATH"].split(os.pathsep): + if os.path.isfile(os.path.join(path, 'java.exe')): + java_exe = os.path.join(path, 'java.exe') + break + + if not java_exe and 'java_sdk' in gen_obj._libraries: + jdk = gen_obj._libraries['java_sdk'] + + if os.path.isfile(os.path.join(jdk.lib_dir, '../bin/java.exe')): + java_exe = os.path.join(jdk.lib_dir, '../bin/java.exe') + + if not java_exe: + print('Java not found. Skipping Java tests') + else: + args = ( + os.path.abspath(java_exe), + '-Dtest.rootdir=' + os.path.join(abs_builddir, 'javahl'), + '-Dtest.srcdir=' + os.path.join(abs_srcdir, + 'subversion/bindings/javahl'), + '-Dtest.rooturl=', + '-Dtest.fstype=' + fs_type , + '-Dtest.tests=', + + '-Djava.library.path=' + + os.path.join(abs_objdir, + 'subversion/bindings/javahl/native'), + '-classpath', + os.path.join(abs_srcdir, 'subversion/bindings/javahl/classes') +';' + + gen_obj.junit_path + ) + + sys.stderr.flush() + print('Running org.apache.subversion tests:') + sys.stdout.flush() + + r = subprocess.call(args + tuple(['org.apache.subversion.javahl.RunTests'])) + sys.stdout.flush() + sys.stderr.flush() + if (r != 0): + print('[Test runner reported failure]') + failed = True + + print('Running org.tigris.subversion tests:') + sys.stdout.flush() + r = subprocess.call(args + tuple(['org.tigris.subversion.javahl.RunTests'])) + sys.stdout.flush() + sys.stderr.flush() + if (r != 0): + print('[Test runner reported failure]') + failed = True +elif test_swig == 'perl': + failed = False + swig_dir = os.path.join(abs_builddir, 'swig') + swig_pl_dir = os.path.join(swig_dir, 'p5lib') + swig_pl_svn = os.path.join(swig_pl_dir, 'SVN') + swig_pl_auto_svn = os.path.join(swig_pl_dir, 'auto', 'SVN') + + create_target_dir(swig_pl_svn) + + for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL): + if isinstance(i, gen_base.TargetSWIG) and i.lang == 'perl': + mod_dir = os.path.join(swig_pl_auto_svn, '_' + i.name[5:].capitalize()) + create_target_dir(mod_dir) + copy_changed_file(os.path.join(abs_objdir, i.filename), to_dir=mod_dir) + + elif isinstance(i, gen_base.TargetSWIGLib) and i.lang == 'perl': + copy_changed_file(os.path.join(abs_objdir, i.filename), + to_dir=abs_builddir) + + pm_src = os.path.join(abs_srcdir, 'subversion', 'bindings', 'swig', 'perl', + 'native') + + tests = [] + + for root, dirs, files in os.walk(pm_src): + for name in files: + if name.endswith('.pm'): + fn = os.path.join(root, name) + copy_changed_file(fn, to_dir=swig_pl_svn) + elif name.endswith('.t'): + tests.append(os.path.relpath(os.path.join(root, name), pm_src)) + + perl5lib = swig_pl_dir + if 'PERL5LIB' in os.environ: + perl5lib += os.pathsep + os.environ['PERL5LIB'] + + perl_exe = 'perl.exe' + + print('-- Running Swig Perl tests --') + old_cwd = os.getcwd() + try: + os.chdir(pm_src) + + os.environ['PERL5LIB'] = perl5lib + os.environ["SVN_DBG_NO_ABORT_ON_ERROR_LEAK"] = 'YES' + + r = subprocess.call([ + perl_exe, + '-MExtUtils::Command::MM', + '-e', 'test_harness()' + ] + tests) + finally: + os.chdir(old_cwd) + if (r != 0): print('[Test runner reported failure]') failed = True + sys.exit(1) +elif test_swig == 'python': + failed = False + swig_dir = os.path.join(abs_builddir, 'swig') + swig_py_dir = os.path.join(swig_dir, 'pylib') + swig_py_libsvn = os.path.join(swig_py_dir, 'libsvn') + swig_py_svn = os.path.join(swig_py_dir, 'svn') + + create_target_dir(swig_py_libsvn) + create_target_dir(swig_py_svn) + + for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL): + if (isinstance(i, gen_base.TargetSWIG) + or isinstance(i, gen_base.TargetSWIGLib)) and i.lang == 'python': + + src = os.path.join(abs_objdir, i.filename) + copy_changed_file(src, to_dir=swig_py_libsvn) + + py_src = os.path.join(abs_srcdir, 'subversion', 'bindings', 'swig', 'python') + + for py_file in os.listdir(py_src): + if py_file.endswith('.py'): + copy_changed_file(os.path.join(py_src, py_file), + to_dir=swig_py_libsvn) + + py_src_svn = os.path.join(py_src, 'svn') + for py_file in os.listdir(py_src_svn): + if py_file.endswith('.py'): + copy_changed_file(os.path.join(py_src_svn, py_file), + to_dir=swig_py_svn) + + print('-- Running Swig Python tests --') + + pythonpath = swig_py_dir + if 'PYTHONPATH' in os.environ: + pythonpath += os.pathsep + os.environ['PYTHONPATH'] + + python_exe = 'python.exe' + old_cwd = os.getcwd() + try: + os.environ['PYTHONPATH'] = pythonpath + + r = subprocess.call([ + python_exe, + os.path.join(py_src, 'tests', 'run_all.py') + ]) + finally: + os.chdir(old_cwd) + + if (r != 0): + print('[Test runner reported failure]') + failed = True + +elif test_swig == 'ruby': + failed = False + + if 'ruby' not in gen_obj._libraries: + print('Ruby not found. Skipping Ruby tests') + else: + ruby_lib = gen_obj._libraries['ruby'] + + ruby_exe = 'ruby.exe' + ruby_subdir = os.path.join('subversion', 'bindings', 'swig', 'ruby') + ruby_args = [ + '-I', os.path.join(abs_srcdir, ruby_subdir), + os.path.join(abs_srcdir, ruby_subdir, 'test', 'run-test.rb'), + '--verbose' + ] + + print('-- Running Swig Ruby tests --') + old_cwd = os.getcwd() + try: + os.chdir(ruby_subdir) + + os.environ["BUILD_TYPE"] = objdir + os.environ["SVN_DBG_NO_ABORT_ON_ERROR_LEAK"] = 'YES' + r = subprocess.call([ruby_exe] + ruby_args) + finally: + os.chdir(old_cwd) + + sys.stdout.flush() + sys.stderr.flush() + if (r != 0): + print('[Test runner reported failure]') + failed = True # Stop service daemon, if any if daemon:
