-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 [please consider making your mailer wrap long lines]
According to Tim on 7/14/2009 12:02 PM: > Hi, > > I wonder how to in a bash script(or other languages if more convenient and/or fast) manage independent jobs(executables with command line arguments) and make them run in parallel? Is there a way to create several sub-process without waiting them to finish? Short answer - yes, bash can manage multiple parallel worker tasks. Long answer - you may want to read up on how current autoconf.git implements parallel testsuites. Doing it portably in shell is a nightmare (as so many shells out there have bugs in their trap handlers), but bash at least shines in this area. > > Which one is faster: multi-threading or multi-process? Multi-threading is inherently faster than multi-process, as fewer resources must be managed when swapping between workers. But how much faster depends on the particular OS. Bash can only do multi-process. Here's the core driver loop from a testsuite created by current autoconf.git (take it with a grain of salt - this is a relatively new feature of autotest, and portability improvements are welcome): if (set -m && set +m) >/dev/null 2>&1; then at_job_control_on='set -m' at_job_control_off='set +m' at_job_group=- else at_job_control_on=: at_job_control_off=: at_job_group= fi for at_signal in 1 2 15; do trap 'set +x; set +e $at_job_control_off at_signal='"$at_signal"' echo stop > "$at_stop_file" trap "" $at_signal at_pgids= for at_pgid in `jobs -p 2>/dev/null`; do at_pgids="$at_pgids $at_job_group$at_pgid" done test -z "$at_pgids" || kill -$at_signal $at_pgids 2>/dev/null wait if test "$at_jobs" -eq 1 || test -z "$at_verbose"; then echo >&2 fi at_signame=`kill -l $at_signal 2>&1 || echo $at_signal` set x $at_signame test 0 -gt 2 && at_signame=$at_signal { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: caught signal $at_signame, bailing out" >&5 $as_echo "$as_me: WARNING: caught signal $at_signame, bailing out" >&2;} as_fn_arith 128 + $at_signal && exit_status=$as_val as_fn_exit $exit_status' $at_signal done rm -f "$at_stop_file" at_first=: if test $at_jobs -ne 1 && rm -f "$at_job_fifo" && test -n "$at_job_group" && ( mkfifo "$at_job_fifo" && trap 'exit 1' PIPE STOP TSTP ) 2>/dev/null then # FIFO job dispatcher. trap 'at_pids= for at_pid in `jobs -p`; do at_pids="$at_pids $at_job_group$at_pid" done if test -n "$at_pids"; then at_sig=TSTP test "${TMOUT+set}" = set && at_sig=STOP kill -$at_sig $at_pids 2>/dev/null fi kill -STOP $$ test -z "$at_pids" || kill -CONT $at_pids 2>/dev/null' TSTP echo # Turn jobs into a list of numbers, starting from 1. at_joblist=`$as_echo " $at_groups_all " | \ sed 's/\( '$at_jobs'\) .*/\1/'` set X $at_joblist shift for at_group in $at_groups; do $at_job_control_on 2>/dev/null ( # Start one test group. $at_job_control_off exec 6>"$at_job_fifo" trap 'set +x; set +e trap "" PIPE echo stop > "$at_stop_file" echo token >&6 as_fn_exit 141' PIPE at_fn_group_prepare if cd "$at_group_dir" && at_fn_test $at_group && . "$at_test_source" # AT_JOB_FIFO_FD>&- then :; else { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unable to parse test group: $at_group" >&5 $as_echo "$as_me: WARNING: unable to parse test group: $at_group" >&2;} at_failed=: fi at_fn_group_postprocess echo token >&6 ) & $at_job_control_off if $at_first; then at_first=false exec 6<"$at_job_fifo" fi shift # Consume one token. if test $# -gt 0; then :; else read at_token <&6 || break set x $* fi test -f "$at_stop_file" && break done # Read back the remaining ($at_jobs - 1) tokens. set X $at_joblist shift if test $# -gt 0; then shift for at_job do read at_token done <&6 fi exec 6<&- wait else # Run serially, avoid forks and other potential surprises. for at_group in $at_groups; do at_fn_group_prepare if cd "$at_group_dir" && at_fn_test $at_group && . "$at_test_source"; then :; else { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unable to parse test group: $at_group" >&5 $as_echo "$as_me: WARNING: unable to parse test group: $at_group" >&2;} at_failed=: fi at_fn_group_postprocess test -f "$at_stop_file" && break at_first=false done fi - -- Don't work too hard, make some time for fun as well! Eric Blake e...@byu.net -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.9 (Cygwin) Comment: Public key at home.comcast.net/~ericblake/eblake.gpg Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org iEYEARECAAYFAkpdJkAACgkQ84KuGfSFAYBqiACcDOvuNPb8Y19OxzDAzdNBaLFX HuwAn1aZ8nuAvrQjDHCDQ67kT7/qV5KK =GhEE -----END PGP SIGNATURE-----