Re: [ast-users] suggestion: parallel sh

Cyrille Lefevre Mon, 28 Jun 2010 17:42:42 -0700


Le 27/06/2010 06:42, David Korn a écrit :

ksh93t aleady has this feature.  If you set the variable MAXJOBS=4,
then ksh will block until a job completes if you background more than
four jobs.

You can use the SIGCHLD trap to find out which job as completed
and get the exit status.


thanks David, I thought about it, you did it... :-)
exactly what I do, I try to anticipate the needs...

some weeks ago, I wrote this one for ksh88 and bash :

# tmout isn't implemented, yet !

function intr {
        typeset rc=$?
        exec 6>&2 2> /dev/null
        for pid in ${pids}; do
                kill "${pid}"
        done
        exec 2>&6 6>&-
        exit ${rc}
}

trap intr HUP INT TERM

function loop {
        typeset maxpid=$1
        typeset pid= newpids=
        typeset -i sleep=0
        while (( npid >= maxpid )); do
                newpids= sleep=1
                for pid in ${pids}; do
                        if kill -0 "${pid}" 2> /dev/null; then
                                newpids="${newpids} ${pid}"
                        else
                                sleep=0
                                (( npid -= 1 ))
                                wait "${pid}"
                                echo "${pid}: done"
                        fi
                done
                pids=${newpids}
                (( sleep != 0 )) && sleep 1
        done
}

function parallel {
        typeset ncpu=$1 tmout=$2
        #shift 2
        typeset -i npid=0
        typeset pids=
        while read -r; do
                case ${REPLY} in
                ''|'#'*)
                        continue
                        ;;
                ncpu=*|tmout=*)
                        echo ${REPLY}
                        eval ${REPLY}
                        continue
                        ;;
                rset)
                        echo rset
                        ncpu=${_ncpu}
                        tmout=${_tmout}
                        continue
                        ;;
                wait)
                        echo wait
                        loop 1
                        continue
                        ;;
                esac
                loop "${ncpu}"
                ${REPLY} &
                (( npid += 1 ))
                pids="${pids} $!"
                echo "$!: $REPLY"
        done
        loop 1
}

_ncpu=$1
if (( ${_ncpu:-0} < 1 )); then
        case $(uname) in
        AIX)
                _ncpu=$(LC_ALL=C lsdev -c processor | grep -c Avail)
                ;;
        Darwin)
                #noht#_ncpu=$(sysctl -n hw.physicalcpu)
                _ncpu=$(sysctl -n hw.availcpu) # was logicalcpu
                ;;
        FreeBSD)
                _ncpu=$(sysctl -n hw.ncpu)
                ;;
        HP-UX)
                _ncpu=$(ioscan -fkC processor | grep -c processor)
                ;;
        CYGWIN*)
                # _ncpu=${NUMBER_OF_PROCESSORS}
                _ncpu=$(grep -c processor /proc/cpuinfo)
                ;;
        Linux)

#noht#_ncpu=$(grep 'physical id' /proc/cpuinfo | sort-u | wc -l

)
                _ncpu=$(grep -c processor /proc/cpuinfo)
                ;;
        SunOS)
                _ncpu=$(LC_ALL=C psrinfo -v | grep -c on-line)
                ;;
        esac
fi

_coef=$2
if (( ${_coef:-0} > 1 )); then
        (( _ncpu *= _coef ))
fi

_tmout=$3

if (( $# <= 3 )); then
        shift $#
fi

_time0=${SECONDS}
parallel "${_ncpu:-1}" "${_tmout:-0}" "$@"
(( _time0 -= -${SECONDS} ))
echo elapsed: ${_time0}

I've also just retrieved this old one implementation :

#!/usr/bin/sh

# based onhttp://web.archive.org/web/20080513235232/http://itb.biologie.hu-berlin.de/~benda/software/bash.html


# the number of currently running jobs:
PARALLELNUM=0

# a list with the PIDs of the currently running jobs:
PARALLELQUEUE=

# the number of CPUs available on the machine:
case $(uname) in
AIX) PARALLELCPUS=$(LANG=C lsdev -c processor | grep -c Available) ;;
HP-UX) PARALLELCPUS=$(ioscan -fnkC processor | grep -c processor) ;;
Linux) PARALLELCPUS=$(grep -c vendor_id /proc/cpuinfo) ;;
SunOS) PARALLELCPUS=$(psrinfo | grep -c on-line) ;;
*) PARALLELCPUS=1 ;;
esac

# this function adds the most recently started child process to the queue
# and waits until you can start another job:
function addqueue
{
  # add recent child process to the list:
  PARALLELQUEUE="$PARALLELQUEUE $!"
  let PARALLELNUM+=1
  delqueue
}

function delqueue
{
  typeset OLDPARALLELQUEUE= PREVPARALLELQUEUE= P= PID=
  # wait until one process has finished:
  while [ $PARALLELNUM -ge $PARALLELCPUS ]; do
    OLDPARALLELQUEUE="$PARALLELQUEUE"
    for PID in $OLDPARALLELQUEUE; do
      # remove process from queue if it does not exist anymore:
      if ! kill -0 $PID 2> /dev/null; then
        wait $PID 2 /dev/null
        local PREVPARALLELQUEUE="$PARALLELQUEUE"
        PARALLELQUEUE=""
        for P in $PREVPARALLELQUEUE; do
          [ "$P" != "$PID" ] && PARALLELQUEUE="$PARALLELQUEUE $P"
        done
        let PARALLELNUM-=1
      fi
    done
    sleep 1
  done
}

# wait for all the remaining processes in the queue to terminate:
function waitqueue
{
  delqueue
  # just to be sure:
  PARALLELNUM=0
  PARALLELQUEUE=
}

# This is a stupid example demonstrating the use of the waitqueue function.
# It starts 4 processes:
while read -r REPLY; do
  eval $REPLY &
  addqueue
done

# you might want to wait for the remaining processes to terminate:
waitqueue

Regards,

Cyrille Lefevre
--
mailto:cyrille.lefevre-li...@laposte.net




_______________________________________________
ast-users mailing list
ast-users@research.att.com
https://mailman.research.att.com/mailman/listinfo/ast-users

Re: [ast-users] suggestion: parallel sh

Reply via email to