The following patch allows to execute jobs from a queue in parallel.
The number of jobs executed in parallel is specified with the newly
introduced -P option to the queue command.
The patch should definitely be reviewed carefully before inclusion since
not understanding fully what was going on I used a bit of trial and
error to get it working.
The critical bit is really only the very small CmdExec.cc change. The
rest is just for setting the number of parallel jobs and should be
fairly obvious.
There was a choice where to put the variable for the number of parallel
jobs. I used the QueueFeeder class which is somewhat questionable
because QueueFeeder really does not do anything with that value it is
really used by CmdExec however putting it there would be less than
optimal as well since it is only meaningful if there is a queue attached
to the CmdExec. Basically I went with the less resource intensive of the
two suboptimal choices.
--
Markus
diff -ur lftp-3.5.4-orig/src/CmdExec.cc lftp-3.5.4/src/CmdExec.cc
--- lftp-3.5.4-orig/src/CmdExec.cc 2006-08-02 15:44:26.000000000 +0200
+++ lftp-3.5.4/src/CmdExec.cc 2006-08-20 14:16:00.320167069 +0200
@@ -590,7 +590,9 @@
}
if(status_line && status_line->CanShowNow())
ShowRunStatus(status_line); // this is only for top level CmdExec.
- return m;
+ if(m != STALL || interactive || !feeder || !queue_feeder ||
+ waiting_num >= queue_feeder->GetParallelJobNum())
+ return m;
}
if(!interactive)
@@ -644,6 +646,8 @@
FeedCmd("exit;");
return MOVED;
}
+ if(waiting_num > 0)
+ return m;
RemoveFeeder();
m=MOVED;
goto try_get_cmd;
diff -ur lftp-3.5.4-orig/src/commands.cc lftp-3.5.4/src/commands.cc
--- lftp-3.5.4-orig/src/commands.cc 2006-08-07 10:55:21.000000000 +0200
+++ lftp-3.5.4/src/commands.cc 2006-08-19 14:29:01.000000000 +0200
@@ -395,6 +395,9 @@
"Move the given items before the given queue index, or to the end if
no\n"
"destination is given.\n"
"\n"
+ " queue --parallel|-P num\n\n"
+ "Set the number of jobs executed in parallel from the current queue to
num.\n"
+ "\n"
"Options:\n"
" -q Be quiet.\n"
" -v Be verbose.\n"
@@ -1118,9 +1121,10 @@
{"quiet",no_argument,0,'q'},
{"verbose",no_argument,0,'v'},
{"queue",required_argument,0,'Q'},
+ {"parallel",required_argument,0,'P'},
{0,0,0,0}
};
- enum { ins, del, move } mode = ins;
+ enum { ins, del, move, parallel } mode = ins;
const char *arg = NULL;
/* position to insert at (ins only) */
@@ -1128,7 +1132,7 @@
int verbose = -1; /* default */
int opt;
- while((opt=args->getopt_long("+dm:n:qvQw",queue_options,0))!=EOF)
+ while((opt=args->getopt_long("+dm:n:qvQwP:",queue_options,0))!=EOF)
{
switch(opt)
{
@@ -1155,6 +1159,11 @@
mode = del;
break;
+ case 'P':
+ mode = parallel;
+ arg = optarg;
+ break;
+
case 'q':
verbose = 0;
break;
@@ -1278,6 +1287,24 @@
exit_code=0;
}
break;
+
+ case parallel: {
+ CmdExec *queue=GetQueue(false);
+ if(!queue) {
+ eprintf(_("%s: No queue is active.\n"), args->a0());
+ break;
+ }
+
+ const int nParallel = atoi(arg);
+ if(nParallel > 0)
+ queue->queue_feeder->SetParallelJobNum(nParallel);
+ else {
+ eprintf(_("%s: -p: positive number expected. "), args->a0());
+ goto err;
+ }
+ exit_code = 0;
+ }
+ break;
}
return 0;
diff -ur lftp-3.5.4-orig/src/QueueFeeder.h lftp-3.5.4/src/QueueFeeder.h
--- lftp-3.5.4-orig/src/QueueFeeder.h 2002-07-31 11:05:00.000000000 +0200
+++ lftp-3.5.4/src/QueueFeeder.h 2006-08-20 14:16:00.320167069 +0200
@@ -37,6 +37,8 @@
char *buffer;
+ int parallel_job_num;
+
/* remove the given job from the list */
void unlink_job(QueueJob *job);
@@ -74,11 +76,21 @@
bool MoveJob(int from, int to, int v = 0);
bool MoveJob(const char *cmd, int to, int v = 0);
+ /* Set/Get number of jobs that should be executed in parallel from this
queue. */
+ void SetParallelJobNum(int n)
+ {
+ parallel_job_num = n;
+ }
+ int GetParallelJobNum() const
+ {
+ return parallel_job_num;
+ }
+
enum { PrintRequeue = 9999 };
void PrintStatus(int v,const char *prefix="\t") const;
QueueFeeder(const char *pwd, const char *lpwd):
- jobs(0), lastjob(0), cur_pwd(0), cur_lpwd(0), buffer(0)
+ jobs(0), lastjob(0), cur_pwd(0), cur_lpwd(0), buffer(0),
parallel_job_num(1)
{
cur_pwd = xstrdup(pwd);
cur_lpwd = xstrdup(lpwd);
