Hi all,
Following a thread earlier this year
(http://lists.gnu.org/archive/html/help-make/2006-04/msg00014.html<http://imp4-g19.free.fr/horde/util/go.php?url=http%3A%2F%2Flists.gnu.org%2Farchive%2Fhtml%2Fhelp-make%2F2006-04%2Fmsg00014.html&Horde=a60e4626d41a2b264996a046a8eb817f>),
I
have decided to work myself, and I propose a patch as attached to this
email.
It is based upon work from Alexey Neyman (see
http://lists.gnu.org/archive/html/help-make/2005-04/msg00087.html<http://imp4-g19.free.fr/horde/util/go.php?url=http%3A%2F%2Flists.gnu.org%2Farchive%2Fhtml%2Fhelp-make%2F2005-04%2Fmsg00087.html&Horde=a60e4626d41a2b264996a046a8eb817f>
).
In summary, it adds a news special dependency '.WAIT' which is used to
stop parallelization: any of the dependencies _after_ .WAIT wait for
the dependency before .WAIT before starting to be updated. (Note that
they wait only for the dependency immediately before .WAIT, not for
all the preceding ones)
Attached is a simple test Makefile to show the results.
I would like to get your feedback/comments on this patch.
Best regards,
Christophe.
foo1: x1 x3 .WAIT x2
foo2: x2 .WAIT x3
foo3: x1 x2 x3
foo4: y1 y2 y3
1: A .WAIT B
2: B .WAIT C
3: C .WAIT A
4: C A
x2 x3 y1 y2 y3 A B C:
echo $@ ; sleep 2 ; echo finished $@
x1:
echo $@ ; sleep 4 ; echo finished $@
all: a b $(WAIT) c d s-e s-f
all2: a b c $(WAIT) d e $(WAIT) f g h $(WAIT) i $(WAIT) $(WAIT) j k
all3: a b sub3 i j
all4: a b sub4 i j
.PHONY: all1 all2 all3 sub3 clean
sub3: c d e f g h
.NOTPARALLEL: sub3
sub4: c .WAIT d .WAIT e .WAIT f .WAIT g .WAIT h
clean:
rm -f [a-k] [stu]-*
a b c d e f g h i j k:
@echo Starting update of $@; sleep 2; echo touch $@; echo Finished
update of $@
s-%: u-% $(WAIT) t-%
@echo Starting update of $@; sleep 2; echo touch $@; echo Finished
update of $@
t-%:
@echo Starting update of $@; sleep 2; echo touch $@; echo Finished
update of $@
u-%:
@echo Starting update of $@; sleep 2; echo touch $@; echo Finished
update of $@
$ ../make-3.81-wait/make -j WAIT=.WAIT all
Starting update of a
Starting update of b
touch a
Finished update of a
touch b
Finished update of b
Starting update of c
Starting update of d
Starting update of u-e
Starting update of u-f
touch c
Finished update of c
touch u-e
Finished update of u-e
touch d
Finished update of d
Starting update of t-e
touch u-f
Finished update of u-f
Starting update of t-f
touch t-f
Finished update of t-f
touch t-e
Finished update of t-e
Starting update of s-e
Starting update of s-f
touch s-e
Finished update of s-e
touch s-f
Finished update of s-f
$ ../make-3.81-wait/make -j WAIT=.WAIT all2
Starting update of b
Starting update of c
Starting update of a
touch c
Finished update of c
touch b
Finished update of b
touch a
Finished update of a
Starting update of d
Starting update of e
touch d
touch e
Finished update of d
Finished update of e
Starting update of f
Starting update of g
Starting update of h
touch f
touch g
Finished update of f
Finished update of g
touch h
Finished update of h
Starting update of i
touch i
Finished update of i
Starting update of j
Starting update of k
touch j
Finished update of j
touch k
Finished update of k
$ ../make-3.81-wait/make -j WAIT=.WAIT all3
Starting update of b
Starting update of a
Starting update of c
Starting update of i
Starting update of j
touch a
Finished update of a
touch i
Finished update of i
touch b
Finished update of b
touch j
Finished update of j
touch c
Finished update of c
Starting update of d
touch d
Finished update of d
Starting update of e
touch e
Finished update of e
Starting update of f
touch f
Finished update of f
Starting update of g
touch g
Finished update of g
Starting update of h
touch h
Finished update of h
diff -r -u make-3.81/default.c make-3.81.patch-wait/default.c
--- make-3.81/default.c 2006-02-11 23:16:04.000000000 +0100
+++ make-3.81.patch-wait/default.c 2006-12-03 21:30:25.000000000 +0100
@@ -531,7 +531,7 @@
char *p = default_suffixes;
suffix_file->deps = (struct dep *)
multi_glob (parse_file_seq (&p, '\0', sizeof (struct dep), 1),
- sizeof (struct dep));
+ sizeof (struct dep), NULL);
(void) define_variable ("SUFFIXES", 8, default_suffixes, o_default, 0);
}
}
diff -r -u make-3.81/dep.h make-3.81.patch-wait/dep.h
--- make-3.81/dep.h 2006-03-17 15:24:20.000000000 +0100
+++ make-3.81.patch-wait/dep.h 2006-12-03 21:30:56.000000000 +0100
@@ -43,6 +43,7 @@
unsigned int ignore_mtime : 1;
unsigned int staticpattern : 1;
unsigned int need_2nd_expansion : 1;
+ unsigned int wait : 1 ;
};
@@ -55,7 +56,7 @@
};
-extern struct nameseq *multi_glob PARAMS ((struct nameseq *chain, unsigned int
size));
+extern struct nameseq *multi_glob PARAMS ((struct nameseq *chain, unsigned int
size, int *seen_special));
#ifdef VMS
extern struct nameseq *parse_file_seq ();
#else
diff -r -u make-3.81/file.c make-3.81.patch-wait/file.c
--- make-3.81/file.c 2006-03-17 15:24:20.000000000 +0100
+++ make-3.81.patch-wait/file.c 2006-12-03 21:36:58.000000000 +0100
@@ -409,12 +409,40 @@
}
}
+/* Notice special dependencies. */
+void
+notice_special_deps(struct dep **d_ptr)
+{
+ struct dep *d;
+
+ restart_loop:
+ for (; *d_ptr; d_ptr = &(*d_ptr)->next)
+ {
+ d = *d_ptr;
+ if (!strcmp(d->name, ".WAIT"))
+ {
+ /* Found a .WAIT dependency: record this on the next dep,
+ and remove this dep from the chain */
+ free(d->name);
+ if (d->next)
+ d->next->wait = 1;
+ *d_ptr = d->next;
+ free(d);
+ goto restart_loop;
+ }
+ }
+}
+
struct dep *
parse_prereqs (char *p)
{
+ /* Nonzero if we saw possibly special dependencies (e.g. .WAIT) while
+ parsing the dependencies for that rule. */
+ int seen_special = 0;
+
struct dep *new = (struct dep *)
multi_glob (parse_file_seq (&p, '|', sizeof (struct dep), 1),
- sizeof (struct dep));
+ sizeof (struct dep), &seen_special);
if (*p)
{
@@ -425,7 +453,7 @@
++p;
ood = (struct dep *)
multi_glob (parse_file_seq (&p, '\0', sizeof (struct dep), 1),
- sizeof (struct dep));
+ sizeof (struct dep), &seen_special);
if (! new)
new = ood;
@@ -441,6 +469,12 @@
ood->ignore_mtime = 1;
}
+ /* Before dependencies are entered as files, treat special
+ dependencies. Need to do this before entering them as files
+ as treatment might involve deletion of a dependency. */
+ if (seen_special)
+ notice_special_deps(&new);
+
return new;
}
@@ -463,6 +497,7 @@
char *file_stem = f->stem;
unsigned int last_dep_has_cmds = f->updating;
int initialized = 0;
+ int seen_special = 0;
f->updating = 0;
f->deps = 0;
@@ -574,6 +609,7 @@
}
/* Enter them as files. */
+ struct dep* prev = NULL;
for (d1 = new; d1 != 0; d1 = d1->next)
{
d1->file = lookup_file (d1->name);
@@ -584,6 +620,8 @@
d1->name = 0;
d1->staticpattern = 0;
d1->need_2nd_expansion = 0;
+
+ prev = d1;
}
/* Add newly parsed deps to f->deps. If this is the last dependency
@@ -630,7 +668,7 @@
{
struct file *f;
struct file *f2;
- struct dep *d;
+ struct dep *d, *d2;
struct file **file_slot_0;
struct file **file_slot;
struct file **file_end;
@@ -734,9 +772,23 @@
f2->command_flags |= COMMANDS_SILENT;
}
+
+ /* .NOTPARALLEL with deps makes these targets non-parallel (i.e. all
+ their dependencies are serialized). Without deps, it forbids
+ parallel execution at all. */
f = lookup_file (".NOTPARALLEL");
if (f != 0 && f->is_target)
- not_parallel = 1;
+ {
+ if (f->deps == 0)
+ not_parallel = 1;
+ else
+ for (d = f->deps; d != 0; d = d->next)
+ for (f2 = d->file; f2 != 0; f2 = f2->prev)
+ for (d2 = f2->deps; d2 != 0; d2 = d2->next) {
+ d2->wait = 1;
+ }
+ }
+
#ifndef NO_MINUS_C_MINUS_O
/* If .POSIX was defined, remove OUTPUT_OPTION to comply. */
diff -r -u make-3.81/filedef.h make-3.81.patch-wait/filedef.h
--- make-3.81/filedef.h 2006-02-11 23:16:04.000000000 +0100
+++ make-3.81.patch-wait/filedef.h 2006-12-03 21:35:39.000000000 +0100
@@ -61,6 +61,11 @@
the same file. Otherwise this is null. */
struct file *double_colon;
+ /* Target that file is waiting for before starting, or nil if
+ there isn't one. */
+ struct file *wait;
+ unsigned int waiting:1;
+
short int update_status; /* Status of the last attempt to update,
or -1 if none has been made. */
diff -r -u make-3.81/function.c make-3.81.patch-wait/function.c
--- make-3.81/function.c 2006-04-01 08:36:40.000000000 +0200
+++ make-3.81.patch-wait/function.c 2006-12-03 21:35:50.000000000 +0100
@@ -352,7 +352,7 @@
That would break examples like:
$(patsubst ./%.c,obj/%.o,$(wildcard ./?*.c)). */
0),
- sizeof (struct nameseq));
+ sizeof (struct nameseq), NULL);
if (result == 0)
{
diff -r -u make-3.81/implicit.c make-3.81.patch-wait/implicit.c
--- make-3.81/implicit.c 2006-04-01 08:36:40.000000000 +0200
+++ make-3.81.patch-wait/implicit.c 2006-12-03 21:40:53.000000000 +0100
@@ -74,6 +74,7 @@
char *intermediate_pattern; /* pattern for intermediate file */
unsigned char had_stem; /* had % substituted with stem */
unsigned char ignore_mtime; /* ignore_mtime flag */
+ unsigned char wait;
};
static void
@@ -182,6 +183,31 @@
return beg;
}
+/* Handle special dependencies in implicit rules. */
+static void
+notice_special_ideps(struct idep **id_ptr)
+{
+ struct idep *id;
+
+ restart_loop:
+ for (; *id_ptr; id_ptr = &(*id_ptr)->next)
+ {
+ id = *id_ptr;
+ if (!strcmp(id->name, ".WAIT"))
+ {
+ /* Found a .WAIT dependency: record this on the next dep,
+ and remove this dep from the chain */
+ free(id->name);
+ if (id->next)
+ id->next->wait = 1;
+ *id_ptr = id->next;
+ free(id);
+ goto restart_loop;
+ }
+ }
+}
+
+
/* Search the pattern rules for a rule with an existing dependency to make
FILE. If a rule is found, the appropriate commands and deps are put in FILE
and 1 is returned. If not, 0 is returned.
@@ -258,6 +284,10 @@
that is not just `%'. */
int specific_rule_matched = 0;
+ /* Nonzero if we saw possibly special dependencies (e.g. .WAIT) while
+ parsing the dependencies for that rule. */
+ int seen_special = 0;
+
unsigned int i = 0; /* uninit checks OK */
struct rule *rule;
struct dep *dep, *expl_d;
@@ -336,6 +366,9 @@
/* It can't possibly match. */
continue;
+ /* This rule hasn't encountered special dependencies (so far). */
+ seen_special = 0;
+
/* From the lengths of the filename and the pattern parts,
find the stem: the part of the filename that matches the %. */
stem = filename + (suffix - target - 1);
@@ -599,7 +632,8 @@
parse_file_seq (&p2,
order_only ? '\0' : '|',
sizeof (struct idep),
- 1), sizeof (struct idep));
+ 1), sizeof (struct idep),
+ &seen_special);
/* @@ It would be nice to teach parse_file_seq or
multi_glob to add prefix. This would save us some
@@ -650,6 +684,10 @@
file->stem = 0;
+ /* If there were special dependencies, handle them. */
+ if (seen_special)
+ notice_special_ideps(&deps);
+
/* @@ This loop can be combined with the previous one. I do
it separately for now for transparency.*/
@@ -820,6 +858,7 @@
expl_d = file->deps; /* We will add them at the end. */
d_ptr = &file->deps;
+ char* prev_name = NULL;
for (d = deps; d != 0; d = d->next)
{
register char *s;
@@ -872,6 +911,7 @@
dep = alloc_dep ();
dep->ignore_mtime = d->ignore_mtime;
+ dep->wait = d->wait;
s = d->name; /* Hijacking the name. */
d->name = 0;
if (recursions == 0)
@@ -905,6 +945,24 @@
*d_ptr = dep;
d_ptr = &dep->next;
+
+ /* Record the actual file we are waiting for, if any */
+ if (prev_name && d->wait) {
+ if (recursions != 0) {
+ dep->file = lookup_file(s);
+ }
+ dep->file->wait = lookup_file(prev_name);
+ }
+
+ if (d->intermediate_file) {
+ prev_name = d->intermediate_file->name;
+ } else {
+ if (recursions == 0) {
+ prev_name = dep->file->name;
+ } else {
+ prev_name = dep->name;
+ }
+ }
}
*d_ptr = expl_d;
diff -r -u make-3.81/main.c make-3.81.patch-wait/main.c
--- make-3.81/main.c 2006-03-20 03:36:37.000000000 +0100
+++ make-3.81.patch-wait/main.c 2006-12-03 21:41:08.000000000 +0100
@@ -2162,7 +2162,7 @@
ns = multi_glob (
parse_file_seq (&p, '\0', sizeof (struct nameseq), 1),
- sizeof (struct nameseq));
+ sizeof (struct nameseq), NULL);
/* .DEFAULT_GOAL should contain one target. */
if (ns->next != 0)
diff -r -u make-3.81/read.c make-3.81.patch-wait/read.c
--- make-3.81/read.c 2006-03-17 15:24:20.000000000 +0100
+++ make-3.81.patch-wait/read.c 2006-12-03 21:41:40.000000000 +0100
@@ -809,7 +809,7 @@
files = multi_glob (parse_file_seq (&p2, '\0',
sizeof (struct nameseq),
1),
- sizeof (struct nameseq));
+ sizeof (struct nameseq), NULL);
free (p);
/* Save the state of conditionals and start
@@ -1001,7 +1001,7 @@
filenames = multi_glob (parse_file_seq (&p2, '\0',
sizeof (struct nameseq),
1),
- sizeof (struct nameseq));
+ sizeof (struct nameseq), NULL);
*p2 = ':';
if (!filenames)
@@ -3013,7 +3013,7 @@
that have room for additional info. */
struct nameseq *
-multi_glob (struct nameseq *chain, unsigned int size)
+multi_glob (struct nameseq *chain, unsigned int size, int *seen_special)
{
extern void dir_setup_glob ();
register struct nameseq *new = 0;
@@ -3041,6 +3041,11 @@
}
}
+ if (seen_special && old->name[0] == '.'
+ && isalpha (old->name[1]) && isupper (old->name[1])) {
+ *seen_special = 1;
+ }
+
#ifndef NO_ARCHIVES
if (ar_name (old->name))
{
diff -r -u make-3.81/remake.c make-3.81.patch-wait/remake.c
--- make-3.81/remake.c 2006-03-20 03:36:37.000000000 +0100
+++ make-3.81.patch-wait/remake.c 2006-12-03 22:02:26.000000000 +0100
@@ -68,6 +68,24 @@
static FILE_TIMESTAMP name_mtime PARAMS ((char *name));
static int library_search PARAMS ((char **lib, FILE_TIMESTAMP *mtime_ptr));
+/* Check for loops in wait dependencies */
+static int check_circular_wait(struct file* chain, struct file *file)
+{
+ int circular = 0;
+
+ for (; chain && chain->wait; chain = chain->wait) {
+ if (chain->wait == file) {
+ error (NILF, _("Circular %s <- %s wait chain dropped."),
+ file->name, chain->wait->name);
+ circular = 1;
+ break;
+ }
+ }
+
+ return circular;
+}
+
+
/* Remake all the goals in the `struct dep' chain GOALS. Return -1 if nothing
was done, 0 if all goals were updated successfully, or 1 if a goal failed.
@@ -134,6 +152,7 @@
unsigned int ocommands_started;
int x;
check_renamed (file);
+
if (rebuilding_makefiles)
{
if (file->cmd_target)
@@ -364,6 +383,23 @@
error (NILF, msg_parent, "*** ", file->name, file->parent->name, ".");
}
+/* Mark the dependencies as considered in this pass (to avoid skipping them
+ on the next pass). */
+static void
+consider_deps (struct dep *d)
+{
+ for (; d; d = d->next)
+ d->file->considered = considered;
+}
+
+/* Mark all the next dependencies as waiting in this pass. */
+static void
+wait_deps (struct dep *d)
+{
+ for (; d; d = d->next)
+ d->file->waiting = 1;
+}
+
/* Consider a single `struct file' and update it as appropriate. */
static int
@@ -468,12 +504,54 @@
lastd = 0;
d = file->deps;
+ struct dep* prev = NULL;
while (d != 0)
{
FILE_TIMESTAMP mtime;
int maybe_make;
int dontcare = 0;
+ /* Record .WAIT dependencies only when they are actually used. */
+ if (prev && d->wait && (d->file->wait == NULL)) {
+ d->file->wait = prev->file;
+ if (check_circular_wait(prev->file, d->file)) {
+ d->file->wait = NULL;
+ }
+ }
+ if (d->file->wait) {
+ if (running && (d->file->wait == prev->file)) {
+ /* Waiting for the prev dependency, which is currently
+ running, so wait */
+ consider_deps (d);
+ wait_deps (d);
+ d->file->waiting = 1;
+ break;
+ } else if (d->file->waiting) {
+ if (d->file->wait->command_state == cs_finished) {
+ /* Already waiting, but the file we were waiting for has
+ been rebuilt: stop waiting */
+ d->file->waiting = 0;
+ } else {
+ /* Already waiting, for a dep other than the previous one
+ (from another dep chain). Keep waiting */
+ consider_deps (d);
+ wait_deps (d);
+ break;
+ }
+ } else {
+ /* Not yet waiting, and the dep we should wait for has not
+ started yet: start waiting */
+ if (d->file->wait->command_state == cs_not_started) {
+ consider_deps (d);
+ wait_deps (d);
+ d->file->waiting = 1;
+ break;
+ }
+ }
+ }
+
+ prev = d;
+
check_renamed (d->file);
mtime = file_mtime (d->file);
@@ -547,24 +625,62 @@
/* Now we know whether this target needs updating.
If it does, update all the intermediate files we depend on. */
+ prev = NULL;
if (must_make || always_make_flag)
{
for (d = file->deps; d != 0; d = d->next)
- if (d->file->intermediate)
- {
- int dontcare = 0;
+ {
+ if (d->file->wait) {
- FILE_TIMESTAMP mtime = file_mtime (d->file);
- check_renamed (d->file);
- d->file->parent = file;
+ if (prev && (d->file->wait == prev->file)
+ && (d->file->wait->command_state == cs_deps_running)) {
+ /* Waiting for the previous file, whose deps are
+ currently running: start waiting */
+ running |= 1;
+ consider_deps (d);
+ wait_deps (d);
+ d->file->waiting = 1;
+ break;
+ } else if (d->file->waiting) {
+ if (d->file->wait->command_state == cs_finished) {
+ /* Stop waiting if the deps we were waiting for has
+ been built */
+ d->file->waiting = 0;
+ } else {
+ /* Keep waiting if needed, as well as the next deps */
+ running |= 1;
+ consider_deps (d);
+ wait_deps (d);
+ break;
+ }
+ } else {
+ if (d->file->wait->command_state == cs_not_started) {
+ /* Start waiting if deps have not started yet */
+ running |= 1;
+ consider_deps (d);
+ wait_deps (d);
+ d->file->waiting = 1;
+ break;
+ }
+ }
+ }
- /* Inherit dontcare flag from our parent. */
- if (rebuilding_makefiles)
- {
- dontcare = d->file->dontcare;
- d->file->dontcare = file->dontcare;
- }
+ prev = d;
+ if (d->file->intermediate)
+ {
+ int dontcare = 0;
+
+ FILE_TIMESTAMP mtime = file_mtime (d->file);
+ check_renamed (d->file);
+ d->file->parent = file;
+
+ /* Inherit dontcare flag from our parent. */
+ if (rebuilding_makefiles)
+ {
+ dontcare = d->file->dontcare;
+ d->file->dontcare = file->dontcare;
+ }
dep_status |= update_file (d->file, depth);
@@ -594,6 +710,7 @@
d->changed = ((file->phony && file->cmds != 0)
|| file_mtime (d->file) != mtime);
}
+ }
}
finish_updating (file);
@@ -987,10 +1104,40 @@
lastd = 0;
d = file->deps;
+ struct dep* prev = NULL;
while (d != 0)
{
int maybe_make;
+
+ if (d->file->wait) {
+ if (prev && (d->file->wait == prev->file)
+ && prev->file->command_state == cs_running) {
+ /* Waiting for previous */
+ consider_deps (d);
+ wait_deps (d);
+ d->file->waiting = 1;
+ break;
+ } else if (d->file->waiting) {
+ if (d->file->wait->command_state == cs_finished) {
+ /* Stop waiting if dep finished */
+ d->file->waiting = 0;
+ } else {
+ consider_deps (d);
+ wait_deps (d);
+ break;
+ }
+ } else {
+ if (d->file->wait->command_state == cs_not_started) {
+ consider_deps (d);
+ wait_deps (d);
+ d->file->waiting = 1;
+ break;
+ }
+ }
+ }
+ prev = d;
+
if (is_updating (d->file))
{
error (NILF, _("Circular %s <- %s dependency dropped."),
diff -r -u make-3.81/rule.c make-3.81.patch-wait/rule.c
--- make-3.81/rule.c 2006-03-17 15:24:20.000000000 +0100
+++ make-3.81.patch-wait/rule.c 2006-12-03 22:03:12.000000000 +0100
@@ -361,7 +361,7 @@
{
register struct rule *r;
char *ptr;
-
+ int seen_special = 0;
r = (struct rule *) xmalloc (sizeof (struct rule));
r->targets = (char **) xmalloc (2 * sizeof (char *));
@@ -386,7 +386,10 @@
ptr = p->dep;
r->deps = (struct dep *) multi_glob (parse_file_seq (&ptr, '\0',
sizeof (struct dep), 1),
- sizeof (struct dep));
+ sizeof (struct dep), &seen_special);
+
+ if (seen_special)
+ notice_special_deps(&r->deps);
if (new_pattern_rule (r, 0))
{_______________________________________________
Help-make mailing list
[email protected]
http://lists.gnu.org/mailman/listinfo/help-make
