expand: Do not reprocess data when expanding words

2018-05-29 Thread Herbert Xu
Currently various paths will reprocess data when performing word
expansion.  For example, expari will skip backwards looking for
the start of the arithmetic expansion, while evalvar will skip
unexpanded words manually.

This is cumbersome and error-prone.  This patch fixes this by
making word expansions proceed in a linear fashion.  This means
changing argstr and the various expansion functions such as expari
and subevalvar to return the next character to be expanded.

This is inspired by similar code from FreeBSD.  However, we take
things one step further and completely remove the manual word
skipping in evalvar.  This is accomplished by introducing a new
EXP_DISCARD flag that tells argstr to only parse and not produce
any actual expansions.

Incidentally, argstr will now always NUL-terminate the expansion
unless the EXP_WORD flag is set.  This is because all but one
caller of argstr wants the result to be NUL-termianted.

Signed-off-by: Herbert Xu 

diff --git a/src/expand.c b/src/expand.c
index 7ed1bc0..3a1bf94 100644
--- a/src/expand.c
+++ b/src/expand.c
@@ -110,10 +110,10 @@ static struct ifsregion *ifslastp;
 /* holds expanded arg list */
 static struct arglist exparg;
 
-STATIC void argstr(char *, int);
-STATIC char *exptilde(char *, char *, int);
+static char *argstr(char *p, int flag);
+static char *exptilde(char *startp, int flag);
+static char *expari(char *start, int flag);
 STATIC void expbackq(union node *, int);
-STATIC const char *subevalvar(char *, char *, int, int, int, int, int);
 STATIC char *evalvar(char *, int);
 static size_t strtodest(const char *p, int flags);
 static void memtodest(const char *p, size_t len, int flags);
@@ -192,13 +192,11 @@ expandarg(union node *arg, struct arglist *arglist, int 
flag)
argbackq = arg->narg.backquote;
STARTSTACKSTR(expdest);
argstr(arg->narg.text, flag);
-   p = _STPUTC('\0', expdest);
-   expdest = p - 1;
if (arglist == NULL) {
/* here document expanded */
goto out;
}
-   p = grabstackstr(p);
+   p = grabstackstr(expdest);
exparg.lastp = 
/*
 * TODO - EXP_REDIR
@@ -232,8 +230,7 @@ out:
  * $@ like $* since no splitting will be performed.
  */
 
-STATIC void
-argstr(char *p, int flag)
+static char *argstr(char *p, int flag)
 {
static const char spclchars[] = {
'=',
@@ -243,6 +240,7 @@ argstr(char *p, int flag)
CTLESC,
CTLVAR,
CTLBACKQ,
+   CTLARI,
CTLENDARI,
0
};
@@ -253,35 +251,41 @@ argstr(char *p, int flag)
size_t length;
int startloc;
 
-   if (!(flag & EXP_VARTILDE)) {
-   reject += 2;
-   } else if (flag & EXP_VARTILDE2) {
-   reject++;
-   }
+   reject += !!(flag & EXP_VARTILDE2);
+   reject += flag & EXP_VARTILDE ? 0 : 2;
inquotes = 0;
length = 0;
if (flag & EXP_TILDE) {
-   char *q;
-
flag &= ~EXP_TILDE;
 tilde:
-   q = p;
-   if (*q == '~')
-   p = exptilde(p, q, flag);
+   if (*p == '~')
+   p = exptilde(p, flag);
}
 start:
startloc = expdest - (char *)stackblock();
for (;;) {
+   int end;
+
length += strcspn(p + length, reject);
+   end = 0;
c = (signed char)p[length];
-   if (c && (!(c & 0x80) || c == CTLENDARI)) {
-   /* c == '=' || c == ':' || c == CTLENDARI */
+   if (!(c & 0x80) || c == CTLENDARI || c == CTLENDVAR) {
+   /*
+* c == '=' || c == ':' || c == '\0' ||
+* c == CTLENDARI || c == CTLENDVAR
+*/
length++;
+   /* c == '\0' || c == CTLENDARI || c == CTLENDVAR */
+   end = !!((c - 1) & 0x80);
}
-   if (length > 0) {
+   if (length > 0 && !(flag & EXP_DISCARD)) {
int newloc;
-   expdest = stnputs(p, length, expdest);
-   newloc = expdest - (char *)stackblock();
+   char *q;
+
+   q = stnputs(p, length, expdest);
+   q[-1] &= end - 1;
+   expdest = q - (flag & EXP_WORD ? end : 0);
+   newloc = expdest - (char *)stackblock() - end;
if (breakall && !inquotes && newloc > startloc) {
recordregion(startloc, newloc, 0);
}
@@ -290,14 +294,11 @@ start:
p += length + 1;
length = 0;
 
+   if (end)
+   break;
+
switch (c) {
-   case '\0':
-   goto breakloop;
  

Re: [PATCH] [BUILD] fix parallel build

2018-05-29 Thread Herbert Xu
Yann E. MORIN  wrote:
> When neither token.h nor token_var.h exist, and we are building in
> parallel, it is possible that make will spawn two mktokens at roughly
> the same time, because of two different rules, one that needs token.h ad
> another that need token_vars.h.
> 
> For example, make can have a dependency chain toward just token.h, from
> syntax.c, and another dependency chain toweard token_vars.h from the
> generic BUILT_SOURCES.
> 
> So, make will "quickly" decide that it needs token.h, then spawn
> mktoken. A bit later, while that mktoken still runs but hasn't yet
> created token_vars.h, make will need it, and thus spawn a second
> mktoken.
> 
> While the second one runs, the first terminates. However, the token.h it
> had generated has been in the meantime overwritten by the second mktoken
> that is still generating it, and token.h is still empty.
> 
> But make proceeds to the rule that required token.h in the first place,
> and the still-empty token.h gets icluded from a C file, and the build
> fails because of missing defines.
> 
> For example:
>
> http://autobuild.buildroot.org/results/fc4/fc4e4ab47455ac47dd4a3a60083cec2848e74dbb/build-end.log
> 
> Fix that by serialising both headers.
> 
> Reported-by: Baruch Siach 
> Signed-off-by: "Yann E. MORIN" 
> Cc: Baruch Siach 
> ---
> src/Makefile.am | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
> 
> diff --git a/src/Makefile.am b/src/Makefile.am
> index 46399c7..5bf5a52 100644
> --- a/src/Makefile.am
> +++ b/src/Makefile.am
> @@ -44,7 +44,8 @@ EXTRA_DIST = \
>mktokens mkbuiltins builtins.def.in mkinit.c \
>mknodes.c nodetypes nodes.c.pat mksyntax.c mksignames.c
> 
> -token.h token_vars.h: mktokens
> +token_vars.h: token.h
> +token.h: mktokens
>$(SHELL) $^

I don't think this is always going to work.  If you do a make and
then remove token_vars.h wouldn't this fail?

Also there is no guarantee that the two timestamps will be in the
order that you expect.

The same problem also applies to all the other rules in the Makefile
with multiple targets.

Until there is a good solution to rules with multiple targets,
perhaps we should just disable parallel building in dash.

Thanks,
-- 
Email: Herbert Xu 
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
--
To unsubscribe from this list: send the line "unsubscribe dash" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html