On Sat, May 12, 2007 at 07:48:04PM -0400, Rob Landley wrote:
> On Wednesday 09 May 2007 5:05 pm, Harald van Dijk wrote:
> > On Tue, May 08, 2007 at 02:33:13AM +0200, Harald van Dijk wrote:
> > > Hi,
> > >
> > > Here's a patch to handle spacing during stringizing
> >
> > and here's a patch to be applied on top of it. It fixes stringizing in
> > cases involving macro or macro argument substitutions where the
> > replacement is empty. If this happens, whitespace before and after the
> > replacement needs to be combined. An example:
>
> I applied the previous patch (minus the hunk that failed). I'm not convinced
> the dropped hunk isn't doing _anything_ useful, could you confirm?
Yes, it really didn't do anything important. It merely made sure
print_defines continued to work, but since print_defines is gone,
there's no longer a need for that. And nothing else in the patch relied
on print_defines anyway.
> Also, this patch had another failing hunk. Could you please send me a
> version
> against the current repository?
That hunk was also for print_defines. :-)
Here's an updated patch. It combines the previous two patches, minus the
modifications to print_defines, and minus the modification to the
Makefile, and changes the new tcc_preprocess function to update
next_tok_flags instead of tok_flags.
--- tinycc-1eefdf914c2d/tccasm.c
+++ tinycc-1eefdf914c2d/tccasm.c
@@ -618,7 +618,7 @@
/* XXX: undefine C labels */
ch = file->buf_ptr[0];
- tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
+ next_tok_flags = TOK_FLAG_BOW | TOK_FLAG_BOL | TOK_FLAG_BOF;
parse_flags = PARSE_FLAG_ASM_COMMENTS;
if (do_preprocess)
parse_flags |= PARSE_FLAG_PREPROCESS;
--- tinycc-1eefdf914c2d/tcc.c
+++ tinycc-1eefdf914c2d/tcc.c
@@ -55,7 +55,7 @@
static CValue tokc;
static CString tokcstr; /* current parsed string, if any */
/* additional informations about token */
-static int tok_flags;
+static int tok_flags, next_tok_flags;
static int *macro_ptr, *macro_ptr_allocated;
static int *unget_saved_macro_ptr;
@@ -1511,19 +1511,20 @@
return str;
}
-static void tok_str_add(TokenString *s, int t)
+static void tok_str_add(TokenString *s, int t, int tf)
{
int len, *str;
len = s->len;
str = s->str;
- if (len >= s->allocated_len)
+ if (len+1 >= s->allocated_len)
str = tok_str_realloc(s);
str[len++] = t;
+ str[len++] = tf;
s->len = len;
}
-static void tok_str_add2(TokenString *s, int t, CValue *cv)
+static void tok_str_add2(TokenString *s, int t, int tf, CValue *cv)
{
int len, *str;
@@ -1534,6 +1535,7 @@
if (len + TOK_MAX_SIZE > s->allocated_len)
str = tok_str_realloc(s);
str[len++] = t;
+ str[len++] = tf;
switch(t) {
case TOK_CINT:
case TOK_CUINT:
@@ -1596,9 +1598,9 @@
if (file->line_num != s->last_line_num) {
s->last_line_num = file->line_num;
cval.i = s->last_line_num;
- tok_str_add2(s, TOK_LINENUM, &cval);
+ tok_str_add2(s, TOK_LINENUM, 0, &cval);
}
- tok_str_add2(s, tok, &tokc);
+ tok_str_add2(s, tok, tok_flags, &tokc);
}
#if LDOUBLE_SIZE == 12
@@ -1617,9 +1619,10 @@
/* get a token from an integer array and increment pointer
accordingly. we code it as a macro to avoid pointer aliasing. */
-#define TOK_GET(t, p, cv) \
+#define TOK_GET(t, tf, p, cv) \
{ \
t = *p++; \
+ tf = *p++; \
switch(t) { \
case TOK_CINT: \
case TOK_CUINT: \
@@ -1779,8 +1782,8 @@
}
tok_str_add_tok(&str);
}
- tok_str_add(&str, -1); /* simulate end of file */
+ tok_str_add(&str, -1, 0); /* simulate end of file */
- tok_str_add(&str, 0);
+ tok_str_add(&str, 0, 0);
/* now evaluate C constant expression */
macro_ptr = str.str;
next();
@@ -1793,11 +1796,11 @@
#if defined(PARSE_DEBUG) || defined(PP_DEBUG)
static void tok_print(int *str)
{
- int t;
+ int t, tf;
CValue cval;
while (1) {
- TOK_GET(t, str, cval);
+ TOK_GET(t, tf, str, cval);
if (!t)
break;
printf(" %s", get_tok_str(t, &cval));
@@ -1853,10 +1856,10 @@
next_nomacro();
/* EOF testing necessary for '-D' handling */
while (tok != TOK_LINEFEED && tok != TOK_EOF) {
- tok_str_add2(&str, tok, &tokc);
+ tok_str_add2(&str, tok, tok_flags, &tokc);
next_nomacro();
}
- tok_str_add(&str, 0);
+ tok_str_add(&str, 0, 0);
#ifdef PP_DEBUG
printf("define %s %d: ", get_tok_str(v, NULL), t);
tok_print(str.str);
@@ -2120,7 +2123,7 @@
if (do_debug) {
put_stabs(file->filename, N_BINCL, 0, 0, 0);
}
- tok_flags |= TOK_FLAG_BOF | TOK_FLAG_BOL;
+ next_tok_flags |= TOK_FLAG_BOW | TOK_FLAG_BOF | TOK_FLAG_BOL;
ch = file->buf_ptr[0];
goto the_end;
}
@@ -2191,7 +2194,7 @@
file->ifndef_macro = 0;
while (tok != TOK_LINEFEED)
next_nomacro();
- tok_flags |= TOK_FLAG_ENDIF;
+ next_tok_flags |= TOK_FLAG_ENDIF;
goto the_end;
}
break;
@@ -2673,6 +2676,7 @@
case '\v':
case '\r':
p++;
+ next_tok_flags |= TOK_FLAG_BOW;
goto redo_no_start;
case '\\':
@@ -2706,7 +2710,7 @@
/* test if previous '#endif' was after a #ifdef at
start of file */
- if (tok_flags & TOK_FLAG_ENDIF) {
+ if (next_tok_flags & TOK_FLAG_ENDIF) {
#ifdef INC_DEBUG
printf("#endif %s\n",
get_tok_str(file->ifndef_macro_saved, NULL));
#endif
@@ -2733,7 +2737,7 @@
tok = TOK_LINEFEED;
} else {
file->line_num++;
- tok_flags |= TOK_FLAG_BOL;
+ next_tok_flags |= TOK_FLAG_BOW | TOK_FLAG_BOL;
p++;
goto redo_no_start;
}
@@ -2742,10 +2746,10 @@
case '#':
/* XXX: simplify */
PEEKC(c, p);
- if ((tok_flags & TOK_FLAG_BOL) &&
+ if ((next_tok_flags & TOK_FLAG_BOL) &&
(parse_flags & PARSE_FLAG_PREPROCESS)) {
file->buf_ptr = p;
- preprocess(tok_flags & TOK_FLAG_BOF);
+ preprocess(next_tok_flags & TOK_FLAG_BOF);
p = file->buf_ptr;
goto redo_no_start;
} else {
@@ -3034,9 +3038,11 @@
PEEKC(c, p);
if (c == '*') {
p = parse_comment(p);
+ next_tok_flags |= TOK_FLAG_BOW;
goto redo_no_start;
} else if (c == '/') {
p = parse_line_comment(p);
+ next_tok_flags |= TOK_FLAG_BOW;
goto redo_no_start;
} else if (c == '=') {
p++;
@@ -3068,7 +3074,8 @@
break;
}
file->buf_ptr = p;
+ tok_flags = next_tok_flags;
- tok_flags = 0;
+ next_tok_flags = 0;
#if defined(PARSE_DEBUG)
printf("token = %s\n", get_tok_str(tok, &tokc));
#endif
@@ -3082,7 +3089,7 @@
redo:
tok = *macro_ptr;
if (tok) {
- TOK_GET(tok, macro_ptr, tokc);
+ TOK_GET(tok, tok_flags, macro_ptr, tokc);
if (tok == TOK_LINENUM) {
file->line_num = tokc.i;
goto redo;
@@ -3096,7 +3103,7 @@
/* substitute args in macro_str and return allocated string */
static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args)
{
- int *st, last_tok, t, notfirst;
+ int *st, last_tok, t, tf, save_tf, notfirst;
Sym *s;
CValue cval;
TokenString str;
@@ -3104,13 +3111,17 @@
tok_str_new(&str);
last_tok = 0;
+ save_tf = 0;
while(1) {
- TOK_GET(t, macro_str, cval);
+ TOK_GET(t, tf, macro_str, cval);
+ tf |= save_tf;
+ save_tf = 0;
if (!t)
break;
if (t == '#') {
/* stringize */
+ save_tf = tf;
- TOK_GET(t, macro_str, cval);
+ TOK_GET(t, tf, macro_str, cval);
if (!t)
break;
s = sym_find2(args, t);
@@ -3119,9 +3130,9 @@
st = (int *)s->c;
notfirst = 0;
while (*st) {
- if (notfirst)
- cstr_ccat(&cstr, ' ');
- TOK_GET(t, st, cval);
+ TOK_GET(t, tf, st, cval);
+ if (notfirst && tf & TOK_FLAG_BOW)
+ cstr_ccat(&cstr, ' ');
cstr_cat(&cstr, get_tok_str(t, &cval));
notfirst = 1;
}
@@ -3131,15 +3142,17 @@
#endif
/* add string */
cval.cstr = &cstr;
- tok_str_add2(&str, TOK_STR, &cval);
+ tok_str_add2(&str, TOK_STR, save_tf, &cval);
cstr_free(&cstr);
} else {
- tok_str_add2(&str, t, &cval);
+ tok_str_add2(&str, t, save_tf, &cval);
}
+ save_tf = 0;
} else if (t >= TOK_IDENT) {
s = sym_find2(args, t);
if (s) {
st = (int *)s->c;
+ st[1] = save_tf = tf;
/* if '##' is present before or after, no arg substitution */
if (*macro_str == TOK_TWOSHARPS || last_tok == TOK_TWOSHARPS) {
/* special case for var arg macros : ## eats the
@@ -3149,39 +3162,42 @@
problems */
if (gnu_ext && s->type.t &&
last_tok == TOK_TWOSHARPS &&
- str.len >= 2 && str.str[str.len - 2] == ',') {
+ str.len >= 4 && str.str[str.len - 4] == ',') {
if (*st == 0) {
/* suppress ',' '##' */
- str.len -= 2;
+ str.len -= 4;
+ next_tok_flags = tok_flags;
} else {
/* suppress '##' and add variable */
- str.len--;
+ str.len -= 2;
goto add_var;
}
} else {
int t1;
add_var:
+ save_tf = 0;
for(;;) {
- TOK_GET(t1, st, cval);
+ TOK_GET(t1, tf, st, cval);
if (!t1)
break;
- tok_str_add2(&str, t1, &cval);
+ tok_str_add2(&str, t1, tf, &cval);
}
}
} else {
/* NOTE: the stream cannot be read when macro
substituing an argument */
- macro_subst(&str, nested_list, st, NULL);
+ if (macro_subst(&str, nested_list, st, NULL) > 0)
+ save_tf = 0;
}
} else {
- tok_str_add(&str, t);
+ tok_str_add(&str, t, tf);
}
} else {
- tok_str_add2(&str, t, &cval);
+ tok_str_add2(&str, t, tf, &cval);
}
last_tok = t;
}
- tok_str_add(&str, 0);
+ tok_str_add(&str, 0, 0);
return str.str;
}
@@ -3193,8 +3209,8 @@
/* do macro substitution of current token with macro 's' and add
result to (tok_str,tok_len). 'nested_list' is the list of all
- macros we got inside to avoid recursing. Return non zero if no
+ macros we got inside to avoid recursing. Return -1 if no
- substitution needs to be done */
+ substitution needs to be done, else the number of added tokens. */
static int macro_subst_tok(TokenString *tok_str,
Sym **nested_list, Sym *s, struct macro_level
**can_read_stream)
{
@@ -3237,10 +3253,12 @@
cstr_cat(&cstr, cstrval);
cstr_ccat(&cstr, '\0');
cval.cstr = &cstr;
- tok_str_add2(tok_str, t1, &cval);
+ tok_str_add2(tok_str, t1, tok_flags, &cval);
cstr_free(&cstr);
+ return 1;
} else {
mstr = (int *)s->c;
+ mstr[1] = tok_flags;
mstr_allocated = 0;
if (s->type.t == MACRO_FUNC) {
/* NOTE: we do not use next_nomacro to avoid eating the
@@ -3295,10 +3313,10 @@
parlevel++;
else if (tok == ')')
parlevel--;
- tok_str_add2(&str, tok, &tokc);
+ tok_str_add2(&str, tok, tok_flags, &tokc);
next_nomacro();
}
- tok_str_add(&str, 0);
+ tok_str_add(&str, 0, 0);
sym_push2(&args, sa->v & ~SYM_FIELD, sa->type.t, (int)str.str);
sa = sa->next;
if (tok == ')') {
@@ -3331,15 +3349,15 @@
mstr_allocated = 1;
}
sym_push2(nested_list, s->v, 0, 0);
- macro_subst(tok_str, nested_list, mstr, can_read_stream);
+ t = macro_subst(tok_str, nested_list, mstr, can_read_stream);
/* pop nested defined symbol */
sa1 = *nested_list;
*nested_list = sa1->prev;
sym_free(sa1);
if (mstr_allocated)
tok_str_free(mstr);
+ return t;
}
- return 0;
}
/* handle the '##' operator. Return NULL if no '##' seen. Otherwise
@@ -3348,7 +3366,7 @@
{
TokenSym *ts;
const int *macro_ptr1, *start_macro_ptr, *ptr, *saved_macro_ptr;
- int t;
+ int t, tf;
const char *p1, *p2;
CValue cval;
TokenString macro_str1;
@@ -3358,7 +3376,7 @@
/* we search the first '##' */
for(;;) {
macro_ptr1 = macro_str;
- TOK_GET(t, macro_str, cval);
+ TOK_GET(t, tf, macro_str, cval);
/* nothing more to do if end of string */
if (t == 0)
return NULL;
@@ -3370,23 +3388,24 @@
cstr_new(&cstr);
tok_str_new(¯o_str1);
tok = t;
+ tok_flags = tf;
tokc = cval;
/* add all tokens seen so far */
for(ptr = start_macro_ptr; ptr < macro_ptr1;) {
- TOK_GET(t, ptr, cval);
+ TOK_GET(t, tf, ptr, cval);
- tok_str_add2(¯o_str1, t, &cval);
+ tok_str_add2(¯o_str1, t, tf, &cval);
}
saved_macro_ptr = macro_ptr;
/* XXX: get rid of the use of macro_ptr here */
macro_ptr = (int *)macro_str;
for(;;) {
while (*macro_ptr == TOK_TWOSHARPS) {
- macro_ptr++;
+ macro_ptr+=2;
macro_ptr1 = macro_ptr;
t = *macro_ptr;
if (t) {
- TOK_GET(t, macro_ptr, cval);
+ TOK_GET(t, tf, macro_ptr, cval);
/* We concatenate the two tokens if we have an
identifier or a preprocessing number */
cstr_reset(&cstr);
@@ -3460,7 +3479,7 @@
p2 = get_tok_str(t, &cval);
warning("pasting \"%s\" and \"%s\" does not give a
valid preprocessing token", cstr.data, p2);
/* cannot merge tokens: just add them separately */
- tok_str_add2(¯o_str1, tok, &tokc);
+ tok_str_add2(¯o_str1, tok, tok_flags, &tokc);
/* XXX: free associated memory ? */
tok = t;
tokc = cval;
@@ -3468,28 +3487,28 @@
}
}
}
- tok_str_add2(¯o_str1, tok, &tokc);
+ tok_str_add2(¯o_str1, tok, tok_flags, &tokc);
next_nomacro();
if (tok == 0)
break;
}
macro_ptr = (int *)saved_macro_ptr;
cstr_free(&cstr);
- tok_str_add(¯o_str1, 0);
+ tok_str_add(¯o_str1, 0, 0);
return macro_str1.str;
}
/* do macro substitution of macro_str and add result to
(tok_str,tok_len). 'nested_list' is the list of all macros we got
- inside to avoid recursing. */
+ inside to avoid recursing. Return the number of added tokens. */
-static void macro_subst(TokenString *tok_str, Sym **nested_list,
+static int macro_subst(TokenString *tok_str, Sym **nested_list,
- const int *macro_str, struct macro_level **
can_read_stream)
+ const int *macro_str, struct macro_level **
can_read_stream)
{
Sym *s;
int *macro_str1;
const int *ptr;
- int t, ret;
+ int t, tf, save_tf, len, ret;
CValue cval;
struct macro_level ml;
@@ -3498,12 +3517,18 @@
macro_str1 = macro_twosharps(ptr);
if (macro_str1)
ptr = macro_str1;
+
+ save_tf = 0;
+ len = 0;
while (1) {
/* NOTE: ptr == NULL can only happen if tokens are read from
file stream due to a macro function call */
if (ptr == NULL)
break;
- TOK_GET(t, ptr, cval);
+ TOK_GET(t, tf, ptr, cval);
+ tf |= save_tf;
+ save_tf = 0;
+
if (t == 0)
break;
s = define_find(t);
@@ -3516,20 +3541,27 @@
ml.prev = *can_read_stream, *can_read_stream = &ml;
macro_ptr = (int *)ptr;
tok = t;
+ tok_flags = tf;
ret = macro_subst_tok(tok_str, nested_list, s, can_read_stream);
ptr = (int *)macro_ptr;
macro_ptr = ml.p;
if (can_read_stream && *can_read_stream == &ml)
*can_read_stream = ml.prev;
- if (ret != 0)
+ if (ret < 0)
goto no_subst;
+ if (ret == 0)
+ save_tf = tf;
+ if (ret > 0)
+ len += ret;
} else {
no_subst:
- tok_str_add2(tok_str, t, &cval);
+ tok_str_add2(tok_str, t, tf, &cval);
+ len += 1;
}
}
if (macro_str1)
tok_str_free(macro_str1);
+ return len;
}
@@ -3553,9 +3585,9 @@
tok_str_new(&str);
nested_list = NULL;
ml = NULL;
- if (macro_subst_tok(&str, &nested_list, s, &ml) == 0) {
+ if (macro_subst_tok(&str, &nested_list, s, &ml) >= 0) {
/* substitution done, NOTE: maybe empty */
- tok_str_add(&str, 0);
+ tok_str_add(&str, 0, 0);
macro_ptr = str.str;
macro_ptr_allocated = str.str;
goto redo;
@@ -7952,8 +7984,8 @@
next();
}
}
- tok_str_add(&init_str, -1);
+ tok_str_add(&init_str, -1, 0);
- tok_str_add(&init_str, 0);
+ tok_str_add(&init_str, 0, 0);
/* compute size */
save_parse_state(&saved_parse_state);
@@ -8380,8 +8412,8 @@
break;
}
}
- tok_str_add(&func_str, -1);
+ tok_str_add(&func_str, -1, 0);
- tok_str_add(&func_str, 0);
+ tok_str_add(&func_str, 0, 0);
sym->r = (int)func_str.str;
} else {
/* compute text section */
@@ -8535,7 +8567,7 @@
s1->error_set_jmp_enabled = 1;
ch = file->buf_ptr[0];
- tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
+ next_tok_flags = TOK_FLAG_BOW | TOK_FLAG_BOL | TOK_FLAG_BOF;
parse_flags = PARSE_FLAG_PREPROCESS | PARSE_FLAG_TOK_NUM;
next();
decl(VT_CONST);
@@ -8572,7 +8604,7 @@
define_start = define_stack;
ch = file->buf_ptr[0];
- tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF;
+ next_tok_flags = TOK_FLAG_BOW | TOK_FLAG_BOL | TOK_FLAG_BOF;
parse_flags = PARSE_FLAG_ASM_COMMENTS | PARSE_FLAG_PREPROCESS |
PARSE_FLAG_LINEFEED;
last_is_space = 1;
--- tinycc-1eefdf914c2d/tcc.h
+++ tinycc-1eefdf914c2d/tcc.h
@@ -121,7 +121,7 @@
#define TOK_HASH_SIZE 8192 /* must be a power of two */
#define TOK_ALLOC_INCR 512 /* must be a power of two */
-#define TOK_MAX_SIZE 4 /* token max size in int unit when stored in
string */
+#define TOK_MAX_SIZE 5 /* token max size in int unit when stored in
string */
/* token symbol management */
typedef struct TokenSym {
@@ -310,9 +310,10 @@
#define CACHED_INCLUDES_HASH_SIZE 512
/* additional information about token */
+#define TOK_FLAG_BOW 0x0001 /* beginning of word before */
-#define TOK_FLAG_BOL 0x0001 /* beginning of line before */
+#define TOK_FLAG_BOL 0x0002 /* beginning of line before */
-#define TOK_FLAG_BOF 0x0002 /* beginning of file before */
+#define TOK_FLAG_BOF 0x0004 /* beginning of file before */
-#define TOK_FLAG_ENDIF 0x0004 /* a endif was found matching starting #ifdef */
+#define TOK_FLAG_ENDIF 0x0008 /* a endif was found matching starting #ifdef */
#define PARSE_FLAG_PREPROCESS 0x0001 /* activate preprocessing */
#define PARSE_FLAG_TOK_NUM 0x0002 /* return numbers instead of TOK_PPNUM */
@@ -701,8 +702,8 @@
int *p;
};
-static void macro_subst(TokenString *tok_str, Sym **nested_list,
+static int macro_subst(TokenString *tok_str, Sym **nested_list,
- const int *macro_str, struct macro_level
**can_read_stream);
+ const int *macro_str, struct macro_level
**can_read_stream);
void gen_op(int op);
void force_charshort_cast(int t);
static void gen_cast(CType *type);
_______________________________________________
Tinycc-devel mailing list
[email protected]
http://lists.nongnu.org/mailman/listinfo/tinycc-devel
