Author: kjs
Date: Mon Mar 26 04:10:58 2007
New Revision: 17764
Modified:
trunk/compilers/pirc/doc/design.pod
trunk/compilers/pirc/src/pirlexer.c
trunk/compilers/pirc/src/pirlexer.h
trunk/compilers/pirc/src/pirparser.c
Log:
compilers/pirc:
* added binary and hex constants to lexer
* removed 2 unused token types
* fixed pod error in design.pod
* some code refactoring in parser.
Modified: trunk/compilers/pirc/doc/design.pod
==============================================================================
--- trunk/compilers/pirc/doc/design.pod (original)
+++ trunk/compilers/pirc/doc/design.pod Mon Mar 26 04:10:58 2007
@@ -16,11 +16,11 @@
=head2 OVERVIEW
The lexer is defined in C<pirlexer.c>. The header file lists all tokens that
-may be returned by the lexer.
+may be returned by the lexer.
The lexer reads the complete file contents into a buffer, from which it reads
the individual words, or I<tokens>. A buffer is much faster than using
C<getc()>
-for each character, as I/O is relatively slow.
+for each character, as I/O is relatively slow.
The lexer has three functions that may be invoked to receive the next token:
@@ -42,6 +42,7 @@
C<read_macro()> is used to read a macro body. It reads up to the C<.endm> word,
and then returns the T_ENDM token, which can then be matched by the parser.
+=back
=head2 IMPLEMENTATION
@@ -52,10 +53,10 @@
The structure looks like this:
typedef struct lexer_state {
- struct file_buffer *curfile;
- char *token_chars;
- char *charptr;
-
+ struct file_buffer *curfile;
+ char *token_chars;
+ char *charptr;
+
} lexer_state;
It has three pointers: the first one is a pointer to a file_buffer structure,
which
@@ -63,24 +64,24 @@
This is a buffer in which the characters for the current token are stored.
The field C<charptr> acts like an index to add and delete characters from
C<token_chars>.
-As mentioned, the lexer reads the source file and returns tokens. The source
file is
+As mentioned, the lexer reads the source file and returns tokens. The source
file is
represented by another structure, called C<file_buffer>. It looks like this:
typedef struct file_buffer {
- char *filename;
- char *buffer;
- char *curchar;
- unsigned filesize;
- unsigned int line;
- unsigned short linepos;
- char lastchar;
- struct file_buffer *prevbuffer;
-
+ char *filename;
+ char *buffer;
+ char *curchar;
+ unsigned filesize;
+ unsigned int line;
+ unsigned short linepos;
+ char lastchar;
+ struct file_buffer *prevbuffer;
+
} file_buffer;
-
-The first field C<filename> holds the name of the file that is being scanned.
+
+The first field C<filename> holds the name of the file that is being scanned.
The second field C<buffer> is a pointer to the contents of the file. The field
-C<curchar> points into this buffer to the current position, and can be
considered
+C<curchar> points into this buffer to the current position, and can be
considered
the I<cursor>.
Then, C<filesize> contains the length of the file; C<line> keeps track of what
line
is being processed; linepos keeps track of the number of characters since the
last
@@ -100,7 +101,7 @@
The parser communicates with the lexer through the lexer's accessor function.
Of
these, the C<next_token()> function is most important: it requests the next
token
-from the lexer.
+from the lexer.
The parser does not know anything about the spelling of tokens, although it can
request these through C<find_keyword()>.
@@ -112,17 +113,17 @@
shown below:
typedef struct parser_state {
- struct lexer_state *lexer;
- token curtoken;
- char *heredoc_ids[10];
- int heredoc_index;
- unsigned parse_errors;
-
+ struct lexer_state *lexer;
+ token curtoken;
+ char *heredoc_ids[10];
+ int heredoc_index;
+ unsigned parse_errors;
+
} parser_state;
It consists of the following fields: a pointer to a lexer_state object; the
current
token as returned by the lexer; a list to store heredoc identifiers, which is
used
-when parsing heredocs as subroutine arguments; an index for this list; and
finally a
+when parsing heredocs as subroutine arguments; an index for this list; and
finally a
counter to keep track of parse errors.
=head1 AUTHOR
Modified: trunk/compilers/pirc/src/pirlexer.c
==============================================================================
--- trunk/compilers/pirc/src/pirlexer.c (original)
+++ trunk/compilers/pirc/src/pirlexer.c Mon Mar 26 04:10:58 2007
@@ -186,7 +186,6 @@
"'string'", /* T_SINGLE_QUOTED_STRING, */
"'literal'", /* T_LITERAL, */
"invocant id", /* T_INVOCANT_IDENT, */
- "'number'", /* T_NUMBER, */
"'error'", /* T_ERROR, */
"**", /* T_POWER, */
"**=", /* T_POWER_ASSIGN, */
@@ -194,7 +193,6 @@
"+=", /* T_PLUS_ASSIGN, */
"-=", /* T_MINUS_ASSIGN, */
".=", /* T_CONCAT_ASSIGN */
- "'register'", /* T_REGISTER, */
"/=", /* T_DIVIDE_ASSIGN, */
"//=", /* T_FDIVIDE_ASSIGN, */
"%=", /* T_MODULO_ASSIGN, */
@@ -247,15 +245,20 @@
the current file being read, a buffer holding the current
token, and a pointer to add characters to the token buffer.
+ typedef struct lexer_state {
+ struct file_buffer *curfile; -- pointer to the current file
+ char *token_chars; -- characters of the current token
+ char *charptr; -- used for adding/removing token chars
+
+ } lexer_state;
+
=cut
*/
typedef struct lexer_state {
- struct file_buffer *curfile; /* pointer to the current file */
- char *token_chars; /* characters of the current token */
- char *charptr; /* used for adding/removing token chars */
- /*Parrot_Interp interp;
- */
+ struct file_buffer *curfile;
+ char *token_chars;
+ char *charptr;
} lexer_state;
@@ -278,11 +281,10 @@
if ((t >= 0) && (t <= MAX_TOKEN)) {
return dictionary[t];
}
- else {
+ else { /* this should never happen; error in dictionary */
fprintf(stderr, "FATAL: invalid token in find_keyword()\n");
fprintf(stderr, "No entry for token %d\n", t);
- if (t-1 <= MAX_TOKEN) fprintf(stderr, "Previous token: %s\n",
dictionary[t - 1]);
- return "ERROR";
+ exit(1);
}
}
@@ -342,16 +344,17 @@
*/
void
print_error_context(struct lexer_state *s) {
- /* print context of size ERROR_CONTEXT_SIZE */
+ /* print context of max. size ERROR_CONTEXT_SIZE */
int read_chars = s->curfile->curchar - s->curfile->buffer;
int context_length = read_chars > ERROR_CONTEXT_SIZE ? ERROR_CONTEXT_SIZE
: read_chars;
char *start = s->curfile->curchar - context_length;
char *end = s->curfile->curchar;
+ /* print all characters from start to end */
while (start < end ) {
fprintf(stderr, "%c", *start++);
}
- /* print an indicator like "^" */
+ /* print an indicator like "^" on the next line */
fprintf(stderr, "\n%*s\n", s->curfile->linepos - 1, "^");
}
@@ -391,13 +394,12 @@
/* store previous character, but only if not at first character of file */
if (buf->curchar > buf->buffer) buf->lastchar = *(buf->curchar - 1);
- else buf->lastchar = '\n';
+ else buf->lastchar = '\n'; /* makes the check for prev. char. at start of
file successful */
buf->curchar++; /* update pointer to current char. */
buf->linepos++; /* update the token position */
return c;
-
}
/*
@@ -405,14 +407,12 @@
=item unread_char()
Push back the last read character.
-It was never removed from the buffer, so just
-decrement the pointer in the buffer.
=cut
*/
static void
-unread_char(file_buffer *buf, char c) {
+unread_char(file_buffer *buf) {
--buf->curchar;
--buf->linepos;
}
@@ -681,7 +681,7 @@
c = read_char(lexer->curfile);
count++;
}
- unread_char(lexer->curfile, c); /* no digit, put it back */
+ unread_char(lexer->curfile); /* no digit, put it back */
return count;
}
@@ -766,7 +766,7 @@
}
while (c != '\n');
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
continue; /* with main loop */
}
@@ -862,7 +862,7 @@
else { /* not a label or invocant put the last read char. back */
token tmp;
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
tmp = check_dictionary(lexer, dictionary); /* look up the id */
@@ -889,7 +889,7 @@
if (c == '.') return T_DOTDOT; /* ".." */
if ( isspace(c) ) { /* a dot followed by a space */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_CONCAT;
}
@@ -898,7 +898,7 @@
c = read_char(lexer->curfile);
if (c == EOF_MARKER) break;
}
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
tmp = check_dictionary(lexer, dictionary);
/* if not found, then it is a macro id */
@@ -911,18 +911,40 @@
}
else if (isdigit(c) ) { /* check for numbers */
buffer_char(lexer, c);
- read_digits(lexer);
-
c = read_char(lexer->curfile);
- if (c == '.') { /* floating point number */
+
+ if (isdigit(c)) { /* integer or float */
+ do {
+ buffer_char(lexer, c);
+ c = read_char(lexer->curfile);
+ }
+ while (isdigit(c));
+
+ /* it is a different char, either '.', ' ' or something else */
+ if (c == '.') { /* floating point number */
+ buffer_char(lexer, c);
+ read_digits(lexer);
+ return T_NUMBER_CONSTANT;
+ }
+ else {
+ unread_char(lexer->curfile); /* put back last read char. */
+ return T_INTEGER_CONSTANT;
+ }
+ }
+ else if (c == 'b' || c == 'B') { /* 0b<digit>+ or 0B<digit>+ */
buffer_char(lexer, c);
read_digits(lexer);
- return T_NUMBER_CONSTANT;
+ return T_INTEGER_CONSTANT;
}
- else {
- unread_char(lexer->curfile, c); /* put back last read char. */
+ else if (c == 'x' || c == 'X') { /* 0x<digit>+ or 0X<digit>+ */
+ buffer_char(lexer, c);
+ read_digits(lexer);
return T_INTEGER_CONSTANT;
}
+ else { /* 1 digit */
+ return T_INTEGER_CONSTANT;
+ }
+
}
else if (c == '$') { /* parse PIR register or macro label */
token regtype;
@@ -954,7 +976,7 @@
return T_MACRO_LABEL;
}
else { /* $ ??? */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_ERROR;
}
}
@@ -1012,7 +1034,7 @@
=head3 Augmented operators
- **= *= %= /= //= += -= .= >>= >>>= <<= &= |= ~=
+ **= *= %= /= //= += -= .= >>= >>>= <<= &= |= ~=
=head3 Conditional operators
@@ -1034,7 +1056,7 @@
case '=': return T_POWER_ASSIGN; /* **= */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_MULTIPLY; /* * */
}
}
@@ -1044,27 +1066,27 @@
case '=': return T_MODULO_ASSIGN; /* %= */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_MODULO; /* % */
}
}
else if (c == '/') {
c = read_char(lexer->curfile);
switch(c) {
- case '/':
+ case '/':
c = read_char(lexer->curfile);
if (c == '=') {
return T_FDIVIDE_ASSIGN; /* //= */
}
else {
- unread_char(lexer->curfile, c);
- return T_FDIVIDE;
+ unread_char(lexer->curfile);
+ return T_FDIVIDE;
}
break;
case '=': return T_DIVIDE_ASSIGN; /* /= */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_DIVIDE; /* / */
}
}
@@ -1074,7 +1096,7 @@
case '=': return T_PLUS_ASSIGN; /* += */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_PLUS; /* + */
}
}
@@ -1085,7 +1107,7 @@
case '=': return T_MINUS_ASSIGN; /* -= */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_MINUS;
}
}
@@ -1095,7 +1117,7 @@
case '=': return T_NE; /* != */
case EOF_MARKER: return T_EOF;
default: /* ! */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_NOT;
}
}
@@ -1106,7 +1128,7 @@
case '=': return T_EQ; /* == */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_ASSIGN; /* = */
}
}
@@ -1121,7 +1143,7 @@
return T_LOG_RSHIFT_ASSIGN;
}
else { /* >>> */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_LOG_RSHIFT;
}
}
@@ -1129,13 +1151,13 @@
return T_RSHIFT_ASSIGN;
}
else { /* >> */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_RSHIFT;
}
case '=': return T_GE; /* >= */
case EOF_MARKER: return T_EOF;
default: /* > */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_GT;
}
}
@@ -1156,17 +1178,17 @@
}
else { /* no heredoc */
if (c == '=') { /* <<= */
- return T_LSHIFT_ASSIGN;
+ return T_LSHIFT_ASSIGN;
}
else { /* << */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_LSHIFT;
}
}
break;
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_LT;
}
}
@@ -1177,7 +1199,7 @@
case '=': return T_BXOR_ASSIGN; /* ~= */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c); /* ~ */
+ unread_char(lexer->curfile); /* ~ */
return T_BXOR;
}
}
@@ -1188,18 +1210,18 @@
case '=': return T_BAND_ASSIGN; /* &= */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c); /* & */
+ unread_char(lexer->curfile); /* & */
return T_BAND;
}
}
- else if (c == '|') {
+ else if (c == '|') {
c = read_char(lexer->curfile);
switch(c) {
case '|': return T_OR; /* || */
case '=': return T_BOR_ASSIGN; /* |= */
case EOF_MARKER: return T_EOF;
default:
- unread_char(lexer->curfile, c); /* | */
+ unread_char(lexer->curfile); /* | */
return T_BOR;
}
}
@@ -1217,7 +1239,7 @@
while (isspace(c));
/* the last read char. was not space/newline, put it back */
- unread_char(lexer->curfile, c);
+ unread_char(lexer->curfile);
return T_NEWLINE;
}
@@ -1265,7 +1287,7 @@
}
while ( isalnum(c) );
- unread_char(lexer->curfile, c); /* push back last character not
needed */
+ unread_char(lexer->curfile); /* push back last character not
needed */
tmp = check_dictionary(lexer, dictionary);
/* if not found, then no valid flag found */
@@ -1333,7 +1355,7 @@
/* the loop broke, but why? */
if (*heredoc_iter == '\0' && isspace(c)) { /* loop broke because
heredoc label was fully iterated; success! */
- unread_char(lexer->curfile, c); /* we read 1 char too many;
put back last read character */
+ unread_char(lexer->curfile); /* we read 1 char too many; put
back last read character */
return T_HEREDOC_STRING; /* return success */
}
}
Modified: trunk/compilers/pirc/src/pirlexer.h
==============================================================================
--- trunk/compilers/pirc/src/pirlexer.h (original)
+++ trunk/compilers/pirc/src/pirlexer.h Mon Mar 26 04:10:58 2007
@@ -124,7 +124,6 @@
T_SINGLE_QUOTED_STRING, /* "'string'", */
T_LITERAL, /* "'literal'", */
T_INVOCANT_IDENT, /* "invocant id", */
- T_NUMBER, /* "'number'", */
T_ERROR, /* "'error'", */
T_POWER, /* "**", */
T_POWER_ASSIGN, /* "**=", */
@@ -132,7 +131,6 @@
T_PLUS_ASSIGN, /* "+=", */
T_MINUS_ASSIGN, /* "-=", */
T_CONCAT_ASSIGN, /* ".=" */
- T_REGISTER, /* "'register'", */
T_DIVIDE_ASSIGN, /* "/=", */
T_FDIVIDE_ASSIGN, /* "//=", */
T_MODULO_ASSIGN, /* "%=", */
Modified: trunk/compilers/pirc/src/pirparser.c
==============================================================================
--- trunk/compilers/pirc/src/pirparser.c (original)
+++ trunk/compilers/pirc/src/pirparser.c Mon Mar 26 04:10:58 2007
@@ -516,7 +516,7 @@
=item methodcall()
- methodcall -> INVOCANT_IDENT method arguments '\n'
+ methodcall -> INVOCANT_IDENT method arguments
=cut
@@ -584,8 +584,8 @@
| 'global' stringconstant
| heredocstring
| methodcall
- | 'null'
- ) '\n'
+ | 'null' )
+ '\n'
unop -> '-' | '!' | '~'
@@ -673,15 +673,11 @@
}
-
-
-
-
/*
=item return_statement()
- return_statement -> '.return' ( arguments | target arguments | methodcall )
'\n'
+ return_statement -> '.return' ( arguments | target ['->' method] arguments |
methodcall ) '\n'
=cut
@@ -690,13 +686,13 @@
return_statement(parser_state *p) {
match(p, T_RETURN);
switch (p->curtoken) {
- case T_LPAREN:
+ case T_LPAREN: /* return_statement -> '.return' '(' [args] ')' */
arguments(p);
break;
- case T_INVOCANT_IDENT: /* tail method call */
+ case T_INVOCANT_IDENT: /* '.return' methodcall */
methodcall(p);
break;
- default: /* normal sub tailcall */
+ default: /* '.return' target ['->' method] arguments */
target(p);
if (p->curtoken == T_PTR) {
next(p);
@@ -1643,13 +1639,28 @@
parameters(parser_state *p) {
while (p->curtoken == T_PARAM) {
next(p); /* skip '.param */
- if (p->curtoken == T_REGISTER) { /* parameter -> '.param' register */
- next(p);
- }
- else { /* parameter -> '.param' type IDENT param_flag */
- type(p);
- match(p, T_IDENTIFIER);
- param_flags(p);
+ switch (p->curtoken) {
+ case T_INT:
+ case T_NUM:
+ case T_PMC:
+ case T_STRING:
+ type(p);
+ match(p, T_IDENTIFIER);
+ param_flags(p);
+ break;
+ case T_PREG:
+ case T_NREG:
+ case T_SREG:
+ case T_IREG:
+ case T_PASM_PREG:
+ case T_PASM_NREG:
+ case T_PASM_SREG:
+ case T_PASM_IREG:
+ next(p);
+ break;
+ default:
+ syntax_error(p, 1, "type or register expected");
+ break;
}
match(p, T_NEWLINE);
}
