Le 11 oct. 2012 à 15:59, Akim Demaille a écrit : > Previous discussions with Joel have established that they are not > related, so let's treat them separately. Simplest first: %define > lex-symbols.
[…] > The "token" subcategory seems to fit nicely. Something like > > %define api.token.constructor > > would make sense. Yet Joel reminded us that we would like to follow > two rules of thumb: avoid Boolean variables, and avoid plurals (so > it's api.token, not api.tokens). But I don't see well what names > would allow use to avoid a mere Boolean value. > > If someone has a better idea, I'd be happy to read it! I will install the following patches in master. While working on them, I realized that there might be a generalization that would avoid the Boolean status here: instead of %define api.token.constructor make it %define api.constructor tokens since we could imagine also providing constructors for the nonterminals. I don't know if it would be truly used, so anyway, for a start, I'll stick to the appended patches. commit 62cb570a8dc4895110c087dbfa5810cd0bc316a8 Author: Akim Demaille <[email protected]> Date: Thu Nov 1 17:50:37 2012 +0100 bison.m4: support b4_*_if macros whose name differ from their variable * data/bison.m4 (b4_percent_define_if_define_, b4_percent_define_if_define): Accept a second argument. diff --git a/data/bison.m4 b/data/bison.m4 index 0d2a39e..c3ae128 100644 --- a/data/bison.m4 +++ b/data/bison.m4 @@ -697,17 +697,18 @@ m4_define([b4_percent_define_default], m4_define([b4_percent_define_syncline(]$1[)], [[]])])]) -# b4_percent_define_if_define(VARIABLE) -# ------------------------------------- -# Define b4_VARIABLE_if that executes its $1 or $2 depending whether +# b4_percent_define_if_define(NAME, [VARIABLE = NAME]) +# ---------------------------------------------------- +# Define b4_NAME_if that executes its $1 or $2 depending whether # VARIABLE was %defined. The characters `.' and `-' in VARIABLE are mapped # to `_'. m4_define([b4_percent_define_if_define_], [m4_define(m4_bpatsubst([b4_$1_if], [[-.]], [_]), - [b4_percent_define_flag_if([$1], [$2], [$3])])]) + [b4_percent_define_flag_if(m4_default([$2], [$1]), + [$3], [$4])])]) m4_define([b4_percent_define_if_define], -[b4_percent_define_default([[$1]], [[false]]) -b4_percent_define_if_define_([$1], $[1], $[2])]) +[b4_percent_define_default([m4_default([$2], [$1])], [[false]]) +b4_percent_define_if_define_([$1], [$2], $[1], $[2])]) # b4_percent_define_check_values(VALUES) commit 0b3287025df27584045a91a006eb63665909cab9 Author: Akim Demaille <[email protected]> Date: Thu Nov 1 17:52:41 2012 +0100 examples: simplify/improve * examples/variant.yy: Put yylex in yy::, and simplify accordingly. Minor formatting changes. diff --git a/examples/variant.yy b/examples/variant.yy index 39a57ba..1288a4a 100644 --- a/examples/variant.yy +++ b/examples/variant.yy @@ -38,7 +38,10 @@ typedef std::list<std::string> strings_type; #include <sstream> // Prototype of the yylex function providing subsequent tokens. - static yy::parser::symbol_type yylex (); + namespace yy + { + static parser::symbol_type yylex (); + } // Printing a list of strings. // Koening look up will look into std, since that's an std::list. @@ -91,41 +94,44 @@ item: ; %% -// The yylex function providing subsequent tokens: -// TEXT "I have three numbers for you:" -// NUMBER 1 -// NUMBER 2 -// NUMBER 3 -// TEXT " and that's all!" -// END_OF_FILE - -static -yy::parser::symbol_type -yylex () +namespace yy { - static int stage = -1; - ++stage; - yy::parser::location_type loc(0, stage + 1, stage + 1); - switch (stage) + // The yylex function providing subsequent tokens: + // TEXT "I have three numbers for you." + // NUMBER 1 + // NUMBER 2 + // NUMBER 3 + // TEXT "And that's all!" + // END_OF_FILE + + static + parser::symbol_type + yylex () { - case 0: - return yy::parser::make_TEXT ("I have three numbers for you.", loc); - case 1: - case 2: - case 3: - return yy::parser::make_NUMBER (stage, loc); - case 4: - return yy::parser::make_TEXT ("And that's all!", loc); - default: - return yy::parser::make_END_OF_FILE (loc); + static int stage = -1; + ++stage; + parser::location_type loc(0, stage + 1, stage + 1); + switch (stage) + { + case 0: + return parser::make_TEXT ("I have three numbers for you.", loc); + case 1: + case 2: + case 3: + return parser::make_NUMBER (stage, loc); + case 4: + return parser::make_TEXT ("And that's all!", loc); + default: + return parser::make_END_OF_FILE (loc); + } } -} -// Mandatory error function -void -yy::parser::error (const yy::parser::location_type& loc, const std::string& msg) -{ - std::cerr << loc << ": " << msg << std::endl; + // Mandatory error function + void + parser::error (const parser::location_type& loc, const std::string& msg) + { + std::cerr << loc << ": " << msg << std::endl; + } } int commit e36ec1f41ffbe9f00db405c775201dbbc384c45c Author: Akim Demaille <[email protected]> Date: Thu Nov 1 17:54:13 2012 +0100 lalr1.cc: rename lex_symbol as api.token.constructor * data/bison.m4 (b4_lex_symbol_if): Rename as... (b4_token_ctor_if): this. Depend upon api.token.constructor. * data/c++.m4, data/lalr1.cc: Adjust. * doc/bison.texi: Fix all the occurrences of lex_symbol. * etc/bench.pl.in: Adjust. * examples/variant.yy: Likewise. * tests/local.at (AT_BISON_OPTION_PUSHDEFS, AT_BISON_OPTION_POPDEFS): Handle AT_TOKEN_CTOR_IF. * tests/c++.at: Adjust to using api.token.constructor and AT_TOKEN_CTOR_IF. Simplify the test of both build call styles. (AT_CHECK_VARIANTS): Rename as... (AT_TEST): this. And undef when done. diff --git a/data/bison.m4 b/data/bison.m4 index c3ae128..c652b2e 100644 --- a/data/bison.m4 +++ b/data/bison.m4 @@ -796,10 +796,10 @@ m4_percent_define_default([[api.token.prefix]], [[]]) # b4_parse_assert_if([IF-ASSERTIONS-ARE-USED], [IF-NOT]) # b4_parse_trace_if([IF-DEBUG-TRACES-ARE-ENABLED], [IF-NOT]) -# b4_lex_symbol_if([IF-YYLEX-RETURNS-A-COMPLETE-SYMBOL], [IF-NOT]) +# b4_token_ctor_if([IF-YYLEX-RETURNS-A-TOKEN], [IF-NOT]) # b4_variant_if([IF-VARIANT-ARE-USED], [IF-NOT]) # ---------------------------------------------- -b4_percent_define_if_define([lex_symbol]) +b4_percent_define_if_define([token_ctor], [api.token.constructor]) b4_percent_define_if_define([locations]) # Whether locations are tracked. b4_percent_define_if_define([parse.assert]) b4_percent_define_if_define([parse.trace]) diff --git a/data/c++.m4 b/data/c++.m4 index f4235ab..10855df 100644 --- a/data/c++.m4 +++ b/data/c++.m4 @@ -300,7 +300,7 @@ m4_define([b4_public_types_define], { return type; } -]b4_lex_symbol_if([[ +]b4_token_ctor_if([[ inline ]b4_parser_class_name[::token_type ]b4_parser_class_name[::symbol_type::token () const @@ -335,7 +335,7 @@ m4_define([b4_symbol_constructor_define], []) m4_define([b4_yytranslate_define], [[ // Symbol number corresponding to token number t. ]b4_parser_class_name[::token_number_type - ]b4_parser_class_name[::yytranslate_ (]b4_lex_symbol_if([token_type], + ]b4_parser_class_name[::yytranslate_ (]b4_token_ctor_if([token_type], [int])[ t) { static diff --git a/data/lalr1.cc b/data/lalr1.cc index 3168758..7e5baeb 100644 --- a/data/lalr1.cc +++ b/data/lalr1.cc @@ -239,7 +239,7 @@ b4_location_define])])])[ #endif // ]b4_api_PREFIX[DEBUG /// Convert a scanner token number \a t to a symbol number. - static inline token_number_type yytranslate_ (]b4_lex_symbol_if([token_type], [int])[ t); + static inline token_number_type yytranslate_ (]b4_token_ctor_if([token_type], [int])[ t); #if ]b4_api_PREFIX[DEBUG /// \brief Display a symbol type, value and location. @@ -320,7 +320,7 @@ b4_location_define])])])[ ]b4_parse_param_vars[ }; -]b4_lex_symbol_if([b4_yytranslate_define +]b4_token_ctor_if([b4_yytranslate_define b4_public_types_define])[ ]b4_namespace_close[ @@ -500,7 +500,7 @@ m4_if(b4_prefix, [yy], [], | Symbol types. | `---------------*/ -]b4_lex_symbol_if([], [b4_public_types_define])[ +]b4_token_ctor_if([], [b4_public_types_define])[ // stack_symbol_type. ]b4_parser_class_name[::stack_symbol_type::stack_symbol_type () @@ -726,7 +726,7 @@ b4_dollar_popdef])[]dnl YYCDEBUG << "Reading a token: "; try { -]b4_lex_symbol_if( +]b4_token_ctor_if( [ yyla = b4_function_call([yylex], [symbol_type], m4_ifdef([b4_lex_param], b4_lex_param));], [ yyla.type = yytranslate_ (b4_function_call([yylex], [int], @@ -1138,7 +1138,7 @@ b4_error_verbose_if([state_type yystate, int yytoken], } #endif // ]b4_api_PREFIX[DEBUG -]b4_lex_symbol_if([], [b4_yytranslate_define])[ +]b4_token_ctor_if([], [b4_yytranslate_define])[ ]b4_namespace_close[ ]b4_epilogue[]dnl m4_divert_pop(0) diff --git a/doc/bison.texi b/doc/bison.texi index 3b9f2da..848bca4 100644 --- a/doc/bison.texi +++ b/doc/bison.texi @@ -5547,6 +5547,30 @@ More user feedback will help to stabilize it.) +@c ================================================== api.token.constructor +@item api.token.constructor +@findex %define api.token.constructor + +@itemize @bullet +@item Language(s): +C++ + +@item Purpose: +When variant-based semantic values are enabled (@pxref{C++ Variants}), +request that symbols be handled as a whole (type, value, and possibly +location) in the scanner. @xref{Complete Symbols}, for details. + +@item Accepted Values: +Boolean. + +@item Default Value: +@code{false} +@item History: +introduced in Bison 2.8 +@end itemize +@c api.token.constructor + + @c ================================================== api.token.prefix @item api.token.prefix @findex %define api.token.prefix @@ -5587,28 +5611,6 @@ introduced in Bison 2.8 @c api.token.prefix -@c ================================================== lex_symbol -@item lex_symbol -@findex %define lex_symbol - -@itemize @bullet -@item Language(s): -C++ - -@item Purpose: -When variant-based semantic values are enabled (@pxref{C++ Variants}), -request that symbols be handled as a whole (type, value, and possibly -location) in the scanner. @xref{Complete Symbols}, for details. - -@item Accepted Values: -Boolean. - -@item Default Value: -@code{false} -@end itemize -@c lex_symbol - - @c ================================================== lr.default-reduction @item lr.default-reduction @@ -10194,7 +10196,8 @@ or @node Complete Symbols @subsubsection Complete Symbols -If you specified both @code{%define variant} and @code{%define lex_symbol}, +If you specified both @code{%define variant} and +@code{%define api.token.constructor}, the @code{parser} class also defines the class @code{parser::symbol_type} which defines a @emph{complete} symbol, aggregating its type (i.e., the traditional value returned by @code{yylex}), its semantic value (i.e., the @@ -10456,18 +10459,18 @@ the grammar for. @end example @noindent +@findex %define api.token.constructor @findex %define variant -@findex %define lex_symbol This example will use genuine C++ objects as semantic values, therefore, we require the variant-based interface. To make sure we properly use it, we enable assertions. To fully benefit from type-safety and more natural -definition of ``symbol'', we enable @code{lex_symbol}. +definition of ``symbol'', we enable @code{api.token.constructor}. @comment file: calc++-parser.yy @example -%define variant +%define api.token.constructor %define parse.assert -%define lex_symbol +%define variant @end example @noindent diff --git a/etc/bench.pl.in b/etc/bench.pl.in index 5d83fc7..6b1e87d 100755 --- a/etc/bench.pl.in +++ b/etc/bench.pl.in @@ -579,7 +579,7 @@ sub generate_grammar_list ($$@) my ($base, $max, @directive) = @_; my $directives = directives ($base, @directive); my $variant = grep { /%define "?variant"?/ } @directive; - my $lex_symbol = grep { /%define "?lex_symbol"?/ } @directive; + my $token_ctor = grep { /%define "?api.token.constructor"?/ } @directive; my $out = new IO::File ">$base.y" or die; print $out <<EOF; @@ -601,12 +601,12 @@ $directives #define STAGE_MAX ($max * 10) // max = $max -#define USE_LEX_SYMBOL $lex_symbol +#define USE_TOKEN_CTOR $token_ctor #define USE_VARIANTS $variant // Prototype of the yylex function providing subsequent tokens. static -#if USE_LEX_SYMBOL +#if USE_TOKEN_CTOR yy::parser::symbol_type yylex(); #else yy::parser::token_type yylex(yy::parser::semantic_type* yylval, @@ -678,7 +678,7 @@ EOF # static -#if USE_LEX_SYMBOL +#if USE_TOKEN_CTOR yy::parser::symbol_type yylex() #else yy::parser::token_type yylex(yy::parser::semantic_type* yylval, @@ -691,7 +691,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval, ++stage; if (stage == STAGE_MAX) { -#if USE_LEX_SYMBOL +#if USE_TOKEN_CTOR return yy::parser::make_END_OF_FILE (location_type ()); #else *yylloc = location_type (); @@ -700,7 +700,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval, } else if (stage % 2) { -#if USE_LEX_SYMBOL +#if USE_TOKEN_CTOR return yy::parser::make_NUMBER (stage, location_type ()); #else # if defined ONE_STAGE_BUILD @@ -716,7 +716,7 @@ yy::parser::token_type yylex(yy::parser::semantic_type* yylval, } else { -#if USE_LEX_SYMBOL +#if USE_TOKEN_CTOR return yy::parser::make_TEXT ("A string.", location_type ()); #else # if defined ONE_STAGE_BUILD @@ -914,7 +914,7 @@ sub bench_variant_parser () [ %d variant & - [ #d ONE_STAGE_BUILD | %d lex_symbol ] + [ #d ONE_STAGE_BUILD | %d api.token.constructor ] ] ) ); diff --git a/examples/variant.yy b/examples/variant.yy index 1288a4a..25f476a 100644 --- a/examples/variant.yy +++ b/examples/variant.yy @@ -18,9 +18,9 @@ %debug %skeleton "lalr1.cc" %defines +%define api.token.constructor %define parse.assert %define variant -%define lex_symbol %locations %code requires // *.hh diff --git a/tests/c++.at b/tests/c++.at index d5e7596..21ebafe 100644 --- a/tests/c++.at +++ b/tests/c++.at @@ -22,12 +22,13 @@ AT_BANNER([[C++ Features.]]) ## Variants. ## ## ---------- ## -# AT_CHECK_VARIANTS([DIRECTIVES]) -# ------------------------------- +# AT_TEST([DIRECTIVES]) +# --------------------- # Check the support of variants in C++, with the additional DIRECTIVES. -m4_define([AT_CHECK_VARIANTS], +m4_pushdef([AT_TEST], [AT_SETUP([Variants $1]) +AT_BISON_OPTION_PUSHDEFS([%skeleton "lalr1.cc" $1]) # Store strings and integers in a list of strings. AT_DATA_GRAMMAR([list.yy], [[%debug @@ -50,13 +51,13 @@ typedef std::list<std::string> strings_type; #include <iostream> #include <sstream> - static -#if defined USE_LEX_SYMBOL - yy::parser::symbol_type yylex (); -#else - yy::parser::token_type yylex (yy::parser::semantic_type* yylval, - yy::parser::location_type* yylloc); -#endif + namespace yy + { + static]AT_TOKEN_CTOR_IF([[ + parser::symbol_type yylex ()]], [[ + parser::token_type yylex (parser::semantic_type* yylval, + parser::location_type* yylloc)]])[; + } // Printing a list of strings (for %printer). // Koening look up will look into std, since that's an std::list. @@ -118,66 +119,51 @@ item: ; %% -#define STAGE_MAX 5 -static -#if defined USE_LEX_SYMBOL -yy::parser::symbol_type yylex () +#ifdef TWO_STAGE_BUILD +# define BUILD(Type, Value) build<Type> () = Value #else -yy::parser::token_type yylex (yy::parser::semantic_type* yylval, - yy::parser::location_type* yylloc) -#endif -{ -#ifndef USE_LEX_SYMBOL - typedef yy::parser::token token; +# define BUILD(Type, Value) build (Value) #endif - typedef yy::parser::location_type location_type; - static int stage = -1; - ++stage; - if (stage == STAGE_MAX) - { -#if defined USE_LEX_SYMBOL - return yy::parser::make_END_OF_FILE (location_type ()); -#else - *yylloc = location_type (); - return token::END_OF_FILE; -#endif - } - else if (stage % 2) - { -#if defined USE_LEX_SYMBOL - return yy::parser::make_NUMBER (stage, location_type ()); -#else -# if defined ONE_STAGE_BUILD - yylval->build (stage); -# else - yylval->build<int>() = stage; -# endif - *yylloc = location_type (); - return token::NUMBER; -#endif - } - else - { -#if defined USE_LEX_SYMBOL - return yy::parser::make_TEXT (string_cast (stage), location_type ()); -#else -# if defined ONE_STAGE_BUILD - yylval->build (string_cast (stage)); -# else - yylval->build<std::string>() = string_cast (stage); -# endif - *yylloc = location_type (); - return token::TEXT; -#endif - } - abort (); -} -void -yy::parser::error (const yy::parser::location_type&, - const std::string& message) +#define STAGE_MAX 5 +namespace yy { - std::cerr << message << std::endl; + static]AT_TOKEN_CTOR_IF([[ + parser::symbol_type yylex ()]], [[ + parser::token_type yylex (parser::semantic_type* yylval, + parser::location_type* yylloc)]])[ + { + typedef parser::location_type location; + static int stage = -1; + ++stage; + if (stage == STAGE_MAX) + {]AT_TOKEN_CTOR_IF([[ + return parser::make_END_OF_FILE (location ());]], [[ + *yylloc = location (); + return parser::token::END_OF_FILE;]])[ + } + else if (stage % 2) + {]AT_TOKEN_CTOR_IF([[ + return parser::make_NUMBER (stage, location ());]], [[ + yylval->BUILD (int, stage); + *yylloc = location (); + return parser::token::NUMBER;]])[ + } + else + {]AT_TOKEN_CTOR_IF([[ + return parser::make_TEXT (string_cast (stage), location ());]], [[ + yylval->BUILD (std::string, string_cast (stage)); + *yylloc = location (); + return parser::token::TEXT;]])[ + } + abort (); + } + + void + parser::error (const parser::location_type&, const std::string& message) + { + std::cerr << message << std::endl; + } } int @@ -195,14 +181,17 @@ AT_CHECK([./list], 0, [(0, 1, 2, 4) ]) +AT_BISON_OPTION_POPDEFS AT_CLEANUP ]) -AT_CHECK_VARIANTS([]) -AT_CHECK_VARIANTS([%define parse.assert]) -AT_CHECK_VARIANTS([[%define parse.assert %code {\n#define ONE_STAGE_BUILD\n}]]) -AT_CHECK_VARIANTS([[%define parse.assert %define lex_symbol %code {\n#define USE_LEX_SYMBOL\n}]]) -AT_CHECK_VARIANTS([[%define parse.assert %define lex_symbol %code {\n#define USE_LEX_SYMBOL\n} %define api.token.prefix "TOK_"]]) +AT_TEST([]) +AT_TEST([%define parse.assert]) +AT_TEST([[%define parse.assert %code {\n#define TWO_STAGE_BUILD\n}]]) +AT_TEST([[%define parse.assert %define api.token.constructor]]) +AT_TEST([[%define parse.assert %define api.token.constructor %define api.token.prefix "TOK_"]]) + +m4_popdef([AT_TEST]) ## ----------------------- ## diff --git a/tests/local.at b/tests/local.at index b619338..9fd641c 100644 --- a/tests/local.at +++ b/tests/local.at @@ -152,6 +152,8 @@ m4_pushdef([AT_NAME_PREFIX], [m4_bmatch([$3], [\(%define api\.prefix\|%name-prefix\) ".*"], [m4_bregexp([$3], [\(%define api\.prefix\|%name-prefix\) "\([^""]*\)"], [\2])], [yy])]) +m4_pushdef([AT_TOKEN_CTOR_IF], +[m4_bmatch([$3], [%define api.token.constructor], [$1], [$2])]) m4_pushdef([AT_TOKEN_PREFIX], [m4_bmatch([$3], [%define api.token.prefix ".*"], [m4_bregexp([$3], [%define api.token.prefix "\(.*\)"], [\1])])]) @@ -241,6 +243,8 @@ m4_popdef([AT_YYERROR_SEES_LOC_IF]) m4_popdef([AT_YYERROR_ARG_LOC_IF]) m4_popdef([AT_API_PREFIX]) m4_popdef([AT_API_prefix]) +m4_popdef([AT_TOKEN_PREFIX]) +m4_popdef([AT_TOKEN_CTOR_IF]) m4_popdef([AT_NAME_PREFIX]) m4_popdef([AT_GLR_OR_PARAM_IF]) m4_popdef([AT_PURE_AND_LOC_IF])
