changeset e21e9ab5fad0 in /z/repo/m5 details: http://repo.m5sim.org/m5?cmd=changeset;node=e21e9ab5fad0 description: ply: update PLY to version 3.2
diffstat: 164 files changed, 9123 insertions(+), 3820 deletions(-) ext/ply/ANNOUNCE | 19 ext/ply/CHANGES | 332 ++ ext/ply/COPYING | 504 --- ext/ply/README | 101 ext/ply/TODO | 16 ext/ply/doc/internal.html | 874 +++++ ext/ply/doc/makedoc.py | 10 ext/ply/doc/ply.html | 1099 ++++-- ext/ply/example/BASIC/basic.py | 9 ext/ply/example/BASIC/basiclex.py | 16 ext/ply/example/BASIC/basiclog.py | 79 ext/ply/example/BASIC/basinterp.py | 125 ext/ply/example/BASIC/basparse.py | 26 ext/ply/example/GardenSnake/GardenSnake.py | 28 ext/ply/example/ansic/clex.py | 20 ext/ply/example/ansic/cparse.py | 10 ext/ply/example/calc/calc.py | 24 ext/ply/example/calcdebug/calc.py | 113 ext/ply/example/classcalc/calc.py | 21 ext/ply/example/closurecalc/calc.py | 130 ext/ply/example/hedit/hedit.py | 12 ext/ply/example/newclasscalc/calc.py | 23 ext/ply/example/optcalc/README | 2 ext/ply/example/optcalc/calc.py | 20 ext/ply/example/unicalc/calc.py | 9 ext/ply/example/yply/ylex.py | 12 ext/ply/example/yply/yparse.py | 14 ext/ply/ply/cpp.py | 898 +++++ ext/ply/ply/ctokens.py | 133 ext/ply/ply/lex.py | 986 +++--- ext/ply/ply/yacc.py | 4422 +++++++++++++++++----------- ext/ply/setup.py | 25 ext/ply/test/calclex.py | 10 ext/ply/test/cleanup.sh | 2 ext/ply/test/lex_closure.py | 54 ext/ply/test/lex_doc1.exp | 1 ext/ply/test/lex_doc1.py | 8 ext/ply/test/lex_dup1.exp | 2 ext/ply/test/lex_dup1.py | 6 ext/ply/test/lex_dup2.exp | 2 ext/ply/test/lex_dup2.py | 6 ext/ply/test/lex_dup3.exp | 2 ext/ply/test/lex_dup3.py | 6 ext/ply/test/lex_empty.exp | 1 ext/ply/test/lex_empty.py | 6 ext/ply/test/lex_error1.exp | 1 ext/ply/test/lex_error1.py | 6 ext/ply/test/lex_error2.exp | 1 ext/ply/test/lex_error2.py | 6 ext/ply/test/lex_error3.exp | 2 ext/ply/test/lex_error3.py | 6 ext/ply/test/lex_error4.exp | 2 ext/ply/test/lex_error4.py | 6 ext/ply/test/lex_hedit.exp | 3 ext/ply/test/lex_hedit.py | 12 ext/ply/test/lex_ignore.exp | 7 ext/ply/test/lex_ignore.py | 4 ext/ply/test/lex_ignore2.exp | 1 ext/ply/test/lex_ignore2.py | 6 ext/ply/test/lex_literal1.py | 25 ext/ply/test/lex_literal2.py | 25 ext/ply/test/lex_many_tokens.py | 27 ext/ply/test/lex_module.py | 10 ext/ply/test/lex_module_import.py | 42 ext/ply/test/lex_nowarn.py | 30 ext/ply/test/lex_object.py | 55 ext/ply/test/lex_opt_alias.py | 54 ext/ply/test/lex_optimize.py | 50 ext/ply/test/lex_optimize2.py | 50 ext/ply/test/lex_optimize3.py | 52 ext/ply/test/lex_re1.exp | 7 ext/ply/test/lex_re1.py | 6 ext/ply/test/lex_re2.exp | 7 ext/ply/test/lex_re2.py | 6 ext/ply/test/lex_re3.exp | 8 ext/ply/test/lex_re3.py | 6 ext/ply/test/lex_rule1.exp | 2 ext/ply/test/lex_rule1.py | 8 ext/ply/test/lex_rule2.py | 29 ext/ply/test/lex_rule3.py | 27 ext/ply/test/lex_state1.exp | 7 ext/ply/test/lex_state1.py | 10 ext/ply/test/lex_state2.exp | 8 ext/ply/test/lex_state2.py | 10 ext/ply/test/lex_state3.exp | 8 ext/ply/test/lex_state3.py | 12 ext/ply/test/lex_state4.exp | 7 ext/ply/test/lex_state4.py | 14 ext/ply/test/lex_state5.exp | 7 ext/ply/test/lex_state5.py | 12 ext/ply/test/lex_state_noerror.exp | 1 ext/ply/test/lex_state_noerror.py | 12 ext/ply/test/lex_state_norule.exp | 7 ext/ply/test/lex_state_norule.py | 12 ext/ply/test/lex_state_try.exp | 7 ext/ply/test/lex_state_try.py | 13 ext/ply/test/lex_token1.exp | 1 ext/ply/test/lex_token1.py | 6 ext/ply/test/lex_token2.exp | 1 ext/ply/test/lex_token2.py | 5 ext/ply/test/lex_token3.exp | 2 ext/ply/test/lex_token3.py | 7 ext/ply/test/lex_token4.exp | 2 ext/ply/test/lex_token4.py | 6 ext/ply/test/lex_token5.exp | 1 ext/ply/test/lex_token5.py | 6 ext/ply/test/lex_token_dup.py | 29 ext/ply/test/rununit.py | 62 ext/ply/test/testlex.py | 604 +++ ext/ply/test/testyacc.py | 348 +- ext/ply/test/yacc_badargs.exp | 3 ext/ply/test/yacc_badargs.py | 8 ext/ply/test/yacc_badid.py | 77 ext/ply/test/yacc_badprec.exp | 1 ext/ply/test/yacc_badprec.py | 11 ext/ply/test/yacc_badprec2.exp | 3 ext/ply/test/yacc_badprec2.py | 11 ext/ply/test/yacc_badprec3.py | 68 ext/ply/test/yacc_badrule.exp | 5 ext/ply/test/yacc_badrule.py | 11 ext/ply/test/yacc_badtok.exp | 1 ext/ply/test/yacc_badtok.py | 12 ext/ply/test/yacc_dup.exp | 4 ext/ply/test/yacc_dup.py | 11 ext/ply/test/yacc_error1.exp | 1 ext/ply/test/yacc_error1.py | 11 ext/ply/test/yacc_error2.exp | 1 ext/ply/test/yacc_error2.py | 13 ext/ply/test/yacc_error3.exp | 1 ext/ply/test/yacc_error3.py | 11 ext/ply/test/yacc_error4.py | 72 ext/ply/test/yacc_inf.exp | 5 ext/ply/test/yacc_inf.py | 9 ext/ply/test/yacc_literal.py | 69 ext/ply/test/yacc_misplaced.py | 68 ext/ply/test/yacc_missing1.exp | 2 ext/ply/test/yacc_missing1.py | 11 ext/ply/test/yacc_nested.py | 33 ext/ply/test/yacc_nodoc.exp | 2 ext/ply/test/yacc_nodoc.py | 11 ext/ply/test/yacc_noerror.exp | 2 ext/ply/test/yacc_noerror.py | 9 ext/ply/test/yacc_nop.exp | 2 ext/ply/test/yacc_nop.py | 11 ext/ply/test/yacc_notfunc.exp | 4 ext/ply/test/yacc_notfunc.py | 11 ext/ply/test/yacc_notok.exp | 1 ext/ply/test/yacc_notok.py | 11 ext/ply/test/yacc_prec1.py | 68 ext/ply/test/yacc_rr.exp | 2 ext/ply/test/yacc_rr.py | 11 ext/ply/test/yacc_rr_unused.py | 30 ext/ply/test/yacc_simple.exp | 1 ext/ply/test/yacc_simple.py | 11 ext/ply/test/yacc_sr.exp | 2 ext/ply/test/yacc_sr.py | 11 ext/ply/test/yacc_term1.exp | 2 ext/ply/test/yacc_term1.py | 11 ext/ply/test/yacc_unused.exp | 4 ext/ply/test/yacc_unused.py | 11 ext/ply/test/yacc_unused_rule.py | 72 ext/ply/test/yacc_uprec.exp | 2 ext/ply/test/yacc_uprec.py | 11 ext/ply/test/yacc_uprec2.py | 63 diffs (truncated from 17065 to 300 lines): diff -r 64bf776c5e70 -r e21e9ab5fad0 ext/ply/ANNOUNCE --- a/ext/ply/ANNOUNCE Thu Aug 13 10:37:37 2009 -0500 +++ b/ext/ply/ANNOUNCE Sun Aug 16 13:39:58 2009 -0700 @@ -1,12 +1,13 @@ -February 19, 2007 +March 24, 2009 - Announcing : PLY-2.3 (Python Lex-Yacc) + Announcing : PLY-3.2 (Python Lex-Yacc) http://www.dabeaz.com/ply I'm pleased to announce a significant new update to PLY---a 100% Python -implementation of the common parsing tools lex and yacc. PLY-2.3 is -a minor bug fix release, but also features improved performance. +implementation of the common parsing tools lex and yacc. PLY-3.2 adds +compatibility for Python 2.6 and 3.0, provides some new customization +options, and cleans up a lot of internal implementation details. If you are new to PLY, here are a few highlights: @@ -29,19 +30,11 @@ problems. Currently, PLY can build its parsing tables using either SLR or LALR(1) algorithms. -- PLY can be used to build parsers for large programming languages. - Although it is not ultra-fast due to its Python implementation, - PLY can be used to parse grammars consisting of several hundred - rules (as might be found for a language like C). The lexer and LR - parser are also reasonably efficient when parsing normal - sized programs. - More information about PLY can be obtained on the PLY webpage at: http://www.dabeaz.com/ply -PLY is freely available and is licensed under the terms of the Lesser -GNU Public License (LGPL). +PLY is freely available. Cheers, diff -r 64bf776c5e70 -r e21e9ab5fad0 ext/ply/CHANGES --- a/ext/ply/CHANGES Thu Aug 13 10:37:37 2009 -0500 +++ b/ext/ply/CHANGES Sun Aug 16 13:39:58 2009 -0700 @@ -1,3 +1,335 @@ + +Version 3.2 +----------------------------- +03/24/09: beazley + Added an extra check to not print duplicated warning messages + about reduce/reduce conflicts. + +03/24/09: beazley + Switched PLY over to a BSD-license. + +03/23/09: beazley + Performance optimization. Discovered a few places to make + speedups in LR table generation. + +03/23/09: beazley + New warning message. PLY now warns about rules never + reduced due to reduce/reduce conflicts. Suggested by + Bruce Frederiksen. + +03/23/09: beazley + Some clean-up of warning messages related to reduce/reduce errors. + +03/23/09: beazley + Added a new picklefile option to yacc() to write the parsing + tables to a filename using the pickle module. Here is how + it works: + + yacc(picklefile="parsetab.p") + + This option can be used if the normal parsetab.py file is + extremely large. For example, on jython, it is impossible + to read parsing tables if the parsetab.py exceeds a certain + threshold. + + The filename supplied to the picklefile option is opened + relative to the current working directory of the Python + interpreter. If you need to refer to the file elsewhere, + you will need to supply an absolute or relative path. + + For maximum portability, the pickle file is written + using protocol 0. + +03/13/09: beazley + Fixed a bug in parser.out generation where the rule numbers + where off by one. + +03/13/09: beazley + Fixed a string formatting bug with one of the error messages. + Reported by Richard Reitmeyer + +Version 3.1 +----------------------------- +02/28/09: beazley + Fixed broken start argument to yacc(). PLY-3.0 broke this + feature by accident. + +02/28/09: beazley + Fixed debugging output. yacc() no longer reports shift/reduce + or reduce/reduce conflicts if debugging is turned off. This + restores similar behavior in PLY-2.5. Reported by Andrew Waters. + +Version 3.0 +----------------------------- +02/03/09: beazley + Fixed missing lexer attribute on certain tokens when + invoking the parser p_error() function. Reported by + Bart Whiteley. + +02/02/09: beazley + The lex() command now does all error-reporting and diagonistics + using the logging module interface. Pass in a Logger object + using the errorlog parameter to specify a different logger. + +02/02/09: beazley + Refactored ply.lex to use a more object-oriented and organized + approach to collecting lexer information. + +02/01/09: beazley + Removed the nowarn option from lex(). All output is controlled + by passing in a logger object. Just pass in a logger with a high + level setting to suppress output. This argument was never + documented to begin with so hopefully no one was relying upon it. + +02/01/09: beazley + Discovered and removed a dead if-statement in the lexer. This + resulted in a 6-7% speedup in lexing when I tested it. + +01/13/09: beazley + Minor change to the procedure for signalling a syntax error in a + production rule. A normal SyntaxError exception should be raised + instead of yacc.SyntaxError. + +01/13/09: beazley + Added a new method p.set_lineno(n,lineno) that can be used to set the + line number of symbol n in grammar rules. This simplifies manual + tracking of line numbers. + +01/11/09: beazley + Vastly improved debugging support for yacc.parse(). Instead of passing + debug as an integer, you can supply a Logging object (see the logging + module). Messages will be generated at the ERROR, INFO, and DEBUG + logging levels, each level providing progressively more information. + The debugging trace also shows states, grammar rule, values passed + into grammar rules, and the result of each reduction. + +01/09/09: beazley + The yacc() command now does all error-reporting and diagnostics using + the interface of the logging module. Use the errorlog parameter to + specify a logging object for error messages. Use the debuglog parameter + to specify a logging object for the 'parser.out' output. + +01/09/09: beazley + *HUGE* refactoring of the the ply.yacc() implementation. The high-level + user interface is backwards compatible, but the internals are completely + reorganized into classes. No more global variables. The internals + are also more extensible. For example, you can use the classes to + construct a LALR(1) parser in an entirely different manner than + what is currently the case. Documentation is forthcoming. + +01/07/09: beazley + Various cleanup and refactoring of yacc internals. + +01/06/09: beazley + Fixed a bug with precedence assignment. yacc was assigning the precedence + each rule based on the left-most token, when in fact, it should have been + using the right-most token. Reported by Bruce Frederiksen. + +11/27/08: beazley + Numerous changes to support Python 3.0 including removal of deprecated + statements (e.g., has_key) and the additional of compatibility code + to emulate features from Python 2 that have been removed, but which + are needed. Fixed the unit testing suite to work with Python 3.0. + The code should be backwards compatible with Python 2. + +11/26/08: beazley + Loosened the rules on what kind of objects can be passed in as the + "module" parameter to lex() and yacc(). Previously, you could only use + a module or an instance. Now, PLY just uses dir() to get a list of + symbols on whatever the object is without regard for its type. + +11/26/08: beazley + Changed all except: statements to be compatible with Python2.x/3.x syntax. + +11/26/08: beazley + Changed all raise Exception, value statements to raise Exception(value) for + forward compatibility. + +11/26/08: beazley + Removed all print statements from lex and yacc, using sys.stdout and sys.stderr + directly. Preparation for Python 3.0 support. + +11/04/08: beazley + Fixed a bug with referring to symbols on the the parsing stack using negative + indices. + +05/29/08: beazley + Completely revamped the testing system to use the unittest module for everything. + Added additional tests to cover new errors/warnings. + +Version 2.5 +----------------------------- +05/28/08: beazley + Fixed a bug with writing lex-tables in optimized mode and start states. + Reported by Kevin Henry. + +Version 2.4 +----------------------------- +05/04/08: beazley + A version number is now embedded in the table file signature so that + yacc can more gracefully accomodate changes to the output format + in the future. + +05/04/08: beazley + Removed undocumented .pushback() method on grammar productions. I'm + not sure this ever worked and can't recall ever using it. Might have + been an abandoned idea that never really got fleshed out. This + feature was never described or tested so removing it is hopefully + harmless. + +05/04/08: beazley + Added extra error checking to yacc() to detect precedence rules defined + for undefined terminal symbols. This allows yacc() to detect a potential + problem that can be really tricky to debug if no warning message or error + message is generated about it. + +05/04/08: beazley + lex() now has an outputdir that can specify the output directory for + tables when running in optimize mode. For example: + + lexer = lex.lex(optimize=True, lextab="ltab", outputdir="foo/bar") + + The behavior of specifying a table module and output directory are + more aligned with the behavior of yacc(). + +05/04/08: beazley + [Issue 9] + Fixed filename bug in when specifying the modulename in lex() and yacc(). + If you specified options such as the following: + + parser = yacc.yacc(tabmodule="foo.bar.parsetab",outputdir="foo/bar") + + yacc would create a file "foo.bar.parsetab.py" in the given directory. + Now, it simply generates a file "parsetab.py" in that directory. + Bug reported by cptbinho. + +05/04/08: beazley + Slight modification to lex() and yacc() to allow their table files + to be loaded from a previously loaded module. This might make + it easier to load the parsing tables from a complicated package + structure. For example: + + import foo.bar.spam.parsetab as parsetab + parser = yacc.yacc(tabmodule=parsetab) + + Note: lex and yacc will never regenerate the table file if used + in the form---you will get a warning message instead. + This idea suggested by Brian Clapper. + + +04/28/08: beazley + Fixed a big with p_error() functions being picked up correctly + when running in yacc(optimize=1) mode. Patch contributed by + Bart Whiteley. + +02/28/08: beazley + Fixed a bug with 'nonassoc' precedence rules. Basically the + non-precedence was being ignored and not producing the correct + run-time behavior in the parser. + +02/16/08: beazley + Slight relaxation of what the input() method to a lexer will + accept as a string. Instead of testing the input to see + if the input is a string or unicode string, it checks to see + if the input object looks like it contains string data. + This change makes it possible to pass string-like objects + in as input. For example, the object returned by mmap. + + import mmap, os + data = mmap.mmap(os.open(filename,os.O_RDONLY), + os.path.getsize(filename), + access=mmap.ACCESS_READ) + lexer.input(data) + + +11/29/07: beazley + Modification of ply.lex to allow token functions to aliased. + This is subtle, but it makes it easier to create libraries and + to reuse token specifications. For example, suppose you defined + a function like this: + + def number(t): + r'\d+' + t.value = int(t.value) + return t _______________________________________________ m5-dev mailing list m5-dev@m5sim.org http://m5sim.org/mailman/listinfo/m5-dev