Hello community,
here is the log from the commit of package python3-spark_parser for
openSUSE:Factory checked in at 2017-02-22 13:56:35
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python3-spark_parser (Old)
and /work/SRC/openSUSE:Factory/.python3-spark_parser.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python3-spark_parser"
Changes:
--------
---
/work/SRC/openSUSE:Factory/python3-spark_parser/python3-spark_parser.changes
2016-12-06 14:27:17.000000000 +0100
+++
/work/SRC/openSUSE:Factory/.python3-spark_parser.new/python3-spark_parser.changes
2017-02-22 13:56:37.513932318 +0100
@@ -1,0 +2,17 @@
+Tue Feb 21 05:10:05 UTC 2017 - [email protected]
+
+- specfile:
+ * update copyright year
+ * require click
+ * added binary spark-parser-coverage
+
+- update to version 1.6.0:
+ * Add ability to track grammar coverage
+ * Add ability to remove grammar rules
+
+- changes from version 1.5.2:
+ * Fix bug in dumpGrammar()
+ * print routines take an optional I/O parameter
+ * Correct Jay Earley's name
+
+-------------------------------------------------------------------
Old:
----
spark_parser-1.5.1.tar.gz
New:
----
spark_parser-1.6.0.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python3-spark_parser.spec ++++++
--- /var/tmp/diff_new_pack.1E8rZZ/_old 2017-02-22 13:56:38.117846406 +0100
+++ /var/tmp/diff_new_pack.1E8rZZ/_new 2017-02-22 13:56:38.121845838 +0100
@@ -1,7 +1,7 @@
#
# spec file for package python3-spark_parser
#
-# Copyright (c) 2016 SUSE LINUX GmbH, Nuernberg, Germany.
+# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany.
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -17,16 +17,18 @@
Name: python3-spark_parser
-Version: 1.5.1
+Version: 1.6.0
Release: 0
Summary: An Early-Algorithm Context-free grammar Parser
License: MIT
Group: Development/Languages/Python
Url: https://github.com/rocky/python3-spark/
Source:
https://files.pythonhosted.org/packages/source/s/spark_parser/spark_parser-%{version}.tar.gz
+BuildRequires: python3-click
BuildRequires: python3-devel
BuildRequires: python3-nose
BuildRequires: python3-setuptools
+Requires: python3-click
BuildRoot: %{_tmppath}/%{name}-%{version}-build
BuildArch: noarch
@@ -58,6 +60,7 @@
%files
%defattr(-,root,root,-)
%doc LICENSE ChangeLog README.rst
+%{_bindir}/spark-parser-coverage
%{python3_sitelib}/*
%changelog
++++++ spark_parser-1.5.1.tar.gz -> spark_parser-1.6.0.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/ChangeLog
new/spark_parser-1.6.0/ChangeLog
--- old/spark_parser-1.5.1/ChangeLog 2016-11-28 13:09:23.000000000 +0100
+++ new/spark_parser-1.6.0/ChangeLog 2017-02-01 00:58:47.000000000 +0100
@@ -1,6 +1,76 @@
+2017-01-31 rocky <[email protected]>
+
+ * ChangeLog: Get ready for release
+
+2017-01-30 rocky <[email protected]>
+
+ * NEW-FEATURES.rst, NEWS, spark_parser/spark.py,
+ spark_parser/version.py: Add the ability to remove a rule Go over new
features and over NEWS
+
+2017-01-29 rocky <[email protected]>
+
+ * spark_parser/spark.py: Reinstate extra debug_reduce parameters
+
+2017-01-29 rocky <[email protected]>
+
+ * spark_parser/spark.py: function parameter mismatch
+
+2017-01-29 rocky <[email protected]>
+
+ * bin/spark-parser-coverage, spark_parser/spark.py: Fix bugs in
+ accumulating coverage counts
+
+2017-01-29 rocky <[email protected]>
+
+ * bin/spark-parser-coverage, setup.py, spark_parser/spark.py,
+ test/test_spark.py: More coverage code
+
+2017-01-28 rocky <[email protected]>
+
+ * spark_parser/spark.py, test/test_spark.py: Banc on coverage a
+ little more
+
+2017-01-28 rocky <[email protected]>
+
+ * spark_parser/__init__.py, spark_parser/spark.py,
+ test/test_spark.py: Start tracking grammar coverage
+
+2017-01-27 rocky <[email protected]>
+
+ * setup.py: Works on 3.6.0
+
+2017-01-27 rocky <[email protected]>
+
+ * ChangeLog, NEWS, example/expr2/eval.py,
+ example/python2/test/format/if.py,
+ example/python2/test/parse/if.py,
+ example/python2/test/parse/if.right, spark_parser/version.py: Get
+ ready for release 1.5.2 Some lint changes
+
+2017-01-27 rocky <[email protected]>
+
+ * spark_parser/spark.py, test/test_misc.py: Fix bug in dumpGrammar()
+
+2016-12-08 R. Bernstein <[email protected]>
+
+ * NEW-FEATURES.rst: Update NEW-FEATURES.rst
+
+2016-12-08 R. Bernstein <[email protected]>
+
+ * NEW-FEATURES.rst: Update NEW-FEATURES.rst
+
+2016-12-08 rocky <[email protected]>
+
+ * README.rst, setup.py: Correct Jay EarlEy's name
+
2016-11-28 rocky <[email protected]>
- * setup.py, spark_parser/version.py: Get ready for release 1.5.1
+ * ChangeLog, NEWS, example/expr2/eval.py,
+ example/python2/py2_format.py, example/python2/py2_parser.py,
+ example/python2/test/scan/indent1.py,
+ example/python2/test/scan/indent1.right, setup.py,
+ spark_parser/version.py, test/test_misc.py: Get ready for release
+ 1.5.1 Lint some files
2016-11-28 R. Bernstein <[email protected]>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/NEW-FEATURES.rst
new/spark_parser-1.6.0/NEW-FEATURES.rst
--- old/spark_parser-1.5.1/NEW-FEATURES.rst 2016-11-28 12:49:08.000000000
+0100
+++ new/spark_parser-1.6.0/NEW-FEATURES.rst 2017-02-01 01:11:28.000000000
+0100
@@ -1,4 +1,7 @@
-The original version of this Early parser, circa 2000, was pretty awesome for
its
+Introduction
+============
+
+The original version of this Earley parser, circa 2000, was pretty awesome for
its
age. It was remarkably fast, and small: one Python file. Some care
was put into making it run fast.
@@ -13,6 +16,9 @@
Many of the changes I've added come from using the program. I list
features aside from the packaging, tests, and examples mentioned above.
+Comments in Grammar
+===================
+
The first thing I desired when working with a large complex grammar
and then sets of grammars that I found desirable was the ability to
comment the grammar. For example:
@@ -23,6 +29,9 @@
term := term MULT_OP atom
+AST Tree Display
+================
+
Next, I needed better AST tree display routines. For example:
.. code-block::
@@ -43,19 +52,32 @@
2. atom
type: NUMBER, value: '2'
-After that it was useful to have grammar checking routines,
-specifically, the ability to find unused left-hand-side nonterminals
-that are not either the start symbol or used on the right-hand side.
-Likewise unused nonterminals (lower-case symbols) that appear on the
-right-hand side that are not defined as a rule. Of course, tokens or
-upper-case symbols are ok.
+Grammar Checking
+================
+
+In a large project like uncompyle6_ there are lots of grammar rules:
+over 600 rules for each of the 13 or so versions of Python.
+
+It is very easy to create nonsensical grammar rules, so we need to
+have a way to check the grammar. Partuclarly useful is the ability to
+find unused left-hand-side nonterminals that are not either the start
+symbol or used on the right-hand side. Likewise unused nonterminals
+(lower-case symbols) that appear on the right-hand side that are not
+defined as a rule. Of course, tokens or upper-case symbols are ok.
+
+Checking for duplicate rules is also handy. Also finding immediate
+recursion rules. e.g. `expr ::= expr`.
-Checking for duplicate rules was also handy.
+Parser Error State
+==================
The original code showed you the how far you had parsed and that was
useful. But in production code you often want more. So I added the
list of rule states of the current state. I won't show that here.
+Reduce Rule Tracing
+===================
+
However also added was the ability to dump rules as reductions
occurred. Here is an example of that from uncompyle6:
@@ -107,12 +129,38 @@
printing program. The same can be done for duplicate-rule printing
and other things like that.
+Custom Additional Reduction Rule Checks
+=======================================
+
More recently, I the ability to callback before each reduction so
additional checks can be peformed before a reduction. In an ambiguous
grammar useful as it helps distinguish which rule should be used among
many.
-Lastly, I've added a little syntactic sugar for the Kleene closure
+Here are some little examples from the project *uncompyle6* which
+deparses Python bytecode. There is a rule in the grammar for a keyword
+argument that's used in a parameter list of a function.
+for example the `path=` in `os.path.exists(path='/etc/hosts')`
+
+This grammar rule is:
+
+.. code-block::
+
+ kwarg ::= LOAD_CONST expr
+
+
+But there is an additional restriction that the value in the
+`LOAD_CONST` can't be any old value; it must be a "string" (which
+would have the value "path") in the previous example.
+
+The reduction rule checking can work at a strickly token level, or it
+can work on and AST tree that would be generated if the reduction were done.
+
+
+Limited Grammar Shorthands: \+, \*, ?
+=====================================
+
+I also added a little syntactic sugar for the Kleene closure
operators `+`, `*` and optional suffix `?`. It is limited to only one
nonterminal on the right-hand side, but that does come up often and
helps a little. So you can now do things like:
@@ -143,6 +191,27 @@
.. code-block::
opt_comma ::= COMMA
- ratings ::=
+ opt_comma ::=
respectively.
+
+Tracking Grammar Coverage
+==========================
+
+Again in *uncompyle6* there are lots of grammar rules, so it is very
+easy to have dead grammar rules that never get used. And
+grammar constructs from one version of Python can easily bleed into
+another version. By looking at grammar coverage over a large set of
+parses, I can prune grammar rules or segregate them. I can also craft
+smaller parse tests which cover more of the grammar in fewer Python
+statements
+
+Removing Grammar Rules
+======================
+
+This may sound like a weird thing to want. But in a program like
+*uncompyle6* where there is a lot of grammar sharing via inheritance
+sometimes the grammar inherited is too large. This gives me a way
+to prune the grammar back down.
+
+.. _uncompyle6: https://pypi.python.org/pypi/uncompyle6/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/PKG-INFO
new/spark_parser-1.6.0/PKG-INFO
--- old/spark_parser-1.5.1/PKG-INFO 2016-11-28 13:30:14.000000000 +0100
+++ new/spark_parser-1.6.0/PKG-INFO 2017-02-01 01:13:19.000000000 +0100
@@ -1,7 +1,7 @@
Metadata-Version: 1.1
Name: spark_parser
-Version: 1.5.1
-Summary: An Early-Algorithm Context-free grammar Parser Toolkit
+Version: 1.6.0
+Summary: An Earley-Algorithm Context-free grammar Parser Toolkit
Home-page: https://github.com/rocky/python-spark/
Author: Rocky Bernstein
Author-email: [email protected]
@@ -12,7 +12,7 @@
=====
SPARK stands for Scanning, Parsing, and Rewriting Kit. It uses Jay
- Early's algorithm for parsing context free grammars, and comes with
+ Earley's algorithm for parsing context free grammars, and comes with
some generic Abstract Syntax Tree routines. There is also a prototype
scanner which does its job by combining Python regular expressions.
@@ -22,7 +22,7 @@
current incarnation of this code is maintained (or not) by Rocky
Bernstein.
- Note: Early algorithm parsers are almost linear when given an LR
grammar.
+ Note: Earley algorithm parsers are almost linear when given an LR
grammar.
These are grammars which are left-recursive.
Installation
@@ -71,5 +71,6 @@
Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
Classifier: Topic :: Software Development :: Code Generators
Classifier: Topic :: Software Development :: Libraries :: Python Modules
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/README.rst
new/spark_parser-1.6.0/README.rst
--- old/spark_parser-1.5.1/README.rst 2016-11-28 13:06:42.000000000 +0100
+++ new/spark_parser-1.6.0/README.rst 2017-01-30 07:40:22.000000000 +0100
@@ -4,7 +4,7 @@
=====
SPARK stands for Scanning, Parsing, and Rewriting Kit. It uses Jay
-Early's algorithm for parsing context free grammars, and comes with
+Earley's algorithm for parsing context free grammars, and comes with
some generic Abstract Syntax Tree routines. There is also a prototype
scanner which does its job by combining Python regular expressions.
@@ -14,7 +14,7 @@
current incarnation of this code is maintained (or not) by Rocky
Bernstein.
-Note: Early algorithm parsers are almost linear when given an LR grammar.
+Note: Earley algorithm parsers are almost linear when given an LR grammar.
These are grammars which are left-recursive.
Installation
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/bin/spark-parser-coverage
new/spark_parser-1.6.0/bin/spark-parser-coverage
--- old/spark_parser-1.5.1/bin/spark-parser-coverage 1970-01-01
01:00:00.000000000 +0100
+++ new/spark_parser-1.6.0/bin/spark-parser-coverage 2017-01-30
07:40:22.000000000 +0100
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+from __future__ import print_function
+import click
+import pickle
+
+def sort_profile_info(path, max_count=1000):
+ profile_info = pickle.load(open(path, "rb"))
+ items = sorted(profile_info.items(),
+ key=lambda kv: kv[1],
+ reverse=False)
+ return [item for item in items if item[1] <= max_count]
+
+DEFAULT_COVERAGE_FILE = "/tmp/spark-grammar.cover",
+DEFAULT_COUNT = 100
[email protected]()
[email protected]('--path', type=str, default=DEFAULT_COVERAGE_FILE,
+ help=("grammar coverage file (default %s)" %
DEFAULT_COVERAGE_FILE))
[email protected]('--max-count', type=int, default=DEFAULT_COUNT,
+ help=("limit output to rules having no more than this many hits
(default %d)" % DEFAULT_COUNT))
+def run(path, max_count):
+ """Print grammar reduce statistics for a series of spark-parser parses
+ """
+ for rule, count in sort_profile_info(path, max_count):
+ print("%d: %s" % (count, rule))
+ pass
+ return
+
+if __name__ == '__main__':
+ run()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/example/expr2/eval.py
new/spark_parser-1.6.0/example/expr2/eval.py
--- old/spark_parser-1.5.1/example/expr2/eval.py 2016-11-28
13:24:41.000000000 +0100
+++ new/spark_parser-1.6.0/example/expr2/eval.py 2017-01-30
07:40:22.000000000 +0100
@@ -91,7 +91,7 @@
parsed = parse_expr(expr_str, show_tokens=show_tokens,
parser_debug=parser_debug)
if showast:
- print (parsed)
+ print(parsed)
assert parsed == 'expr', 'Should have parsed grammar start'
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_format.cpython-33.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_format.cpython-33.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_format.cpython-34.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_format.cpython-34.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_format.cpython-35.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_format.cpython-35.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_format.cpython-36.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_format.cpython-36.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_parser.cpython-33.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_parser.cpython-33.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_parser.cpython-34.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_parser.cpython-34.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_parser.cpython-35.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_parser.cpython-35.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_parser.cpython-36.pyc
and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_parser.cpython-36.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_scan.cpython-33.pyc and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_scan.cpython-33.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_scan.cpython-34.pyc and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_scan.cpython-34.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_scan.cpython-35.pyc and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_scan.cpython-35.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_scan.cpython-36.pyc and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_scan.cpython-36.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/__pycache__/py2_token.cpython-36.pyc and
new/spark_parser-1.6.0/example/python2/__pycache__/py2_token.cpython-36.pyc
differ
Binary files old/spark_parser-1.5.1/example/python2/py2_format.pyc and
new/spark_parser-1.6.0/example/python2/py2_format.pyc differ
Binary files old/spark_parser-1.5.1/example/python2/py2_parser.pyc and
new/spark_parser-1.6.0/example/python2/py2_parser.pyc differ
Binary files old/spark_parser-1.5.1/example/python2/py2_scan.pyc and
new/spark_parser-1.6.0/example/python2/py2_scan.pyc differ
Binary files
old/spark_parser-1.5.1/example/python2/test/__pycache__/helper.cpython-33.pyc
and
new/spark_parser-1.6.0/example/python2/test/__pycache__/helper.cpython-33.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/test/__pycache__/helper.cpython-34.pyc
and
new/spark_parser-1.6.0/example/python2/test/__pycache__/helper.cpython-34.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/test/__pycache__/helper.cpython-35.pyc
and
new/spark_parser-1.6.0/example/python2/test/__pycache__/helper.cpython-35.pyc
differ
Binary files
old/spark_parser-1.5.1/example/python2/test/__pycache__/helper.cpython-36.pyc
and
new/spark_parser-1.6.0/example/python2/test/__pycache__/helper.cpython-36.pyc
differ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/example/python2/test/format/if.py
new/spark_parser-1.6.0/example/python2/test/format/if.py
--- old/spark_parser-1.5.1/example/python2/test/format/if.py 2016-11-28
12:49:08.000000000 +0100
+++ new/spark_parser-1.6.0/example/python2/test/format/if.py 2017-01-30
07:40:22.000000000 +0100
@@ -12,13 +12,13 @@
pass
if True:
- pass
+ pass
elif False:
- pass
+ pass
if True:
- pass
+ pass
elif False:
- pass
+ pass
else:
- pass
+ pass
Binary files old/spark_parser-1.5.1/example/python2/test/helper.pyc and
new/spark_parser-1.6.0/example/python2/test/helper.pyc differ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/example/python2/test/parse/if.py
new/spark_parser-1.6.0/example/python2/test/parse/if.py
--- old/spark_parser-1.5.1/example/python2/test/parse/if.py 2016-11-28
12:49:08.000000000 +0100
+++ new/spark_parser-1.6.0/example/python2/test/parse/if.py 2017-01-30
07:40:22.000000000 +0100
@@ -10,13 +10,13 @@
pass
if True:
- pass
+ pass
elif False:
- pass
+ pass
if True:
- pass
+ pass
elif False:
- pass
+ pass
else:
- pass
+ pass
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/spark_parser-1.5.1/example/python2/test/parse/if.right
new/spark_parser-1.6.0/example/python2/test/parse/if.right
--- old/spark_parser-1.5.1/example/python2/test/parse/if.right 2016-11-28
12:49:08.000000000 +0100
+++ new/spark_parser-1.6.0/example/python2/test/parse/if.right 2017-01-30
07:40:22.000000000 +0100
@@ -105,11 +105,11 @@
3. suite (5)
0. L12.8: NEWLINE: '\n'
1. indent
- L13.3: INDENT: ' '
+ L13.4: INDENT: ' '
2. stmt_plus
pass_stmt
- L13.7: PASS: 'pass'
- 3. L13.8: NEWLINE: '\n'
+ L13.8: PASS: 'pass'
+ 3. L13.9: NEWLINE: '\n'
4. L14.4: DEDENT: ''
4. elif_suites (5)
0. elif_suites
@@ -124,11 +124,11 @@
4. suite (5)
0. L14.11: NEWLINE: '\n'
1. indent
- L15.3: INDENT: ' '
+ L15.4: INDENT: ' '
2. stmt_plus
pass_stmt
- L15.7: PASS: 'pass'
- 3. L15.8: NEWLINE: '\n'
+ L15.8: PASS: 'pass'
+ 3. L15.9: NEWLINE: '\n'
4. L17.0: DEDENT: ''
5. else_suite_opt
12. newline_or_stmt
@@ -146,11 +146,11 @@
3. suite (5)
0. L17.8: NEWLINE: '\n'
1. indent
- L18.3: INDENT: ' '
+ L18.4: INDENT: ' '
2. stmt_plus
pass_stmt
- L18.7: PASS: 'pass'
- 3. L18.8: NEWLINE: '\n'
+ L18.8: PASS: 'pass'
+ 3. L18.9: NEWLINE: '\n'
4. L19.4: DEDENT: ''
4. elif_suites (5)
0. elif_suites
@@ -165,11 +165,11 @@
4. suite (5)
0. L19.11: NEWLINE: '\n'
1. indent
- L20.3: INDENT: ' '
+ L20.4: INDENT: ' '
2. stmt_plus
pass_stmt
- L20.7: PASS: 'pass'
- 3. L20.8: NEWLINE: '\n'
+ L20.8: PASS: 'pass'
+ 3. L20.9: NEWLINE: '\n'
4. L21.4: DEDENT: ''
5. else_suite_opt (3)
0. L21.4: ELSE: 'else'
@@ -177,10 +177,10 @@
2. suite (5)
0. L21.6: NEWLINE: '\n'
1. indent
- L22.3: INDENT: ' '
+ L22.4: INDENT: ' '
2. stmt_plus
pass_stmt
- L22.7: PASS: 'pass'
- 3. L22.8: NEWLINE: '\n'
+ L22.8: PASS: 'pass'
+ 3. L22.9: NEWLINE: '\n'
4. L23.1: DEDENT: ''
1. L23.1: ENDMARKER: '\x04'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/setup.cfg
new/spark_parser-1.6.0/setup.cfg
--- old/spark_parser-1.5.1/setup.cfg 2016-11-28 13:30:14.000000000 +0100
+++ new/spark_parser-1.6.0/setup.cfg 2017-02-01 01:13:19.000000000 +0100
@@ -1,5 +1,5 @@
[egg_info]
+tag_svn_revision = 0
tag_build =
tag_date = 0
-tag_svn_revision = 0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/setup.py
new/spark_parser-1.6.0/setup.py
--- old/spark_parser-1.5.1/setup.py 2016-11-28 13:08:46.000000000 +0100
+++ new/spark_parser-1.6.0/setup.py 2017-01-30 07:40:22.000000000 +0100
@@ -26,6 +26,7 @@
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
'Topic :: Software Development :: Code Generators',
'Topic :: Software Development :: Libraries :: Python Modules',
]
@@ -39,7 +40,7 @@
modname = 'spark_parser'
name = 'spark_parser'
py_modules = None
-short_desc = 'An Early-Algorithm Context-free grammar Parser Toolkit'
+short_desc = 'An Earley-Algorithm Context-free grammar Parser Toolkit'
web = 'https://github.com/rocky/python-spark/'
# tracebacks in zip files are funky and not debuggable
@@ -63,7 +64,7 @@
setup(
classifiers = classifiers,
description = short_desc,
- # install_requires = install_requires,
+ install_requires = ['click'],
license = license,
long_description = long_description,
maintainer = maintainer,
@@ -71,6 +72,7 @@
packages = find_packages(),
py_modules = py_modules,
name = name,
+ scripts = ['bin/spark-parser-coverage'],
test_suite = 'nose.collector',
url = web,
tests_require = ['nose>=1.0'],
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/spark_parser/__init__.py
new/spark_parser-1.6.0/spark_parser/__init__.py
--- old/spark_parser-1.5.1/spark_parser/__init__.py 2016-06-11
14:14:15.000000000 +0200
+++ new/spark_parser-1.6.0/spark_parser/__init__.py 2017-01-30
07:40:22.000000000 +0100
@@ -6,21 +6,11 @@
PYTHON3 = (sys.version_info >= (3, 0))
-if PYTHON3:
- from spark_parser.ast import AST as AST
- from spark_parser.ast import GenericASTTraversal as GenericASTTraversal
- from spark_parser.ast import GenericASTTraversalPruningException as
GenericASTTraversalPruningException
- from spark_parser.spark import DEFAULT_DEBUG
- from spark_parser.spark import GenericParser as GenericParser
- from spark_parser.spark import GenericASTBuilder as GenericASTBuilder
- from spark_parser.scanner import GenericScanner as GenericScanner
- from spark_parser.scanner import GenericToken as GenericToken
-else:
- from ast import AST as AST
- from ast import GenericASTTraversal as GenericASTTraversal
- from ast import GenericASTTraversalPruningException as
GenericASTTraversalPruningException
- from spark import DEFAULT_DEBUG
- from spark import GenericParser as GenericParser
- from spark import GenericASTBuilder as GenericASTBuilder
- from scanner import GenericScanner as GenericScanner
- from scanner import GenericToken as GenericToken
+from spark_parser.ast import AST as AST
+from spark_parser.ast import GenericASTTraversal as GenericASTTraversal
+from spark_parser.ast import GenericASTTraversalPruningException as
GenericASTTraversalPruningException
+from spark_parser.spark import DEFAULT_DEBUG
+from spark_parser.spark import GenericParser as GenericParser
+from spark_parser.spark import GenericASTBuilder as GenericASTBuilder
+from spark_parser.scanner import GenericScanner as GenericScanner
+from spark_parser.scanner import GenericToken as GenericToken
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/spark_parser/spark.py
new/spark_parser-1.6.0/spark_parser/spark.py
--- old/spark_parser-1.5.1/spark_parser/spark.py 2016-11-28
13:06:42.000000000 +0100
+++ new/spark_parser-1.6.0/spark_parser/spark.py 2017-01-30
07:41:02.000000000 +0100
@@ -1,5 +1,5 @@
"""
-Copyright (c) 2015-2016 Rocky Bernstein
+Copyright (c) 2015-2017 Rocky Bernstein
Copyright (c) 1998-2002 John Aycock
Permission is hereby granted, free of charge, to any person obtaining
@@ -22,7 +22,7 @@
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
-import os, re, sys
+import os, pickle, re, sys
if sys.version[0:3] <= '2.3':
from sets import Set as set
@@ -44,7 +44,7 @@
return namelist
def rule2str(rule):
- return "%s ::= %s" % (rule[0], ' '.join(rule[1]))
+ return ("%s ::= %s" % (rule[0], ' '.join(rule[1]))).rstrip()
class _State:
'''
@@ -73,11 +73,28 @@
Parsing", unpublished paper, 2001.
'''
- def __init__(self, start, debug=DEFAULT_DEBUG):
+ def __init__(self, start, debug=DEFAULT_DEBUG,
+ coverage_path=None):
+ """_start_ : grammar start symbol;
+ _debug_ : produce optional parsing debug information
+ _profile_ : if not None should be a file path to open
+ with where to store profile is stored
+ """
+
self.rules = {}
self.rule2func = {}
self.rule2name = {}
+ # grammar coverage information
+ self.coverage_path = coverage_path
+ if coverage_path:
+ self.profile_info = {}
+ if isinstance(coverage_path, str):
+ if os.path.exists(coverage_path):
+ self.profile_info = pickle.load(open(coverage_path, "rb"))
+ else:
+ self.profile_info = None
+
# When set, shows additional debug output
self.debug = debug
@@ -121,15 +138,15 @@
#
# XXX - should find a better way to do this..
#
- changes = 1
+ changes = True
while changes:
- changes = 0
+ changes = False
for k, v in list(self.edges.items()):
if v is None:
state, sym = k
if state in self.states:
self.goto(state, sym)
- changes = 1
+ changes = True
rv = self.__dict__.copy()
for s in list(self.states.values()):
del s.items
@@ -222,9 +239,39 @@
self.rule2func[rule] = fn
self.rule2name[rule] = func.__name__[2:]
self.ruleschanged = True
+
+ if self.profile_info is not None:
+ rule_str = self.reduce_string(rule)
+ if rule_str not in self.profile_info:
+ self.profile_info[rule_str] = 0
pass
return
+ def remove_rule(self, doc):
+ """Remove a grammar rules from _self.rules_, _self.rule2func_,
+ and _self.rule2name_
+ """
+ rules = doc.split()
+ index = []
+ for i in range(len(rules)):
+ if rules[i] == '::=':
+ index.append(i-1)
+ index.append(len(rules))
+ for i in range(len(index)-1):
+ lhs = rules[index[i]]
+ rhs = rules[index[i]+2:index[i+1]]
+ rule = (lhs, tuple(rhs))
+
+ if lhs not in self.rules:
+ return
+
+ self.rules[lhs].remove(rule)
+ del self.rule2func[rule]
+ del self.rule2name[rule]
+ self.ruleschanged = True
+ return
+
+
def collectRules(self):
for name in _namelist(self):
if name[:2] == 'p_':
@@ -407,6 +454,9 @@
else:
self.error(None, None)
+ if self.profile_info is not None:
+ self.dump_profile_info()
+
return self.buildTree(self._START, finalitem,
tokens, len(sets)-2)
@@ -575,6 +625,8 @@
lhs, rhs = rule
if self.debug['reduce']:
self.debug_reduce(rule, tokens, parent, i)
+ if self.profile_info is not None:
+ self.profile_rule(rule)
if lhs in self.check_reduce and tokens:
if self.check_reduce[lhs] == 'AST':
ast = self.reduce_ast(rule, tokens, item, i, sets)
@@ -767,29 +819,29 @@
'''
return list[0]
- def dumpGrammar(self):
+ def dumpGrammar(self, out=sys.stdout):
"""
Print grammar rules
"""
for rule in sorted(self.rule2name.items()):
- print("%s" % rule2str(rule))
+ out.write("%s\n" % rule2str(rule[0]))
return
- def checkGrammar(self):
+ def checkGrammar(self, out=sys.stderr):
'''
Check grammar
'''
lhs, rhs, tokens, right_recursive = self.checkSets()
if len(lhs) > 0:
- print("LHS symbols not used on the RHS:")
- print(sorted(lhs))
+ out.write("LHS symbols not used on the RHS:\n")
+ out.write(sorted(lhs), "\n")
if len(rhs) > 0:
- print("RHS symbols not used on the LHS:")
- print(sorted(rhs))
+ out.write("RHS symbols not used on the LHS:\n")
+ out.write(sorted(rhs, "\n"))
if len(right_recursive) > 0:
- print("Right recursive rules:")
+ out.write("Right recursive rules:\n")
for rule in right_recursive:
- print("%s ::= %s" % (rule[0], ' '.join(rule[1])))
+ out.write("%s ::= %s\n" % (rule[0], ' '.join(rule[1])))
pass
pass
@@ -824,8 +876,38 @@
missing_rhs = rhs_set - lhs_set
return (missing_lhs, missing_rhs, token_set, right_recursive)
+ def reduce_string(self, rule):
+ return "%s ::= %s" % (rule[0], ' '.join(rule[1]))
+
+ # Note the unused parameters here are used in subclassed
+ # routines that need more information
def debug_reduce(self, rule, tokens, parent, i):
- print("%s ::= %s" % (rule[0], ' '.join(rule[1])))
+ print(self.reduce_string(rule))
+
+ def profile_rule(self, rule):
+ """Bump count of the number of times _rule_ was used"""
+ rule_str = self.reduce_string(rule)
+ if rule_str not in self.profile_info:
+ self.profile_info[rule_str] = 1
+ else:
+ self.profile_info[rule_str] += 1
+
+ def get_profile_info(self):
+ """Show the accumulated results of how many times each rule was used"""
+ return sorted(self.profile_info.items(),
+ key=lambda kv: kv[1],
+ reverse=False)
+ return
+
+ def dump_profile_info(self):
+ if isinstance(self.coverage_path, str):
+ with open(self.coverage_path, 'wb') as fp:
+ pickle.dump(self.profile_info, fp)
+ else:
+ for rule, count in self.get_profile_info():
+ self.coverage_path.write("%s -- %d\n" % (rule, count))
+ pass
+ self.coverage_path.write("-" * 40 + "\n")
def reduce_ast(self, rule, tokens, item, k, sets):
rhs = rule[1]
@@ -860,7 +942,12 @@
class GenericASTBuilder(GenericParser):
def __init__(self, AST, start, debug=DEFAULT_DEBUG):
- GenericParser.__init__(self, start, debug=debug)
+ if 'SPARK_PARSER_COVERAGE' in os.environ:
+ coverage_path = os.environ['SPARK_PARSER_COVERAGE']
+ else:
+ coverage_path = None
+ GenericParser.__init__(self, start, debug=debug,
+ coverage_path=coverage_path)
self.AST = AST
def preprocess(self, rule, func):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/spark_parser/version.py
new/spark_parser-1.6.0/spark_parser/version.py
--- old/spark_parser-1.5.1/spark_parser/version.py 2016-11-28
13:07:14.000000000 +0100
+++ new/spark_parser-1.6.0/spark_parser/version.py 2017-01-30
08:15:15.000000000 +0100
@@ -1,3 +1,3 @@
# This file is suitable for sourcing inside bash as
# well as importing into Python
-VERSION='1.5.1'
+VERSION='1.6.0'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/spark_parser.egg-info/PKG-INFO
new/spark_parser-1.6.0/spark_parser.egg-info/PKG-INFO
--- old/spark_parser-1.5.1/spark_parser.egg-info/PKG-INFO 2016-11-28
13:30:14.000000000 +0100
+++ new/spark_parser-1.6.0/spark_parser.egg-info/PKG-INFO 2017-02-01
01:13:19.000000000 +0100
@@ -1,7 +1,7 @@
Metadata-Version: 1.1
Name: spark-parser
-Version: 1.5.1
-Summary: An Early-Algorithm Context-free grammar Parser Toolkit
+Version: 1.6.0
+Summary: An Earley-Algorithm Context-free grammar Parser Toolkit
Home-page: https://github.com/rocky/python-spark/
Author: Rocky Bernstein
Author-email: [email protected]
@@ -12,7 +12,7 @@
=====
SPARK stands for Scanning, Parsing, and Rewriting Kit. It uses Jay
- Early's algorithm for parsing context free grammars, and comes with
+ Earley's algorithm for parsing context free grammars, and comes with
some generic Abstract Syntax Tree routines. There is also a prototype
scanner which does its job by combining Python regular expressions.
@@ -22,7 +22,7 @@
current incarnation of this code is maintained (or not) by Rocky
Bernstein.
- Note: Early algorithm parsers are almost linear when given an LR
grammar.
+ Note: Earley algorithm parsers are almost linear when given an LR
grammar.
These are grammars which are left-recursive.
Installation
@@ -71,5 +71,6 @@
Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
Classifier: Topic :: Software Development :: Code Generators
Classifier: Topic :: Software Development :: Libraries :: Python Modules
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/spark_parser.egg-info/SOURCES.txt
new/spark_parser-1.6.0/spark_parser.egg-info/SOURCES.txt
--- old/spark_parser-1.5.1/spark_parser.egg-info/SOURCES.txt 2016-11-28
13:30:14.000000000 +0100
+++ new/spark_parser-1.6.0/spark_parser.egg-info/SOURCES.txt 2017-02-01
01:13:19.000000000 +0100
@@ -6,6 +6,7 @@
README.rst
TODO.rst
setup.py
+bin/spark-parser-coverage
example/README.md
example/expr/README.md
example/expr/expr.py
@@ -40,15 +41,19 @@
example/python2/__pycache__/py2_format.cpython-33.pyc
example/python2/__pycache__/py2_format.cpython-34.pyc
example/python2/__pycache__/py2_format.cpython-35.pyc
+example/python2/__pycache__/py2_format.cpython-36.pyc
example/python2/__pycache__/py2_parser.cpython-33.pyc
example/python2/__pycache__/py2_parser.cpython-34.pyc
example/python2/__pycache__/py2_parser.cpython-35.pyc
+example/python2/__pycache__/py2_parser.cpython-36.pyc
example/python2/__pycache__/py2_scan.cpython-33.pyc
example/python2/__pycache__/py2_scan.cpython-34.pyc
example/python2/__pycache__/py2_scan.cpython-35.pyc
+example/python2/__pycache__/py2_scan.cpython-36.pyc
example/python2/__pycache__/py2_token.cpython-33.pyc
example/python2/__pycache__/py2_token.cpython-34.pyc
example/python2/__pycache__/py2_token.cpython-35.pyc
+example/python2/__pycache__/py2_token.cpython-36.pyc
example/python2/test/Makefile
example/python2/test/helper.py
example/python2/test/helper.pyc
@@ -63,6 +68,7 @@
example/python2/test/__pycache__/helper.cpython-33.pyc
example/python2/test/__pycache__/helper.cpython-34.pyc
example/python2/test/__pycache__/helper.cpython-35.pyc
+example/python2/test/__pycache__/helper.cpython-36.pyc
example/python2/test/format/assert.py
example/python2/test/format/assert.right
example/python2/test/format/def-bug.py-notyet
@@ -114,10 +120,14 @@
spark_parser.egg-info/PKG-INFO
spark_parser.egg-info/SOURCES.txt
spark_parser.egg-info/dependency_links.txt
+spark_parser.egg-info/requires.txt
spark_parser.egg-info/top_level.txt
spark_parser.egg-info/zip-safe
test/test_checker.py
test/test_checker.pyc
+test/test_grammar.py
+test/test_grammar.pyc
+test/test_grammar.py~
test/test_misc.py
test/test_misc.pyc
test/test_spark.py
@@ -125,9 +135,15 @@
test/__pycache__/test_checker.cpython-33.pyc
test/__pycache__/test_checker.cpython-34.pyc
test/__pycache__/test_checker.cpython-35.pyc
+test/__pycache__/test_checker.cpython-36.pyc
+test/__pycache__/test_grammar.cpython-33.pyc
+test/__pycache__/test_grammar.cpython-34.pyc
+test/__pycache__/test_grammar.cpython-35.pyc
test/__pycache__/test_misc.cpython-33.pyc
test/__pycache__/test_misc.cpython-34.pyc
test/__pycache__/test_misc.cpython-35.pyc
+test/__pycache__/test_misc.cpython-36.pyc
test/__pycache__/test_spark.cpython-33.pyc
test/__pycache__/test_spark.cpython-34.pyc
-test/__pycache__/test_spark.cpython-35.pyc
\ No newline at end of file
+test/__pycache__/test_spark.cpython-35.pyc
+test/__pycache__/test_spark.cpython-36.pyc
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/spark_parser-1.5.1/spark_parser.egg-info/requires.txt
new/spark_parser-1.6.0/spark_parser.egg-info/requires.txt
--- old/spark_parser-1.5.1/spark_parser.egg-info/requires.txt 1970-01-01
01:00:00.000000000 +0100
+++ new/spark_parser-1.6.0/spark_parser.egg-info/requires.txt 2017-02-01
01:13:19.000000000 +0100
@@ -0,0 +1 @@
+click
Binary files
old/spark_parser-1.5.1/test/__pycache__/test_checker.cpython-36.pyc and
new/spark_parser-1.6.0/test/__pycache__/test_checker.cpython-36.pyc differ
Binary files
old/spark_parser-1.5.1/test/__pycache__/test_grammar.cpython-33.pyc and
new/spark_parser-1.6.0/test/__pycache__/test_grammar.cpython-33.pyc differ
Binary files
old/spark_parser-1.5.1/test/__pycache__/test_grammar.cpython-34.pyc and
new/spark_parser-1.6.0/test/__pycache__/test_grammar.cpython-34.pyc differ
Binary files
old/spark_parser-1.5.1/test/__pycache__/test_grammar.cpython-35.pyc and
new/spark_parser-1.6.0/test/__pycache__/test_grammar.cpython-35.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_misc.cpython-33.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_misc.cpython-33.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_misc.cpython-34.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_misc.cpython-34.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_misc.cpython-35.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_misc.cpython-35.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_misc.cpython-36.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_misc.cpython-36.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_spark.cpython-33.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_spark.cpython-33.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_spark.cpython-34.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_spark.cpython-34.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_spark.cpython-35.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_spark.cpython-35.pyc differ
Binary files old/spark_parser-1.5.1/test/__pycache__/test_spark.cpython-36.pyc
and new/spark_parser-1.6.0/test/__pycache__/test_spark.cpython-36.pyc differ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/test/test_grammar.py
new/spark_parser-1.6.0/test/test_grammar.py
--- old/spark_parser-1.5.1/test/test_grammar.py 1970-01-01 01:00:00.000000000
+0100
+++ new/spark_parser-1.6.0/test/test_grammar.py 2017-02-01 01:06:03.000000000
+0100
@@ -0,0 +1,67 @@
+import unittest
+from spark_parser import (GenericParser, PYTHON3)
+
+if PYTHON3:
+ from io import StringIO
+else:
+ from StringIO import StringIO
+
+
+class Expr(GenericParser):
+ """Testing DumpGrammar, adding a rule and removing a rule"""
+ def p_rules(self, args):
+ """
+ expr ::= expr ADD_OP term
+ expr ::= term
+ term ::= term MULT_OP factor
+ term ::= factor
+ factor ::= INTEGER
+ """
+ return
+ pass
+
+nop_func = lambda self, args: None
+
+class TestGrammar(unittest.TestCase):
+
+ def test_basic(self):
+ parser = Expr('expr')
+ f = StringIO()
+ parser.dumpGrammar(f)
+ expect = """START ::= |- expr
+expr ::= expr ADD_OP term
+expr ::= term
+factor ::= INTEGER
+term ::= factor
+term ::= term MULT_OP factor
+"""
+ self.assertEqual(expect, f.getvalue())
+
+ parser.addRule("expr ::= expr SUB_OP term", nop_func)
+ expect = """START ::= |- expr
+expr ::= expr ADD_OP term
+expr ::= expr SUB_OP term
+expr ::= term
+factor ::= INTEGER
+term ::= factor
+term ::= term MULT_OP factor
+"""
+ f = StringIO()
+ parser.dumpGrammar(f)
+ self.assertEqual(expect, f.getvalue())
+
+ parser.remove_rule("expr ::= expr ADD_OP term")
+ expect = """START ::= |- expr
+expr ::= expr SUB_OP term
+expr ::= term
+factor ::= INTEGER
+term ::= factor
+term ::= term MULT_OP factor
+"""
+ f = StringIO()
+ parser.dumpGrammar(f)
+ self.assertEqual(expect, f.getvalue())
+
+
+if __name__ == '__main__':
+ unittest.main()
Binary files old/spark_parser-1.5.1/test/test_grammar.pyc and
new/spark_parser-1.6.0/test/test_grammar.pyc differ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/test/test_grammar.py~
new/spark_parser-1.6.0/test/test_grammar.py~
--- old/spark_parser-1.5.1/test/test_grammar.py~ 1970-01-01
01:00:00.000000000 +0100
+++ new/spark_parser-1.6.0/test/test_grammar.py~ 2017-01-30
07:41:02.000000000 +0100
@@ -0,0 +1,64 @@
+import unittest
+from spark_parser.spark import GenericParser
+
+from StringIO import StringIO
+
+
+class Expr(GenericParser):
+ """Testing DumpGrammar, adding a rule and removing a rule"""
+ def p_rules(self, args):
+ """
+ expr ::= expr ADD_OP term
+ expr ::= term
+ term ::= term MULT_OP factor
+ term ::= factor
+ factor ::= INTEGER
+ """
+ return
+ pass
+
+nop_func = lambda self, args: None
+
+class TestGrammar(unittest.TestCase):
+
+ def test_basic(self):
+ parser = Expr('expr')
+ f = StringIO()
+ parser.dumpGrammar(f)
+ expect = """START ::= |- expr
+expr ::= expr ADD_OP term
+expr ::= term
+factor ::= INTEGER
+term ::= factor
+term ::= term MULT_OP factor
+"""
+ self.assertEqual(expect, f.getvalue())
+
+ parser.addRule("expr ::= expr SUB_OP term", nop_func)
+ expect = """START ::= |- expr
+expr ::= expr ADD_OP term
+expr ::= expr SUB_OP term
+expr ::= term
+factor ::= INTEGER
+term ::= factor
+term ::= term MULT_OP factor
+"""
+ f = StringIO()
+ parser.dumpGrammar(f)
+ self.assertEqual(expect, f.getvalue())
+
+ parser.remove_rule("expr ::= expr ADD_OP term")
+ expect = """START ::= |- expr
+expr ::= expr SUB_OP term
+expr ::= term
+factor ::= INTEGER
+term ::= factor
+term ::= term MULT_OP factor
+"""
+ f = StringIO()
+ parser.dumpGrammar(f)
+ self.assertEqual(expect, f.getvalue())
+
+
+if __name__ == '__main__':
+ unittest.main()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/test/test_misc.py
new/spark_parser-1.6.0/test/test_misc.py
--- old/spark_parser-1.5.1/test/test_misc.py 2016-11-28 13:14:51.000000000
+0100
+++ new/spark_parser-1.6.0/test/test_misc.py 2017-01-30 07:40:22.000000000
+0100
@@ -1,5 +1,11 @@
import unittest
from spark_parser.spark import GenericParser
+from spark_parser import PYTHON3
+
+if PYTHON3:
+ from io import StringIO
+else:
+ from StringIO import StringIO
class Rules(GenericParser):
"""Testing duplicate rules"""
@@ -54,6 +60,17 @@
(('stmts', ('stmt',)), 'rules'),
(('stmts', ('stmts', 'stmt')), 'rules'),
(('x', ('TOKEN',)), 'rules')])
+ f = StringIO()
+ expect = \
+"""START ::= |- x
+ratings ::=
+ratings ::= ratings STARS
+stmts ::= stmt
+stmts ::= stmts stmt
+x ::= TOKEN
+"""
+ parser.dumpGrammar(f)
+ self.assertEqual(f.getvalue(), expect)
# Check Invalid rule
try:
@@ -79,6 +96,5 @@
(('opt_period', ()), 'rules'),
(('opt_period', ('PERIOD',)), 'rules'), ])
-
if __name__ == '__main__':
unittest.main()
Binary files old/spark_parser-1.5.1/test/test_misc.pyc and
new/spark_parser-1.6.0/test/test_misc.pyc differ
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/spark_parser-1.5.1/test/test_spark.py
new/spark_parser-1.6.0/test/test_spark.py
--- old/spark_parser-1.5.1/test/test_spark.py 2016-06-21 06:51:23.000000000
+0200
+++ new/spark_parser-1.6.0/test/test_spark.py 2017-01-30 07:40:22.000000000
+0100
@@ -10,7 +10,7 @@
class ExprScanner(GenericScanner):
- def __init__(self):
+ def __init__(self, coverage_path=None):
GenericScanner.__init__(self)
def tokenize(self, input):
@@ -61,8 +61,8 @@
by SPARK.
"""
- def __init__(self, start='expr', debug=DEFAULT_DEBUG):
- GenericParser.__init__(self, start, debug)
+ def __init__(self, start='expr', debug=DEFAULT_DEBUG, coverage_path=None):
+ GenericParser.__init__(self, start, debug=debug,
coverage_path=coverage_path)
def p_expr_add_term(self, args):
' expr ::= expr ADD_OP term '
@@ -110,7 +110,11 @@
test_term6 = AST('multiply', [test_term_to_factor2,
test_factor_to_integer3])
test_expr7 = AST('add', [test_expr_to_term1, test_term6])
+ # import sys
+ # parser = ExprParser(coverage_path=sys.stdout)
+ # parser = ExprParser(coverage_path="/tmp/spark-grammar.cover")
parser = ExprParser()
+
lhs, rhs, tokens, right_recursive = parser.checkSets()
self.assertEqual(len(lhs), 0)
self.assertEqual(len(rhs), 0)
Binary files old/spark_parser-1.5.1/test/test_spark.pyc and
new/spark_parser-1.6.0/test/test_spark.pyc differ
