This is an automated email from the git hooks/post-receive script. js pushed a commit to tag PEVANS in repository libparser-mgc-perl.
commit 059907fea15b933ebbb8659e341e82d42afc4b3a Author: Paul Evans <leon...@leonerd.org.uk> Date: Fri Feb 25 16:47:44 2011 +0000 Import of PEVANS/Parser-MGC-0.05 from CPAN. gitpan-cpan-distribution: Parser-MGC gitpan-cpan-version: 0.05 gitpan-cpan-path: PEVANS/Parser-MGC-0.05.tar.gz gitpan-cpan-author: PEVANS gitpan-cpan-maturity: released --- Changes | 4 ++ MANIFEST | 6 ++- META.yml | 4 +- README | 24 +++++++++++ lib/Parser/MGC.pm | 92 +++++++++++++++++++++++++++++++++++++----- t/03reader.t | 32 +++++++++++++++ t/{03where.t => 04where.t} | 0 t/{04comment.t => 05comment.t} | 0 t/31scope_level.t | 30 ++++++++++++++ 9 files changed, 179 insertions(+), 13 deletions(-) diff --git a/Changes b/Changes index 18985a7..5b7312c 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,9 @@ Revision history for Parser-MGC +0.05 CHANGES: + * Added ->scope_level + * Added ->from_reader as a new potential source of string input + 0.04 CHANGES: * Added ->token_float * Optionally parse 0o... ad octal integers diff --git a/MANIFEST b/MANIFEST index af37bd3..d2bcc68 100644 --- a/MANIFEST +++ b/MANIFEST @@ -11,8 +11,9 @@ README t/00use.t t/01base.t t/02expect.t -t/03where.t -t/04comment.t +t/03reader.t +t/04where.t +t/05comment.t t/10token_int.t t/11token_float.t t/12token_string.t @@ -24,4 +25,5 @@ t/22list_of.t t/23sequence_of.t t/24one_of.t t/30commit.t +t/31scope_level.t t/99pod.t diff --git a/META.yml b/META.yml index 912d436..4a88122 100644 --- a/META.yml +++ b/META.yml @@ -15,9 +15,9 @@ name: Parser-MGC provides: Parser::MGC: file: lib/Parser/MGC.pm - version: 0.04 + version: 0.05 requires: File::Slurp: 0 resources: license: http://dev.perl.org/licenses/ -version: 0.04 +version: 0.05 diff --git a/README b/README index 4024ed1..6b211da 100644 --- a/README +++ b/README @@ -93,6 +93,24 @@ METHODS Parse the given file, which may be a pathname in a string, or an opened IO handle, and return the result from the "parse" method. + $result = $parser->from_reader( \&reader ) + Parse the input which is read by the "reader" function. This function + will be called in scalar context to generate portions of string to + parse, being passed the $parser object. The function should return + "undef" when it has no more string to return. + + $reader->( $parser ) + + Note that because it is not generally possible to detect exactly when + more input may be required due to failed regexp parsing, the reader + function is only invoked during searching for skippable whitespace. This + makes it suitable for reading lines of a file in the common case where + lines are considered as skippable whitespace, or for reading lines of + input interractively from a user. It cannot be used in all cases (for + example, reading fixed-size buffers from a file) because two successive + invocations may split a single token across the buffer boundaries, and + cause parse failures parse failures. + ( $lineno, $col, $text ) = $parser->where Returns the current parse position, as a line and column number, and the entire current line of text. The first line is numbered 1, and the first @@ -106,6 +124,9 @@ METHODS $eos = $parser->at_eos Returns true if the input string is at the end of the string. + $level = $parser->scope_level + Returns the number of nested "scope_of" calls that have been made. + STRUCTURE-FORMING METHODS The following methods may be used to build a grammatical structure out of the defined basic token-parsing methods. Each takes at least one code @@ -259,6 +280,9 @@ TODO * Easy ability for subclasses to define more token types + * Investigate how well "from_reader" can cope with buffer splitting + across other tokens than simply skippable whitespace + AUTHOR Paul Evans <leon...@leonerd.org.uk> diff --git a/lib/Parser/MGC.pm b/lib/Parser/MGC.pm index 73d442f..235ec8d 100644 --- a/lib/Parser/MGC.pm +++ b/lib/Parser/MGC.pm @@ -1,14 +1,14 @@ # You may distribute under the terms of either the GNU General Public License # or the Artistic License (the same terms as Perl itself) # -# (C) Paul Evans, 2010 -- leon...@leonerd.org.uk +# (C) Paul Evans, 2010-2011 -- leon...@leonerd.org.uk package Parser::MGC; use strict; use warnings; -our $VERSION = '0.04'; +our $VERSION = '0.05'; use Carp; @@ -146,6 +146,7 @@ sub new my $self = bless { patterns => {}, + scope_level => 0, }, $class; $self->{patterns}{$_} = $args{patterns}{$_} || $self->${\"pattern_$_"} for @patterns; @@ -201,6 +202,44 @@ sub from_file $self->from_string( scalar(slurp $filename) ); } +=head2 $result = $parser->from_reader( \&reader ) + +Parse the input which is read by the C<reader> function. This function will be +called in scalar context to generate portions of string to parse, being passed +the C<$parser> object. The function should return C<undef> when it has no more +string to return. + + $reader->( $parser ) + +Note that because it is not generally possible to detect exactly when more +input may be required due to failed regexp parsing, the reader function is +only invoked during searching for skippable whitespace. This makes it suitable +for reading lines of a file in the common case where lines are considered as +skippable whitespace, or for reading lines of input interractively from a +user. It cannot be used in all cases (for example, reading fixed-size buffers +from a file) because two successive invocations may split a single token +across the buffer boundaries, and cause parse failures parse failures. + +=cut + +sub from_reader +{ + my $self = shift; + my ( $reader ) = @_; + + local $self->{reader} = $reader; + + $self->{str} = ""; + pos $self->{str} = 0; + + my $result = $self->parse; + + $self->at_eos or + $self->fail( "Expected end of input" ); + + return $result; +} + =head2 ( $lineno, $col, $text ) = $parser->where Returns the current parse position, as a line and column number, and @@ -273,6 +312,18 @@ sub at_eos return $at_eos; } +=head2 $level = $parser->scope_level + +Returns the number of nested C<scope_of> calls that have been made. + +=cut + +sub scope_level +{ + my $self = shift; + return $self->{scope_level}; +} + =head1 STRUCTURE-FORMING METHODS The following methods may be used to build a grammatical structure out of the @@ -353,6 +404,7 @@ sub scope_of $self->expect( $start ); local $self->{endofscope} = $stop; + local $self->{scope_level} = $self->{scope_level} + 1; my $ret = $code->( $self ); @@ -519,8 +571,26 @@ sub skip_ws my $ws = $self->{patterns}{ws}; my $c = $self->{patterns}{comment}; - 1 while $self->{str} =~ m/\G$ws/gc or - ( $c and $self->{str} =~ m/\G$c/gc ); + { + 1 while $self->{str} =~ m/\G$ws/gc or + ( $c and $self->{str} =~ m/\G$c/gc ); + + return if pos( $self->{str} ) < length $self->{str}; + + return unless $self->{reader}; + + my $more = $self->{reader}->( $self ); + if( defined $more ) { + my $pos = pos( $self->{str} ); + $self->{str} .= $more; + pos( $self->{str} ) = $pos; + + redo; + } + + undef $self->{reader}; + return; + } } =head2 $parser->expect( $string ) @@ -694,11 +764,6 @@ sub STRING # Provide fallback operators for cmp, eq, etc... use overload fallback => 1; -# Keep perl happy; keep Britain tidy -1; - -__END__ - =head1 TODO =over 4 @@ -711,8 +776,17 @@ Unescaping of string constants; customisable Easy ability for subclasses to define more token types +=item * + +Investigate how well C<from_reader> can cope with buffer splitting across +other tokens than simply skippable whitespace + =back =head1 AUTHOR Paul Evans <leon...@leonerd.org.uk> + +=cut + +0x55AA; diff --git a/t/03reader.t b/t/03reader.t new file mode 100644 index 0000000..5118fc8 --- /dev/null +++ b/t/03reader.t @@ -0,0 +1,32 @@ +#!/usr/bin/perl -w + +use strict; + +use Test::More tests => 1; + +package TestParser; +use base qw( Parser::MGC ); + +sub parse +{ + my $self = shift; + + my @tokens; + push @tokens, $self->expect( qr/[a-z]+/ ) while !$self->at_eos; + + return \@tokens; +} + +package main; + +my $parser = TestParser->new; + +my @strings = ( + "here is a list ", + "of some more ", + "tokens" +); + +is_deeply( $parser->from_reader( sub { return shift @strings } ), + [qw( here is a list of some more tokens )], + 'tokens from reader' ); diff --git a/t/03where.t b/t/04where.t similarity index 100% rename from t/03where.t rename to t/04where.t diff --git a/t/04comment.t b/t/05comment.t similarity index 100% rename from t/04comment.t rename to t/05comment.t diff --git a/t/31scope_level.t b/t/31scope_level.t new file mode 100644 index 0000000..06e87ac --- /dev/null +++ b/t/31scope_level.t @@ -0,0 +1,30 @@ +#!/usr/bin/perl -w + +use strict; + +use Test::More tests => 3; + +package TestParser; +use base qw( Parser::MGC ); + +sub parse +{ + my $self = shift; + + $self->sequence_of( + sub { + $self->one_of( + sub { $self->expect( qr/[a-z]+/ ) . "/" . $self->scope_level }, + sub { $self->scope_of( "(", \&parse, ")" ) }, + ); + }, + ); +} + +package main; + +my $parser = TestParser->new; + +is_deeply( $parser->from_string( "a" ), [ "a/0" ], 'a' ); +is_deeply( $parser->from_string( "(b)" ), [ [ "b/1" ] ], '(b)' ); +is_deeply( $parser->from_string( "c (d) e" ), [ "c/0", [ "d/1" ], "e/0" ], 'c (d) e' ); -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-perl/packages/libparser-mgc-perl.git _______________________________________________ Pkg-perl-cvs-commits mailing list Pkg-perl-cvs-commits@lists.alioth.debian.org http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-perl-cvs-commits