The scanner didn't tokenize ##(1/2)) correctly. After scanning the last $) has not been processed yet. Count the number of optimized tokens and consume an equal amount of $). >>#scanLiteralArrayParts will only parse the $) when there is an unclosed compile time constant.
2013-12-21 Holger Hans Peter Freyther <[email protected]> * RBScannerTests.st: Add new tests. * RBParser.st: Change parsing of optimized tokens. --- packages/stinst/parser/ChangeLog | 5 ++ packages/stinst/parser/Makefile.frag | 2 +- packages/stinst/parser/RBParser.st | 12 +++-- packages/stinst/parser/RBScannerTests.st | 79 ++++++++++++++++++++++++++++++++ packages/stinst/parser/package.xml | 2 + 5 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 packages/stinst/parser/RBScannerTests.st diff --git a/packages/stinst/parser/ChangeLog b/packages/stinst/parser/ChangeLog index 2bd7eeb..d32a0cc 100644 --- a/packages/stinst/parser/ChangeLog +++ b/packages/stinst/parser/ChangeLog @@ -1,3 +1,8 @@ +2013-12-21 Holger Hans Peter Freyther <[email protected]> + + * RBScannerTests.st: Add new tests. + * RBParser.st: Change parsing of optimized tokens. + 2013-12-19 Holger Hans Peter Freyther <[email protected]> * STCompiler.st: Rename STCompiler>>#addPool: to diff --git a/packages/stinst/parser/Makefile.frag b/packages/stinst/parser/Makefile.frag index b70055b..f86626f 100644 --- a/packages/stinst/parser/Makefile.frag +++ b/packages/stinst/parser/Makefile.frag @@ -1,5 +1,5 @@ Parser_FILES = \ -packages/stinst/parser/ChangeLog packages/stinst/parser/DebugInformationTests.st packages/stinst/parser/Exporter.st packages/stinst/parser/Extensions.st packages/stinst/parser/GSTParser.st packages/stinst/parser/GSTParserTests.st packages/stinst/parser/NewSyntaxExporter.st packages/stinst/parser/OldSyntaxExporter.st packages/stinst/parser/OrderedSet.st packages/stinst/parser/ParseTreeSearcher.st packages/stinst/parser/PoolResolutionTests.st packages/stinst/parser/RBFormatter.st packages/stinst/parser/RBParseNodes.st packages/stinst/parser/RBParser.st packages/stinst/parser/RBToken.st packages/stinst/parser/RewriteTests.st packages/stinst/parser/SIFParser.st packages/stinst/parser/SqueakExporter.st packages/stinst/parser/SqueakParser.st packages/stinst/parser/STCompiler.st packages/stinst/parser/STCompilerTests.st packages/stinst/parser/STCompLit.st packages/stinst/parser/STDecompiler.st packages/stinst/parser/STEvaluationDriver.st packages/stinst/parser/STFileParser.st packages/stin st/parser/STLoaderObjs.st packages/stinst/parser/STLoaderObjsTests.st packages/stinst/parser/STLoader.st packages/stinst/parser/STSymTable.st +packages/stinst/parser/ChangeLog packages/stinst/parser/DebugInformationTests.st packages/stinst/parser/Exporter.st packages/stinst/parser/Extensions.st packages/stinst/parser/GSTParser.st packages/stinst/parser/GSTParserTests.st packages/stinst/parser/NewSyntaxExporter.st packages/stinst/parser/OldSyntaxExporter.st packages/stinst/parser/OrderedSet.st packages/stinst/parser/ParseTreeSearcher.st packages/stinst/parser/PoolResolutionTests.st packages/stinst/parser/RBFormatter.st packages/stinst/parser/RBParseNodes.st packages/stinst/parser/RBParser.st packages/stinst/parser/RBScannerTests.st packages/stinst/parser/RBToken.st packages/stinst/parser/RewriteTests.st packages/stinst/parser/SIFParser.st packages/stinst/parser/SqueakExporter.st packages/stinst/parser/SqueakParser.st packages/stinst/parser/STCompiler.st packages/stinst/parser/STCompilerTests.st packages/stinst/parser/STCompLit.st packages/stinst/parser/STDecompiler.st packages/stinst/parser/STEvaluationDriver.st packages/st inst/parser/STFileParser.st packages/stinst/parser/STLoaderObjs.st packages/stinst/parser/STLoaderObjsTests.st packages/stinst/parser/STLoader.st packages/stinst/parser/STSymTable.st $(Parser_FILES): $(srcdir)/packages/stinst/parser/stamp-classes: $(Parser_FILES) touch $(srcdir)/packages/stinst/parser/stamp-classes diff --git a/packages/stinst/parser/RBParser.st b/packages/stinst/parser/RBParser.st index b953762..77e4832 100644 --- a/packages/stinst/parser/RBParser.st +++ b/packages/stinst/parser/RBParser.st @@ -1215,16 +1215,20 @@ Stream subclass: RBScanner [ scanLiteralArray [ <category: 'private-scanning'> - | arrayStream start | + | arrayStream start optimizedNodes | arrayStream := WriteStream on: (Array new: 10). self step. start := tokenStart. + optimizedNodes := 0. [self stripSeparators. tokenStart := stream position. - currentCharacter == $)] + currentCharacter == $) and: [optimizedNodes = 0]] whileFalse: - [arrayStream nextPut: self scanLiteralArrayParts. + [ |res| + res := arrayStream nextPut: self scanLiteralArrayParts. + res isOptimized ifTrue: [optimizedNodes := optimizedNodes + 1]. + res isSpecial ifTrue: [optimizedNodes := optimizedNodes - 1]. buffer reset]. self step. ^RBLiteralToken @@ -1254,6 +1258,8 @@ Stream subclass: RBScanner [ currentCharacter == $$ ifTrue: [^self scanLiteralCharacter]. currentCharacter == $( ifTrue: [^self scanLiteralArray]. currentCharacter == $[ ifTrue: [^self scanByteArray]. + "In the case of RBOptimizedToken we scan for $) as well" + currentCharacter == $) ifTrue: [^self scanSpecialCharacter]. ^self scannerError: 'Unknown character in literal array' ] diff --git a/packages/stinst/parser/RBScannerTests.st b/packages/stinst/parser/RBScannerTests.st new file mode 100644 index 0000000..ba84813 --- /dev/null +++ b/packages/stinst/parser/RBScannerTests.st @@ -0,0 +1,79 @@ +"====================================================================== +| +| Smalltalk in Smalltalk RBScanner tests +| +| + ======================================================================" + +"====================================================================== +| +| Copyright 2013 Free Software Foundation, Inc. +| Written by Holger Hans Peter Freyther. +| +| This file is part of GNU Smalltalk. +| +| GNU Smalltalk is free software; you can redistribute it and/or modify it +| under the terms of the GNU General Public License as published by the Free +| Software Foundation; either version 2, or (at your option) any later version. +| +| GNU Smalltalk is distributed in the hope that it will be useful, but WITHOUT +| ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +| FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +| details. +| +| You should have received a copy of the GNU General Public License along with +| GNU Smalltalk; see the file COPYING. If not, write to the Free Software +| Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +| + ======================================================================" + + +TestCase subclass: TestRBScanner [ + <comment: 'I do test the RBScanner/Tokenizer.'> + + testEmbeddedCompiletimeConstant [ + | scanner token value | + scanner := RBScanner on: '#(##(1))' readStream. + + "Token is the literal" + token := scanner next. + self assert: token isLiteral. + + "And the literal should be RBOptimizedToken RBNumberLiteralToken RBSpecialCharacterToken" + value := token value. + value inspect. + self assert: value size equals: 3. + self assert: value first isOptimized. + self assert: value second isLiteral. + self assert: value second value equals: 1. + self assert: value third isSpecial. + self assert: value third value equals: $). + + "And we are at the end" + scanner atEnd printNl. + self assert: scanner atEnd. + ] + + testDirectCompiletimeConstant [ + | scanner token | + scanner := RBScanner on: '##(1)' readStream. + + "Token is for optimized code" + token := scanner next. + self assert: token isOptimized. + + "Token is the '1'" + token := scanner next. + self assert: token isLiteral. + self assert: token value equals: 1. + + "Token is the ')'" + token := scanner next. + self assert: token isSpecial. + self assert: token value equals: $). + + "And we are at the end" + self assert: scanner atEnd. + ] +] + diff --git a/packages/stinst/parser/package.xml b/packages/stinst/parser/package.xml index 6102af7..1bbe77c 100644 --- a/packages/stinst/parser/package.xml +++ b/packages/stinst/parser/package.xml @@ -38,12 +38,14 @@ <sunit>STInST.Tests.TestSTLoaderObjects</sunit> <sunit>STInST.Tests.TestGSTParser</sunit> <sunit>STInST.Tests.TestCompiler</sunit> + <sunit>STInST.Tests.TestRBScanner</sunit> <filein>RewriteTests.st</filein> <filein>PoolResolutionTests.st</filein> <filein>DebugInformationTests.st</filein> <filein>STLoaderObjsTests.st</filein> <filein>GSTParserTests.st</filein> <filein>STCompilerTests.st</filein> + <filein>RBScannerTests.st</filein> </test> <file>ChangeLog</file> -- 1.8.5.1 _______________________________________________ help-smalltalk mailing list [email protected] https://lists.gnu.org/mailman/listinfo/help-smalltalk
