edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/IronPython/IronPython/Compiler/Tokenizer.cs;C490025 File: Tokenizer.cs =================================================================== --- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/IronPython/IronPython/Compiler/Tokenizer.cs;C490025 (server) 7/14/2008 5:01 PM +++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/IronPython/IronPython/Compiler/Tokenizer.cs;TokenizerFixes @@ -131,13 +131,13 @@ public SourceLocation TokenStart { get { - return _buffer.TokenStart; + return _sourceUnit.MakeLocation(_buffer.TokenStart); } } public SourceLocation TokenEnd { get { - return _buffer.TokenEnd; + return _sourceUnit.MakeLocation(_buffer.TokenEnd); } } =================================================================== edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Ast/AstFactory.cs;C490874 File: AstFactory.cs =================================================================== --- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Ast/AstFactory.cs;C490874 (server) 7/14/2008 7:05 PM +++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Ast/AstFactory.cs;TokenizerFixes @@ -21,13 +21,13 @@ using System.Scripting.Runtime; using System.Scripting.Utils; using Ruby.Runtime; -using AstUtils = Microsoft.Scripting.Ast.Utils; -using MSA = System.Linq.Expressions; +using Ruby.Runtime.Calls; namespace Ruby.Compiler.Ast { using Ast = System.Linq.Expressions.Expression; - using Ruby.Runtime.Calls; - + using AstUtils = Microsoft.Scripting.Ast.Utils; + using MSA = System.Linq.Expressions; + internal static class AstFactory { public static readonly MSA.Expression/*!*/ EmptyStatement = Ast.Empty(); @@ -395,5 +395,14 @@ args = CollectionUtils.GetRange(yieldArgs.Expressions, 0, yieldArgs.Expressions.Count - (hasSplattedArgument ? 1 : 0)); splatArg = hasSplattedArgument ? yieldArgs.Expressions[yieldArgs.Expressions.Count - 1] : null; } + + internal static MSA.TryStatementBuilder/*!*/ FinallyIf(this MSA.TryStatementBuilder/*!*/ builder, bool ifdef, params MSA.Expression[]/*!*/ body) { + return ifdef ? builder.Finally(body) : builder; + } + + internal static MSA.TryStatementBuilder/*!*/ FilterIf(this MSA.TryStatementBuilder/*!*/ builder, bool ifdef, + Type/*!*/ type, MSA.VariableExpression/*!*/ holder, MSA.Expression/*!*/ condition, params MSA.Expression[]/*!*/ body) { + return ifdef ? builder.Filter(type, holder, condition, body) : builder; + } } } =================================================================== edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Ast/Body.cs;C478584 File: Body.cs =================================================================== --- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Ast/Body.cs;C478584 (server) 7/14/2008 7:07 PM +++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Ast/Body.cs;TokenizerFixes @@ -18,10 +18,11 @@ using System.Scripting; using System.Scripting.Utils; using Ruby.Runtime; -using MSA = System.Linq.Expressions; +using Ruby.Compiler.Ast; namespace Ruby.Compiler.Ast { using Ast = System.Linq.Expressions.Expression; + using MSA = System.Linq.Expressions; public partial class Body : Node { // begin/class/def @@ -142,7 +143,7 @@ transformedBody = gen.TransformStatements(_statements, (_elseStatements != null) ? ResultOperation.Ignore : resultOperation); MSA.Expression setInRescueFlag = null, clearInRescueFlag = null; - MSA.LabelTarget label = Ast.Label(); + var label = Ast.Label(); // make rescue clause: MSA.Expression transformedRescue; @@ -157,7 +158,7 @@ gen.EnterRescueClause(retryingVariable, label); - MSA.IfStatementTest[] handlers = new MSA.IfStatementTest[_rescueClauses.Count]; + var handlers = new MSA.IfStatementTest[_rescueClauses.Count]; for (int i = 0; i < handlers.Length; i++) { handlers[i] = _rescueClauses[i].Transform(gen, resultOperation); } @@ -184,7 +185,7 @@ )); } - MSA.Expression result = AstFactory.Infinite(label, + var result = AstFactory.Infinite(label, Ast.Assign(exceptionThrownVariable, Ast.False()), Ast.Assign(exceptionRethrowVariable, Ast.False()), Ast.Assign(retryingVariable, Ast.False()), @@ -201,7 +202,7 @@ AstFactory.OpCall("SetCurrentExceptionAndStackTrace", gen.CurrentScopeVariable, exceptionVariable), transformedRescue - ).SkipIf(_rescueClauses == null).Finally( + ).FinallyIf((_rescueClauses != null), // restore previous exception if the current one has been handled: Ast.Unless(exceptionRethrowVariable, AstFactory.OpCall("SetCurrentException", gen.CurrentScopeVariable, oldExceptionVariable) @@ -211,8 +212,8 @@ // unless (exception_thrown) do end transformedElse - ).SkipIf(_rescueClauses == null && _elseStatements == null) - .Filter(typeof(Exception), exceptionVariable, gen.RfcCall("CanRescue", exceptionVariable), + ).FilterIf((_rescueClauses != null || _elseStatements != null), + typeof(Exception), exceptionVariable, gen.RfcCall("CanRescue", exceptionVariable), AstFactory.OpCall("SetCurrentExceptionAndStackTrace", gen.CurrentScopeVariable, exceptionVariable), Ast.Assign(exceptionRethrowVariable, Ast.True()) =================================================================== edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Symbols.cs;C491229 File: Symbols.cs =================================================================== --- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Symbols.cs;C491229 (server) 7/14/2008 7:27 PM +++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Symbols.cs;TokenizerFixes @@ -21,6 +21,7 @@ // TODO: public static readonly SymbolId None = SymbolTable.StringToId(""); public static readonly SymbolId Error = SymbolTable.StringToId("#error"); + public static readonly SymbolId ErrorVariable = SymbolTable.StringToId("error__"); public static readonly SymbolId RestArgsLocal = SymbolTable.StringToId("?rest?"); public static readonly SymbolId MethodMissing = SymbolTable.StringToId("method_missing"); =================================================================== edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Terminators.cs;C468100 File: Terminators.cs =================================================================== --- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Terminators.cs;C468100 (server) 7/14/2008 7:33 PM +++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Terminators.cs;TokenizerFixes @@ -20,8 +20,10 @@ public abstract class Terminator { } + // TODO: remove [Flags] - public enum StringType { + public enum OldStringType { + str_squote = 0, str_dquote = STR_FUNC_EXPAND, str_xquote = STR_FUNC_EXPAND, @@ -41,38 +43,66 @@ FinalWordSeparator = 0x100, } - public class StringTerminator : Terminator { - public StringType _func; + [Flags] + public enum StringType { + Default = 0, + ExpandsEmbedded = 1, + RegularExpression = 2, + Words = 4, + Symbol = 8, + IndentedHeredoc = 16, + FinalWordSeparator = 32, + } + public sealed class StringTerminator : Terminator { + private StringType _properties; + public int _nestingLevel; + // the character terminating the string: - public readonly int _terminator; + private readonly char _terminator; // parenthesis opening the string; if non-zero parenthesis of this kind is balanced before the string is closed: - public readonly int _openingParenthesis; - public int _nestingLevel; + private readonly char _openingParenthesis; - public StringTerminator(StringType func, int term, int paren) { - _func = func; - _terminator = term; - _openingParenthesis = paren; + public StringTerminator(StringType properties, char terminator) + : this(properties, terminator, '\0') { + } + + public StringTerminator(StringType properties, char terminator, char openingParenthesis) { + _properties = properties; + _terminator = terminator; + _openingParenthesis = openingParenthesis; _nestingLevel = 0; } + public StringType Properties { + get { return _properties; } + set { _properties = value; } + } + + public char TerminatingCharacter { + get { return _terminator; } + } + + public char OpeningParenthesis { + get { return _openingParenthesis; } + } + public override string ToString() { - return String.Format("StringTerminator({0},{1},{2},{3},{4})", (int)_func, _terminator, _openingParenthesis, 0, _nestingLevel); + return String.Format("StringTerminator({0},{1},{2},{3},{4})", _properties, (int)_terminator, (int)_openingParenthesis, 0, _nestingLevel); } } - public class HEREDOC : Terminator { - public readonly StringType _func; + public sealed class Heredoc : Terminator { + public readonly StringType _properties; public readonly string _label; public readonly int _resumePosition; public readonly string _resumeLine; public readonly int _firstLine; public readonly int _firstLineIndex; - public HEREDOC(StringType func, string label, int resume_position, String resume_line, int firstLine, int firstLineIndex) { - _func = func; + public Heredoc(StringType properties, string label, int resume_position, String resume_line, int firstLine, int firstLineIndex) { + _properties = properties; _label = label; _resumePosition = resume_position; _resumeLine = resume_line; @@ -81,7 +111,7 @@ } public override string ToString() { - return String.Format("Heredoc('{0}',{1},'{2}')", _label, _resumePosition, _resumeLine); + return String.Format("Heredoc({0},'{1}',{2},'{2}')", _properties, _label, _resumePosition, _resumeLine); } } } =================================================================== edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Tokenizer.cs;C491229 File: Tokenizer.cs =================================================================== --- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Tokenizer.cs;C491229 (server) 7/14/2008 5:33 PM +++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Compiler/Parser/Tokenizer.cs;TokenizerFixes @@ -253,8 +253,6 @@ private int _heredocEndLineIndex = -1; private SourceLocation _initialLocation; - private int lex_gets_ptr; - private int _cmdArgStack = 0; private int _condStack = 0; @@ -459,24 +457,6 @@ #region Buffer Operations - private String lex_get_str(string str) { - int beg, end, pend; - - beg = 0; - if (lex_gets_ptr != 0) { - if (str.Length == lex_gets_ptr) - return null; - beg += lex_gets_ptr; - } - pend = str.Length; - end = beg; - while (end < pend) { - if (str[end++] == '\n') break; - } - lex_gets_ptr = end; - return str.Substring(beg, end - beg); - } - // reads a line including eoln (any of \r, \n, \r\n) // returns null if no characters read (end of sream) // doesn't return an empty string @@ -692,7 +672,6 @@ private void MarkTokenStart() { _currentTokenStart = GetCurrentLocation(); - newtok(); } private SourceLocation GetCurrentLocation() { @@ -769,7 +748,7 @@ } private Tokens TokenizeCurrentString() { - HEREDOC heredoc = _currentString as HEREDOC; + Heredoc heredoc = _currentString as Heredoc; if (heredoc != null) { Tokens token = TokenizeHeredoc(heredoc); if (token == Tokens.StringEnd) { @@ -1229,14 +1208,14 @@ // String: "... private Tokens ReadDoubleQuote() { - _currentString = new StringTerminator(StringType.str_dquote, '"', 0); + _currentString = new StringTerminator(StringType.ExpandsEmbedded, '"'); _tokenValue.SetStringTerminator(_currentString); return Tokens.StringBeg; } // String: '... private Tokens ReadSingleQuote() { - _currentString = new StringTerminator(StringType.str_squote, '\'', 0); + _currentString = new StringTerminator(StringType.Default, '\''); _tokenValue.SetStringTerminator(_currentString); return Tokens.StringBeg; } @@ -1420,6 +1399,7 @@ // $0[A-Za-z0-9_] are invalid: SkipVariableName(); ReportError(Errors.InvalidGlobalVariableName, _lineBuffer.Substring(start - 1, _bufferPos - start)); + _tokenValue.SetSymbol(Symbols.ErrorVariable); return Tokens.GlobalVariable; } pushback(c); @@ -1615,7 +1595,7 @@ // Literals: /... (regex start) private Tokens ReadSlash(bool whitespaceSeen) { if (_lexicalState == LexicalState.EXPR_BEG || _lexicalState == LexicalState.EXPR_MID) { - _currentString = new StringTerminator(StringType.str_regexp, '/', 0); + _currentString = new StringTerminator(StringType.RegularExpression | StringType.ExpandsEmbedded, '/'); _tokenValue.SetStringTerminator(_currentString); return Tokens.RegexpBeg; } @@ -1631,7 +1611,7 @@ if (IS_ARG() && whitespaceSeen) { if (!IsWhiteSpace(c)) { ReportWarning(Errors.AmbiguousFirstArgument); - _currentString = new StringTerminator(StringType.str_regexp, '/', 0); + _currentString = new StringTerminator(StringType.RegularExpression | StringType.ExpandsEmbedded, '/'); _tokenValue.SetStringTerminator(_currentString); return Tokens.RegexpBeg; } @@ -1675,11 +1655,11 @@ switch (c) { case '\'': - _currentString = new StringTerminator(StringType.str_ssym, c, 0); + _currentString = new StringTerminator(StringType.Symbol, '\''); break; case '"': - _currentString = new StringTerminator(StringType.str_dsym, c, 0); + _currentString = new StringTerminator(StringType.Symbol | StringType.ExpandsEmbedded, '"'); break; default: @@ -1865,7 +1845,7 @@ return (Tokens)'`'; } - _currentString = new StringTerminator(StringType.str_xquote, '`', 0); + _currentString = new StringTerminator(StringType.ExpandsEmbedded, '`'); _tokenValue.SetStringTerminator(_currentString); return Tokens.ShellStringBegin; } @@ -2119,40 +2099,21 @@ int c = nextc(); switch (c) { - case '\\': /* Backslash */ - return '\\'; + case '\\': return '\\'; + case 'n': return '\n'; + case 't': return '\t'; + case 'r': return '\r'; + case 'f': return '\f'; + case 'v': return '\v'; + case 'a': return '\a'; + case 'e': return (char)27; + case 'b': return '\b'; + case 's': return ' '; - case 'n': /* newline */ - return '\n'; - - case 't': /* horizontal tab */ - return '\t'; - - case 'r': /* carriage-return */ - return '\r'; - - case 'f': /* form-feed */ - return '\f'; - - case 'v': /* vertical tab */ - return '\v'; - - case 'a': /* alarm(bell) */ - return '\a'; - - //case 'e': /* escape */ fixme - //return '\e'; fixme - - case 'b': /* backspace */ - return '\b'; //'\010'; - - case 's': /* space */ - return ' '; - case '0': case '1': case '2': - case '3': /* octal constant */ + case '3': case '4': case '5': case '6': @@ -2322,70 +2283,76 @@ return 0; } - private int ReadStringContent(StringType func, int term, int paren, ref int nest) { - int c; + private int ReadStringContent(StringType stringType, int terminator, int openingParenthesis, ref int nestingLevel) { + while (true) { + int c = nextc(); + if (c == -1) { + return -1; + } - while ((c = nextc()) != -1) { - if (paren != 0 && c == paren) { - nest++; - } else if (c == term) { - if (nest == 0) { + if (openingParenthesis != 0 && c == openingParenthesis) { + nestingLevel++; + } else if (c == terminator) { + if (nestingLevel == 0) { pushback(c); - break; + return c; } - nest--; - } else if (((func & StringType.STR_FUNC_EXPAND) != 0) && c == '#' && _bufferPos < _lineBuffer.Length) { + nestingLevel--; + } else if (((stringType & StringType.ExpandsEmbedded) != 0) && c == '#' && _bufferPos < _lineBuffer.Length) { int c2 = _lineBuffer[_bufferPos]; if (c2 == '$' || c2 == '@' || c2 == '{') { pushback(c); - break; + return c; } } else if (c == '\\') { c = nextc(); + switch (c) { case '\n': - if ((func & StringType.STR_FUNC_QWORDS) != 0) break; - if ((func & StringType.STR_FUNC_EXPAND) != 0) continue; + if ((stringType & StringType.Words) != 0) break; + if ((stringType & StringType.ExpandsEmbedded) != 0) continue; tokadd('\\'); break; case '\\': - if ((func & StringType.STR_FUNC_ESCAPE) != 0) tokadd((char)c); + if ((stringType & StringType.RegularExpression) != 0) { + tokadd('\\'); + } break; default: - if ((func & StringType.STR_FUNC_REGEXP) != 0) { + if ((stringType & StringType.RegularExpression) != 0) { pushback(c); - if (TokenizeEscapeAppend(term) < 0) { + if (TokenizeEscapeAppend(terminator) < 0) { return -1; } continue; - } else if ((func & StringType.STR_FUNC_EXPAND) != 0) { + } else if ((stringType & StringType.ExpandsEmbedded) != 0) { pushback(c); - if ((func & StringType.STR_FUNC_ESCAPE) != 0) tokadd('\\'); + if ((stringType & StringType.RegularExpression) != 0) { + tokadd('\\'); + } c = ReadEscape(); - } else if ((func & StringType.STR_FUNC_QWORDS) != 0 && IsWhiteSpace(c)) { + } else if ((stringType & StringType.Words) != 0 && IsWhiteSpace(c)) { /* ignore backslashed spaces in %w */ - } else if (c != term && !(paren != 0 && c == paren)) { + } else if (c != terminator && !(openingParenthesis != 0 && c == openingParenthesis)) { tokadd('\\'); } break; } - } else if ((func & StringType.STR_FUNC_QWORDS) != 0 && IsWhiteSpace(c)) { + } else if ((stringType & StringType.Words) != 0 && IsWhiteSpace(c)) { pushback(c); - break; + return c; } - if (c == 0 && (func & StringType.STR_FUNC_SYMBOL) != 0) { - func &= ~StringType.STR_FUNC_SYMBOL; + if (c == 0 && (stringType & StringType.Symbol) != 0) { ReportError("symbol cannot contain '\\0'"); continue; } tokadd((char)c); } - return c; } // @@ -2397,9 +2364,9 @@ // - StringContent ... string data // private Tokens TokenizeString(StringTerminator/*!*/ info) { - StringType stringKind = info._func; - int term = info._terminator; - int paren = info._openingParenthesis; + StringType stringKind = info.Properties; + int term = info.TerminatingCharacter; + char paren = info.OpeningParenthesis; bool whitespaceSeen = false; // final separator in the list of words (see grammar): @@ -2423,7 +2390,7 @@ c = nextc(); // skip whitespace in word list: - if ((stringKind & StringType.STR_FUNC_QWORDS) != 0 && IsWhiteSpace(c)) { + if ((stringKind & StringType.Words) != 0 && IsWhiteSpace(c)) { do { c = nextc(); } while (IsWhiteSpace(c)); @@ -2435,15 +2402,15 @@ if (c == term && info._nestingLevel == 0) { // end of words: - if ((stringKind & StringType.STR_FUNC_QWORDS) != 0) { + if ((stringKind & StringType.Words) != 0) { // final separator in the list of words (see grammar): - info._func = StringType.FinalWordSeparator; + info.Properties = StringType.FinalWordSeparator; MarkMultiLineTokenEnd(); return Tokens.WordSeparator; } // end of regex: - if ((stringKind & StringType.STR_FUNC_REGEXP) != 0) { + if ((stringKind & StringType.RegularExpression) != 0) { _tokenValue.SetRegexOptions(ReadRegexOptions()); MarkSingleLineTokenEnd(); return Tokens.RegexpEnd; @@ -2464,7 +2431,7 @@ newtok(); // start of #$variable, #@variable, #{expression} in a string: - if ((stringKind & StringType.STR_FUNC_EXPAND) != 0 && c == '#') { + if ((stringKind & StringType.ExpandsEmbedded) != 0 && c == '#') { c = nextc(); switch (c) { case '$': @@ -2490,24 +2457,20 @@ private Tokens TokenizeHeredocLabel() { int c = nextc(); int term; - StringType func = StringType.str_squote; + StringType func = StringType.Default; if (c == '-') { c = nextc(); - func = StringType.STR_FUNC_INDENT; + func = StringType.IndentedHeredoc; } switch (c) { case '\'': - func |= StringType.str_squote; goto quoted; case '"': - func |= StringType.str_dquote; - goto quoted; - case '`': - func |= StringType.str_xquote; + func |= StringType.ExpandsEmbedded; quoted: newtok(); @@ -2544,13 +2507,13 @@ default: if (!IsIdentifier((char)c)) { pushback(c); - if ((func & StringType.STR_FUNC_INDENT) != 0) { + if ((func & StringType.IndentedHeredoc) != 0) { pushback('-'); } return Tokens.None; } term = '"'; - func |= StringType.str_dquote; + func |= StringType.ExpandsEmbedded; newtok(); do { tokadd((char)c); @@ -2565,13 +2528,13 @@ // skip the rest of the line (the content is stored in heredoc string terminal and tokenized upon restore) int resume = _bufferPos; _bufferPos = _lineBuffer.Length; - _currentString = new HEREDOC(func, tok(), resume, _lineBuffer, _currentLine, _currentLineIndex); + _currentString = new Heredoc(func, tok(), resume, _lineBuffer, _currentLine, _currentLineIndex); _tokenValue.SetStringTerminator(_currentString); return term == '`' ? Tokens.ShellStringBegin : Tokens.StringBeg; } - private void HeredocRestore(HEREDOC/*!*/ here) { + private void HeredocRestore(Heredoc/*!*/ here) { String line = here._resumeLine; _lineBuffer = line; _bufferPos = here._resumePosition; @@ -2581,9 +2544,9 @@ _currentLineIndex = here._firstLineIndex; } - private Tokens TokenizeHeredoc(HEREDOC/*!*/ heredoc) { - StringType stringKind = heredoc._func; - bool isIndented = (stringKind & StringType.STR_FUNC_INDENT) != 0; + private Tokens TokenizeHeredoc(Heredoc/*!*/ heredoc) { + StringType stringKind = heredoc._properties; + bool isIndented = (stringKind & StringType.IndentedHeredoc) != 0; int c = peekc(); MarkTokenStart(); @@ -2621,7 +2584,7 @@ return Tokens.StringEnd; } - if ((stringKind & StringType.STR_FUNC_EXPAND) == 0) { + if ((stringKind & StringType.ExpandsEmbedded) == 0) { string str = ReadNonexpandingHeredocContent(heredoc); @@ -2641,8 +2604,8 @@ //return Tokens.StringContent; } - private string ReadNonexpandingHeredocContent(HEREDOC/*!*/ heredoc) { - bool isIndented = (heredoc._func & StringType.STR_FUNC_INDENT) != 0; + private string ReadNonexpandingHeredocContent(Heredoc/*!*/ heredoc) { + bool isIndented = (heredoc._properties & StringType.IndentedHeredoc) != 0; string str = null; // reads lines until the line contains heredoc label @@ -2689,7 +2652,7 @@ return str; } - private Tokens TokenizeExpandingHeredocContent(HEREDOC/*!*/ heredoc) { + private Tokens TokenizeExpandingHeredocContent(Heredoc/*!*/ heredoc) { newtok(); int c = nextc(); @@ -2711,14 +2674,14 @@ tokadd('#'); } - bool isIndented = (heredoc._func & StringType.STR_FUNC_INDENT) != 0; + bool isIndented = (heredoc._properties & StringType.IndentedHeredoc) != 0; pushback(c); do { // read string content upto the end of the line: int tmp = 0; - c = ReadStringContent(heredoc._func, '\n', 0, ref tmp); + c = ReadStringContent(heredoc._properties, '\n', 0, ref tmp); if (c == -1 || c != '\n') { break; @@ -2749,52 +2712,52 @@ // c is the character following % switch (c) { case 'Q': - type = StringType.str_dquote; + type = StringType.ExpandsEmbedded; token = Tokens.StringBeg; terminator = nextc(); break; case 'q': - type = StringType.str_squote; + type = StringType.Default; token = Tokens.StringBeg; terminator = nextc(); break; case 'W': - type = StringType.str_dquote | StringType.STR_FUNC_QWORDS; + type = StringType.Words | StringType.ExpandsEmbedded; token = Tokens.WordsBeg; terminator = nextc(); SkipWhitespace(); break; case 'w': - type = StringType.str_squote | StringType.STR_FUNC_QWORDS; + type = StringType.Words; token = Tokens.VerbatimWordsBegin; terminator = nextc(); SkipWhitespace(); break; case 'x': - type = StringType.str_xquote; + type = StringType.ExpandsEmbedded; token = Tokens.ShellStringBegin; terminator = nextc(); break; case 'r': - type = StringType.str_regexp; + type = StringType.RegularExpression | StringType.ExpandsEmbedded; token = Tokens.RegexpBeg; terminator = nextc(); break; case 's': - type = StringType.str_ssym; + type = StringType.Symbol; token = Tokens.Symbeg; terminator = nextc(); _lexicalState = LexicalState.EXPR_FNAME; break; default: - type = StringType.str_dquote; + type = StringType.ExpandsEmbedded; token = Tokens.StringBeg; terminator = c; break; @@ -2820,7 +2783,7 @@ break; } - _currentString = new StringTerminator(type, terminator, parenthesis); + _currentString = new StringTerminator(type, (char)terminator, (char)parenthesis); _tokenValue.SetStringTerminator(_currentString); return token; } @@ -2848,10 +2811,7 @@ // private Tokens ReadUnsignedNumber(int c) { _lexicalState = LexicalState.EXPR_END; - - // TODO: remove - newtok(); - + if (c == '0') { switch (peekc()) { case 'x': ===================================================================