edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/IronRuby.Tests/Helpers.cs;C633889
File: Helpers.cs
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/IronRuby.Tests/Helpers.cs;C633889 (server) 2/5/2009 10:50 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/IronRuby.Tests/Helpers.cs;re
@@ -289,5 +289,10 @@
internal static void Assert(bool condition) {
if (!condition) throw new Exception("Assertion failed");
}
+
+ internal static void Assert(bool condition, string expected, string actual) {
+ if (!condition) throw new Exception(String.Format("Assertion failed: Expected:{0} Actual:{1}", expected, actual));
+ }
+
}
}
===================================================================
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/IronRuby.Tests/Runtime/RegexTests.cs;C633889
File: RegexTests.cs
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/IronRuby.Tests/Runtime/RegexTests.cs;C633889 (server) 2/5/2009 10:48 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/IronRuby.Tests/Runtime/RegexTests.cs;re
@@ -13,6 +13,7 @@
*
* ***************************************************************************/
+using System;
using System.Text.RegularExpressions;
using IronRuby.Builtins;
@@ -108,13 +109,13 @@
for (int i = 0; i < incorrectPatterns.Length; i += 2) {
string expected = incorrectPatterns[i + 1];
string actual = StringRegex.TransformPattern(incorrectPatterns[i], RubyRegexOptions.NONE);
- Assert(actual == expected);
+ Assert(actual == expected, expected, actual);
}
for (int i = 0; i < correctPatterns.Length; i += 2) {
string expected = correctPatterns[i + 1];
string actual = StringRegex.TransformPattern(correctPatterns[i], RubyRegexOptions.NONE);
- Assert(actual == expected);
+ Assert(actual == expected, expected, actual);
new Regex(expected);
}
}
===================================================================
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Ruby.csproj;C720249
File: Ruby.csproj
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Ruby.csproj;C720249 (server) 2/5/2009 3:05 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Ruby.csproj;re
@@ -110,6 +110,7 @@
+
===================================================================
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Ruby.csproj.vspscc;C390406
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/GenericRegex.cs;C574574
File: GenericRegex.cs
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/GenericRegex.cs;C574574 (server) 2/4/2009 9:03 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/GenericRegex.cs;re
@@ -29,6 +29,9 @@
public abstract bool IsEmpty { get; }
public RubyRegexOptions Options { get { return _options; } }
public abstract MutableString/*!*/ GetPattern();
+#if DEBUG
+ public abstract string/*!*/ GetTransformedPattern();
+#endif
public abstract Match/*!*/ Match(MutableString/*!*/ input, int start, int count);
public abstract Match/*!*/ ReverseMatch(MutableString/*!*/ input, int start);
@@ -84,6 +87,11 @@
return MutableString.CreateBinary(_pattern);
}
+#if DEBUG
+ public override string/*!*/ GetTransformedPattern() {
+ throw new NotImplementedException();
+ }
+#endif
public override MutableString[]/*!*/ Split(MutableString/*!*/ input, int count, int start) {
throw new NotImplementedException();
}
@@ -97,6 +105,8 @@
public class StringRegex : GenericRegex {
internal static readonly StringRegex Empty = new StringRegex(new Regex(String.Empty, RubyRegex.ToClrOptions(RubyRegexOptions.NONE)));
+ private const int EndOfPattern = -1;
+
private readonly Regex/*!*/ _regex;
private readonly string/*!*/ _pattern;
@@ -105,7 +115,7 @@
Assert.NotNull(pattern);
_pattern = pattern;
- string transformed = TransformPattern(pattern, options);
+ string transformed = RegexpTransformer.Transform(pattern, options);
try {
_regex = new Regex(transformed, RubyRegex.ToClrOptions(options));
} catch (ArgumentException e) {
@@ -149,6 +159,11 @@
return MutableString.Create(_pattern);
}
+#if DEBUG
+ public override string/*!*/ GetTransformedPattern() {
+ return _regex.ToString();
+ }
+#endif
public override MutableString[]/*!*/ Split(MutableString/*!*/ input, int count, int start) {
return MutableString.MakeArray(_regex.Split(input.ConvertToString(), count, start));
}
@@ -198,7 +213,7 @@
}
escaped = '\0';
- return -1;
+ return EndOfPattern;
}
internal static string/*!*/ Escape(string/*!*/ pattern) {
@@ -211,7 +226,7 @@
char escaped;
int i = SkipNonSpecial(pattern, 0, out escaped);
- if (i == -1) {
+ if (i == EndOfPattern) {
return null;
}
@@ -236,141 +251,8 @@
return result;
}
- private static int SkipWellEscaped(string/*!*/ pattern, int i) {
- while (i < pattern.Length - 1) {
- if (pattern[i] == '\\') {
- switch (pattern[i + 1]) {
- // metacharacters:
- case '$':
- case '^':
- case '|':
- case '[':
- case ']':
- case '(':
- case ')':
- case '\\':
- case '.':
- case '#':
- case '-':
-
- case '{':
- case '}':
- case '*':
- case '+':
- case '?':
-
- case '0': // octal
- case 't':
- case 'v':
- case 'n':
- case 'r':
- case 'f':
- case 'a':
- case 'e': // characters
-
- case 'b': // word boundary or backslash in character group
- case 'B': // not word boundary
- case 'A': // beginning of string
- case 'Z': // end of string, or before newline at the end
- case 'z': // end of string
- case 'G':
-
- case 'd':
- case 'D':
- case 's':
- case 'S':
- case 'w': // word character
- case 'W': // non word char
- // TODO: may need non-Unicode adjustment
-
- case 'C': // control characters
- case 'M': // meta characters
- // TODO: replace
-
- // Oniguruma + .NET - character classes, they don't match so some fixups would be needed
- // MRI: doesn't support, but is also an error since it is followed by {name}, which is illegal
- case 'p':
- case 'P':
- // keep
- break;
-
- default:
- return i;
- }
- i += 2;
- } else {
- i += 1;
- }
- }
- return -1;
- }
-
- // fixes escapes
- // - unescapes non-special characters
- // - fixes \xF -> \x0F
internal static string/*!*/ TransformPattern(string/*!*/ pattern, RubyRegexOptions options) {
-
- int first = 0;
- int i = SkipWellEscaped(pattern, 0);
-
- // trailing backslash is an error in both MRI, .NET
- if (i == -1) {
- return pattern;
- }
-
- StringBuilder result = new StringBuilder(pattern.Length);
-
- do {
- Debug.Assert(i + 1 < pattern.Length);
- Debug.Assert(pattern[i] == '\\');
-
- result.Append(pattern, first, i - first);
- i++;
-
- char c = pattern[i++];
- switch (c) {
- case 'x':
- result.Append('\\');
- result.Append('x');
-
- // error:
- if (i == pattern.Length) {
- break;
- }
-
- // fix single digit:
- c = pattern[i++];
- if (i == pattern.Length || !Tokenizer.IsHexadecimalDigit(pattern[i])) {
- result.Append('0');
- }
- result.Append(c);
- break;
-
- case 'h': // Oniguruma only: [0-9A-Fa-f]
- case 'H': // Oniguruma only: [^0-9A-Fa-f]
- case 'g': // Oniguruma only
- case 'k': // Oniguruma, .NET: named backreference, MRI not supported
- // remove backslash
-
- default:
- if (Tokenizer.IsDecimalDigit(c)) {
- // TODO:
- // \([1-9][0-9]*) where there is no group of such number (replace by an empty string)
- result.Append('\\');
- }
-
- // .NET throws invalid escape exception, remove backslash:
- result.Append(c);
- break;
- }
- Debug.Assert(i <= pattern.Length);
-
- first = i;
- i = SkipWellEscaped(pattern, i);
- } while (i >= 0);
-
- result.Append(pattern, first, pattern.Length - first);
- return result.ToString();
+ return RegexpTransformer.Transform(pattern, options);
}
}
}
===================================================================
add: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/RegexpTransformer.cs
File: RegexpTransformer.cs
===================================================================
--- [no source file]
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/RegexpTransformer.cs;re
@@ -1,0 +1,285 @@
+?/* ****************************************************************************
+ *
+ * Copyright (c) Microsoft Corporation.
+ *
+ * This source code is subject to terms and conditions of the Microsoft Public License. A
+ * copy of the license can be found in the License.html file at the root of this distribution. If
+ * you cannot locate the Microsoft Public License, please send an email to
+ * ironruby@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
+ * by the terms of the Microsoft Public License.
+ *
+ * You must not remove this notice, or any other, from this software.
+ *
+ *
+ * ***************************************************************************/
+
+using System;
+using System.Diagnostics;
+using Microsoft.Scripting.Runtime;
+using Microsoft.Scripting.Utils;
+using System.Text;
+using System.Text.RegularExpressions;
+using IronRuby.Compiler;
+using IronRuby.Runtime;
+
+namespace IronRuby.Builtins {
+ ///
+ /// Converts a Ruby regexp pattern to a CLR pattern
+ ///
+ internal class RegexpTransformer {
+ [Flags]
+ enum PatternState {
+ Normal,
+ InEscapeSequence,
+ InCharacterClass
+ }
+
+ string/*!*/ _rubyPattern;
+ PatternState _state = PatternState.Normal;
+ int _index;
+ StringBuilder/*!*/ _sb;
+
+ internal static string Transform(string/*!*/ rubyPattern, RubyRegexOptions options) {
+ RegexpTransformer transformer = new RegexpTransformer(rubyPattern);
+ return transformer.Transform();
+ }
+
+ private RegexpTransformer(string/*!*/ rubyPattern) {
+ _rubyPattern = rubyPattern;
+ _sb = new StringBuilder(rubyPattern.Length);
+ }
+
+ private bool InEscapeSequence { get { return (_state & PatternState.InEscapeSequence) != 0; } }
+ private bool InCharacterClass { get { return (_state & PatternState.InCharacterClass) != 0; } }
+
+ private bool HasMoreCharacters { get { return (_index + 1) < _rubyPattern.Length; } }
+
+ private char CurrentCharacter {
+ get {
+ Debug.Assert(_index < _rubyPattern.Length);
+ return _rubyPattern[_index];
+ }
+ }
+
+ private char NextCharacter {
+ get {
+ Debug.Assert(HasMoreCharacters);
+ return _rubyPattern[_index + 1];
+ }
+ }
+
+ private void LeaveEscapeSequence() {
+ _state &= ~PatternState.InEscapeSequence;
+ }
+
+ private void AppendEscapedChar(char c) {
+ Debug.Assert(InEscapeSequence);
+ Debug.Assert(_rubyPattern[_index - 1] == '\\');
+ _sb.Append('\\');
+ _sb.Append(c);
+ LeaveEscapeSequence();
+ }
+
+ private void OnBackSlash() {
+ if (InEscapeSequence) {
+ AppendEscapedChar('\\');
+ } else {
+ _state |= PatternState.InEscapeSequence;
+ }
+ }
+
+ private static readonly string[][] _PredefinedCharacterClasses = new string[][] {
+ new string[] { "[:alnum:]", "A-Za-z0-9_" },
+ new string[] { "[:alpha:]", "A-Za-z_" },
+ new string[] { "[:blank:]", " \t" },
+ new string[] { "[:cntrl:]", "\\c0" }, // TODO
+ new string[] { "[:digit:]", "0-9" },
+ new string[] { "[:graph:]", "\\g" }, // TODO
+ new string[] { "[:lower:]", "a-z" },
+ new string[] { "[:print:]", " " }, // TODO
+ new string[] { "[:punct:]", ",.?" }, // TODO
+ new string[] { "[:space:]", " \t\f\n\r\v" },
+ new string[] { "[:upper:]", "A-Z" },
+ new string[] { "[:xdigit:]", "0-9A-Fa-f" },
+ };
+
+ private bool CheckReplacePredefinedCharacterClass() {
+ string remainingString = _rubyPattern.Substring(_index);
+ foreach (string[] predefinedCharacterClass in _PredefinedCharacterClasses) {
+ if (remainingString.StartsWith(predefinedCharacterClass[0])) {
+ _sb.Append(predefinedCharacterClass[1]);
+ _index += predefinedCharacterClass[0].Length - 1;
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private void OnOpenBracket() {
+ if (InEscapeSequence) {
+ AppendEscapedChar('[');
+ } else if (InCharacterClass) {
+ if (!CheckReplacePredefinedCharacterClass()) {
+ // TODO - Ruby 1.9 allows "/[a[b]]/". Not sure what this means. So we really will need to keep nesting count.
+ // We need the nesting count anyway for "/[a&&[b]]/"
+ _sb.Append('[');
+ }
+ } else {
+ _sb.Append('[');
+ _state |= PatternState.InCharacterClass;
+ }
+ }
+
+ private void OnCloseBracket() {
+ if (InEscapeSequence) {
+ AppendEscapedChar(']');
+ } else {
+ _sb.Append(']');
+ _state &= ~PatternState.InCharacterClass;
+ }
+ }
+
+ private string/*!*/ Transform() {
+ for (/**/; _index < _rubyPattern.Length; _index++) {
+ char c = _rubyPattern[_index];
+ switch (c) {
+ case '\\': OnBackSlash(); break;
+ case '[': OnOpenBracket(); break;
+ case ']': OnCloseBracket(); break;
+ default: OnChar(c); break;
+ }
+ }
+
+ if (InEscapeSequence) {
+ // error case
+ _sb.Append('\\');
+ LeaveEscapeSequence();
+ }
+
+ return _sb.ToString();
+ }
+
+ private static bool IsMetaCharacterWithDirectMapping(char c) {
+ switch (c) {
+ // metacharacters:
+ case '$':
+ case '^':
+ case '|':
+ case '[':
+ case ']':
+ case '(':
+ case ')':
+ case '\\':
+ case '.':
+ case '#':
+ case '-':
+
+ case '{':
+ case '}':
+ case '*':
+ case '+':
+ case '?':
+
+ case '0': // octal
+ case 't':
+ case 'v':
+ case 'n':
+ case 'r':
+ case 'f':
+ case 'a':
+ case 'e': // characters
+
+ case 'b': // word boundary or backslash in character group
+ case 'B': // not word boundary
+ case 'A': // beginning of string
+ case 'Z': // end of string, or before newline at the end
+ case 'z': // end of string
+ case 'G':
+
+ case 'd':
+ case 'D':
+ case 's':
+ case 'S':
+ case 'w': // word character
+ case 'W': // non word char
+ // TODO: may need non-Unicode adjustment
+
+ case 'M': // meta characters
+ // TODO: replace
+
+ // Oniguruma + .NET - character classes, they don't match so some fixups would be needed
+ // MRI: doesn't support, but is also an error since it is followed by {name}, which is illegal
+ case 'p':
+ case 'P':
+ return true;
+ }
+
+ return false;
+ }
+
+ // fixes escapes
+ // - unescapes non-special characters
+ private void OnChar(char c) {
+ if (!InEscapeSequence) {
+ _sb.Append(c);
+ return;
+ }
+
+ if (IsMetaCharacterWithDirectMapping(c)) {
+ // keep
+ AppendEscapedChar(c);
+ return;
+ }
+
+ // Some characters need more processing to be mapped
+
+ switch (c) {
+ case 'c':
+ AppendEscapedChar('c');
+ // \c# -> \cC
+ if (HasMoreCharacters && NextCharacter == '#') {
+ _index++;
+ _sb.Append('C');
+ }
+ break;
+
+ case 'x':
+ // We need to do the following mappings:
+ // \xF... -> \x0F
+ // \xF1234... -> \x0F1234
+ // \x1 -> \x01 (why do we need this?)
+ AppendEscapedChar('x');
+
+ // error:
+ if (!HasMoreCharacters) {
+ break;
+ }
+
+ // fix single digit
+ c = _rubyPattern[++_index];
+ if (!HasMoreCharacters || !Tokenizer.IsHexadecimalDigit(CurrentCharacter)) {
+ _sb.Append('0');
+ }
+ _sb.Append(c);
+ break;
+
+ case 'h': // Oniguruma only: [0-9A-Fa-f]
+ case 'H': // Oniguruma only: [^0-9A-Fa-f]
+ case 'g': // Oniguruma only
+ case 'k': // Oniguruma, .NET: named backreference, MRI not supported
+ default:
+ if (Tokenizer.IsDecimalDigit(c)) {
+ // TODO:
+ // \([1-9][0-9]*) where there is no group of such number (replace by an empty string)
+ AppendEscapedChar(c);
+ } else {
+ // .NET throws invalid escape exception, remove backslash
+ _sb.Append(c);
+ LeaveEscapeSequence();
+ }
+ break;
+ }
+ }
+ }
+}
===================================================================
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/RubyRegex.cs;C641155
File: RubyRegex.cs
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/RubyRegex.cs;C641155 (server) 1/29/2009 3:53 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Builtins/RubyRegex.cs;re
@@ -87,6 +87,11 @@
return _regex.GetPattern();
}
+#if DEBUG
+ public string/*!*/ GetTransformedPattern() {
+ return _regex.GetTransformedPattern();
+ }
+#endif
public bool Equals(RubyRegex other) {
return ReferenceEquals(this, other) || other != null && _regex.Equals(other._regex);
}
@@ -178,6 +183,11 @@
public static RegexOptions ToClrOptions(RubyRegexOptions options) {
RegexOptions result = RegexOptions.Multiline;
+#if DEBUG
+ if (RubyOptions.CompileRegexps) {
+ result |= RegexOptions.Compiled;
+ }
+#endif
if ((options & RubyRegexOptions.IgnoreCase) != 0) {
result |= RegexOptions.IgnoreCase;
}
===================================================================
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Hosting/RubyOptionsParser.cs;C691193
File: RubyOptionsParser.cs
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Hosting/RubyOptionsParser.cs;C691193 (server) 1/29/2009 3:53 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Hosting/RubyOptionsParser.cs;re
@@ -138,6 +138,10 @@
case "-useThreadAbortForSyncRaise":
LanguageSetup.Options["UseThreadAbortForSyncRaise"] = true;
break;
+
+ case "-compileRegexps":
+ LanguageSetup.Options["CompileRegexps"] = true;
+ break;
#endif
case "-I":
_loadPaths.AddRange(PopNextArg().Split(Path.PathSeparator));
@@ -192,6 +196,7 @@
string [,] rubyOptions = new string[,] {
{ "-opt", "dummy" },
{ "-useThreadAbortForSyncRaise", "For testing purposes" },
+ { "-compileRegexps", "Faster throughput, slower startup" },
};
// Append the Ruby-specific options and the standard options
===================================================================
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Runtime/RubyOps.cs;C721344
File: RubyOps.cs
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Runtime/RubyOps.cs;C721344 (server) 2/5/2009 10:57 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Runtime/RubyOps.cs;re
@@ -961,7 +961,12 @@
[Emitted]
public static RubyRegex/*!*/ CreateRegexB(string/*!*/ str1, RubyRegexOptions options) {
- return new RubyRegex(str1, options);
+ try {
+ return new RubyRegex(str1, options);
+ } catch (RegexpError e) {
+ // Ideally, this should be thrown during parsing of the source
+ throw new SyntaxError(e.Message);
+ }
}
[Emitted]
===================================================================
edit: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Runtime/RubyOptions.cs;C691193
File: RubyOptions.cs
===================================================================
--- $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Runtime/RubyOptions.cs;C691193 (server) 1/29/2009 3:53 PM
+++ Shelved Change: $/Dev10/feature/vs_langs01/Merlin/Main/Languages/Ruby/Ruby/Runtime/RubyOptions.cs;re
@@ -35,6 +35,7 @@
private readonly RubyCompatibility _compatibility;
#if DEBUG
private static bool _UseThreadAbortForSyncRaise;
+ private static bool _CompileRegexps;
#endif
public ReadOnlyCollection/*!*/ Arguments {
@@ -81,6 +82,10 @@
public static bool UseThreadAbortForSyncRaise {
get { return _UseThreadAbortForSyncRaise; }
}
+
+ public static bool CompileRegexps {
+ get { return _CompileRegexps; }
+ }
#endif
public RubyOptions(IDictionary/*!*/ options)
@@ -98,6 +103,7 @@
_compatibility = GetCompatibility(options, "Compatibility", RubyCompatibility.Default);
#if DEBUG
_UseThreadAbortForSyncRaise = GetOption(options, "UseThreadAbortForSyncRaise", false);
+ _CompileRegexps = GetOption(options, "CompileRegexps", false);
#endif
}
===================================================================