This is an automated email from the git hooks/post-receive script. henrich pushed a commit to branch debian/sid in repository jruby-joni.
commit 061676e4a115849e6ca1cc8abbb9dd30d5b57fa8 Author: Marcin Mielzynski <[email protected]> Date: Fri Apr 10 01:17:53 2015 +0200 Support for \X --- src/org/joni/Lexer.java | 3 ++ src/org/joni/Parser.java | 30 +++++++++++ src/org/joni/Syntax.java | 5 +- src/org/joni/constants/TokenType.java | 1 + test/org/joni/test/TestA.java | 2 + .../org/joni/test/TestAU.java | 61 ++++++++++++---------- test/org/joni/test/TestJoni.java | 3 ++ test/org/joni/test/TestU8.java | 2 + 8 files changed, 76 insertions(+), 31 deletions(-) diff --git a/src/org/joni/Lexer.java b/src/org/joni/Lexer.java index 2287e77..f447059 100644 --- a/src/org/joni/Lexer.java +++ b/src/org/joni/Lexer.java @@ -1125,6 +1125,9 @@ class Lexer extends ScannerSupport { case 'R': if (syntax.op2EscCapitalRLinebreak()) token.type = TokenType.LINEBREAK; break; + case 'X': + if (syntax.op2EscCapitalXExtendedGraphemeCluster()) token.type = TokenType.EXTENDED_GRAPHEME_CLUSTER; + break; default: unfetch(); int num = fetchEscapedValue(); diff --git a/src/org/joni/Parser.java b/src/org/joni/Parser.java index 2f01a1c..3891416 100644 --- a/src/org/joni/Parser.java +++ b/src/org/joni/Parser.java @@ -27,6 +27,8 @@ import static org.joni.Option.isIgnoreCase; import org.jcodings.Ptr; import org.jcodings.constants.CharacterType; import org.jcodings.constants.PosixBracket; +import org.jcodings.specific.ASCIIEncoding; +import org.jcodings.unicode.UnicodeEncoding; import org.joni.ast.AnchorNode; import org.joni.ast.AnyCharNode; import org.joni.ast.BackRefNode; @@ -759,6 +761,34 @@ class Parser extends Lexer { node = en; break; + case EXTENDED_GRAPHEME_CLUSTER: + if (Config.USE_UNICODE_PROPERTIES) { + if (enc instanceof UnicodeEncoding) { + int ctype = enc.propertyNameToCType(new byte[]{(byte)'M'}, 0, 1); + if (ctype > 0) { + CClassNode cc1 = new CClassNode(); /* \P{M} */ + cc1.addCType(ctype, false, env, this); + cc1.setNot(); + CClassNode cc2 = new CClassNode(); /* \p{M}* */ + cc1.addCType(ctype, false, env, this); + QuantifierNode qn = new QuantifierNode(0, QuantifierNode.REPEAT_INFINITE, false); + qn.setTarget(cc2); + /* \P{M}\p{M}* */ + ConsAltNode list2 = ConsAltNode.newListNode(qn, null); + ConsAltNode list1 = ConsAltNode.newListNode(cc1, list2); + EncloseNode en2 = new EncloseNode(EncloseType.STOP_BACKTRACK); + en2.setTarget(list1); + node = en2; + } + } + } + if (node == null) { + AnyCharNode np1 = new AnyCharNode(); + EncloseNode on = new EncloseNode(bsOnOff(env.option, Option.MULTILINE, false), 0); + on.setTarget(np1); + node = np1; + } + break; case STRING: return parseExpTkByte(group); // tk_byte: diff --git a/src/org/joni/Syntax.java b/src/org/joni/Syntax.java index baea7c6..47f069d 100644 --- a/src/org/joni/Syntax.java +++ b/src/org/joni/Syntax.java @@ -420,8 +420,9 @@ public final class Syntax implements SyntaxProperties{ OP2_CCLASS_SET_OP | OP2_ESC_CAPITAL_C_BAR_CONTROL | OP2_ESC_CAPITAL_M_BAR_META | OP2_ESC_V_VTAB | OP2_ESC_H_XDIGIT | - OP2_ESC_CAPITAL_R_LINEBREAK | - OP2_QMARK_LPAREN_CONDITION), + OP2_ESC_CAPITAL_X_EXTENDED_GRAPHEME_CLUSTER | + OP2_QMARK_LPAREN_CONDITION | + OP2_ESC_CAPITAL_R_LINEBREAK), ( GNU_REGEX_BV | ALLOW_INTERVAL_LOW_ABBREV | diff --git a/src/org/joni/constants/TokenType.java b/src/org/joni/constants/TokenType.java index f9a18bd..3dc8386 100644 --- a/src/org/joni/constants/TokenType.java +++ b/src/org/joni/constants/TokenType.java @@ -40,6 +40,7 @@ public enum TokenType { QUOTE_OPEN, CHAR_PROPERTY, /* \p{...}, \P{...} */ LINEBREAK, + EXTENDED_GRAPHEME_CLUSTER, /* in cc */ CC_CLOSE, CC_RANGE, diff --git a/test/org/joni/test/TestA.java b/test/org/joni/test/TestA.java index 7b5f689..de5e3b5 100644 --- a/test/org/joni/test/TestA.java +++ b/test/org/joni/test/TestA.java @@ -526,6 +526,8 @@ public class TestA extends Test { x2s("\\R", "\n", 0, 1); x2s("\\R", "\r", 0, 1); x2s("\\R{3}", "\r\r\n\n", 0, 4); + + x2s("\\X{5}", "ăăab\n", 0, 5); } public static void main(String[] args) throws Throwable{ diff --git a/src/org/joni/constants/TokenType.java b/test/org/joni/test/TestAU.java similarity index 62% copy from src/org/joni/constants/TokenType.java copy to test/org/joni/test/TestAU.java index f9a18bd..0d37a88 100644 --- a/src/org/joni/constants/TokenType.java +++ b/test/org/joni/test/TestAU.java @@ -17,33 +17,36 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -package org.joni.constants; - -public enum TokenType { - EOT, /* end of token */ - RAW_BYTE, - CHAR, - STRING, - CODE_POINT, - ANYCHAR, - CHAR_TYPE, - BACKREF, - CALL, - ANCHOR, - OP_REPEAT, - INTERVAL, - ANYCHAR_ANYTIME, /* SQL '%' == .* */ - ALT, - SUBEXP_OPEN, - SUBEXP_CLOSE, - CC_OPEN, - QUOTE_OPEN, - CHAR_PROPERTY, /* \p{...}, \P{...} */ - LINEBREAK, - /* in cc */ - CC_CLOSE, - CC_RANGE, - POSIX_BRACKET_OPEN, - CC_AND, /* && */ - CC_CC_OPEN /* [ */ +package org.joni.test; + +import org.jcodings.Encoding; +import org.jcodings.specific.ASCIIEncoding; +import org.joni.Option; +import org.joni.Syntax; + +public class TestAU extends Test { + + public int option() { + return Option.DEFAULT; + } + + public Encoding encoding() { + return ASCIIEncoding.INSTANCE; + } + + public String testEncoding() { + return "utf-8"; + } + + public Syntax syntax() { + return Syntax.DEFAULT; + } + + public void test() throws InterruptedException { + x2s("\\X{5}", "あいab\n", 0, 5); + } + + public static void main(String[] args) throws Throwable { + new TestU8().run(); + } } diff --git a/test/org/joni/test/TestJoni.java b/test/org/joni/test/TestJoni.java index d10924d..d7f4fe6 100644 --- a/test/org/joni/test/TestJoni.java +++ b/test/org/joni/test/TestJoni.java @@ -31,6 +31,7 @@ public class TestJoni extends TestCase { private Test testu8; private Test testInterrupt; private Test testPerl; + private Test testAsciiViaUtf; protected void setUp() { testa = new TestA(); @@ -41,6 +42,7 @@ public class TestJoni extends TestCase { testLookBehind = new TestLookBehind(); testInterrupt = new TestInterrupt(); testPerl = new TestPerl(); + testAsciiViaUtf = new TestAU(); } protected void tearDown() { @@ -54,6 +56,7 @@ public class TestJoni extends TestCase { public void testAscii() { testJoniTest(testa); + testJoniTest(testAsciiViaUtf); } public void testEUCJP() { diff --git a/test/org/joni/test/TestU8.java b/test/org/joni/test/TestU8.java index 86c2812..9723100 100644 --- a/test/org/joni/test/TestU8.java +++ b/test/org/joni/test/TestU8.java @@ -84,6 +84,8 @@ public class TestU8 extends Test { x2s("\\R", "\u0085", 0, 2); x2s("\\R", "\u2028", 0, 3); x2s("\\R", "\u2029", 0, 3); + + x2s("\\X", "\u306F\u309A\n", 0, 3); } public static void main(String[] args) throws Throwable { -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-java/jruby-joni.git _______________________________________________ pkg-java-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-java-commits
