[webkit-changes] [221508] trunk/Tools

Fri, 01 Sep 2017 16:55:37 -0700

Title: [221508] trunk/Tools
Revision
221508
Author
mmaxfi...@apple.com
Date
2017-09-01 16:54:23 -0700 (Fri, 01 Sep 2017)

Log Message

WSL's lexer will never emit keyword tokens
https://bugs.webkit.org/show_bug.cgi?id=176248

Reviewed by Filip Pizlo.

Because all tokens are also identified as idents, we need to handle them together.

* WebGPUShadingLanguageRI/Lexer.js:
(Lexer.prototype.next):
(Lexer):
* WebGPUShadingLanguageRI/Test.js:
(doLex):
(checkLexerToken):
(TEST_lexerKeyword):

Modified Paths

Diff

Modified: trunk/Tools/ChangeLog (221507 => 221508)


--- trunk/Tools/ChangeLog	2017-09-01 23:43:42 UTC (rev 221507)
+++ trunk/Tools/ChangeLog	2017-09-01 23:54:23 UTC (rev 221508)
@@ -1,3 +1,20 @@
+2017-09-01  Myles C. Maxfield  <mmaxfi...@apple.com>
+
+        WSL's lexer will never emit keyword tokens
+        https://bugs.webkit.org/show_bug.cgi?id=176248
+
+        Reviewed by Filip Pizlo.
+
+        Because all tokens are also identified as idents, we need to handle them together.
+
+        * WebGPUShadingLanguageRI/Lexer.js:
+        (Lexer.prototype.next):
+        (Lexer):
+        * WebGPUShadingLanguageRI/Test.js:
+        (doLex):
+        (checkLexerToken):
+        (TEST_lexerKeyword):
+
 2017-09-01  Alex Christensen  <achristen...@webkit.org>
 
         Rename WebKit2 API tests after directory rename

Modified: trunk/Tools/WebGPUShadingLanguageRI/Lexer.js (221507 => 221508)


--- trunk/Tools/WebGPUShadingLanguageRI/Lexer.js	2017-09-01 23:43:42 UTC (rev 221507)
+++ trunk/Tools/WebGPUShadingLanguageRI/Lexer.js	2017-09-01 23:54:23 UTC (rev 221508)
@@ -102,8 +102,11 @@
             return null;
         
         // FIXME: Make this do Unicode.
-        if (/^[^\d\W]\w*/.test(relevantText))
+        if (/^[^\d\W]\w*/.test(relevantText)) {
+            if (["struct", "protocol", "typedef", "if", "else", "enum", "continue", "break", "switch", "case", "default", "for", "while", "do", "return", "sizeof", "constant", "device", "threadgroup", "thread", "operator", "null"].includes(RegExp.lastMatch))
+                return result("keyword");
             return result("identifier");
+        }
 
         if (/^[0-9]+u/.test(relevantText))
             return result("uintLiteral");
@@ -115,9 +118,6 @@
         if (/^([0-9]*\.[0-9]+)|([0-9]+\.[0-9]*)/.test(relevantText))
             return result("doubleLiteral");
         
-        if (/^(struct|protocol|typedef|if|else|enum|continue|break|switch|case|default|for|while|do|return|sizeof|constant|device|threadgroup|thread|operator|null)/.test(relevantText))
-            return result("keyword");
-        
         if (/^([{}()\[\]?:=+*\/,.%!~^&|<>\\;-]|->|=>|<=|==|!=|\+=|-=|\*=|\/=|%=|^=|\|=|&=)/.test(relevantText))
             return result("punctuation");

Modified: trunk/Tools/WebGPUShadingLanguageRI/Test.js (221507 => 221508)


--- trunk/Tools/WebGPUShadingLanguageRI/Test.js	2017-09-01 23:43:42 UTC (rev 221507)
+++ trunk/Tools/WebGPUShadingLanguageRI/Test.js	2017-09-01 23:54:23 UTC (rev 221508)
@@ -31,6 +31,19 @@
     return prepare("<test>", 0, code);
 }
 
+function doLex(code)
+{
+    let lexer = new Lexer("<test>", 0, code);
+    var result = [];
+    for (;;) {
+        let next = lexer.next();
+        if (!next)
+            return result;
+        result.push(next);
+    }
+    return result;
+}
+
 function makeInt(program, value)
 {
     return TypedValue.box(program.intrinsics.int32, value);
@@ -43,6 +56,15 @@
     if (result.value != expected)
         throw new Error("Wrong result: " + result + " (expected " + expected + ")");
 }
+function checkLexerToken(result, expectedIndex, expectedKind, expectedText)
+{
+    if (result._index != expectedIndex)
+        throw new Error("Wrong lexer index; result: " + result._index + " (expected " + expectedIndex + ")");
+    if (result._kind != expectedKind)
+        throw new Error("Wrong lexer kind; result: " + result._kind + " (expected " + expectedKind + ")");
+    if (result._text != expectedText)
+        throw new Error("Wrong lexer text; result: " + result._text + " (expected " + expectedText + ")");
+}
 
 function checkFail(callback, predicate)
 {
@@ -314,6 +336,26 @@
         (e) => e instanceof WTypeError && e.message.indexOf("native int32 operator+<>(int32,int32)") != -1);
 }
 
+function TEST_lexerKeyword()
+{
+    let result = doLex("ident for while 123 123u { } {asd asd{ 1a3");
+    if (result.length != 13)
+        throw new Error("Lexer emitted an incorrect number of tokens (expected 12): " + result.length);
+    checkLexerToken(result[0],  0,  "identifier", "ident");
+    checkLexerToken(result[1],  6,  "keyword",     "for");
+    checkLexerToken(result[2],  10, "keyword",     "while");
+    checkLexerToken(result[3],  16, "intLiteral",  "123");
+    checkLexerToken(result[4],  20, "uintLiteral", "123u");
+    checkLexerToken(result[5],  25, "punctuation", "{");
+    checkLexerToken(result[6],  27, "punctuation", "}");
+    checkLexerToken(result[7],  29, "punctuation", "{");
+    checkLexerToken(result[8],  30, "identifier",  "asd");
+    checkLexerToken(result[9],  34, "identifier",  "asd");
+    checkLexerToken(result[10], 37, "punctuation", "{");
+    checkLexerToken(result[11], 39, "intLiteral",  "1");
+    checkLexerToken(result[12], 40, "identifier",  "a3");
+}
+
 let before = preciseTime();
 
 let filter = /.*/; // run everything by default

_______________________________________________
webkit-changes mailing list
webkit-changes@lists.webkit.org
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to