Diff
Modified: trunk/Source/WTF/ChangeLog (289798 => 289799)
--- trunk/Source/WTF/ChangeLog 2022-02-15 09:23:39 UTC (rev 289798)
+++ trunk/Source/WTF/ChangeLog 2022-02-15 09:37:07 UTC (rev 289799)
@@ -1,3 +1,15 @@
+2022-02-15 Robin Morisset <[email protected]>
+
+ [WGSL] Implement enough of the lexer for the simplest shaders
+ https://bugs.webkit.org/show_bug.cgi?id=233276
+ <rdar://problem/85732675>
+
+ Reviewed by Myles Maxfield.
+
+ Add WTF::Unicode::formFeed and WTF::Unicode::verticalTabulation.
+
+ * wtf/unicode/CharacterNames.h:
+
2022-02-14 Gustavo Noronha Silva <[email protected]>
[Linux/aarch64] Move page size ceiling to 16k
Modified: trunk/Source/WTF/wtf/unicode/CharacterNames.h (289798 => 289799)
--- trunk/Source/WTF/wtf/unicode/CharacterNames.h 2022-02-15 09:23:39 UTC (rev 289798)
+++ trunk/Source/WTF/wtf/unicode/CharacterNames.h 2022-02-15 09:37:07 UTC (rev 289799)
@@ -62,6 +62,7 @@
constexpr UChar horizontalEllipsis = 0x2026;
constexpr UChar hyphen = 0x2010;
constexpr UChar enDash = 0x2013;
+constexpr UChar formFeed = 0x000C;
constexpr UChar hyphenMinus = 0x002D;
constexpr UChar ideographicComma = 0x3001;
constexpr UChar ideographicFullStop = 0x3002;
@@ -103,6 +104,7 @@
constexpr UChar tibetanMarkIntersyllabicTsheg = 0x0F0B;
constexpr UChar32 ugariticWordDivider = 0x1039F;
constexpr UChar upArrowhead = 0x2303;
+constexpr UChar verticalTabulation = 0x000b;
constexpr UChar whiteBullet = 0x25E6;
constexpr UChar whiteCircle = 0x25CB;
constexpr UChar whiteSesameDot = 0xFE46;
@@ -143,6 +145,7 @@
using WTF::Unicode::horizontalEllipsis;
using WTF::Unicode::hyphen;
using WTF::Unicode::enDash;
+using WTF::Unicode::formFeed;
using WTF::Unicode::hyphenMinus;
using WTF::Unicode::ideographicComma;
using WTF::Unicode::ideographicFullStop;
@@ -182,6 +185,7 @@
using WTF::Unicode::tibetanMarkIntersyllabicTsheg;
using WTF::Unicode::ugariticWordDivider;
using WTF::Unicode::upArrowhead;
+using WTF::Unicode::verticalTabulation;
using WTF::Unicode::whiteBullet;
using WTF::Unicode::whiteCircle;
using WTF::Unicode::whiteSesameDot;
Modified: trunk/Source/WebGPU/ChangeLog (289798 => 289799)
--- trunk/Source/WebGPU/ChangeLog 2022-02-15 09:23:39 UTC (rev 289798)
+++ trunk/Source/WebGPU/ChangeLog 2022-02-15 09:37:07 UTC (rev 289799)
@@ -1,3 +1,77 @@
+2022-02-15 Robin Morisset <[email protected]>
+
+ [WGSL] Implement enough of the lexer for the simplest shaders
+ https://bugs.webkit.org/show_bug.cgi?id=233276
+ <rdar://problem/85732675>
+
+ Reviewed by Myles Maxfield.
+
+ There is just enough of the lexer in this patch to pass the unit tests (also added by the patch).
+ The lexer is templated by the character size, following the code from JSC's lexer.
+ I did not use a lexer generator like lex for a few reasons:
+ - It would have made the build system(s) even more of a nightmare
+ - It would have made it harder to give accurate and customized error messages
+ - If we ever decide to do anything exotic to the grammar of the language, it could have made it much harder
+
+ The lexing of floating point literals is a bit hacky, and probably has some precision issues in corner cases, but it passed all simple unit tests,
+ so for now I just left it as is with a FIXME, to be fixed once we can run the full CTS.
+
+ The TokenType includes an `Invalid` value. I could instead have used std::optional<Token> everywhere.
+ I made this choice for two reasons:
+ - space efficiency: we don't use an extra word of memory for the variant's tag
+ - (although this part could be solved by using https://github.com/akrzemi1/markable)
+ - ease of use and time efficiency: everywhere that we check for a given TokenType, we would have to first check that the Token is not nullopt, and then check the TokenType.
+
+ * Configurations/WGSLUnitTests.xcconfig:
+ * WGSL/Lexer.cpp: Added.
+ (WGSL::Lexer<T>::lex):
+ (WGSL::Lexer<T>::shift):
+ (WGSL::Lexer<T>::peek):
+ (WGSL::Lexer<T>::skipWhitespace):
+ (WGSL::Lexer<T>::isAtEndOfFile const):
+ (WGSL::Lexer<T>::parseDecimalInteger):
+ (WGSL::Lexer<T>::parseDecimalFloatExponent):
+ (WGSL::Lexer<T>::parseIntegerLiteralSuffix):
+ (WGSL::Lexer<LChar>::isWhiteSpace):
+ (WGSL::Lexer<LChar>::isIdentifierStart):
+ (WGSL::Lexer<LChar>::isValidIdentifierCharacter):
+ (WGSL::Lexer<LChar>::isDecimal):
+ (WGSL::Lexer<LChar>::isHexadecimal):
+ (WGSL::Lexer<LChar>::readDecimal):
+ (WGSL::Lexer<LChar>::readHexadecimal):
+ (WGSL::Lexer<UChar>::isWhiteSpace):
+ (WGSL::Lexer<UChar>::isIdentifierStart):
+ (WGSL::Lexer<UChar>::isValidIdentifierCharacter):
+ (WGSL::Lexer<UChar>::isDecimal):
+ (WGSL::Lexer<UChar>::isHexadecimal):
+ (WGSL::Lexer<UChar>::readDecimal):
+ (WGSL::Lexer<UChar>::readHexadecimal):
+ * WGSL/Lexer.h: Added.
+ (WGSL::Lexer::Lexer):
+ (WGSL::Lexer::currentPosition const):
+ (WGSL::Lexer::currentOffset const):
+ (WGSL::Lexer::currentTokenLength const):
+ (WGSL::Lexer::makeToken):
+ (WGSL::Lexer::makeLiteralToken):
+ (WGSL::Lexer::makeIdentifierToken):
+ * WGSL/SourceSpan.h: Added.
+ (WGSL::SourceSpan::SourceSpan):
+ * WGSL/Token.cpp: Added.
+ (WGSL::toString):
+ * WGSL/Token.h: Added.
+ (WGSL::Token::Token):
+ (WGSL::Token::operator=):
+ (WGSL::Token::~Token):
+ * WGSL/WGSL.cpp:
+ (WGSL::staticCheck):
+ * WGSLUnitTests/WGSLLexerTests.mm: Added.
+ (-[WGSLLexerTests testLexerOnSingleTokens]):
+ (-[WGSLLexerTests testLexerOnComputeShader]):
+ (-[WGSLLexerTests testLexerOnGraphicsShader]):
+ * WGSLUnitTests/WGSLUnitTests.mm: Removed.
+ (-[WGSLUnitTests testExample]): Deleted.
+ * WebGPU.xcodeproj/project.pbxproj:
+
2022-01-26 Myles C. Maxfield <[email protected]>
[WebGPU] Make sure WGSLUnitTests can call into WGSL
Added: trunk/Source/WebGPU/WGSL/Lexer.cpp (0 => 289799)
--- trunk/Source/WebGPU/WGSL/Lexer.cpp (rev 0)
+++ trunk/Source/WebGPU/WGSL/Lexer.cpp 2022-02-15 09:37:07 UTC (rev 289799)
@@ -0,0 +1,419 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "Lexer.h"
+
+#include <wtf/unicode/CharacterNames.h>
+
+namespace WGSL {
+
+template <typename T>
+Token Lexer<T>::lex()
+{
+ skipWhitespace();
+
+ m_tokenStartingPosition = m_currentPosition;
+
+ if (isAtEndOfFile())
+ return makeToken(TokenType::EndOfFile);
+
+ switch (m_current) {
+ case '(':
+ shift();
+ return makeToken(TokenType::ParenLeft);
+ case ')':
+ shift();
+ return makeToken(TokenType::ParenRight);
+ case '{':
+ shift();
+ return makeToken(TokenType::BraceLeft);
+ case '}':
+ shift();
+ return makeToken(TokenType::BraceRight);
+ case '[':
+ shift();
+ return makeToken(TokenType::BracketLeft);
+ case ']':
+ shift();
+ return makeToken(TokenType::BracketRight);
+ case ':':
+ shift();
+ return makeToken(TokenType::Colon);
+ case ',':
+ shift();
+ return makeToken(TokenType::Comma);
+ case ';':
+ shift();
+ return makeToken(TokenType::Semicolon);
+ case '=':
+ shift();
+ return makeToken(TokenType::Equal);
+ case '>':
+ shift();
+ return makeToken(TokenType::GT);
+ case '<':
+ shift();
+ return makeToken(TokenType::LT);
+ case '@':
+ shift();
+ return makeToken(TokenType::Attribute);
+ case '.': {
+ shift();
+ unsigned offset = currentOffset();
+ std::optional<uint64_t> postPeriod = parseDecimalInteger();
+ if (!postPeriod)
+ return makeToken(TokenType::Period);
+ double literalValue = postPeriod.value();
+ // FIXME: verify that there is no unnaceptable precision loss
+ // It should be tested in the CTS, for now let's get something that works
+ // Also the same code appears in a few places below.
+ literalValue /= pow(10, currentOffset() - offset);
+ std::optional<int64_t> exponent = parseDecimalFloatExponent();
+ if (exponent)
+ literalValue *= pow(10, exponent.value());
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ }
+ case '-':
+ shift();
+ if (m_current == '>') {
+ shift();
+ return makeToken(TokenType::Arrow);
+ }
+ break;
+ case '0': {
+ shift();
+ double literalValue = 0;
+ if (m_current == 'x') {
+ // FIXME: add support for hexadecimal floating point literals
+ shift();
+ bool hexNumberIsEmpty = true;
+ while (isHexadecimal(m_current)) {
+ literalValue *= 16;
+ literalValue += readHexadecimal(m_current);
+ shift();
+ hexNumberIsEmpty = false;
+ }
+ if (hexNumberIsEmpty)
+ break;
+ return parseIntegerLiteralSuffix(literalValue);
+ }
+
+ bool isFloatingPoint = false;
+ if (isDecimal(m_current) || m_current == '.' || m_current == 'e' || m_current == 'E') {
+ std::optional<uint64_t> integerPart = parseDecimalInteger();
+ if (integerPart)
+ literalValue = integerPart.value();
+ if (m_current == '.') {
+ isFloatingPoint = true;
+ shift();
+ // FIXME: share this code with the [1-9] case
+ unsigned offset = currentOffset();
+ std::optional<uint64_t> postPeriod = parseDecimalInteger();
+ if (postPeriod) {
+ double fractionalPart = postPeriod.value();
+ fractionalPart /= pow(10, currentOffset() - offset);
+ literalValue += fractionalPart;
+ }
+ if (m_current == 'f') {
+ shift();
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ }
+ }
+ if (std::optional<int64_t> exponent = parseDecimalFloatExponent()) {
+ literalValue *= pow(10, exponent.value());
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ }
+ // Decimal integers are not allowed to start with 0.
+ if (!isFloatingPoint)
+ return makeToken(TokenType::Invalid);
+ }
+ if (m_current == 'f') {
+ shift();
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ }
+ if (isFloatingPoint)
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ return parseIntegerLiteralSuffix(literalValue);
+ }
+ default:
+ if (isDecimal (m_current)) {
+ std::optional<uint64_t> value = parseDecimalInteger();
+ if (!value)
+ return makeToken(TokenType::Invalid);
+ double literalValue = value.value();
+ bool isFloatingPoint = false;
+ if (m_current == '.') {
+ isFloatingPoint = true;
+ shift();
+ unsigned offset = currentOffset();
+ std::optional<uint64_t> postPeriod = parseDecimalInteger();
+ if (postPeriod) {
+ double fractionalPart = postPeriod.value();
+ fractionalPart /= pow(10, currentOffset() - offset);
+ literalValue += fractionalPart;
+ }
+ }
+ if (std::optional<int64_t> exponent = parseDecimalFloatExponent()) {
+ literalValue *= pow(10, exponent.value());
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ }
+ if (m_current == 'f') {
+ shift();
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ }
+ if (!isFloatingPoint)
+ return parseIntegerLiteralSuffix(literalValue);
+ return makeLiteralToken(TokenType::DecimalFloatLiteral, literalValue);
+ } else if (isIdentifierStart(m_current)) {
+ const T* startOfToken = m_code;
+ shift();
+ while (isValidIdentifierCharacter(m_current))
+ shift();
+ // FIXME: a trie would be more efficient here, look at _javascript_Core/KeywordLookupGenerator.py for an example of code autogeneration that produces such a trie.
+ StringView view { startOfToken, currentTokenLength() };
+ // FIXME: I don't think that true/false/f32/u32/i32/bool need to be their own tokens, they could just be regular identifiers.
+ if (view == "true")
+ return makeToken(TokenType::LiteralTrue);
+ if (view == "false")
+ return makeToken(TokenType::LiteralFalse);
+ if (view == "bool")
+ return makeToken(TokenType::KeywordBool);
+ if (view == "i32")
+ return makeToken(TokenType::KeywordI32);
+ if (view == "u32")
+ return makeToken(TokenType::KeywordU32);
+ if (view == "f32")
+ return makeToken(TokenType::KeywordF32);
+ if (view == "fn")
+ return makeToken(TokenType::KeywordFn);
+ if (view == "function")
+ return makeToken(TokenType::KeywordFunction);
+ if (view == "private")
+ return makeToken(TokenType::KeywordPrivate);
+ if (view == "read")
+ return makeToken(TokenType::KeywordRead);
+ if (view == "read_write")
+ return makeToken(TokenType::KeywordReadWrite);
+ if (view == "return")
+ return makeToken(TokenType::KeywordReturn);
+ if (view == "storage")
+ return makeToken(TokenType::KeywordStorage);
+ if (view == "struct")
+ return makeToken(TokenType::KeywordStruct);
+ if (view == "uniform")
+ return makeToken(TokenType::KeywordUniform);
+ if (view == "var")
+ return makeToken(TokenType::KeywordVar);
+ if (view == "workgroup")
+ return makeToken(TokenType::KeywordWorkgroup);
+ if (view == "write")
+ return makeToken(TokenType::KeywordWrite);
+ if (view == "asm" || view == "bf16" || view == "const" || view == "do" || view == "enum"
+ || view == "f16" || view == "f64" || view == "handle" || view == "i8" || view == "i16"
+ || view == "i64" || view == "mat" || view == "premerge" || view == "regardless"
+ || view == "typedef" || view == "u8" || view == "u16" || view == "u64" || view == "unless"
+ || view == "using" || view == "vec" || view == "void" || view == "while")
+ return makeToken(TokenType::ReservedWord);
+ return makeIdentifierToken(view);
+ }
+ break;
+ }
+ return makeToken(TokenType::Invalid);
+}
+
+template <typename T>
+void Lexer<T>::shift()
+{
+ // At one point timing showed that setting m_current to 0 unconditionally was faster than an if-else sequence.
+ m_current = 0;
+ ++m_code;
+ ++m_currentPosition.m_offset;
+ ++m_currentPosition.m_lineOffset;
+ if (LIKELY(m_code < m_codeEnd))
+ m_current = *m_code;
+}
+
+template <typename T>
+T Lexer<T>::peek(unsigned i)
+{
+ if (UNLIKELY(m_code + i >= m_codeEnd))
+ return 0;
+ return *(m_code + i);
+}
+
+template <typename T>
+void Lexer<T>::skipWhitespace()
+{
+ while (isWhiteSpace(m_current)) {
+ if (m_current == '\n') {
+ shift();
+ ++m_currentPosition.m_line;
+ m_currentPosition.m_lineOffset = 0;
+ } else
+ shift();
+ }
+}
+
+template <typename T>
+bool Lexer<T>::isAtEndOfFile() const
+{
+ if (m_code == m_codeEnd) {
+ ASSERT(!m_current);
+ return true;
+ }
+ ASSERT(m_code < m_codeEnd);
+ return false;
+}
+
+template <typename T>
+std::optional<uint64_t> Lexer<T>::parseDecimalInteger()
+{
+ if (!isDecimal(m_current))
+ return std::nullopt;
+
+ CheckedUint64 value = 0;
+ while (isDecimal(m_current)) {
+ value *= 10ull;
+ value += readDecimal(m_current);
+ shift();
+ }
+ if (value.hasOverflowed())
+ return std::nullopt;
+ return { value.value() };
+}
+
+// Parse pattern (e|E)(\+|-)?[0-9]+f? if it is present, and return the exponent
+template <typename T>
+std::optional<int64_t> Lexer<T>::parseDecimalFloatExponent()
+{
+ T char1 = peek(1);
+ T char2 = peek(2);
+ // Check for pattern (e|E)(\+|-)?[0-9]+
+ if (m_current != 'e' && m_current != 'E')
+ return std::nullopt;
+ if (char1 == '+' || char1 == '-') {
+ if (!isDecimal(char2))
+ return std::nullopt;
+ } else if (!isDecimal(char1))
+ return std::nullopt;
+ shift();
+
+ bool negateExponent = false;
+ if (m_current == '-') {
+ negateExponent = true;
+ shift();
+ } else if (m_current == '+')
+ shift();
+
+ std::optional<int64_t> exponent = parseDecimalInteger();
+ if (!exponent)
+ return std::nullopt;
+ CheckedInt64 exponentValue = exponent.value();
+ if (negateExponent)
+ exponentValue = - exponentValue;
+ if (exponentValue.hasOverflowed())
+ return std::nullopt;
+ return { exponentValue.value() };
+};
+
+template <typename T>
+Token Lexer<T>::parseIntegerLiteralSuffix(double literalValue)
+{
+ if (m_current == 'i') {
+ shift();
+ return makeLiteralToken(TokenType::IntegerLiteralSigned, literalValue);
+ }
+ if (m_current == 'u') {
+ shift();
+ return makeLiteralToken(TokenType::IntegerLiteralUnsigned, literalValue);
+ }
+ return makeLiteralToken(TokenType::IntegerLiteral, literalValue);
+};
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::isWhiteSpace(T ch)
+{
+ switch (ch) {
+ case WTF::Unicode::space:
+ case WTF::Unicode::tabCharacter:
+ case WTF::Unicode::carriageReturn:
+ case WTF::Unicode::newlineCharacter:
+ case WTF::Unicode::verticalTabulation:
+ case WTF::Unicode::formFeed:
+ return true;
+ default:
+ return false;
+ }
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::isIdentifierStart(T ch)
+{
+ return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::isValidIdentifierCharacter(T ch)
+{
+ return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_';
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::isDecimal(T ch)
+{
+ return (ch >= '0' && ch <= '9');
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::isHexadecimal(T ch)
+{
+ return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
+}
+
+template <typename T>
+ALWAYS_INLINE uint64_t Lexer<T>::readDecimal(T ch)
+{
+ ASSERT(isDecimal(ch));
+ return ch - '0';
+}
+
+template <typename T>
+ALWAYS_INLINE uint64_t Lexer<T>::readHexadecimal(T ch)
+{
+ ASSERT(isHexadecimal(ch));
+ if (ch >= '0' && ch <= '9')
+ return ch - '0';
+ if (ch >= 'a' && ch <= 'f')
+ return ch - 'a';
+ ASSERT(ch >= 'A' && ch <= 'F');
+ return ch - 'A';
+}
+
+template class Lexer<LChar>;
+template class Lexer<UChar>;
+
+}
+
Added: trunk/Source/WebGPU/WGSL/Lexer.h (0 => 289799)
--- trunk/Source/WebGPU/WGSL/Lexer.h (rev 0)
+++ trunk/Source/WebGPU/WGSL/Lexer.h 2022-02-15 09:37:07 UTC (rev 289799)
@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "Token.h"
+#include <wtf/text/StringView.h>
+#include <wtf/text/WTFString.h>
+
+namespace WGSL {
+
+template<typename T>
+class Lexer {
+public:
+ Lexer(const String& wgsl)
+ {
+ if constexpr (std::is_same<T, LChar>::value) {
+ m_code = wgsl.characters8();
+ m_codeEnd = m_code + wgsl.sizeInBytes();
+ } else {
+ static_assert(std::is_same<T, UChar>::value, "The lexer expects its template parameter to be either LChar or UChar");
+ m_code = wgsl.characters16();
+ ASSERT(!(wgsl.sizeInBytes() % 2));
+ m_codeEnd = m_code + wgsl.sizeInBytes() / 2;
+ }
+
+ m_current = (m_code != m_codeEnd) ? *m_code : 0;
+ }
+
+ Token lex();
+ bool isAtEndOfFile() const;
+ SourcePosition currentPosition() const { return m_currentPosition; }
+
+ // Only public so that the UChar version of the Lexer can defer to the LChar version
+ static bool isWhiteSpace(T character);
+ static bool isIdentifierStart(T character);
+ static bool isValidIdentifierCharacter(T character);
+ static bool isDecimal(T character);
+ static bool isHexadecimal(T character);
+ static uint64_t readDecimal(T character);
+ static uint64_t readHexadecimal(T character);
+
+private:
+ unsigned currentOffset() const { return m_currentPosition.m_offset; }
+ unsigned currentTokenLength() const { return currentOffset() - m_tokenStartingPosition.m_offset; }
+
+ Token makeToken(TokenType type)
+ {
+ return { type, m_tokenStartingPosition, currentTokenLength() };
+ }
+ Token makeLiteralToken(TokenType type, double literalValue)
+ {
+ return { type, m_tokenStartingPosition, currentTokenLength(), literalValue };
+ }
+ Token makeIdentifierToken(StringView view)
+ {
+ return { WGSL::TokenType::Identifier, m_tokenStartingPosition, currentTokenLength(), view };
+ }
+
+ void shift();
+ T peek(unsigned);
+ void skipWhitespace();
+
+ // Reads [0-9]+
+ std::optional<uint64_t> parseDecimalInteger();
+ // Parse pattern (e|E)(\+|-)?[0-9]+f? if it is present, and return the exponent
+ std::optional<int64_t> parseDecimalFloatExponent();
+ // Checks whether there is an "i" or "u" coming, and return the right kind of literal token
+ Token parseIntegerLiteralSuffix(double literalValue);
+
+ T m_current;
+ const T* m_code;
+ const T* m_codeEnd;
+ SourcePosition m_currentPosition { 0, 0, 0 };
+ SourcePosition m_tokenStartingPosition { 0, 0, 0 };
+};
+
+}
Added: trunk/Source/WebGPU/WGSL/SourceSpan.h (0 => 289799)
--- trunk/Source/WebGPU/WGSL/SourceSpan.h (rev 0)
+++ trunk/Source/WebGPU/WGSL/SourceSpan.h 2022-02-15 09:37:07 UTC (rev 289799)
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+namespace WGSL {
+
+struct SourcePosition {
+ unsigned m_line;
+ unsigned m_lineOffset;
+ unsigned m_offset;
+};
+
+struct SourceSpan {
+ // FIXME: we could possibly skip m_lineOffset and recompute it only when trying to show an error
+ // This would shrink the AST size by 32 bits per AST node, at the cost of a bit of code complexity in the error toString function.
+ unsigned m_line;
+ unsigned m_lineOffset;
+ unsigned m_offset;
+ unsigned m_length;
+
+ SourceSpan(unsigned line, unsigned lineOffset, unsigned offset, unsigned length)
+ : m_line(line)
+ , m_lineOffset(lineOffset)
+ , m_offset(offset)
+ , m_length(length)
+ {
+ }
+
+ SourceSpan(SourcePosition start, SourcePosition end)
+ : m_line(start.m_line)
+ , m_lineOffset(start.m_lineOffset)
+ , m_offset(start.m_offset)
+ , m_length(end.m_offset - start.m_offset)
+ {
+ }
+};
+
+}
Added: trunk/Source/WebGPU/WGSL/Token.cpp (0 => 289799)
--- trunk/Source/WebGPU/WGSL/Token.cpp (rev 0)
+++ trunk/Source/WebGPU/WGSL/Token.cpp 2022-02-15 09:37:07 UTC (rev 289799)
@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "Token.h"
+
+namespace WGSL {
+
+String toString(TokenType type)
+{
+ switch (type) {
+ case TokenType::Invalid:
+ return "Invalid";
+ case TokenType::EndOfFile:
+ return "EOF";
+ case TokenType::IntegerLiteral:
+ return "IntegerLiteral";
+ case TokenType::IntegerLiteralSigned:
+ return "IntegerLiteralSigned";
+ case TokenType::IntegerLiteralUnsigned:
+ return "IntegerLiteralUnsigned";
+ case TokenType::DecimalFloatLiteral:
+ return "DecimalFloatLiteral";
+ case TokenType::HexFloatLiteral:
+ return "HexFloatLiteral";
+ case TokenType::Identifier:
+ return "Identifier";
+ case TokenType::ReservedWord:
+ return "ReservedWord";
+ case TokenType::KeywordStruct:
+ return "struct";
+ case TokenType::KeywordFn:
+ return "fn";
+ case TokenType::KeywordFunction:
+ return "function";
+ case TokenType::KeywordPrivate:
+ return "private";
+ case TokenType::KeywordRead:
+ return "read";
+ case TokenType::KeywordReadWrite:
+ return "read_write";
+ case TokenType::KeywordReturn:
+ return "return";
+ case TokenType::KeywordStorage:
+ return "storage";
+ case TokenType::KeywordUniform:
+ return "uniform";
+ case TokenType::KeywordVar:
+ return "var";
+ case TokenType::KeywordWorkgroup:
+ return "workgroup";
+ case TokenType::KeywordWrite:
+ return "write";
+ case TokenType::KeywordI32:
+ return "i32";
+ case TokenType::KeywordU32:
+ return "u32";
+ case TokenType::KeywordF32:
+ return "f32";
+ case TokenType::KeywordBool:
+ return "bool";
+ case TokenType::LiteralTrue:
+ return "true";
+ case TokenType::LiteralFalse:
+ return "false";
+ case TokenType::Arrow:
+ return "->";
+ case TokenType::Attribute:
+ return "@";
+ case TokenType::BracketLeft:
+ return "[";
+ case TokenType::BracketRight:
+ return "]";
+ case TokenType::BraceLeft:
+ return "{";
+ case TokenType::BraceRight:
+ return "}";
+ case TokenType::Colon:
+ return ":";
+ case TokenType::Comma:
+ return ",";
+ case TokenType::Equal:
+ return "=";
+ case TokenType::GT:
+ return ">";
+ case TokenType::LT:
+ return "<";
+ case TokenType::Period:
+ return ".";
+ case TokenType::ParenLeft:
+ return "(";
+ case TokenType::ParenRight:
+ return ")";
+ case TokenType::Semicolon:
+ return ";";
+ }
+}
+
+}
Added: trunk/Source/WebGPU/WGSL/Token.h (0 => 289799)
--- trunk/Source/WebGPU/WGSL/Token.h (rev 0)
+++ trunk/Source/WebGPU/WGSL/Token.h 2022-02-15 09:37:07 UTC (rev 289799)
@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "SourceSpan.h"
+#include <wtf/text/StringView.h>
+#include <wtf/text/WTFString.h>
+
+namespace WGSL {
+
+enum class TokenType: uint32_t {
+ // Instead of having this type, we could have a std::optional<Token> everywhere that we currently have a Token.
+ // I made this choice for two reasons:
+ // - space efficiency: we don't use an extra word of memory for the variant's tag
+ // (although this part could be solved by using https://github.com/akrzemi1/markable)
+ // - ease of use and time efficiency: everywhere that we check for a given TokenType, we would have to first check that the Token is not nullopt, and then check the TokenType.
+ Invalid,
+
+ EndOfFile,
+
+ IntegerLiteral,
+ IntegerLiteralSigned,
+ IntegerLiteralUnsigned,
+ DecimalFloatLiteral,
+ HexFloatLiteral,
+
+ Identifier,
+
+ ReservedWord,
+ KeywordFn,
+ KeywordFunction,
+ KeywordPrivate,
+ KeywordRead,
+ KeywordReadWrite,
+ KeywordReturn,
+ KeywordStorage,
+ KeywordStruct,
+ KeywordUniform,
+ KeywordVar,
+ KeywordWorkgroup,
+ KeywordWrite,
+ KeywordI32,
+ KeywordU32,
+ KeywordF32,
+ KeywordBool,
+ LiteralTrue,
+ LiteralFalse,
+ // FIXME: add all the other keywords: see #keyword-summary in the WGSL spec
+
+ Arrow,
+ Attribute,
+ BraceLeft,
+ BraceRight,
+ BracketLeft,
+ BracketRight,
+ Colon,
+ Comma,
+ Equal,
+ GT,
+ LT,
+ Period,
+ ParenLeft,
+ ParenRight,
+ Semicolon,
+ // FIXME: add all the other special tokens
+};
+
+String toString(TokenType);
+
+struct Token {
+ TokenType m_type;
+ SourceSpan m_span;
+ union {
+ double m_literalValue;
+ StringView m_ident;
+ };
+
+ Token(TokenType type, SourcePosition position, unsigned length)
+ : m_type(type)
+ , m_span(position.m_line, position.m_lineOffset, position.m_offset, length)
+ {
+ ASSERT(type != TokenType::Identifier);
+ ASSERT(type != TokenType::IntegerLiteral);
+ ASSERT(type != TokenType::IntegerLiteralSigned);
+ ASSERT(type != TokenType::IntegerLiteralUnsigned);
+ ASSERT(type != TokenType::DecimalFloatLiteral);
+ ASSERT(type != TokenType::HexFloatLiteral);
+ }
+
+ Token(TokenType type, SourcePosition position, unsigned length, double literalValue)
+ : m_type(type)
+ , m_span(position.m_line, position.m_lineOffset, position.m_offset, length)
+ , m_literalValue(literalValue)
+ {
+ ASSERT(type == TokenType::IntegerLiteral
+ || type == TokenType::IntegerLiteralSigned
+ || type == TokenType::IntegerLiteralUnsigned
+ || type == TokenType::DecimalFloatLiteral
+ || type == TokenType::HexFloatLiteral);
+ }
+
+ Token(TokenType type, SourcePosition position, unsigned length, StringView ident)
+ : m_type(type)
+ , m_span(position.m_line, position.m_lineOffset, position.m_offset, length)
+ , m_ident(ident)
+ {
+ ASSERT(type == TokenType::Identifier);
+ }
+
+ Token& operator=(Token&& other)
+ {
+ if (m_type == TokenType::Identifier)
+ m_ident.~StringView();
+
+ m_type = other.m_type;
+ m_span = other.m_span;
+
+ switch (other.m_type) {
+ case TokenType::Identifier:
+ new (NotNull, &m_ident) StringView();
+ m_ident = other.m_ident;
+ break;
+ case TokenType::IntegerLiteral:
+ case TokenType::IntegerLiteralSigned:
+ case TokenType::IntegerLiteralUnsigned:
+ case TokenType::DecimalFloatLiteral:
+ case TokenType::HexFloatLiteral:
+ m_literalValue = other.m_literalValue;
+ break;
+ default:
+ break;
+ }
+
+ return *this;
+ }
+
+ Token(const Token& other)
+ : m_type(other.m_type)
+ , m_span(other.m_span)
+ {
+ switch (other.m_type) {
+ case TokenType::Identifier:
+ new (NotNull, &m_ident) StringView();
+ m_ident = other.m_ident;
+ break;
+ case TokenType::IntegerLiteral:
+ case TokenType::IntegerLiteralSigned:
+ case TokenType::IntegerLiteralUnsigned:
+ case TokenType::DecimalFloatLiteral:
+ case TokenType::HexFloatLiteral:
+ m_literalValue = other.m_literalValue;
+ break;
+ default:
+ break;
+ }
+ }
+
+ ~Token()
+ {
+ if (m_type == TokenType::Identifier)
+ (&m_ident)->~StringView();
+ }
+};
+
+} // namespace WGSL
Modified: trunk/Source/WebGPU/WGSL/WGSL.cpp (289798 => 289799)
--- trunk/Source/WebGPU/WGSL/WGSL.cpp 2022-02-15 09:23:39 UTC (rev 289798)
+++ trunk/Source/WebGPU/WGSL/WGSL.cpp 2022-02-15 09:37:07 UTC (rev 289799)
@@ -27,11 +27,15 @@
#include "WGSL.h"
#include "AST.h"
+#include "Lexer.h"
namespace WGSL {
-std::variant<SuccessfulCheck, FailedCheck> staticCheck(const String&, const std::optional<SourceMap>&)
+std::variant<SuccessfulCheck, FailedCheck> staticCheck(const String& str, const std::optional<SourceMap>&)
{
+ // Making sure that the lexer builds correctly, will be removed in later patches
+ Lexer<LChar> lexer(str);
+ lexer.lex();
return FailedCheck { { }, { } };
}
Added: trunk/Source/WebGPU/WGSLUnitTests/WGSLLexerTests.mm (0 => 289799)
--- trunk/Source/WebGPU/WGSLUnitTests/WGSLLexerTests.mm (rev 0)
+++ trunk/Source/WebGPU/WGSLUnitTests/WGSLLexerTests.mm 2022-02-15 09:37:07 UTC (rev 289799)
@@ -0,0 +1,295 @@
+/*
+ * Copyright (c) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+#import "Lexer.h"
+
+#import <XCTest/XCTest.h>
+
+@interface WGSLLexerTests : XCTestCase
+
+@end
+
+@implementation WGSLLexerTests
+
+- (void)testLexerOnSingleTokens {
+ auto checkSingleToken = [] (const char* string, WGSL::TokenType type) {
+ WGSL::Lexer<LChar> lexer(string);
+ WGSL::Token result = lexer.lex();
+ XCTAssertEqual(result.m_type, type);
+ return result;
+ };
+ auto checkSingleLiteral = [&] (const char* string, WGSL::TokenType type, double literalValue) {
+ WGSL::Token result = checkSingleToken(string, type);
+ XCTAssertEqual(result.m_literalValue, literalValue);
+ };
+
+ checkSingleToken("", WGSL::TokenType::EndOfFile);
+ checkSingleLiteral("1", WGSL::TokenType::IntegerLiteral, 1);
+ checkSingleLiteral("0", WGSL::TokenType::IntegerLiteral, 0);
+ checkSingleLiteral("142", WGSL::TokenType::IntegerLiteral, 142);
+ checkSingleLiteral("1.1", WGSL::TokenType::DecimalFloatLiteral, 1.1);
+ checkSingleLiteral("0.4", WGSL::TokenType::DecimalFloatLiteral, 0.4);
+ checkSingleLiteral("0123.456", WGSL::TokenType::DecimalFloatLiteral, 0123.456);
+ checkSingleToken("0123", WGSL::TokenType::Invalid);
+ checkSingleLiteral("0123.", WGSL::TokenType::DecimalFloatLiteral, 123);
+ checkSingleLiteral(".456", WGSL::TokenType::DecimalFloatLiteral, 0.456);
+ checkSingleToken(".", WGSL::TokenType::Period);
+ checkSingleLiteral("42f", WGSL::TokenType::DecimalFloatLiteral, 42);
+ checkSingleLiteral("42e0f", WGSL::TokenType::DecimalFloatLiteral, 42);
+ checkSingleLiteral("042e0f", WGSL::TokenType::DecimalFloatLiteral, 42);
+ checkSingleToken("042f", WGSL::TokenType::Invalid);
+ checkSingleLiteral("42e-3", WGSL::TokenType::DecimalFloatLiteral, 42e-3);
+ checkSingleLiteral("42e-a", WGSL::TokenType::IntegerLiteral, 42);
+}
+
+- (void) testLexerOnComputeShader {
+ WGSL::Lexer<LChar> lexer("@block struct B {\n"
+ " a: i32;\n"
+ "}\n"
+ "\n"
+ "@group(0) @binding(0)\n"
+ "var<storage, read_write> x: B;\n"
+ "\n"
+ "@stage(compute)\n"
+ "fn main() {\n"
+ " x.a = 42;\n"
+ "}");
+
+ unsigned lineNumber = 0;
+ auto checkNextToken = [&] (WGSL::TokenType type) {
+ WGSL::Token result = lexer.lex();
+ XCTAssertEqual(result.m_type, type);
+ XCTAssertEqual(result.m_span.m_line, lineNumber);
+ return result;
+ };
+ auto checkNextTokenIsIdentifier = [&] (const char* ident) {
+ WGSL::Token result = checkNextToken(WGSL::TokenType::Identifier);
+ XCTAssertEqual(result.m_ident, ident);
+ };
+ auto checkNextTokenIsLiteral = [&] (WGSL::TokenType type, double literalValue) {
+ WGSL::Token result = checkNextToken(type);
+ XCTAssertEqual(result.m_literalValue, literalValue);
+ };
+
+ // @block struct B {
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("block");
+ checkNextToken(WGSL::TokenType::KeywordStruct);
+ checkNextTokenIsIdentifier("B");
+ checkNextToken(WGSL::TokenType::BraceLeft);
+
+ // a: i32;
+ ++lineNumber;
+ checkNextTokenIsIdentifier("a");
+ checkNextToken(WGSL::TokenType::Colon);
+ checkNextToken(WGSL::TokenType::KeywordI32);
+ checkNextToken(WGSL::TokenType::Semicolon);
+
+ // }
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::BraceRight);
+
+ // @group(0) @binding(0)
+ lineNumber += 2;
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("group");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsLiteral(WGSL::TokenType::IntegerLiteral, 0);
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("binding");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsLiteral(WGSL::TokenType::IntegerLiteral, 0);
+ checkNextToken(WGSL::TokenType::ParenRight);
+
+ // var<storage, read_write> x: B;
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::KeywordVar);
+ checkNextToken(WGSL::TokenType::LT);
+ checkNextToken(WGSL::TokenType::KeywordStorage);
+ checkNextToken(WGSL::TokenType::Comma);
+ checkNextToken(WGSL::TokenType::KeywordReadWrite);
+ checkNextToken(WGSL::TokenType::GT);
+ checkNextTokenIsIdentifier("x");
+ checkNextToken(WGSL::TokenType::Colon);
+ checkNextTokenIsIdentifier("B");
+ checkNextToken(WGSL::TokenType::Semicolon);
+
+ // @stage(compute)
+ lineNumber += 2;
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("stage");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsIdentifier("compute");
+ checkNextToken(WGSL::TokenType::ParenRight);
+
+ // fn main() {
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::KeywordFn);
+ checkNextTokenIsIdentifier("main");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextToken(WGSL::TokenType::BraceLeft);
+
+ // x.a = 42;
+ ++lineNumber;
+ checkNextTokenIsIdentifier("x");
+ checkNextToken(WGSL::TokenType::Period);
+ checkNextTokenIsIdentifier("a");
+ checkNextToken(WGSL::TokenType::Equal);
+ checkNextTokenIsLiteral(WGSL::TokenType::IntegerLiteral, 42);
+ checkNextToken(WGSL::TokenType::Semicolon);
+
+ // }
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::BraceRight);
+ checkNextToken(WGSL::TokenType::EndOfFile);
+}
+
+- (void) testLexerOnGraphicsShader {
+ WGSL::Lexer<LChar> lexer("@stage(vertex)\n"
+ "fn vertexShader(@location(0) x: vec4<f32>) -> @builtin(position) vec4<f32> {\n"
+ " return x;\n"
+ "}\n"
+ "\n"
+ "@stage(fragment)\n"
+ "fn fragmentShader() -> @location(0) vec4<f32> {\n"
+ " return vec4<f32>(0.4, 0.4, 0.8, 1.0);\n"
+ "}");
+
+ unsigned lineNumber = 0;
+ auto checkNextToken = [&] (WGSL::TokenType type) {
+ WGSL::Token result = lexer.lex();
+ XCTAssertEqual(result.m_type, type);
+ XCTAssertEqual(result.m_span.m_line, lineNumber);
+ return result;
+ };
+ auto checkNextTokenIsIdentifier = [&] (const char* ident) {
+ WGSL::Token result = checkNextToken(WGSL::TokenType::Identifier);
+ XCTAssertEqual(result.m_ident, ident);
+ };
+ auto checkNextTokenIsLiteral = [&] (WGSL::TokenType type, double literalValue) {
+ WGSL::Token result = checkNextToken(type);
+ XCTAssertEqual(result.m_literalValue, literalValue);
+ };
+
+ // [[stage(vertex)]]
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("stage");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsIdentifier("vertex");
+ checkNextToken(WGSL::TokenType::ParenRight);
+
+ ++lineNumber;
+ // fn vertexShader(@location(0) x: vec4<f32>) -> @builtin(position) vec4<f32> {
+ checkNextToken(WGSL::TokenType::KeywordFn);
+ checkNextTokenIsIdentifier("vertexShader");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("location");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsLiteral(WGSL::TokenType::IntegerLiteral, 0);
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextTokenIsIdentifier("x");
+ checkNextToken(WGSL::TokenType::Colon);
+ checkNextTokenIsIdentifier("vec4");
+ checkNextToken(WGSL::TokenType::LT);
+ checkNextToken(WGSL::TokenType::KeywordF32);
+ checkNextToken(WGSL::TokenType::GT);
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextToken(WGSL::TokenType::Arrow);
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("builtin");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsIdentifier("position");
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextTokenIsIdentifier("vec4");
+ checkNextToken(WGSL::TokenType::LT);
+ checkNextToken(WGSL::TokenType::KeywordF32);
+ checkNextToken(WGSL::TokenType::GT);
+ checkNextToken(WGSL::TokenType::BraceLeft);
+
+ // return x;
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::KeywordReturn);
+ checkNextTokenIsIdentifier("x");
+ checkNextToken(WGSL::TokenType::Semicolon);
+
+ // }
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::BraceRight);
+
+ // @stage(fragment)
+ lineNumber += 2;
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("stage");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsIdentifier("fragment");
+ checkNextToken(WGSL::TokenType::ParenRight);
+
+ // fn fragmentShader() -> @location(0) vec4<f32> {
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::KeywordFn);
+ checkNextTokenIsIdentifier("fragmentShader");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextToken(WGSL::TokenType::Arrow);
+ checkNextToken(WGSL::TokenType::Attribute);
+ checkNextTokenIsIdentifier("location");
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsLiteral(WGSL::TokenType::IntegerLiteral, 0);
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextTokenIsIdentifier("vec4");
+ checkNextToken(WGSL::TokenType::LT);
+ checkNextToken(WGSL::TokenType::KeywordF32);
+ checkNextToken(WGSL::TokenType::GT);
+ checkNextToken(WGSL::TokenType::BraceLeft);
+
+ // return vec4<f32>(0.4, 0.4, 0.8, 1.0);
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::KeywordReturn);
+ checkNextTokenIsIdentifier("vec4");
+ checkNextToken(WGSL::TokenType::LT);
+ checkNextToken(WGSL::TokenType::KeywordF32);
+ checkNextToken(WGSL::TokenType::GT);
+ checkNextToken(WGSL::TokenType::ParenLeft);
+ checkNextTokenIsLiteral(WGSL::TokenType::DecimalFloatLiteral, 0.4);
+ checkNextToken(WGSL::TokenType::Comma);
+ checkNextTokenIsLiteral(WGSL::TokenType::DecimalFloatLiteral, 0.4);
+ checkNextToken(WGSL::TokenType::Comma);
+ checkNextTokenIsLiteral(WGSL::TokenType::DecimalFloatLiteral, 0.8);
+ checkNextToken(WGSL::TokenType::Comma);
+ checkNextTokenIsLiteral(WGSL::TokenType::DecimalFloatLiteral, 1.0);
+ checkNextToken(WGSL::TokenType::ParenRight);
+ checkNextToken(WGSL::TokenType::Semicolon);
+
+ // }
+ ++lineNumber;
+ checkNextToken(WGSL::TokenType::BraceRight);
+ checkNextToken(WGSL::TokenType::EndOfFile);
+}
+
+@end
Deleted: trunk/Source/WebGPU/WGSLUnitTests/WGSLUnitTests.mm (289798 => 289799)
--- trunk/Source/WebGPU/WGSLUnitTests/WGSLUnitTests.mm 2022-02-15 09:23:39 UTC (rev 289798)
+++ trunk/Source/WebGPU/WGSLUnitTests/WGSLUnitTests.mm 2022-02-15 09:37:07 UTC (rev 289799)
@@ -1,50 +0,0 @@
-/*
- * Copyright (c) 2021 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
- * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#import "config.h"
-#import "WGSL.h"
-
-#import <XCTest/XCTest.h>
-#import <vector>
-#import <wtf/DataLog.h>
-#import <wtf/Vector.h>
-
-@interface WGSLUnitTests : XCTestCase
-
-@end
-
-@implementation WGSLUnitTests
-
-- (void)testExample {
- dataLog("asdf\n");
- Vector<int> x { 2, 3, };
- Vector<int> y;
- y.append(2);
- y.append(3);
- XCTAssertEqual(x, y);
- WGSL::staticCheck("", std::nullopt);
-}
-
-@end
Modified: trunk/Source/WebGPU/WebGPU.xcodeproj/project.pbxproj (289798 => 289799)
--- trunk/Source/WebGPU/WebGPU.xcodeproj/project.pbxproj 2022-02-15 09:23:39 UTC (rev 289798)
+++ trunk/Source/WebGPU/WebGPU.xcodeproj/project.pbxproj 2022-02-15 09:37:07 UTC (rev 289799)
@@ -7,7 +7,7 @@
objects = {
/* Begin PBXBuildFile section */
- 1C023D3F27449070001DB734 /* WGSLUnitTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 1C023D3E27449070001DB734 /* WGSLUnitTests.mm */; };
+ 1C023D3F27449070001DB734 /* WGSLLexerTests.mm in Sources */ = {isa = PBXBuildFile; fileRef = 1C023D3E27449070001DB734 /* WGSLLexerTests.mm */; };
1C023D4027449070001DB734 /* libwgsl.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1CEBD7F22716B2CC00A5254D /* libwgsl.a */; };
1C023D4B274495B9001DB734 /* _javascript_Core.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1C023D4A274495B9001DB734 /* _javascript_Core.framework */; };
1C2CEDEE271E8A7300EDC16F /* Metal.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1C2CEDED271E8A7300EDC16F /* Metal.framework */; };
@@ -41,6 +41,11 @@
1CEBD7F82716B34400A5254D /* WGSL.h in Headers */ = {isa = PBXBuildFile; fileRef = 1CEBD7F72716B34400A5254D /* WGSL.h */; settings = {ATTRIBUTES = (Public, ); }; };
1CEBD8032716BF8200A5254D /* WGSL.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1CEBD8022716BF8200A5254D /* WGSL.cpp */; };
1CEBD8262716CACC00A5254D /* libwgsl.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1CEBD7F22716B2CC00A5254D /* libwgsl.a */; };
+ 338BB2CE27B6B60200E066AB /* Token.h in Headers */ = {isa = PBXBuildFile; fileRef = 338BB2CD27B6B60200E066AB /* Token.h */; };
+ 338BB2D027B6B61B00E066AB /* Token.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 338BB2CF27B6B61B00E066AB /* Token.cpp */; };
+ 338BB2D227B6B63F00E066AB /* SourceSpan.h in Headers */ = {isa = PBXBuildFile; fileRef = 338BB2D127B6B63F00E066AB /* SourceSpan.h */; };
+ 338BB2D427B6B66C00E066AB /* Lexer.h in Headers */ = {isa = PBXBuildFile; fileRef = 338BB2D327B6B66C00E066AB /* Lexer.h */; };
+ 338BB2D627B6B68700E066AB /* Lexer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 338BB2D527B6B68700E066AB /* Lexer.cpp */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
@@ -62,7 +67,7 @@
/* Begin PBXFileReference section */
1C023D3C27449070001DB734 /* WGSLUnitTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = WGSLUnitTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
- 1C023D3E27449070001DB734 /* WGSLUnitTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = WGSLUnitTests.mm; sourceTree = "<group>"; };
+ 1C023D3E27449070001DB734 /* WGSLLexerTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = WGSLLexerTests.mm; sourceTree = "<group>"; };
1C023D472744916D001DB734 /* WGSLUnitTests.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = WGSLUnitTests.xcconfig; sourceTree = "<group>"; };
1C023D4A274495B9001DB734 /* _javascript_Core.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = _javascript_Core.framework; sourceTree = BUILT_PRODUCTS_DIR; };
1C2CEDED271E8A7300EDC16F /* Metal.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Metal.framework; path = System/Library/Frameworks/Metal.framework; sourceTree = SDKROOT; };
@@ -135,6 +140,11 @@
1CEBD82B2716CAFB00A5254D /* CoreFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreFoundation.framework; path = System/Library/Frameworks/CoreFoundation.framework; sourceTree = SDKROOT; };
1CEBD82D2716CB1600A5254D /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
1CEBD8302716CB3800A5254D /* libicucore.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libicucore.tbd; path = usr/lib/libicucore.tbd; sourceTree = SDKROOT; };
+ 338BB2CD27B6B60200E066AB /* Token.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Token.h; sourceTree = "<group>"; };
+ 338BB2CF27B6B61B00E066AB /* Token.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = Token.cpp; sourceTree = "<group>"; };
+ 338BB2D127B6B63F00E066AB /* SourceSpan.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SourceSpan.h; sourceTree = "<group>"; };
+ 338BB2D327B6B66C00E066AB /* Lexer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Lexer.h; sourceTree = "<group>"; };
+ 338BB2D527B6B68700E066AB /* Lexer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = Lexer.cpp; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@@ -170,7 +180,7 @@
1C023D3D27449070001DB734 /* WGSLUnitTests */ = {
isa = PBXGroup;
children = (
- 1C023D3E27449070001DB734 /* WGSLUnitTests.mm */,
+ 1C023D3E27449070001DB734 /* WGSLLexerTests.mm */,
);
path = WGSLUnitTests;
sourceTree = "<group>";
@@ -261,8 +271,13 @@
children = (
1CEBD8072716C2C000A5254D /* AST.h */,
1CEBD8042716BFAB00A5254D /* config.h */,
+ 338BB2D327B6B66C00E066AB /* Lexer.h */,
+ 338BB2D527B6B68700E066AB /* Lexer.cpp */,
+ 338BB2D127B6B63F00E066AB /* SourceSpan.h */,
+ 338BB2CD27B6B60200E066AB /* Token.h */,
+ 338BB2CF27B6B61B00E066AB /* Token.cpp */,
+ 1CEBD7F72716B34400A5254D /* WGSL.h */,
1CEBD8022716BF8200A5254D /* WGSL.cpp */,
- 1CEBD7F72716B34400A5254D /* WGSL.h */,
);
path = WGSL;
sourceTree = "<group>";
@@ -312,6 +327,9 @@
isa = PBXHeadersBuildPhase;
buildActionMask = 2147483647;
files = (
+ 338BB2D227B6B63F00E066AB /* SourceSpan.h in Headers */,
+ 338BB2D427B6B66C00E066AB /* Lexer.h in Headers */,
+ 338BB2CE27B6B60200E066AB /* Token.h in Headers */,
1CEBD7F82716B34400A5254D /* WGSL.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
@@ -381,7 +399,6 @@
isa = PBXProject;
attributes = {
BuildIndependentTargetsInParallel = NO;
- DefaultBuildSystemTypeForWorkspace = Original;
LastUpgradeCheck = 1330;
TargetAttributes = {
1C023D3B27449070001DB734 = {
@@ -459,7 +476,7 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
- 1C023D3F27449070001DB734 /* WGSLUnitTests.mm in Sources */,
+ 1C023D3F27449070001DB734 /* WGSLLexerTests.mm in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -497,7 +514,9 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
+ 338BB2D627B6B68700E066AB /* Lexer.cpp in Sources */,
1CEBD8032716BF8200A5254D /* WGSL.cpp in Sources */,
+ 338BB2D027B6B61B00E066AB /* Token.cpp in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
