Diff
Modified: trunk/Source/_javascript_Core/API/JSValueRef.cpp (89183 => 89184)
--- trunk/Source/_javascript_Core/API/JSValueRef.cpp 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/API/JSValueRef.cpp 2011-06-18 04:25:57 UTC (rev 89184)
@@ -234,7 +234,8 @@
{
ExecState* exec = toJS(ctx);
APIEntryShim entryShim(exec);
- LiteralParser parser(exec, string->ustring(), LiteralParser::StrictJSON);
+ UString str = string->ustring();
+ LiteralParser parser(exec, str.characters(), str.length(), LiteralParser::StrictJSON);
return toRef(exec, parser.tryLiteralParse());
}
Modified: trunk/Source/_javascript_Core/ChangeLog (89183 => 89184)
--- trunk/Source/_javascript_Core/ChangeLog 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/ChangeLog 2011-06-18 04:25:57 UTC (rev 89184)
@@ -1,3 +1,50 @@
+2011-06-17 Oliver Hunt <[email protected]>
+
+ Reviewed by Gavin Barraclough.
+
+ JSONP is unnecessarily slow
+ https://bugs.webkit.org/show_bug.cgi?id=62920
+
+ JSONP has unfortunately become a fairly common idiom online, yet
+ it triggers very poor performance in JSC as we end up doing codegen
+ for a large number of property accesses that will
+ * only be run once, so the vast amount of logic we dump to handle
+ caching of accesses is unnecessary.
+ * We are doing codegen that is directly proportional to just
+ creating the object in the first place.
+
+ This patch extends the use of the literal parser to JSONP-like structures
+ in global code, handling a number of different forms I have seen online.
+ In an extreme case this improves performance of JSONP by more than 2x
+ due to removal of code generation and execution time, and a few optimisations
+ that I made to the parser itself.
+
+ * API/JSValueRef.cpp:
+ (JSValueMakeFromJSONString):
+ * interpreter/Interpreter.cpp:
+ (JSC::Interpreter::callEval):
+ (JSC::Interpreter::execute):
+ * parser/Lexer.cpp:
+ (JSC::Lexer::isKeyword):
+ * parser/Lexer.h:
+ * runtime/JSGlobalObjectFunctions.cpp:
+ (JSC::globalFuncEval):
+ * runtime/JSONObject.cpp:
+ (JSC::JSONProtoFuncParse):
+ * runtime/LiteralParser.cpp:
+ (JSC::LiteralParser::tryJSONPParse):
+ (JSC::LiteralParser::makeIdentifier):
+ (JSC::LiteralParser::Lexer::lex):
+ (JSC::LiteralParser::Lexer::next):
+ (JSC::isSafeStringCharacter):
+ (JSC::LiteralParser::Lexer::lexString):
+ (JSC::LiteralParser::Lexer::lexNumber):
+ (JSC::LiteralParser::parse):
+ * runtime/LiteralParser.h:
+ (JSC::LiteralParser::LiteralParser):
+ (JSC::LiteralParser::tryLiteralParse):
+ (JSC::LiteralParser::Lexer::Lexer):
+
2011-06-17 Geoffrey Garen <[email protected]>
Reviewed by Oliver Hunt.
Modified: trunk/Source/_javascript_Core/interpreter/Interpreter.cpp (89183 => 89184)
--- trunk/Source/_javascript_Core/interpreter/Interpreter.cpp 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/interpreter/Interpreter.cpp 2011-06-18 04:25:57 UTC (rev 89184)
@@ -396,7 +396,7 @@
if (!codeBlock->isStrictMode()) {
// FIXME: We can use the preparser in strict mode, we just need additional logic
// to prevent duplicates.
- LiteralParser preparser(callFrame, programSource, LiteralParser::NonStrictJSON);
+ LiteralParser preparser(callFrame, programSource.characters(), programSource.length(), LiteralParser::NonStrictJSON);
if (JSValue parsedObject = preparser.tryLiteralParse())
return parsedObject;
}
@@ -744,6 +744,74 @@
return checkedReturn(throwStackOverflowError(callFrame));
DynamicGlobalObjectScope globalObjectScope(*scopeChain->globalData, scopeChain->globalObject.get());
+ LiteralParser literalParser(callFrame, program->source().data(), program->source().length(), LiteralParser::JSONP);
+ Vector<LiteralParser::JSONPData> JSONPData;
+ if (literalParser.tryJSONPParse(JSONPData)) {
+ JSGlobalObject* globalObject = scopeChain->globalObject.get();
+ JSValue result;
+ for (unsigned entry = 0; entry < JSONPData.size(); entry++) {
+ Vector<LiteralParser::JSONPPathEntry> JSONPPath;
+ JSONPPath.swap(JSONPData[entry].m_path);
+ JSValue JSONPValue = JSONPData[entry].m_value.get();
+ if (JSONPPath.size() == 1 && JSONPPath[0].m_type == LiteralParser::JSONPPathEntryTypeDeclare) {
+ if (globalObject->hasProperty(callFrame, JSONPPath[0].m_pathEntryName)) {
+ PutPropertySlot slot;
+ globalObject->put(callFrame, JSONPPath[0].m_pathEntryName, JSONPValue, slot);
+ } else
+ globalObject->putWithAttributes(callFrame, JSONPPath[0].m_pathEntryName, JSONPValue, DontEnum | DontDelete);
+ // var declarations return undefined
+ result = jsUndefined();
+ continue;
+ }
+ JSValue baseObject(globalObject);
+ for (unsigned i = 0; i < JSONPPath.size() - 1; i++) {
+ ASSERT(JSONPPath[i].m_type != LiteralParser::JSONPPathEntryTypeDeclare);
+ switch (JSONPPath[i].m_type) {
+ case LiteralParser::JSONPPathEntryTypeDot: {
+ if (i == 0) {
+ PropertySlot slot(globalObject);
+ if (!globalObject->getPropertySlot(callFrame, JSONPPath[i].m_pathEntryName, slot))
+ return throwError(callFrame, createUndefinedVariableError(globalObject->globalExec(), JSONPPath[i].m_pathEntryName));
+
+ } else
+ baseObject = baseObject.get(callFrame, JSONPPath[i].m_pathEntryName);
+ if (callFrame->hadException())
+ return jsUndefined();
+ continue;
+ }
+ case LiteralParser::JSONPPathEntryTypeLookup: {
+ baseObject = baseObject.get(callFrame, JSONPPath[i].m_pathIndex);
+ if (callFrame->hadException())
+ return jsUndefined();
+ continue;
+ }
+ default:
+ ASSERT_NOT_REACHED();
+ return jsUndefined();
+ }
+ }
+ PutPropertySlot slot;
+ switch (JSONPPath.last().m_type) {
+ case LiteralParser::JSONPPathEntryTypeDot: {
+ baseObject.put(callFrame, JSONPPath.last().m_pathEntryName, JSONPValue, slot);
+ if (callFrame->hadException())
+ return jsUndefined();
+ break;
+ }
+ case LiteralParser::JSONPPathEntryTypeLookup: {
+ baseObject.put(callFrame, JSONPPath.last().m_pathIndex, JSONPValue);
+ if (callFrame->hadException())
+ return jsUndefined();
+ break;
+ }
+ default:
+ ASSERT_NOT_REACHED();
+ return jsUndefined();
+ }
+ result = JSONPValue;
+ }
+ return result;
+ }
JSObject* error = program->compile(callFrame, scopeChain);
if (error)
Modified: trunk/Source/_javascript_Core/parser/Lexer.cpp (89183 => 89184)
--- trunk/Source/_javascript_Core/parser/Lexer.cpp 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/parser/Lexer.cpp 2011-06-18 04:25:57 UTC (rev 89184)
@@ -481,6 +481,11 @@
return IDENT;
}
+bool Lexer::isKeyword(const Identifier& ident)
+{
+ return m_keywordTable.entry(m_globalData, ident);
+}
+
template <bool shouldBuildStrings> ALWAYS_INLINE bool Lexer::parseString(JSTokenData* tokenData, bool strictMode)
{
int stringQuoteCharacter = m_current;
Modified: trunk/Source/_javascript_Core/parser/Lexer.h (89183 => 89184)
--- trunk/Source/_javascript_Core/parser/Lexer.h 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/parser/Lexer.h 2011-06-18 04:25:57 UTC (rev 89184)
@@ -90,7 +90,9 @@
SourceProvider* sourceProvider() const { return m_source->provider(); }
JSTokenType lexExpectIdentifier(JSTokenData*, JSTokenInfo*, unsigned, bool strictMode);
-
+
+ bool isKeyword(const Identifier&);
+
private:
friend class JSGlobalData;
Modified: trunk/Source/_javascript_Core/runtime/JSGlobalObjectFunctions.cpp (89183 => 89184)
--- trunk/Source/_javascript_Core/runtime/JSGlobalObjectFunctions.cpp 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/runtime/JSGlobalObjectFunctions.cpp 2011-06-18 04:25:57 UTC (rev 89184)
@@ -445,7 +445,7 @@
UString s = x.toString(exec);
- LiteralParser preparser(exec, s, LiteralParser::NonStrictJSON);
+ LiteralParser preparser(exec, s.characters(), s.length(), LiteralParser::NonStrictJSON);
if (JSValue parsedObject = preparser.tryLiteralParse())
return JSValue::encode(parsedObject);
Modified: trunk/Source/_javascript_Core/runtime/JSONObject.cpp (89183 => 89184)
--- trunk/Source/_javascript_Core/runtime/JSONObject.cpp 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/runtime/JSONObject.cpp 2011-06-18 04:25:57 UTC (rev 89184)
@@ -816,7 +816,7 @@
return JSValue::encode(jsNull());
LocalScope scope(exec->globalData());
- LiteralParser jsonParser(exec, source, LiteralParser::StrictJSON);
+ LiteralParser jsonParser(exec, source.characters(), source.length(), LiteralParser::StrictJSON);
JSValue unfiltered = jsonParser.tryLiteralParse();
if (!unfiltered)
return throwVMError(exec, createSyntaxError(exec, "Unable to parse JSON string"));
Modified: trunk/Source/_javascript_Core/runtime/LiteralParser.cpp (89183 => 89184)
--- trunk/Source/_javascript_Core/runtime/LiteralParser.cpp 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/runtime/LiteralParser.cpp 2011-06-18 04:25:57 UTC (rev 89184)
@@ -42,8 +42,91 @@
return c == ' ' || c == 0x9 || c == 0xA || c == 0xD;
}
-LiteralParser::TokenType LiteralParser::Lexer::lex(LiteralParserToken& token)
+bool LiteralParser::tryJSONPParse(Vector<JSONPData>& results)
{
+ if (m_lexer.next() != TokIdentifier)
+ return false;
+ do {
+ Vector<JSONPPathEntry> path;
+ // Unguarded next to start off the lexer
+ Identifier name = Identifier(m_exec, m_lexer.currentToken().start, m_lexer.currentToken().end - m_lexer.currentToken().start);
+ JSONPPathEntry entry;
+ if (name == m_exec->globalData().propertyNames->varKeyword) {
+ if (m_lexer.next() != TokIdentifier)
+ return false;
+ entry.m_type = JSONPPathEntryTypeDeclare;
+ entry.m_pathEntryName = Identifier(m_exec, m_lexer.currentToken().start, m_lexer.currentToken().end - m_lexer.currentToken().start);
+ path.append(entry);
+ } else {
+ entry.m_type = JSONPPathEntryTypeDot;
+ entry.m_pathEntryName = Identifier(m_exec, m_lexer.currentToken().start, m_lexer.currentToken().end - m_lexer.currentToken().start);
+ path.append(entry);
+ }
+ if (m_exec->globalData().lexer->isKeyword(entry.m_pathEntryName))
+ return false;
+ TokenType tokenType = m_lexer.next();
+ while (tokenType != TokAssign) {
+ switch (tokenType) {
+ case TokLBracket: {
+ entry.m_type = JSONPPathEntryTypeLookup;
+ if (m_lexer.next() != TokNumber)
+ return false;
+ double doubleIndex = m_lexer.currentToken().numberToken;
+ int index = (int)doubleIndex;
+ if (index != doubleIndex || index < 0)
+ return false;
+ entry.m_pathIndex = index;
+ if (m_lexer.next() != TokRBracket)
+ return false;
+ break;
+ }
+ case TokDot: {
+ entry.m_type = JSONPPathEntryTypeDot;
+ if (m_lexer.next() != TokIdentifier)
+ return false;
+ entry.m_pathEntryName = Identifier(m_exec, m_lexer.currentToken().start, m_lexer.currentToken().end - m_lexer.currentToken().start);
+ break;
+ }
+ default:
+ return false;
+ }
+ path.append(entry);
+ tokenType = m_lexer.next();
+ }
+ m_lexer.next();
+ results.append(JSONPData());
+ results.last().m_value.set(m_exec->globalData(), parse(StartParseExpression));
+ if (!results.last().m_value)
+ return false;
+ results.last().m_path.swap(path);
+ if (m_lexer.currentToken().type != TokSemi)
+ break;
+ m_lexer.next();
+ } while (m_lexer.currentToken().type == TokIdentifier);
+ return m_lexer.currentToken().type == TokEnd;
+}
+
+ALWAYS_INLINE const Identifier LiteralParser::makeIdentifier(const UChar* characters, size_t length)
+{
+ if (!length)
+ return m_exec->globalData().propertyNames->emptyIdentifier;
+ if (characters[0] >= MaximumCachableCharacter)
+ return Identifier(&m_exec->globalData(), characters, length);
+
+ if (length == 1) {
+ if (!m_shortIdentifiers[characters[0]].isNull())
+ return m_shortIdentifiers[characters[0]];
+ m_shortIdentifiers[characters[0]] = Identifier(&m_exec->globalData(), characters, length);
+ return m_shortIdentifiers[characters[0]];
+ }
+ if (!m_recentIdentifiers[characters[0]].isNull() && Identifier::equal(m_recentIdentifiers[characters[0]].impl(), characters, length))
+ return m_recentIdentifiers[characters[0]];
+ m_recentIdentifiers[characters[0]] = Identifier(&m_exec->globalData(), characters, length);
+ return m_recentIdentifiers[characters[0]];
+}
+
+template <LiteralParser::ParserMode mode> LiteralParser::TokenType LiteralParser::Lexer::lex(LiteralParserToken& token)
+{
while (m_ptr < m_end && isJSONWhiteSpace(*m_ptr))
++m_ptr;
@@ -89,9 +172,7 @@
token.end = ++m_ptr;
return TokColon;
case '"':
- if (m_mode == StrictJSON)
- return lexString<StrictJSON>(token);
- return lexString<NonStrictJSON>(token);
+ return lexString<mode, '"'>(token);
case 't':
if (m_end - m_ptr >= 4 && m_ptr[1] == 'r' && m_ptr[2] == 'u' && m_ptr[3] == 'e') {
m_ptr += 4;
@@ -115,7 +196,7 @@
token.end = m_ptr;
return TokNull;
}
- break;
+ break;
case '-':
case '0':
case '1':
@@ -129,27 +210,69 @@
case '9':
return lexNumber(token);
}
+ if (m_ptr < m_end) {
+ if (*m_ptr == '.') {
+ token.type = TokDot;
+ token.end = ++m_ptr;
+ return TokDot;
+ }
+ if (*m_ptr == '=') {
+ token.type = TokAssign;
+ token.end = ++m_ptr;
+ return TokAssign;
+ }
+ if (*m_ptr == ';') {
+ token.type = TokSemi;
+ token.end = ++m_ptr;
+ return TokAssign;
+ }
+ if (isASCIIAlpha(*m_ptr) || *m_ptr == '_' || *m_ptr == '$') {
+ while (m_ptr < m_end && (isASCIIAlphanumeric(*m_ptr) || *m_ptr == '_' || *m_ptr == '$'))
+ m_ptr++;
+ token.stringToken = token.start;
+ token.stringLength = m_ptr - token.start;
+ token.type = TokIdentifier;
+ token.end = m_ptr;
+ return TokIdentifier;
+ }
+ if (*m_ptr == '\'') {
+ if (mode == StrictJSON)
+ return TokError;
+ return lexString<mode, '\''>(token);
+ }
+ }
return TokError;
}
-template <LiteralParser::ParserMode mode> static inline bool isSafeStringCharacter(UChar c)
+LiteralParser::TokenType LiteralParser::Lexer::next()
{
- return (c >= ' ' && (mode == LiteralParser::StrictJSON || c <= 0xff) && c != '\\' && c != '"') || c == '\t';
+ if (m_mode == NonStrictJSON)
+ return lex<NonStrictJSON>(m_currentToken);
+ if (m_mode == JSONP)
+ return lex<JSONP>(m_currentToken);
+ return lex<StrictJSON>(m_currentToken);
}
+template <LiteralParser::ParserMode mode, UChar terminator> static inline bool isSafeStringCharacter(UChar c)
+{
+ return (c >= ' ' && (mode == LiteralParser::StrictJSON || c <= 0xff) && c != '\\' && c != terminator) || c == '\t';
+}
+
// "inline" is required here to help WINSCW compiler resolve specialized argument in templated functions.
-template <LiteralParser::ParserMode mode> inline LiteralParser::TokenType LiteralParser::Lexer::lexString(LiteralParserToken& token)
+template <LiteralParser::ParserMode mode, UChar terminator> inline LiteralParser::TokenType LiteralParser::Lexer::lexString(LiteralParserToken& token)
{
++m_ptr;
- const UChar* runStart;
+ const UChar* runStart = m_ptr;
UStringBuilder builder;
do {
runStart = m_ptr;
- while (m_ptr < m_end && isSafeStringCharacter<mode>(*m_ptr))
+ while (m_ptr < m_end && isSafeStringCharacter<mode, terminator>(*m_ptr))
++m_ptr;
- if (runStart < m_ptr)
+ if (builder.length())
builder.append(runStart, m_ptr - runStart);
- if ((mode == StrictJSON) && m_ptr < m_end && *m_ptr == '\\') {
+ if ((mode != NonStrictJSON) && m_ptr < m_end && *m_ptr == '\\') {
+ if (builder.isEmpty() && runStart < m_ptr)
+ builder.append(runStart, m_ptr - runStart);
++m_ptr;
if (m_ptr >= m_end)
return TokError;
@@ -199,15 +322,28 @@
break;
default:
+ if (*m_ptr == '\'' && mode != StrictJSON) {
+ builder.append('\'');
+ m_ptr++;
+ break;
+ }
return TokError;
}
}
- } while ((mode == StrictJSON) && m_ptr != runStart && (m_ptr < m_end) && *m_ptr != '"');
+ } while ((mode != NonStrictJSON) && m_ptr != runStart && (m_ptr < m_end) && *m_ptr != terminator);
- if (m_ptr >= m_end || *m_ptr != '"')
+ if (m_ptr >= m_end || *m_ptr != terminator)
return TokError;
- token.stringToken = builder.toUString();
+ if (builder.isEmpty()) {
+ token.stringBuffer = UString();
+ token.stringToken = runStart;
+ token.stringLength = m_ptr - runStart;
+ } else {
+ token.stringBuffer = builder.toUString();
+ token.stringToken = token.stringBuffer.characters();
+ token.stringLength = token.stringBuffer.length();
+ }
token.type = TokString;
token.end = ++m_ptr;
return TokString;
@@ -253,6 +389,22 @@
++m_ptr;
while (m_ptr < m_end && isASCIIDigit(*m_ptr))
++m_ptr;
+ } else if (m_ptr < m_end && (*m_ptr != 'e' && *m_ptr != 'E') && (m_ptr - token.start) < 10) {
+ int result = 0;
+ token.type = TokNumber;
+ token.end = m_ptr;
+ const UChar* digit = token.start;
+ int negative = 1;
+ if (*digit == '-') {
+ negative = -1;
+ digit++;
+ }
+
+ while (digit < m_ptr)
+ result = result * 10 + (*digit++) - '0';
+ result *= negative;
+ token.numberToken = result;
+ return TokNumber;
}
// ([eE][+-]? [0-9]+)?
@@ -337,7 +489,7 @@
objectStack.append(object);
TokenType type = m_lexer.next();
- if (type == TokString) {
+ if (type == TokString || (m_mode != StrictJSON && type == TokIdentifier)) {
Lexer::LiteralParserToken identifierToken = m_lexer.currentToken();
// Check for colon
@@ -345,10 +497,11 @@
return JSValue();
m_lexer.next();
- identifierStack.append(Identifier(m_exec, identifierToken.stringToken));
+ identifierStack.append(makeIdentifier(identifierToken.stringToken, identifierToken.stringLength));
stateStack.append(DoParseObjectEndExpression);
goto startParseExpression;
- } else if (type != TokRBrace)
+ }
+ if (type != TokRBrace)
return JSValue();
m_lexer.next();
lastValue = objectStack.last();
@@ -358,7 +511,7 @@
doParseObjectStartExpression:
case DoParseObjectStartExpression: {
TokenType type = m_lexer.next();
- if (type != TokString)
+ if (type != TokString && (m_mode == StrictJSON || type != TokIdentifier))
return JSValue();
Lexer::LiteralParserToken identifierToken = m_lexer.currentToken();
@@ -367,7 +520,7 @@
return JSValue();
m_lexer.next();
- identifierStack.append(Identifier(m_exec, identifierToken.stringToken));
+ identifierStack.append(makeIdentifier(identifierToken.stringToken, identifierToken.stringLength));
stateStack.append(DoParseObjectEndExpression);
goto startParseExpression;
}
@@ -394,7 +547,7 @@
case TokString: {
Lexer::LiteralParserToken stringToken = m_lexer.currentToken();
m_lexer.next();
- lastValue = jsString(m_exec, stringToken.stringToken);
+ lastValue = jsString(m_exec, makeIdentifier(stringToken.stringToken, stringToken.stringLength).ustring());
break;
}
case TokNumber: {
Modified: trunk/Source/_javascript_Core/runtime/LiteralParser.h (89183 => 89184)
--- trunk/Source/_javascript_Core/runtime/LiteralParser.h 2011-06-18 02:38:45 UTC (rev 89183)
+++ trunk/Source/_javascript_Core/runtime/LiteralParser.h 2011-06-18 04:25:57 UTC (rev 89184)
@@ -26,6 +26,7 @@
#ifndef LiteralParser_h
#define LiteralParser_h
+#include "Identifier.h"
#include "JSGlobalObjectFunctions.h"
#include "JSValue.h"
#include "UString.h"
@@ -34,10 +35,10 @@
class LiteralParser {
public:
- typedef enum { StrictJSON, NonStrictJSON } ParserMode;
- LiteralParser(ExecState* exec, const UString& s, ParserMode mode)
+ typedef enum { StrictJSON, NonStrictJSON, JSONP } ParserMode;
+ LiteralParser(ExecState* exec, const UChar* characters, unsigned length, ParserMode mode)
: m_exec(exec)
- , m_lexer(s, mode)
+ , m_lexer(characters, length, mode)
, m_mode(mode)
{
}
@@ -46,10 +47,32 @@
{
m_lexer.next();
JSValue result = parse(m_mode == StrictJSON ? StartParseExpression : StartParseStatement);
+ if (m_lexer.currentToken().type == TokSemi)
+ m_lexer.next();
if (m_lexer.currentToken().type != TokEnd)
return JSValue();
return result;
}
+
+ enum JSONPPathEntryType {
+ JSONPPathEntryTypeDeclare, // var pathEntryName = JSON
+ JSONPPathEntryTypeDot, // <prior entries>.pathEntryName = JSON
+ JSONPPathEntryTypeLookup // <prior entries>[pathIndex] = JSON
+ };
+
+ struct JSONPPathEntry {
+ JSONPPathEntryType m_type;
+ Identifier m_pathEntryName;
+ int m_pathIndex;
+ };
+
+ struct JSONPData {
+ Vector<JSONPPathEntry> m_path;
+ Strong<Unknown> m_value;
+ };
+
+ bool tryJSONPParse(Vector<JSONPData>&);
+
private:
enum ParserState { StartParseObject, StartParseArray, StartParseExpression,
StartParseStatement, StartParseStatementEndStatement,
@@ -58,29 +81,31 @@
enum TokenType { TokLBracket, TokRBracket, TokLBrace, TokRBrace,
TokString, TokIdentifier, TokNumber, TokColon,
TokLParen, TokRParen, TokComma, TokTrue, TokFalse,
- TokNull, TokEnd, TokError };
-
+ TokNull, TokEnd, TokDot, TokAssign, TokSemi, TokError };
+
class Lexer {
public:
struct LiteralParserToken {
TokenType type;
const UChar* start;
const UChar* end;
- UString stringToken;
- double numberToken;
+ UString stringBuffer;
+ union {
+ double numberToken;
+ struct {
+ const UChar* stringToken;
+ int stringLength;
+ };
+ };
};
- Lexer(const UString& s, ParserMode mode)
- : m_string(s)
- , m_mode(mode)
- , m_ptr(s.characters())
- , m_end(s.characters() + s.length())
+ Lexer(const UChar* characters, unsigned length, ParserMode mode)
+ : m_mode(mode)
+ , m_ptr(characters)
+ , m_end(characters + length)
{
}
- TokenType next()
- {
- return lex(m_currentToken);
- }
+ TokenType next();
const LiteralParserToken& currentToken()
{
@@ -88,9 +113,9 @@
}
private:
- TokenType lex(LiteralParserToken&);
- template <ParserMode mode> TokenType lexString(LiteralParserToken&);
- TokenType lexNumber(LiteralParserToken&);
+ template <ParserMode mode> TokenType lex(LiteralParserToken&);
+ template <ParserMode mode, UChar terminator> ALWAYS_INLINE TokenType lexString(LiteralParserToken&);
+ ALWAYS_INLINE TokenType lexNumber(LiteralParserToken&);
LiteralParserToken m_currentToken;
UString m_string;
ParserMode m_mode;
@@ -104,7 +129,12 @@
ExecState* m_exec;
LiteralParser::Lexer m_lexer;
ParserMode m_mode;
+ static unsigned const MaximumCachableCharacter = 128;
+ FixedArray<Identifier, MaximumCachableCharacter> m_shortIdentifiers;
+ FixedArray<Identifier, MaximumCachableCharacter> m_recentIdentifiers;
+ ALWAYS_INLINE const Identifier makeIdentifier(const UChar* characters, size_t length);
};
+
}
#endif
