I've been working in json_lex() for a little bit to add support for JSONC[0] (patches to come later). In my endeavors to do so, I found json_lex() to be quite beefy. Part of that is due to it being the entry point for lexing when using both the incremental and non-incremental JSON parsers. This leads to large sections of branching that seem to hurt readability.
I've separated each refactor into its own commit such that we could take some, all, or none of them. I know refactoring can be very opinionated. Based on my experience with the upcoming JSONC patches, I have found these changes to allow my future changes to be much more reviewable. Ideally, these changes also make it easier for future readers to understand and contribute in the JSON lexing code. History points to this code being pretty static with the largest change coming when Andrew added the non-recursive (incremental) parser. Maybe others have thought of other ways to make this function a bit more approachable. [0]: https://jsonc.org -- Tristan Partin PostgreSQL Contributors Team AWS (https://aws.amazon.com)
From 156223e995c37c19951e8136033b45847402b0a9 Mon Sep 17 00:00:00 2001 From: Tristan Partin <[email protected]> Date: Mon, 13 Jul 2026 20:16:52 +0000 Subject: [PATCH v1 1/3] Pull some json_lex() incremental code into separate functions The incremental parser and non-incremental parser both go through json_lex(). Within json_lex(), we have a few cases of branching on lex->incremental. We can reduce one of the branches by moving where we instantiate `s`. Additionally, we can reduce the size of the function by pulling the incremental partial token state code into two separate functions: reset_partial_state() and continue_partial_lex(). Signed-off-by: Tristan Partin <[email protected]> --- src/common/jsonapi.c | 501 ++++++++++++++++++++++--------------------- 1 file changed, 260 insertions(+), 241 deletions(-) diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c index d3860197dad..70bf4687c71 100644 --- a/src/common/jsonapi.c +++ b/src/common/jsonapi.c @@ -271,6 +271,8 @@ static td_entry td_parser_table[JSON_NUM_NONTERMINALS][JSON_NUM_TERMINALS] = /* the GOAL production. Not stored in the table, but will be the initial contents of the prediction stack */ static char JSON_PROD_GOAL[] = {JSON_TOKEN_END, JSON_NT_JSON, 0}; +static void reset_partial_state(JsonLexContext *lex); +static JsonParseErrorType continue_partial_lex(JsonLexContext *lex); static inline JsonParseErrorType json_lex_string(JsonLexContext *lex); static inline JsonParseErrorType json_lex_number(JsonLexContext *lex, const char *s, bool *num_err, size_t *total_len); @@ -1570,6 +1572,256 @@ parse_array(JsonLexContext *lex, const JsonSemAction *sem) return JSON_SUCCESS; } +/* + * We just lexed a completed partial token on the last call, so reset everything + */ +static void +reset_partial_state(JsonLexContext *lex) +{ + Assert(lex->incremental); + Assert(lex->inc_state->partial_completed); + + jsonapi_resetStringInfo(&(lex->inc_state->partial_token)); + lex->token_terminator = lex->input; + lex->inc_state->partial_completed = false; +} + +/* + * We have a partial token. Extend it and if completed lex it by a recursive + * call + */ +static JsonParseErrorType +continue_partial_lex(JsonLexContext *lex) +{ + jsonapi_StrValType *ptok; + size_t added = 0; + bool tok_done = false; + JsonLexContext dummy_lex = {0}; + JsonParseErrorType partial_result; + + Assert(lex->incremental); + Assert(lex->inc_state->partial_token.len); + + ptok = &(lex->inc_state->partial_token); + + if (ptok->data[0] == '"') + { + /* + * It's a string. Accumulate characters until we reach an unescaped '"'. + */ + int escapes = 0; + + for (int i = ptok->len - 1; i > 0; i--) + { + /* count the trailing backslashes on the partial token */ + if (ptok->data[i] == '\\') + escapes++; + else + break; + } + + for (size_t i = 0; i < lex->input_length; i++) + { + char c = lex->input[i]; + + jsonapi_appendStringInfoCharMacro(ptok, c); + added++; + if (c == '"' && escapes % 2 == 0) + { + tok_done = true; + break; + } + if (c == '\\') + escapes++; + else + escapes = 0; + } + } + else + { + /* not a string */ + char c = ptok->data[0]; + + if (c == '-' || (c >= '0' && c <= '9')) + { + /* + * Accumulate numeric continuations, respecting JSON number + * grammar: -? int [frac] [exp] + * + * We must track what parts of the number we've already seen so we + * don't over-consume. '.' is valid only once and not after + * 'e'/'E'; 'e'/'E' is valid only once; '+'/'-' are valid only + * immediately after 'e'/'E'. + */ + bool numend = false; + bool seen_dot = false; + bool seen_exp = false; + char prev; + + /* Scan existing partial token for state */ + for (int j = 0; j < ptok->len; j++) + { + char pc = ptok->data[j]; + + if (pc == '.') + seen_dot = true; + else if (pc == 'e' || pc == 'E') + seen_exp = true; + } + prev = ptok->data[ptok->len - 1]; + + for (size_t i = 0; i < lex->input_length && !numend; i++) + { + char cc = lex->input[i]; + + switch (cc) + { + case '+': + case '-': + if (prev != 'e' && prev != 'E') + { + numend = true; + break; + } + jsonapi_appendStringInfoCharMacro(ptok, cc); + added++; + break; + case '.': + if (seen_dot || seen_exp) + { + numend = true; + break; + } + seen_dot = true; + jsonapi_appendStringInfoCharMacro(ptok, cc); + added++; + break; + case 'e': + case 'E': + if (seen_exp) + { + numend = true; + break; + } + seen_exp = true; + jsonapi_appendStringInfoCharMacro(ptok, cc); + added++; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + jsonapi_appendStringInfoCharMacro(ptok, cc); + added++; + break; + default: + numend = true; + } + if (!numend) + prev = cc; + } + } + + /* + * Add any remaining alphanumeric chars. This takes care of the + * {null, false, true} literals as well as any trailing + * alphanumeric junk on non-string tokens. + */ + for (size_t i = added; i < lex->input_length; i++) + { + char cc = lex->input[i]; + + if (JSON_ALPHANUMERIC_CHAR(cc)) + { + jsonapi_appendStringInfoCharMacro(ptok, cc); + added++; + } + else + { + tok_done = true; + break; + } + } + if (added == lex->input_length && lex->inc_state->is_last_chunk) + { + tok_done = true; + } + } + + if (!tok_done) + { + /* We should have consumed the whole chunk in this case. */ + Assert(added == lex->input_length); + + if (!lex->inc_state->is_last_chunk) + return JSON_INCOMPLETE; + + /* json_errdetail() needs access to the accumulated token. */ + lex->token_start = ptok->data; + lex->token_terminator = ptok->data + ptok->len; + return JSON_INVALID_TOKEN; + } + + /* + * Everything up to lex->input[added] has been added to the partial + * token, so move the input past it. + */ + lex->input += added; + lex->input_length -= added; + + dummy_lex.input = dummy_lex.token_terminator = + dummy_lex.line_start = ptok->data; + dummy_lex.line_number = lex->line_number; + dummy_lex.input_length = ptok->len; + dummy_lex.input_encoding = lex->input_encoding; + dummy_lex.incremental = false; + dummy_lex.need_escapes = lex->need_escapes; + dummy_lex.strval = lex->strval; + + partial_result = json_lex(&dummy_lex); + + /* + * We either have a complete token or an error. In either case we need + * to point to the partial token data for the semantic or error + * routines. If it's not an error we'll readjust on the next call to + * json_lex. + */ + lex->token_type = dummy_lex.token_type; + lex->line_number = dummy_lex.line_number; + + /* + * We know the prev_token_terminator must be back in some previous + * piece of input, so we just make it NULL. + */ + lex->prev_token_terminator = NULL; + + /* + * Normally token_start would be ptok->data, but it could be later, + * see json_lex_string's handling of invalid escapes. + */ + lex->token_start = dummy_lex.token_start; + lex->token_terminator = dummy_lex.token_terminator; + if (partial_result == JSON_SUCCESS) + { + /* make sure we've used all the input */ + if (lex->token_terminator - lex->token_start != ptok->len) + { + Assert(false); + return JSON_INVALID_TOKEN; + } + + lex->inc_state->partial_completed = true; + } + + return partial_result; +} + /* * Lex one token from the input stream. * @@ -1598,257 +1850,24 @@ json_lex(JsonLexContext *lex) if (lex->incremental) { if (lex->inc_state->partial_completed) - { - /* - * We just lexed a completed partial token on the last call, so - * reset everything - */ - jsonapi_resetStringInfo(&(lex->inc_state->partial_token)); - lex->token_terminator = lex->input; - lex->inc_state->partial_completed = false; - } + reset_partial_state(lex); #ifdef JSONAPI_USE_PQEXPBUFFER /* Make sure our partial token buffer is valid before using it below. */ if (PQExpBufferDataBroken(lex->inc_state->partial_token)) return JSON_OUT_OF_MEMORY; #endif + + /* + * If we previously stopped lexing because of a partial token, continue + * attempting to lex it + */ + if (lex->inc_state->partial_token.len) + return continue_partial_lex(lex); } s = lex->token_terminator; - if (lex->incremental && lex->inc_state->partial_token.len) - { - /* - * We have a partial token. Extend it and if completed lex it by a - * recursive call - */ - jsonapi_StrValType *ptok = &(lex->inc_state->partial_token); - size_t added = 0; - bool tok_done = false; - JsonLexContext dummy_lex = {0}; - JsonParseErrorType partial_result; - - if (ptok->data[0] == '"') - { - /* - * It's a string. Accumulate characters until we reach an - * unescaped '"'. - */ - int escapes = 0; - - for (int i = ptok->len - 1; i > 0; i--) - { - /* count the trailing backslashes on the partial token */ - if (ptok->data[i] == '\\') - escapes++; - else - break; - } - - for (size_t i = 0; i < lex->input_length; i++) - { - char c = lex->input[i]; - - jsonapi_appendStringInfoCharMacro(ptok, c); - added++; - if (c == '"' && escapes % 2 == 0) - { - tok_done = true; - break; - } - if (c == '\\') - escapes++; - else - escapes = 0; - } - } - else - { - /* not a string */ - char c = ptok->data[0]; - - if (c == '-' || (c >= '0' && c <= '9')) - { - /* - * Accumulate numeric continuations, respecting JSON number - * grammar: -? int [frac] [exp] - * - * We must track what parts of the number we've already seen - * so we don't over-consume. '.' is valid only once and not - * after 'e'/'E'; 'e'/'E' is valid only once; '+'/'-' are - * valid only immediately after 'e'/'E'. - */ - bool numend = false; - bool seen_dot = false; - bool seen_exp = false; - char prev; - - /* Scan existing partial token for state */ - for (int j = 0; j < ptok->len; j++) - { - char pc = ptok->data[j]; - - if (pc == '.') - seen_dot = true; - else if (pc == 'e' || pc == 'E') - seen_exp = true; - } - prev = ptok->data[ptok->len - 1]; - - for (size_t i = 0; i < lex->input_length && !numend; i++) - { - char cc = lex->input[i]; - - switch (cc) - { - case '+': - case '-': - if (prev != 'e' && prev != 'E') - { - numend = true; - break; - } - jsonapi_appendStringInfoCharMacro(ptok, cc); - added++; - break; - case '.': - if (seen_dot || seen_exp) - { - numend = true; - break; - } - seen_dot = true; - jsonapi_appendStringInfoCharMacro(ptok, cc); - added++; - break; - case 'e': - case 'E': - if (seen_exp) - { - numend = true; - break; - } - seen_exp = true; - jsonapi_appendStringInfoCharMacro(ptok, cc); - added++; - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - jsonapi_appendStringInfoCharMacro(ptok, cc); - added++; - break; - default: - numend = true; - } - if (!numend) - prev = cc; - } - } - - /* - * Add any remaining alphanumeric chars. This takes care of the - * {null, false, true} literals as well as any trailing - * alphanumeric junk on non-string tokens. - */ - for (size_t i = added; i < lex->input_length; i++) - { - char cc = lex->input[i]; - - if (JSON_ALPHANUMERIC_CHAR(cc)) - { - jsonapi_appendStringInfoCharMacro(ptok, cc); - added++; - } - else - { - tok_done = true; - break; - } - } - if (added == lex->input_length && - lex->inc_state->is_last_chunk) - { - tok_done = true; - } - } - - if (!tok_done) - { - /* We should have consumed the whole chunk in this case. */ - Assert(added == lex->input_length); - - if (!lex->inc_state->is_last_chunk) - return JSON_INCOMPLETE; - - /* json_errdetail() needs access to the accumulated token. */ - lex->token_start = ptok->data; - lex->token_terminator = ptok->data + ptok->len; - return JSON_INVALID_TOKEN; - } - - /* - * Everything up to lex->input[added] has been added to the partial - * token, so move the input past it. - */ - lex->input += added; - lex->input_length -= added; - - dummy_lex.input = dummy_lex.token_terminator = - dummy_lex.line_start = ptok->data; - dummy_lex.line_number = lex->line_number; - dummy_lex.input_length = ptok->len; - dummy_lex.input_encoding = lex->input_encoding; - dummy_lex.incremental = false; - dummy_lex.need_escapes = lex->need_escapes; - dummy_lex.strval = lex->strval; - - partial_result = json_lex(&dummy_lex); - - /* - * We either have a complete token or an error. In either case we need - * to point to the partial token data for the semantic or error - * routines. If it's not an error we'll readjust on the next call to - * json_lex. - */ - lex->token_type = dummy_lex.token_type; - lex->line_number = dummy_lex.line_number; - - /* - * We know the prev_token_terminator must be back in some previous - * piece of input, so we just make it NULL. - */ - lex->prev_token_terminator = NULL; - - /* - * Normally token_start would be ptok->data, but it could be later, - * see json_lex_string's handling of invalid escapes. - */ - lex->token_start = dummy_lex.token_start; - lex->token_terminator = dummy_lex.token_terminator; - if (partial_result == JSON_SUCCESS) - { - /* make sure we've used all the input */ - if (lex->token_terminator - lex->token_start != ptok->len) - { - Assert(false); - return JSON_INVALID_TOKEN; - } - - lex->inc_state->partial_completed = true; - } - return partial_result; - /* end of partial token processing */ - } - /* Skip leading whitespace. */ while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) { -- Tristan Partin https://tristan.partin.io
From 0abe55f237eafe9ef3520f22ec8454297ecb2919 Mon Sep 17 00:00:00 2001 From: Tristan Partin <[email protected]> Date: Mon, 13 Jul 2026 21:18:46 +0000 Subject: [PATCH v1 2/3] Extract whitespace skip code in json_lex() Signed-off-by: Tristan Partin <[email protected]> --- src/common/jsonapi.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c index 70bf4687c71..36675d97e0b 100644 --- a/src/common/jsonapi.c +++ b/src/common/jsonapi.c @@ -273,6 +273,9 @@ static char JSON_PROD_GOAL[] = {JSON_TOKEN_END, JSON_NT_JSON, 0}; static void reset_partial_state(JsonLexContext *lex); static JsonParseErrorType continue_partial_lex(JsonLexContext *lex); +static inline const char *skip_leading_whitespace(JsonLexContext *lex, + const char *s, + const char *end); static inline JsonParseErrorType json_lex_string(JsonLexContext *lex); static inline JsonParseErrorType json_lex_number(JsonLexContext *lex, const char *s, bool *num_err, size_t *total_len); @@ -1822,6 +1825,28 @@ continue_partial_lex(JsonLexContext *lex) return partial_result; } +/* + * Skip leading whitespace by pointing JsonLexContext::token_start past it + */ +static inline const char * +skip_leading_whitespace(JsonLexContext *lex, + const char *s, + const char *end) +{ + while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) + { + if (*s++ == '\n') + { + ++lex->line_number; + lex->line_start = s; + } + } + + lex->token_start = s; + + return s; +} + /* * Lex one token from the input stream. * @@ -1868,16 +1893,7 @@ json_lex(JsonLexContext *lex) s = lex->token_terminator; - /* Skip leading whitespace. */ - while (s < end && (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) - { - if (*s++ == '\n') - { - ++lex->line_number; - lex->line_start = s; - } - } - lex->token_start = s; + s = skip_leading_whitespace(lex, s, end); /* Determine token type. */ if (s >= end) -- Tristan Partin https://tristan.partin.io
From a9de08cdc3a762c7ecff30d60c8ce98cebed24cb Mon Sep 17 00:00:00 2001 From: Tristan Partin <[email protected]> Date: Mon, 13 Jul 2026 21:19:50 +0000 Subject: [PATCH v1 3/3] Extract code for determining token type into a separate function Signed-off-by: Tristan Partin <[email protected]> --- src/common/jsonapi.c | 278 +++++++++++++++++++++++-------------------- 1 file changed, 147 insertions(+), 131 deletions(-) diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c index 36675d97e0b..137babee5e9 100644 --- a/src/common/jsonapi.c +++ b/src/common/jsonapi.c @@ -276,6 +276,8 @@ static JsonParseErrorType continue_partial_lex(JsonLexContext *lex); static inline const char *skip_leading_whitespace(JsonLexContext *lex, const char *s, const char *end); +static inline JsonParseErrorType determine_token_type(JsonLexContext *lex, const char *s, + const char *end); static inline JsonParseErrorType json_lex_string(JsonLexContext *lex); static inline JsonParseErrorType json_lex_number(JsonLexContext *lex, const char *s, bool *num_err, size_t *total_len); @@ -1847,6 +1849,148 @@ skip_leading_whitespace(JsonLexContext *lex, return s; } +/* + * Determine the type of the next token in the input stream + */ +static JsonParseErrorType +determine_token_type(JsonLexContext *lex, const char *s, const char *end) +{ + JsonParseErrorType result; + + if (s >= end) + { + lex->token_start = NULL; + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s; + lex->token_type = JSON_TOKEN_END; + + return JSON_SUCCESS; + } + + switch (*s) + { + /* Single-character token, some kind of punctuation mark. */ + case '{': + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + lex->token_type = JSON_TOKEN_OBJECT_START; + break; + case '}': + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + lex->token_type = JSON_TOKEN_OBJECT_END; + break; + case '[': + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + lex->token_type = JSON_TOKEN_ARRAY_START; + break; + case ']': + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + lex->token_type = JSON_TOKEN_ARRAY_END; + break; + case ',': + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + lex->token_type = JSON_TOKEN_COMMA; + break; + case ':': + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + lex->token_type = JSON_TOKEN_COLON; + break; + case '"': + /* string */ + result = json_lex_string(lex); + if (result != JSON_SUCCESS) + return result; + lex->token_type = JSON_TOKEN_STRING; + break; + case '-': + /* Negative number. */ + result = json_lex_number(lex, s + 1, NULL, NULL); + if (result != JSON_SUCCESS) + return result; + lex->token_type = JSON_TOKEN_NUMBER; + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + /* Positive number. */ + result = json_lex_number(lex, s, NULL, NULL); + if (result != JSON_SUCCESS) + return result; + lex->token_type = JSON_TOKEN_NUMBER; + break; + default: + { + const char *p; + + /* + * We're not dealing with a string, number, legal + * punctuation mark, or end of string. The only legal + * tokens we might find here are true, false, and null, + * but for error reporting purposes we scan until we see a + * non-alphanumeric character. That way, we can report + * the whole word as an unexpected token, rather than just + * some unintuitive prefix thereof. + */ + for (p = s; p < end && JSON_ALPHANUMERIC_CHAR(*p); p++) + /* skip */ ; + + /* + * We got some sort of unexpected punctuation or an + * otherwise unexpected character, so just complain about + * that one character. + */ + if (p == s) + { + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = s + 1; + return JSON_INVALID_TOKEN; + } + + if (lex->incremental && !lex->inc_state->is_last_chunk && + p == lex->input + lex->input_length) + { + jsonapi_appendBinaryStringInfo(&(lex->inc_state->partial_token), s, end - s); + return JSON_INCOMPLETE; + } + + /* + * We've got a real alphanumeric token here. If it + * happens to be true, false, or null, all is well. If + * not, error out. + */ + lex->prev_token_terminator = lex->token_terminator; + lex->token_terminator = p; + if (p - s == 4) + { + if (memcmp(s, "true", 4) == 0) + lex->token_type = JSON_TOKEN_TRUE; + else if (memcmp(s, "null", 4) == 0) + lex->token_type = JSON_TOKEN_NULL; + else + return JSON_INVALID_TOKEN; + } + else if (p - s == 5 && memcmp(s, "false", 5) == 0) + lex->token_type = JSON_TOKEN_FALSE; + else + return JSON_INVALID_TOKEN; + } + } /* end of switch */ + + return JSON_SUCCESS; +} + /* * Lex one token from the input stream. * @@ -1895,137 +2039,9 @@ json_lex(JsonLexContext *lex) s = skip_leading_whitespace(lex, s, end); - /* Determine token type. */ - if (s >= end) - { - lex->token_start = NULL; - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s; - lex->token_type = JSON_TOKEN_END; - } - else - { - switch (*s) - { - /* Single-character token, some kind of punctuation mark. */ - case '{': - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s + 1; - lex->token_type = JSON_TOKEN_OBJECT_START; - break; - case '}': - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s + 1; - lex->token_type = JSON_TOKEN_OBJECT_END; - break; - case '[': - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s + 1; - lex->token_type = JSON_TOKEN_ARRAY_START; - break; - case ']': - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s + 1; - lex->token_type = JSON_TOKEN_ARRAY_END; - break; - case ',': - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s + 1; - lex->token_type = JSON_TOKEN_COMMA; - break; - case ':': - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s + 1; - lex->token_type = JSON_TOKEN_COLON; - break; - case '"': - /* string */ - result = json_lex_string(lex); - if (result != JSON_SUCCESS) - return result; - lex->token_type = JSON_TOKEN_STRING; - break; - case '-': - /* Negative number. */ - result = json_lex_number(lex, s + 1, NULL, NULL); - if (result != JSON_SUCCESS) - return result; - lex->token_type = JSON_TOKEN_NUMBER; - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - /* Positive number. */ - result = json_lex_number(lex, s, NULL, NULL); - if (result != JSON_SUCCESS) - return result; - lex->token_type = JSON_TOKEN_NUMBER; - break; - default: - { - const char *p; - - /* - * We're not dealing with a string, number, legal - * punctuation mark, or end of string. The only legal - * tokens we might find here are true, false, and null, - * but for error reporting purposes we scan until we see a - * non-alphanumeric character. That way, we can report - * the whole word as an unexpected token, rather than just - * some unintuitive prefix thereof. - */ - for (p = s; p < end && JSON_ALPHANUMERIC_CHAR(*p); p++) - /* skip */ ; - - /* - * We got some sort of unexpected punctuation or an - * otherwise unexpected character, so just complain about - * that one character. - */ - if (p == s) - { - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = s + 1; - return JSON_INVALID_TOKEN; - } - - if (lex->incremental && !lex->inc_state->is_last_chunk && - p == lex->input + lex->input_length) - { - jsonapi_appendBinaryStringInfo(&(lex->inc_state->partial_token), s, end - s); - return JSON_INCOMPLETE; - } - - /* - * We've got a real alphanumeric token here. If it - * happens to be true, false, or null, all is well. If - * not, error out. - */ - lex->prev_token_terminator = lex->token_terminator; - lex->token_terminator = p; - if (p - s == 4) - { - if (memcmp(s, "true", 4) == 0) - lex->token_type = JSON_TOKEN_TRUE; - else if (memcmp(s, "null", 4) == 0) - lex->token_type = JSON_TOKEN_NULL; - else - return JSON_INVALID_TOKEN; - } - else if (p - s == 5 && memcmp(s, "false", 5) == 0) - lex->token_type = JSON_TOKEN_FALSE; - else - return JSON_INVALID_TOKEN; - } - } /* end of switch */ - } + result = determine_token_type(lex, s, end); + if (result != JSON_SUCCESS) + return result; if (lex->incremental && lex->token_type == JSON_TOKEN_END && !lex->inc_state->is_last_chunk) return JSON_INCOMPLETE; -- Tristan Partin https://tristan.partin.io
