I wrote:
> I found a spot that seemed like a reasonable place, and added some
> coverage of the point. Updated patch attached.
Doc patch pushed.
> It seems to me that there are some discrepancies between what the spec
> says and what jsonpath_scan.l actually does, so maybe we should take a
> hard look at that code too. The biggest issue is that jsonpath_scan.l
> seems to allow single- and double-quoted strings interchangeably, which is
> OK per ECMAScript, but then the SQL/JSON spec seems to be saying that only
> double-quoted strings are allowed. I'd rather be conservative about this
> than get out in front of the spec and use syntax space that they might do
> something else with someday.
The attached proposed patch makes these changes:
1. Remove support for single-quoted literals in jsonpath.
2. Treat an unrecognized escape (e.g., "\z") as meaning the escaped
character, rather than throwing an error.
3. A few cosmetic adjustments to make the jsonpath_scan code shorter and
clearer (IMHO).
As for #1, although the SQL/JSON tech report does reference ECMAScript
which allows both single- and double-quoted strings, it seems to me
that their intent is to allow only the double-quoted variant. They
specifically reference JSON string literals at one point, and of course
JSON only allows double-quoted. Also, all of their discussion and
examples use double-quoted. Plus you'd have to be pretty nuts to want
to use single-quoted when writing a jsonpath string literal inside a SQL
literal (and the tech report seems to contemplate that jsonpaths MUST be
string literals, though of course our implementation does not require
that).
As for #2, the existing code throws an error, but this is contrary
to clear statements in every single one of the relevant standards.
regards, tom lane
diff --git a/src/backend/utils/adt/jsonpath_scan.l b/src/backend/utils/adt/jsonpath_scan.l
index 2165ffc..245255f 100644
--- a/src/backend/utils/adt/jsonpath_scan.l
+++ b/src/backend/utils/adt/jsonpath_scan.l
@@ -59,24 +59,23 @@ fprintf_to_ereport(const char *fmt, const char *msg)
%option noyyfree
/*
- * We use exclusive states for quoted, signle-quoted and non-quoted strings,
- * quoted variable names and C-tyle comments.
+ * We use exclusive states for quoted and non-quoted strings,
+ * quoted variable names and C-style comments.
* Exclusive states:
* <xq> - quoted strings
* <xnq> - non-quoted strings
* <xvq> - quoted variable names
- * <xsq> - single-quoted strings
* <xc> - C-style comment
*/
%x xq
%x xnq
%x xvq
-%x xsq
%x xc
-special [\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/]
-any [^\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/\\\"\' \t\n\r\f]
+/* "other" means anything that's not special, blank, or '\' or '"' */
+special [\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/]
+other [^\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/\\\" \t\n\r\f]
blank [ \t\n\r\f]
digit [0-9]
@@ -95,7 +94,7 @@ hex_fail \\x{hex_dig}{0,1}
%%
-<xnq>{any}+ {
+<xnq>{other}+ {
addstring(false, yytext, yyleng);
}
@@ -105,13 +104,12 @@ hex_fail \\x{hex_dig}{0,1}
return checkKeyword();
}
-
<xnq>\/\* {
yylval->str = scanstring;
BEGIN xc;
}
-<xnq>({special}|\"|\') {
+<xnq>({special}|\") {
yylval->str = scanstring;
yyless(0);
BEGIN INITIAL;
@@ -124,39 +122,37 @@ hex_fail \\x{hex_dig}{0,1}
return checkKeyword();
}
-<xnq,xq,xvq,xsq>\\[\"\'\\] { addchar(false, yytext[1]); }
-
-<xnq,xq,xvq,xsq>\\b { addchar(false, '\b'); }
+<xnq,xq,xvq>\\b { addchar(false, '\b'); }
-<xnq,xq,xvq,xsq>\\f { addchar(false, '\f'); }
+<xnq,xq,xvq>\\f { addchar(false, '\f'); }
-<xnq,xq,xvq,xsq>\\n { addchar(false, '\n'); }
+<xnq,xq,xvq>\\n { addchar(false, '\n'); }
-<xnq,xq,xvq,xsq>\\r { addchar(false, '\r'); }
+<xnq,xq,xvq>\\r { addchar(false, '\r'); }
-<xnq,xq,xvq,xsq>\\t { addchar(false, '\t'); }
+<xnq,xq,xvq>\\t { addchar(false, '\t'); }
-<xnq,xq,xvq,xsq>\\v { addchar(false, '\v'); }
+<xnq,xq,xvq>\\v { addchar(false, '\v'); }
-<xnq,xq,xvq,xsq>{unicode}+ { parseUnicode(yytext, yyleng); }
+<xnq,xq,xvq>{unicode}+ { parseUnicode(yytext, yyleng); }
-<xnq,xq,xvq,xsq>{hex_char} { parseHexChar(yytext); }
+<xnq,xq,xvq>{hex_char} { parseHexChar(yytext); }
-<xnq,xq,xvq,xsq>{unicode}*{unicodefail} { yyerror(NULL, "invalid unicode sequence"); }
+<xnq,xq,xvq>{unicode}*{unicodefail} { yyerror(NULL, "invalid unicode sequence"); }
-<xnq,xq,xvq,xsq>{hex_fail} { yyerror(NULL, "invalid hex character sequence"); }
+<xnq,xq,xvq>{hex_fail} { yyerror(NULL, "invalid hex character sequence"); }
-<xnq,xq,xvq,xsq>{unicode}+\\ {
- /* throw back the \\, and treat as unicode */
- yyless(yyleng - 1);
- parseUnicode(yytext, yyleng);
- }
+<xnq,xq,xvq>{unicode}+\\ {
+ /* throw back the \\, and treat as unicode */
+ yyless(yyleng - 1);
+ parseUnicode(yytext, yyleng);
+ }
-<xnq,xq,xvq,xsq>\\. { yyerror(NULL, "escape sequence is invalid"); }
+<xnq,xq,xvq>\\. { addchar(false, yytext[1]); }
-<xnq,xq,xvq,xsq>\\ { yyerror(NULL, "unexpected end after backslash"); }
+<xnq,xq,xvq>\\ { yyerror(NULL, "unexpected end after backslash"); }
-<xq,xvq,xsq><<EOF>> { yyerror(NULL, "unexpected end of quoted string"); }
+<xq,xvq><<EOF>> { yyerror(NULL, "unexpected end of quoted string"); }
<xq>\" {
yylval->str = scanstring;
@@ -170,16 +166,8 @@ hex_fail \\x{hex_dig}{0,1}
return VARIABLE_P;
}
-<xsq>\' {
- yylval->str = scanstring;
- BEGIN INITIAL;
- return STRING_P;
- }
-
<xq,xvq>[^\\\"]+ { addstring(false, yytext, yyleng); }
-<xsq>[^\\\']+ { addstring(false, yytext, yyleng); }
-
<xc>\*\/ { BEGIN INITIAL; }
<xc>[^\*]+ { }
@@ -210,7 +198,7 @@ hex_fail \\x{hex_dig}{0,1}
\> { return GREATER_P; }
-\${any}+ {
+\${other}+ {
addstring(true, yytext + 1, yyleng - 1);
addchar(false, '\0');
yylval->str = scanstring;
@@ -263,27 +251,22 @@ hex_fail \\x{hex_dig}{0,1}
({realfail1}|{realfail2}) { yyerror(NULL, "invalid floating point number"); }
-{any}+ {
- addstring(true, yytext, yyleng);
- BEGIN xnq;
- }
-
\" {
addchar(true, '\0');
BEGIN xq;
}
-\' {
- addchar(true, '\0');
- BEGIN xsq;
- }
-
\\ {
yyless(0);
addchar(true, '\0');
BEGIN xnq;
}
+{other}+ {
+ addstring(true, yytext, yyleng);
+ BEGIN xnq;
+ }
+
<<EOF>> { yyterminate(); }
%%
diff --git a/src/test/regress/expected/jsonpath.out b/src/test/regress/expected/jsonpath.out
index ea42ae3..fc971dc 100644
--- a/src/test/regress/expected/jsonpath.out
+++ b/src/test/regress/expected/jsonpath.out
@@ -171,30 +171,24 @@ select '"\b\f\r\n\t\v\"\''\\"'::jsonpath;
"\b\f\r\n\t\u000b\"'\\"
(1 row)
-select '''\b\f\r\n\t\v\"\''\\'''::jsonpath;
- jsonpath
--------------------------
- "\b\f\r\n\t\u000b\"'\\"
-(1 row)
-
select '"\x50\u0067\u{53}\u{051}\u{00004C}"'::jsonpath;
jsonpath
----------
"PgSQL"
(1 row)
-select '''\x50\u0067\u{53}\u{051}\u{00004C}'''::jsonpath;
- jsonpath
-----------
- "PgSQL"
-(1 row)
-
select '$.foo\x50\u0067\u{53}\u{051}\u{00004C}\t\"bar'::jsonpath;
jsonpath
---------------------
$."fooPgSQL\t\"bar"
(1 row)
+select '"\z"'::jsonpath; -- unrecognized escape is just the literal char
+ jsonpath
+----------
+ "z"
+(1 row)
+
select '$.g ? ($.a == 1)'::jsonpath;
jsonpath
--------------------
diff --git a/src/test/regress/expected/jsonpath_encoding.out b/src/test/regress/expected/jsonpath_encoding.out
index 8db6e47..ecffe09 100644
--- a/src/test/regress/expected/jsonpath_encoding.out
+++ b/src/test/regress/expected/jsonpath_encoding.out
@@ -81,84 +81,6 @@ select '"null \\u0000 escape"'::jsonpath as not_an_escape;
"null \\u0000 escape"
(1 row)
--- checks for single-quoted values
--- basic unicode input
-SELECT E'\'\u\''::jsonpath; -- ERROR, incomplete escape
-ERROR: invalid Unicode escape
-LINE 1: SELECT E'\'\u\''::jsonpath;
- ^
-HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
-SELECT E'\'\u00\''::jsonpath; -- ERROR, incomplete escape
-ERROR: invalid Unicode escape
-LINE 1: SELECT E'\'\u00\''::jsonpath;
- ^
-HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
-SELECT E'\'\u000g\''::jsonpath; -- ERROR, g is not a hex digit
-ERROR: invalid Unicode escape
-LINE 1: SELECT E'\'\u000g\''::jsonpath;
- ^
-HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
-SELECT E'\'\u0000\''::jsonpath; -- OK, legal escape
-ERROR: invalid Unicode escape value at or near "E'\'\u0000"
-LINE 1: SELECT E'\'\u0000\''::jsonpath;
- ^
-SELECT E'\'\uaBcD\''::jsonpath; -- OK, uppercase and lower case both OK
- jsonpath
-----------
- "ꯍ"
-(1 row)
-
--- handling of unicode surrogate pairs
-select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_in_utf8;
- correct_in_utf8
------------------
- "😄🐶"
-(1 row)
-
-select E'\'\ud83d\ud83d\''::jsonpath; -- 2 high surrogates in a row
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83d\ud83d"
-LINE 1: select E'\'\ud83d\ud83d\''::jsonpath;
- ^
-select E'\'\ude04\ud83d\''::jsonpath; -- surrogates in wrong order
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
-LINE 1: select E'\'\ude04\ud83d\''::jsonpath;
- ^
-select E'\'\ud83dX\''::jsonpath; -- orphan high surrogate
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83dX"
-LINE 1: select E'\'\ud83dX\''::jsonpath;
- ^
-select E'\'\ude04X\''::jsonpath; -- orphan low surrogate
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
-LINE 1: select E'\'\ude04X\''::jsonpath;
- ^
---handling of simple unicode escapes
-select E'\'the Copyright \u00a9 sign\''::jsonpath as correct_in_utf8;
- correct_in_utf8
-------------------------
- "the Copyright © sign"
-(1 row)
-
-select E'\'dollar \u0024 character\''::jsonpath as correct_everywhere;
- correct_everywhere
-----------------------
- "dollar $ character"
-(1 row)
-
-select E'\'dollar \\u0024 character\''::jsonpath as not_an_escape;
- not_an_escape
-----------------------
- "dollar $ character"
-(1 row)
-
-select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
-ERROR: invalid Unicode escape value at or near "E'\'null \u0000"
-LINE 1: select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
- ^
-select E'\'null \\u0000 escape\''::jsonpath as not_an_escape;
-ERROR: unsupported Unicode escape sequence
-LINE 1: select E'\'null \\u0000 escape\''::jsonpath as not_an_escape...
- ^
-DETAIL: \u0000 cannot be converted to text.
-- checks for quoted key names
-- basic unicode input
SELECT '$."\u"'::jsonpath; -- ERROR, incomplete escape
diff --git a/src/test/regress/expected/jsonpath_encoding_1.out b/src/test/regress/expected/jsonpath_encoding_1.out
index e6dff25..c8cc217 100644
--- a/src/test/regress/expected/jsonpath_encoding_1.out
+++ b/src/test/regress/expected/jsonpath_encoding_1.out
@@ -78,78 +78,6 @@ select '"null \\u0000 escape"'::jsonpath as not_an_escape;
"null \\u0000 escape"
(1 row)
--- checks for single-quoted values
--- basic unicode input
-SELECT E'\'\u\''::jsonpath; -- ERROR, incomplete escape
-ERROR: invalid Unicode escape
-LINE 1: SELECT E'\'\u\''::jsonpath;
- ^
-HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
-SELECT E'\'\u00\''::jsonpath; -- ERROR, incomplete escape
-ERROR: invalid Unicode escape
-LINE 1: SELECT E'\'\u00\''::jsonpath;
- ^
-HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
-SELECT E'\'\u000g\''::jsonpath; -- ERROR, g is not a hex digit
-ERROR: invalid Unicode escape
-LINE 1: SELECT E'\'\u000g\''::jsonpath;
- ^
-HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
-SELECT E'\'\u0000\''::jsonpath; -- OK, legal escape
-ERROR: invalid Unicode escape value at or near "E'\'\u0000"
-LINE 1: SELECT E'\'\u0000\''::jsonpath;
- ^
-SELECT E'\'\uaBcD\''::jsonpath; -- OK, uppercase and lower case both OK
-ERROR: Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8 at or near "E'\'\uaBcD"
-LINE 1: SELECT E'\'\uaBcD\''::jsonpath;
- ^
--- handling of unicode surrogate pairs
-select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_in_utf8;
-ERROR: Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8 at or near "E'\'\ud83d\ude04"
-LINE 1: select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_...
- ^
-select E'\'\ud83d\ud83d\''::jsonpath; -- 2 high surrogates in a row
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83d\ud83d"
-LINE 1: select E'\'\ud83d\ud83d\''::jsonpath;
- ^
-select E'\'\ude04\ud83d\''::jsonpath; -- surrogates in wrong order
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
-LINE 1: select E'\'\ude04\ud83d\''::jsonpath;
- ^
-select E'\'\ud83dX\''::jsonpath; -- orphan high surrogate
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83dX"
-LINE 1: select E'\'\ud83dX\''::jsonpath;
- ^
-select E'\'\ude04X\''::jsonpath; -- orphan low surrogate
-ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
-LINE 1: select E'\'\ude04X\''::jsonpath;
- ^
---handling of simple unicode escapes
-select E'\'the Copyright \u00a9 sign\''::jsonpath as correct_in_utf8;
-ERROR: Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8 at or near "E'\'the Copyright \u00a9"
-LINE 1: select E'\'the Copyright \u00a9 sign\''::jsonpath as correct...
- ^
-select E'\'dollar \u0024 character\''::jsonpath as correct_everywhere;
- correct_everywhere
-----------------------
- "dollar $ character"
-(1 row)
-
-select E'\'dollar \\u0024 character\''::jsonpath as not_an_escape;
- not_an_escape
-----------------------
- "dollar $ character"
-(1 row)
-
-select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
-ERROR: invalid Unicode escape value at or near "E'\'null \u0000"
-LINE 1: select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
- ^
-select E'\'null \\u0000 escape\''::jsonpath as not_an_escape;
-ERROR: unsupported Unicode escape sequence
-LINE 1: select E'\'null \\u0000 escape\''::jsonpath as not_an_escape...
- ^
-DETAIL: \u0000 cannot be converted to text.
-- checks for quoted key names
-- basic unicode input
SELECT '$."\u"'::jsonpath; -- ERROR, incomplete escape
diff --git a/src/test/regress/sql/jsonpath.sql b/src/test/regress/sql/jsonpath.sql
index 29ea77a..7afe252 100644
--- a/src/test/regress/sql/jsonpath.sql
+++ b/src/test/regress/sql/jsonpath.sql
@@ -30,10 +30,9 @@ select '$.a/+-1'::jsonpath;
select '1 * 2 + 4 % -3 != false'::jsonpath;
select '"\b\f\r\n\t\v\"\''\\"'::jsonpath;
-select '''\b\f\r\n\t\v\"\''\\'''::jsonpath;
select '"\x50\u0067\u{53}\u{051}\u{00004C}"'::jsonpath;
-select '''\x50\u0067\u{53}\u{051}\u{00004C}'''::jsonpath;
select '$.foo\x50\u0067\u{53}\u{051}\u{00004C}\t\"bar'::jsonpath;
+select '"\z"'::jsonpath; -- unrecognized escape is just the literal char
select '$.g ? ($.a == 1)'::jsonpath;
select '$.g ? (@ == 1)'::jsonpath;
diff --git a/src/test/regress/sql/jsonpath_encoding.sql b/src/test/regress/sql/jsonpath_encoding.sql
index a3b5bc3..3a23b72 100644
--- a/src/test/regress/sql/jsonpath_encoding.sql
+++ b/src/test/regress/sql/jsonpath_encoding.sql
@@ -24,29 +24,6 @@ select '"dollar \\u0024 character"'::jsonpath as not_an_escape;
select '"null \u0000 escape"'::jsonpath as not_unescaped;
select '"null \\u0000 escape"'::jsonpath as not_an_escape;
--- checks for single-quoted values
-
--- basic unicode input
-SELECT E'\'\u\''::jsonpath; -- ERROR, incomplete escape
-SELECT E'\'\u00\''::jsonpath; -- ERROR, incomplete escape
-SELECT E'\'\u000g\''::jsonpath; -- ERROR, g is not a hex digit
-SELECT E'\'\u0000\''::jsonpath; -- OK, legal escape
-SELECT E'\'\uaBcD\''::jsonpath; -- OK, uppercase and lower case both OK
-
--- handling of unicode surrogate pairs
-select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_in_utf8;
-select E'\'\ud83d\ud83d\''::jsonpath; -- 2 high surrogates in a row
-select E'\'\ude04\ud83d\''::jsonpath; -- surrogates in wrong order
-select E'\'\ud83dX\''::jsonpath; -- orphan high surrogate
-select E'\'\ude04X\''::jsonpath; -- orphan low surrogate
-
---handling of simple unicode escapes
-select E'\'the Copyright \u00a9 sign\''::jsonpath as correct_in_utf8;
-select E'\'dollar \u0024 character\''::jsonpath as correct_everywhere;
-select E'\'dollar \\u0024 character\''::jsonpath as not_an_escape;
-select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
-select E'\'null \\u0000 escape\''::jsonpath as not_an_escape;
-
-- checks for quoted key names
-- basic unicode input
