On 01.11.21 07:09, Peter Eisentraut wrote:
Here is an updated patch for this. It's the previous patch polished a
bit more, and it contains changes so that numeric literals reject
trailing identifier parts without whitespace in between, as discussed.
Maybe I should split that into incremental patches, but for now I only
have the one. I don't have a patch for the underscores in numeric
literals yet. It's in progress, but not ready.
Here is a progressed version of this work, split into more incremental
patches. The first three patches are harmless code cleanups. Patch 3
has an interesting naming conflict, noted in the commit message; ideas
welcome. Patches 4 and 5 handle the rejection of trailing junk after
numeric literals, as discussed. I have expanded that compared to the v4
patch to also cover non-integer literals. It also comes with more tests
now. Patch 6 is the titular introduction of non-decimal integer
literals, unchanged from before.From 39aed9c0516fcf0a6b3372361ecfcf4874614578 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Wed, 24 Nov 2021 09:10:32 +0100
Subject: [PATCH v5 1/6] Improve some comments in scanner files
---
src/backend/parser/scan.l | 14 ++++++++------
src/fe_utils/psqlscan.l | 14 ++++++++------
src/interfaces/ecpg/preproc/pgc.l | 16 +++++++++-------
3 files changed, 25 insertions(+), 19 deletions(-)
diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index 6e6824faeb..4e02815803 100644
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -174,7 +174,7 @@ extern void core_yyset_column(int column_no, yyscan_t
yyscanner);
* <xb> bit string literal
* <xc> extended C-style comments
* <xd> delimited identifiers (double-quoted identifiers)
- * <xh> hexadecimal numeric string
+ * <xh> hexadecimal byte string
* <xq> standard quoted strings
* <xqs> quote stop (detect continued strings)
* <xe> extended quoted strings (support backslash escape sequences)
@@ -262,7 +262,7 @@ quotecontinuefail {whitespace}*"-"?
xbstart [bB]{quote}
xbinside [^']*
-/* Hexadecimal number */
+/* Hexadecimal byte string */
xhstart [xX]{quote}
xhinside [^']*
@@ -341,7 +341,6 @@ xcstart \/\*{op_chars}*
xcstop \*+\/
xcinside [^*/]+
-digit [0-9]
ident_start [A-Za-z\200-\377_]
ident_cont [A-Za-z\200-\377_0-9\$]
@@ -380,15 +379,18 @@ self [,()\[\].;\:\+\-\*\/\%\^\<\>\=]
op_chars [\~\!\@\#\^\&\|\`\?\+\-\*\/\%\<\>\=]
operator {op_chars}+
-/* we no longer allow unary minus in numbers.
- * instead we pass it separately to parser. there it gets
- * coerced via doNegate() -- Leon aug 20 1999
+/*
+ * Numbers
+ *
+ * Unary minus is not part of a number here. Instead we pass it separately to
+ * parser, and there it gets coerced via doNegate().
*
* {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
*
* {realfail1} and {realfail2} are added to prevent the need for scanner
* backup when the {real} rule fails to match completely.
*/
+digit [0-9]
integer {digit}+
decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
index 0fab48a382..9aac166aa0 100644
--- a/src/fe_utils/psqlscan.l
+++ b/src/fe_utils/psqlscan.l
@@ -112,7 +112,7 @@ extern void psql_yyset_column(int column_no, yyscan_t
yyscanner);
* <xb> bit string literal
* <xc> extended C-style comments
* <xd> delimited identifiers (double-quoted identifiers)
- * <xh> hexadecimal numeric string
+ * <xh> hexadecimal byte string
* <xq> standard quoted strings
* <xqs> quote stop (detect continued strings)
* <xe> extended quoted strings (support backslash escape sequences)
@@ -200,7 +200,7 @@ quotecontinuefail {whitespace}*"-"?
xbstart [bB]{quote}
xbinside [^']*
-/* Hexadecimal number */
+/* Hexadecimal byte string */
xhstart [xX]{quote}
xhinside [^']*
@@ -279,7 +279,6 @@ xcstart \/\*{op_chars}*
xcstop \*+\/
xcinside [^*/]+
-digit [0-9]
ident_start [A-Za-z\200-\377_]
ident_cont [A-Za-z\200-\377_0-9\$]
@@ -318,15 +317,18 @@ self [,()\[\].;\:\+\-\*\/\%\^\<\>\=]
op_chars [\~\!\@\#\^\&\|\`\?\+\-\*\/\%\<\>\=]
operator {op_chars}+
-/* we no longer allow unary minus in numbers.
- * instead we pass it separately to parser. there it gets
- * coerced via doNegate() -- Leon aug 20 1999
+/*
+ * Numbers
+ *
+ * Unary minus is not part of a number here. Instead we pass it separately to
+ * parser, and there it gets coerced via doNegate().
*
* {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
*
* {realfail1} and {realfail2} are added to prevent the need for scanner
* backup when the {real} rule fails to match completely.
*/
+digit [0-9]
integer {digit}+
decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
diff --git a/src/interfaces/ecpg/preproc/pgc.l
b/src/interfaces/ecpg/preproc/pgc.l
index 7a0356638d..7c3bf52bfa 100644
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -130,7 +130,7 @@ static struct _if_value
* <xc> extended C-style comments
* <xd> delimited identifiers (double-quoted identifiers)
* <xdc> double-quoted strings in C
- * <xh> hexadecimal numeric string
+ * <xh> hexadecimal byte string
* <xn> national character quoted strings
* <xq> standard quoted strings
* <xqs> quote stop (detect continued strings)
@@ -223,7 +223,7 @@ quotecontinuefail {whitespace}*"-"?
xbstart [bB]{quote}
xbinside [^']*
-/* Hexadecimal number */
+/* Hexadecimal byte string */
xhstart [xX]{quote}
xhinside [^']*
@@ -305,7 +305,6 @@ xcstart \/\*{op_chars}*
xcstop \*+\/
xcinside [^*/]+
-digit [0-9]
ident_start [A-Za-z\200-\377_]
ident_cont [A-Za-z\200-\377_0-9\$]
@@ -346,15 +345,18 @@ self [,()\[\].;\:\+\-\*\/\%\^\<\>\=]
op_chars [\~\!\@\#\^\&\|\`\?\+\-\*\/\%\<\>\=]
operator {op_chars}+
-/* we no longer allow unary minus in numbers.
- * instead we pass it separately to parser. there it gets
- * coerced via doNegate() -- Leon aug 20 1999
+/*
+ * Numbers
+ *
+ * Unary minus is not part of a number here. Instead we pass it separately to
+ * parser, and there it gets coerced via doNegate().
*
* {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
*
* {realfail1} and {realfail2} are added to prevent the need for scanner
* backup when the {real} rule fails to match completely.
*/
+digit [0-9]
integer {digit}+
decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
@@ -603,7 +605,7 @@ cppline
{space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+
return BCONST;
case xh:
if
(literalbuf[strspn(literalbuf, "0123456789abcdefABCDEF")] != '\0')
-
mmerror(PARSE_ERROR, ET_ERROR, "invalid hex string literal");
+
mmerror(PARSE_ERROR, ET_ERROR, "invalid hexadecimal string literal");
base_yylval.str =
psprintf("x'%s'", literalbuf);
return XCONST;
case xq:
--
2.33.1
From ba0b3390a82901a3dad52267a7b1e36cc7be50b1 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Wed, 24 Nov 2021 12:30:23 +0100
Subject: [PATCH v5 2/6] Remove unused includes
These haven't been needed for a long time.
---
src/backend/utils/adt/cash.c | 1 -
src/backend/utils/adt/formatting.c | 1 -
src/backend/utils/adt/numeric.c | 1 -
src/backend/utils/adt/rangetypes.c | 1 -
4 files changed, 4 deletions(-)
diff --git a/src/backend/utils/adt/cash.c b/src/backend/utils/adt/cash.c
index d093ce8038..f7e78fa105 100644
--- a/src/backend/utils/adt/cash.c
+++ b/src/backend/utils/adt/cash.c
@@ -26,7 +26,6 @@
#include "libpq/pqformat.h"
#include "utils/builtins.h"
#include "utils/cash.h"
-#include "utils/int8.h"
#include "utils/numeric.h"
#include "utils/pg_locale.h"
diff --git a/src/backend/utils/adt/formatting.c
b/src/backend/utils/adt/formatting.c
index a1145e2721..419469fab5 100644
--- a/src/backend/utils/adt/formatting.c
+++ b/src/backend/utils/adt/formatting.c
@@ -92,7 +92,6 @@
#include "utils/datetime.h"
#include "utils/float.h"
#include "utils/formatting.h"
-#include "utils/int8.h"
#include "utils/memutils.h"
#include "utils/numeric.h"
#include "utils/pg_locale.h"
diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c
index 1de744855f..644d7d3d21 100644
--- a/src/backend/utils/adt/numeric.c
+++ b/src/backend/utils/adt/numeric.c
@@ -39,7 +39,6 @@
#include "utils/builtins.h"
#include "utils/float.h"
#include "utils/guc.h"
-#include "utils/int8.h"
#include "utils/numeric.h"
#include "utils/pg_lsn.h"
#include "utils/sortsupport.h"
diff --git a/src/backend/utils/adt/rangetypes.c
b/src/backend/utils/adt/rangetypes.c
index 815175a654..6c23d02c46 100644
--- a/src/backend/utils/adt/rangetypes.c
+++ b/src/backend/utils/adt/rangetypes.c
@@ -37,7 +37,6 @@
#include "miscadmin.h"
#include "utils/builtins.h"
#include "utils/date.h"
-#include "utils/int8.h"
#include "utils/lsyscache.h"
#include "utils/rangetypes.h"
#include "utils/timestamp.h"
--
2.33.1
From ce0109193e49ef3cda84415ed5a117afd3c47521 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Wed, 24 Nov 2021 12:31:07 +0100
Subject: [PATCH v5 3/6] Move scanint8() to numutils.c
Move scanint8() to numutils.c and rename to pg_strtoint64(). We
already have a "16" and "32" version of that, and the code inside the
functions was aligned, so this move makes all three versions
consistent. The API is also changed to no longer provide the errorOK
case. Instead, provide another function for such uses that just wraps
the OS's strtoll() or similar. We already have such a function for
the strtoull() variant, which is called pg_strtouint64(), so this
version will be called pg_strtoint64(), except we already used that
name above. FIXME
---
src/backend/parser/parse_node.c | 12 ++-
src/backend/replication/pgoutput/pgoutput.c | 9 +-
src/backend/utils/adt/int8.c | 90 +------------------
src/backend/utils/adt/numutils.c | 97 +++++++++++++++++++++
src/bin/pgbench/pgbench.c | 4 +-
src/include/utils/builtins.h | 2 +
src/include/utils/int8.h | 25 ------
7 files changed, 117 insertions(+), 122 deletions(-)
delete mode 100644 src/include/utils/int8.h
diff --git a/src/backend/parser/parse_node.c b/src/backend/parser/parse_node.c
index 8cfe6f67c0..bb82439dc9 100644
--- a/src/backend/parser/parse_node.c
+++ b/src/backend/parser/parse_node.c
@@ -26,7 +26,6 @@
#include "parser/parse_relation.h"
#include "parser/parsetree.h"
#include "utils/builtins.h"
-#include "utils/int8.h"
#include "utils/lsyscache.h"
#include "utils/syscache.h"
#include "utils/varbit.h"
@@ -353,7 +352,6 @@ make_const(ParseState *pstate, A_Const *aconst)
{
Const *con;
Datum val;
- int64 val64;
Oid typeid;
int typelen;
bool typebyval;
@@ -384,8 +382,15 @@ make_const(ParseState *pstate, A_Const *aconst)
break;
case T_Float:
+ {
/* could be an oversize integer as well as a float ...
*/
- if (scanint8(aconst->val.fval.val, true, &val64))
+
+ int64 val64;
+ char *endptr;
+
+ errno = 0;
+ val64 = pg_strtoint64xx(aconst->val.fval.val, &endptr,
10);
+ if (!errno && *endptr == '\0')
{
/*
* It might actually fit in int32. Probably
only INT_MIN can
@@ -425,6 +430,7 @@ make_const(ParseState *pstate, A_Const *aconst)
typebyval = false;
}
break;
+ }
case T_String:
diff --git a/src/backend/replication/pgoutput/pgoutput.c
b/src/backend/replication/pgoutput/pgoutput.c
index 6f6a203dea..2f0f40c75d 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -21,7 +21,6 @@
#include "replication/logicalproto.h"
#include "replication/origin.h"
#include "replication/pgoutput.h"
-#include "utils/int8.h"
#include "utils/inval.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
@@ -205,7 +204,8 @@ parse_output_parameters(List *options, PGOutputData *data)
/* Check each param, whether or not we recognize it */
if (strcmp(defel->defname, "proto_version") == 0)
{
- int64 parsed;
+ unsigned long parsed;
+ char *endptr;
if (protocol_version_given)
ereport(ERROR,
@@ -213,12 +213,13 @@ parse_output_parameters(List *options, PGOutputData *data)
errmsg("conflicting or
redundant options")));
protocol_version_given = true;
- if (!scanint8(strVal(defel->arg), true, &parsed))
+ parsed = strtoul(strVal(defel->arg), &endptr, 10);
+ if (errno || *endptr != '\0')
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("invalid
proto_version")));
- if (parsed > PG_UINT32_MAX || parsed < 0)
+ if (parsed > PG_UINT32_MAX)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("proto_version \"%s\"
out of range",
diff --git a/src/backend/utils/adt/int8.c b/src/backend/utils/adt/int8.c
index 2168080dcc..f8f557526f 100644
--- a/src/backend/utils/adt/int8.c
+++ b/src/backend/utils/adt/int8.c
@@ -24,7 +24,6 @@
#include "nodes/supportnodes.h"
#include "optimizer/optimizer.h"
#include "utils/builtins.h"
-#include "utils/int8.h"
typedef struct
@@ -45,99 +44,14 @@ typedef struct
* Formatting and conversion routines.
*---------------------------------------------------------*/
-/*
- * scanint8 --- try to parse a string into an int8.
- *
- * If errorOK is false, ereport a useful error message if the string is bad.
- * If errorOK is true, just return "false" for bad input.
- */
-bool
-scanint8(const char *str, bool errorOK, int64 *result)
-{
- const char *ptr = str;
- int64 tmp = 0;
- bool neg = false;
-
- /*
- * Do our own scan, rather than relying on sscanf which might be broken
- * for long long.
- *
- * As INT64_MIN can't be stored as a positive 64 bit integer, accumulate
- * value as a negative number.
- */
-
- /* skip leading spaces */
- while (*ptr && isspace((unsigned char) *ptr))
- ptr++;
-
- /* handle sign */
- if (*ptr == '-')
- {
- ptr++;
- neg = true;
- }
- else if (*ptr == '+')
- ptr++;
-
- /* require at least one digit */
- if (unlikely(!isdigit((unsigned char) *ptr)))
- goto invalid_syntax;
-
- /* process digits */
- while (*ptr && isdigit((unsigned char) *ptr))
- {
- int8 digit = (*ptr++ - '0');
-
- if (unlikely(pg_mul_s64_overflow(tmp, 10, &tmp)) ||
- unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
- goto out_of_range;
- }
-
- /* allow trailing whitespace, but not other trailing chars */
- while (*ptr != '\0' && isspace((unsigned char) *ptr))
- ptr++;
-
- if (unlikely(*ptr != '\0'))
- goto invalid_syntax;
-
- if (!neg)
- {
- /* could fail if input is most negative number */
- if (unlikely(tmp == PG_INT64_MIN))
- goto out_of_range;
- tmp = -tmp;
- }
-
- *result = tmp;
- return true;
-
-out_of_range:
- if (!errorOK)
- ereport(ERROR,
- (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
- errmsg("value \"%s\" is out of range for type
%s",
- str, "bigint")));
- return false;
-
-invalid_syntax:
- if (!errorOK)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
- errmsg("invalid input syntax for type %s:
\"%s\"",
- "bigint", str)));
- return false;
-}
-
/* int8in()
*/
Datum
int8in(PG_FUNCTION_ARGS)
{
- char *str = PG_GETARG_CSTRING(0);
- int64 result;
+ char *num = PG_GETARG_CSTRING(0);
- (void) scanint8(str, false, &result);
- PG_RETURN_INT64(result);
+ PG_RETURN_INT64(pg_strtoint64(num));
}
diff --git a/src/backend/utils/adt/numutils.c b/src/backend/utils/adt/numutils.c
index b93096f288..ebc2d222a3 100644
--- a/src/backend/utils/adt/numutils.c
+++ b/src/backend/utils/adt/numutils.c
@@ -325,6 +325,90 @@ pg_strtoint32(const char *s)
return 0; /* keep compiler quiet
*/
}
+/*
+ * Convert input string to a signed 64 bit integer.
+ *
+ * Allows any number of leading or trailing whitespace characters. Will throw
+ * ereport() upon bad input format or overflow.
+ *
+ * NB: Accumulate input as a negative number, to deal with two's complement
+ * representation of the most negative number, which can't be represented as a
+ * positive number.
+ */
+int64
+pg_strtoint64(const char *s)
+{
+ const char *ptr = s;
+ int64 tmp = 0;
+ bool neg = false;
+
+ /*
+ * Do our own scan, rather than relying on sscanf which might be broken
+ * for long long.
+ *
+ * As INT64_MIN can't be stored as a positive 64 bit integer, accumulate
+ * value as a negative number.
+ */
+
+ /* skip leading spaces */
+ while (*ptr && isspace((unsigned char) *ptr))
+ ptr++;
+
+ /* handle sign */
+ if (*ptr == '-')
+ {
+ ptr++;
+ neg = true;
+ }
+ else if (*ptr == '+')
+ ptr++;
+
+ /* require at least one digit */
+ if (unlikely(!isdigit((unsigned char) *ptr)))
+ goto invalid_syntax;
+
+ /* process digits */
+ while (*ptr && isdigit((unsigned char) *ptr))
+ {
+ int8 digit = (*ptr++ - '0');
+
+ if (unlikely(pg_mul_s64_overflow(tmp, 10, &tmp)) ||
+ unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+ }
+
+ /* allow trailing whitespace, but not other trailing chars */
+ while (*ptr != '\0' && isspace((unsigned char) *ptr))
+ ptr++;
+
+ if (unlikely(*ptr != '\0'))
+ goto invalid_syntax;
+
+ if (!neg)
+ {
+ /* could fail if input is most negative number */
+ if (unlikely(tmp == PG_INT64_MIN))
+ goto out_of_range;
+ tmp = -tmp;
+ }
+
+ return tmp;
+
+out_of_range:
+ ereport(ERROR,
+ (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
+ errmsg("value \"%s\" is out of range for type %s",
+ s, "bigint")));
+
+invalid_syntax:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("invalid input syntax for type %s: \"%s\"",
+ "bigint", s)));
+
+ return 0; /* keep compiler quiet
*/
+}
+
/*
* pg_itoa: converts a signed 16-bit integer to its string representation
* and returns strlen(a).
@@ -628,3 +712,16 @@ pg_strtouint64(const char *str, char **endptr, int base)
return strtoul(str, endptr, base);
#endif
}
+
+// XXX unfortunate API naming conflict
+int64
+pg_strtoint64xx(const char *str, char **endptr, int base)
+{
+#ifdef _MSC_VER /* MSVC only */
+ return _strtoi64(str, endptr, base);
+#elif defined(HAVE_STRTOLL) && SIZEOF_LONG < 8
+ return strtoll(str, endptr, base);
+#else
+ return strtol(str, endptr, base);
+#endif
+}
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index c12b6f0615..026c5e3083 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -794,8 +794,8 @@ is_an_int(const char *str)
/*
* strtoint64 -- convert a string to 64-bit integer
*
- * This function is a slightly modified version of scanint8() from
- * src/backend/utils/adt/int8.c.
+ * This function is a slightly modified version of pg_strtoint64() from
+ * src/backend/utils/adt/numutils.c.
*
* The function returns whether the conversion worked, and if so
* "*result" is set to the result.
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index 40fcb0ab6d..e8b2abace9 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -46,6 +46,7 @@ extern int namestrcmp(Name name, const char *str);
extern int32 pg_atoi(const char *s, int size, int c);
extern int16 pg_strtoint16(const char *s);
extern int32 pg_strtoint32(const char *s);
+extern int64 pg_strtoint64(const char *s);
extern int pg_itoa(int16 i, char *a);
extern int pg_ultoa_n(uint32 l, char *a);
extern int pg_ulltoa_n(uint64 l, char *a);
@@ -54,6 +55,7 @@ extern int pg_lltoa(int64 ll, char *a);
extern char *pg_ultostr_zeropad(char *str, uint32 value, int32 minwidth);
extern char *pg_ultostr(char *str, uint32 value);
extern uint64 pg_strtouint64(const char *str, char **endptr, int base);
+extern int64 pg_strtoint64xx(const char *str, char **endptr, int base);
/* oid.c */
extern oidvector *buildoidvector(const Oid *oids, int n);
diff --git a/src/include/utils/int8.h b/src/include/utils/int8.h
deleted file mode 100644
index 6571188f90..0000000000
--- a/src/include/utils/int8.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*-------------------------------------------------------------------------
- *
- * int8.h
- * Declarations for operations on 64-bit integers.
- *
- *
- * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
- * Portions Copyright (c) 1994, Regents of the University of California
- *
- * src/include/utils/int8.h
- *
- * NOTES
- * These data types are supported on all 64-bit architectures, and may
- * be supported through libraries on some 32-bit machines. If your machine
- * is not currently supported, then please try to make it so, then post
- * patches to the postgresql.org hackers mailing list.
- *
- *-------------------------------------------------------------------------
- */
-#ifndef INT8_H
-#define INT8_H
-
-extern bool scanint8(const char *str, bool errorOK, int64 *result);
-
-#endif /* INT8_H */
--
2.33.1
From eef02792b933a48634fcea2f3425d32e8a7e19d4 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Thu, 25 Nov 2021 07:44:32 +0100
Subject: [PATCH v5 4/6] Add test case for trailing junk after numeric literals
PostgreSQL currently accepts numeric literals with trailing
non-digits, such as 123abc where the abc is treated as the next token.
This may be a bit surprising. This commit adds test cases for this;
subsequent commits intend to change this behavior.
---
src/test/regress/expected/numerology.out | 55 ++++++++++++++++++++++++
src/test/regress/sql/numerology.sql | 14 ++++++
2 files changed, 69 insertions(+)
diff --git a/src/test/regress/expected/numerology.out
b/src/test/regress/expected/numerology.out
index 44d6c435de..32c6d80c03 100644
--- a/src/test/regress/expected/numerology.out
+++ b/src/test/regress/expected/numerology.out
@@ -3,6 +3,61 @@
-- Test various combinations of numeric types and functions.
--
--
+-- Trailing junk in numeric literals
+--
+SELECT 123abc;
+ abc
+-----
+ 123
+(1 row)
+
+SELECT 0x0o;
+ x0o
+-----
+ 0
+(1 row)
+
+SELECT 1_2_3;
+ _2_3
+------
+ 1
+(1 row)
+
+SELECT 0.a;
+ a
+---
+ 0
+(1 row)
+
+SELECT 0.0a;
+ a
+-----
+ 0.0
+(1 row)
+
+SELECT .0a;
+ a
+-----
+ 0.0
+(1 row)
+
+SELECT 0.0e1a;
+ a
+---
+ 0
+(1 row)
+
+SELECT 0.0e;
+ e
+-----
+ 0.0
+(1 row)
+
+SELECT 0.0e+a;
+ERROR: syntax error at or near "+"
+LINE 1: SELECT 0.0e+a;
+ ^
+--
-- Test implicit type conversions
-- This fails for Postgres v6.1 (and earlier?)
-- so let's try explicit conversions for now - tgl 97/05/07
diff --git a/src/test/regress/sql/numerology.sql
b/src/test/regress/sql/numerology.sql
index fddb58f8fd..70447a95fa 100644
--- a/src/test/regress/sql/numerology.sql
+++ b/src/test/regress/sql/numerology.sql
@@ -3,6 +3,20 @@
-- Test various combinations of numeric types and functions.
--
+--
+-- Trailing junk in numeric literals
+--
+
+SELECT 123abc;
+SELECT 0x0o;
+SELECT 1_2_3;
+SELECT 0.a;
+SELECT 0.0a;
+SELECT .0a;
+SELECT 0.0e1a;
+SELECT 0.0e;
+SELECT 0.0e+a;
+
--
-- Test implicit type conversions
-- This fails for Postgres v6.1 (and earlier?)
--
2.33.1
From 278f30528fa455c16be86b4c8c6b8fe7d9ad26dc Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Thu, 25 Nov 2021 08:41:48 +0100
Subject: [PATCH v5 5/6] Reject trailing junk after numeric literals
After this, the PostgreSQL lexers no longer accept numeric literals
with trailing non-digits, such as 123abc, which would be scanned as
two tokens: 123 and abc. This is undocumented and surprising, and it
might also interfere with some extended numeric literal syntax being
contemplated for the future.
---
src/backend/parser/scan.l | 27 ++++++----
src/fe_utils/psqlscan.l | 21 +++++---
src/interfaces/ecpg/preproc/pgc.l | 4 ++
src/test/regress/expected/numerology.out | 68 +++++++++---------------
4 files changed, 61 insertions(+), 59 deletions(-)
diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index 4e02815803..42646171e5 100644
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -399,6 +399,10 @@ real
({integer}|{decimal})[Ee][-+]?{digit}+
realfail1 ({integer}|{decimal})[Ee]
realfail2 ({integer}|{decimal})[Ee][-+]
+integer_junk {integer}{ident_start}
+decimal_junk {decimal}{ident_start}
+real_junk {real}{ident_start}
+
param \${integer}
other .
@@ -996,19 +1000,24 @@ other .
return FCONST;
}
{realfail1} {
- /*
- * throw back the [Ee], and figure out
whether what
- * remains is an {integer} or {decimal}.
- */
- yyless(yyleng - 1);
SET_YYLLOC();
- return process_integer_literal(yytext,
yylval);
+ yyerror("trailing junk after numeric
literal");
}
{realfail2} {
- /* throw back the [Ee][+-], and proceed
as above */
- yyless(yyleng - 2);
SET_YYLLOC();
- return process_integer_literal(yytext,
yylval);
+ yyerror("trailing junk after numeric
literal");
+ }
+{integer_junk} {
+ SET_YYLLOC();
+ yyerror("trailing junk after numeric
literal");
+ }
+{decimal_junk} {
+ SET_YYLLOC();
+ yyerror("trailing junk after numeric
literal");
+ }
+{real_junk} {
+ SET_YYLLOC();
+ yyerror("trailing junk after numeric
literal");
}
diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
index 9aac166aa0..4cd5e69d00 100644
--- a/src/fe_utils/psqlscan.l
+++ b/src/fe_utils/psqlscan.l
@@ -337,6 +337,10 @@ real
({integer}|{decimal})[Ee][-+]?{digit}+
realfail1 ({integer}|{decimal})[Ee]
realfail2 ({integer}|{decimal})[Ee][-+]
+integer_junk {integer}{ident_start}
+decimal_junk {decimal}{ident_start}
+real_junk {real}{ident_start}
+
param \${integer}
/* psql-specific: characters allowed in variable names */
@@ -856,17 +860,18 @@ other .
ECHO;
}
{realfail1} {
- /*
- * throw back the [Ee], and figure out
whether what
- * remains is an {integer} or {decimal}.
- * (in psql, we don't actually care...)
- */
- yyless(yyleng - 1);
ECHO;
}
{realfail2} {
- /* throw back the [Ee][+-], and proceed
as above */
- yyless(yyleng - 2);
+ ECHO;
+ }
+{integer_junk} {
+ ECHO;
+ }
+{decimal_junk} {
+ ECHO;
+ }
+{real_junk} {
ECHO;
}
diff --git a/src/interfaces/ecpg/preproc/pgc.l
b/src/interfaces/ecpg/preproc/pgc.l
index 7c3bf52bfa..e641095496 100644
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -365,6 +365,10 @@ real
({integer}|{decimal})[Ee][-+]?{digit}+
realfail1 ({integer}|{decimal})[Ee]
realfail2 ({integer}|{decimal})[Ee][-+]
+integer_junk {integer}{ident_start}
+decimal_junk {decimal}{ident_start}
+real_junk {real}{ident_start}
+
param \${integer}
/* special characters for other dbms */
diff --git a/src/test/regress/expected/numerology.out
b/src/test/regress/expected/numerology.out
index 32c6d80c03..2f176ccb52 100644
--- a/src/test/regress/expected/numerology.out
+++ b/src/test/regress/expected/numerology.out
@@ -6,57 +6,41 @@
-- Trailing junk in numeric literals
--
SELECT 123abc;
- abc
------
- 123
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near "123a"
+LINE 1: SELECT 123abc;
+ ^
SELECT 0x0o;
- x0o
------
- 0
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near "0x"
+LINE 1: SELECT 0x0o;
+ ^
SELECT 1_2_3;
- _2_3
-------
- 1
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near "1_"
+LINE 1: SELECT 1_2_3;
+ ^
SELECT 0.a;
- a
----
- 0
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near "0.a"
+LINE 1: SELECT 0.a;
+ ^
SELECT 0.0a;
- a
------
- 0.0
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near "0.0a"
+LINE 1: SELECT 0.0a;
+ ^
SELECT .0a;
- a
------
- 0.0
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near ".0a"
+LINE 1: SELECT .0a;
+ ^
SELECT 0.0e1a;
- a
----
- 0
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near "0.0e1a"
+LINE 1: SELECT 0.0e1a;
+ ^
SELECT 0.0e;
- e
------
- 0.0
-(1 row)
-
+ERROR: trailing junk after numeric literal at or near "0.0e"
+LINE 1: SELECT 0.0e;
+ ^
SELECT 0.0e+a;
-ERROR: syntax error at or near "+"
+ERROR: trailing junk after numeric literal at or near "0.0e+"
LINE 1: SELECT 0.0e+a;
- ^
+ ^
--
-- Test implicit type conversions
-- This fails for Postgres v6.1 (and earlier?)
--
2.33.1
From d9936e79d59e4971aedbe972346de337ac114678 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Sun, 31 Oct 2021 15:42:18 +0100
Subject: [PATCH v5 6/6] Non-decimal integer literals
Add support for hexadecimal, octal, and binary integer literals:
0x42F
0o273
0b100101
per SQL:202x draft.
This adds support in the lexer as well as in the integer type input
functions.
Discussion:
https://www.postgresql.org/message-id/flat/b239564c-cad0-b23e-c57e-166d883cb...@enterprisedb.com
---
doc/src/sgml/syntax.sgml | 26 ++++
src/backend/catalog/information_schema.sql | 6 +-
src/backend/catalog/sql_features.txt | 1 +
src/backend/parser/scan.l | 96 ++++++++++----
src/backend/utils/adt/numutils.c | 140 +++++++++++++++++++++
src/fe_utils/psqlscan.l | 78 +++++++++---
src/interfaces/ecpg/preproc/pgc.l | 93 +++++++-------
src/test/regress/expected/int2.out | 19 +++
src/test/regress/expected/int4.out | 19 +++
src/test/regress/expected/int8.out | 19 +++
src/test/regress/expected/numerology.out | 59 ++++++++-
src/test/regress/sql/int2.sql | 7 ++
src/test/regress/sql/int4.sql | 7 ++
src/test/regress/sql/int8.sql | 7 ++
src/test/regress/sql/numerology.sql | 21 +++-
15 files changed, 508 insertions(+), 90 deletions(-)
diff --git a/doc/src/sgml/syntax.sgml b/doc/src/sgml/syntax.sgml
index d66560b587..a4f04199c6 100644
--- a/doc/src/sgml/syntax.sgml
+++ b/doc/src/sgml/syntax.sgml
@@ -694,6 +694,32 @@ <title>Numeric Constants</title>
</literallayout>
</para>
+ <para>
+ Additionally, non-decimal integer constants can be used in these forms:
+<synopsis>
+0x<replaceable>hexdigits</replaceable>
+0o<replaceable>octdigits</replaceable>
+0b<replaceable>bindigits</replaceable>
+</synopsis>
+ <replaceable>hexdigits</replaceable> is one or more hexadecimal digits
+ (0-9, A-F), <replaceable>octdigits</replaceable> is one or more octal
+ digits (0-7), <replaceable>bindigits</replaceable> is one or more binary
+ digits (0 or 1). Hexadecimal digits and the radix prefixes can be in
+ upper or lower case. Note that only integers can have non-decimal forms,
+ not numbers with fractional parts.
+ </para>
+
+ <para>
+ These are some examples of this:
+<literallayout>0b100101
+0B10011001
+0o273
+0O755
+0x42f
+0XFFFF
+</literallayout>
+ </para>
+
<para>
<indexterm><primary>integer</primary></indexterm>
<indexterm><primary>bigint</primary></indexterm>
diff --git a/src/backend/catalog/information_schema.sql
b/src/backend/catalog/information_schema.sql
index 11d9dd60c2..ce88c483a2 100644
--- a/src/backend/catalog/information_schema.sql
+++ b/src/backend/catalog/information_schema.sql
@@ -119,7 +119,7 @@ CREATE FUNCTION _pg_numeric_precision(typid oid, typmod
int4) RETURNS integer
WHEN 1700 /*numeric*/ THEN
CASE WHEN $2 = -1
THEN null
- ELSE (($2 - 4) >> 16) & 65535
+ ELSE (($2 - 4) >> 16) & 0xFFFF
END
WHEN 700 /*float4*/ THEN 24 /*FLT_MANT_DIG*/
WHEN 701 /*float8*/ THEN 53 /*DBL_MANT_DIG*/
@@ -147,7 +147,7 @@ CREATE FUNCTION _pg_numeric_scale(typid oid, typmod int4)
RETURNS integer
WHEN $1 IN (1700) THEN
CASE WHEN $2 = -1
THEN null
- ELSE ($2 - 4) & 65535
+ ELSE ($2 - 4) & 0xFFFF
END
ELSE null
END;
@@ -163,7 +163,7 @@ CREATE FUNCTION _pg_datetime_precision(typid oid, typmod
int4) RETURNS integer
WHEN $1 IN (1083, 1114, 1184, 1266) /* time, timestamp, same + tz */
THEN CASE WHEN $2 < 0 THEN 6 ELSE $2 END
WHEN $1 IN (1186) /* interval */
- THEN CASE WHEN $2 < 0 OR $2 & 65535 = 65535 THEN 6 ELSE $2 & 65535
END
+ THEN CASE WHEN $2 < 0 OR $2 & 0xFFFF = 0xFFFF THEN 6 ELSE $2 &
0xFFFF END
ELSE null
END;
diff --git a/src/backend/catalog/sql_features.txt
b/src/backend/catalog/sql_features.txt
index 9f424216e2..d6359503f3 100644
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -526,6 +526,7 @@ T652 SQL-dynamic statements in SQL routines
NO
T653 SQL-schema statements in external routines YES
T654 SQL-dynamic statements in external routines NO
T655 Cyclically dependent routines YES
+T661 Non-decimal integer literals YES SQL:202x draft
T811 Basic SQL/JSON constructor functions NO
T812 SQL/JSON: JSON_OBJECTAGG NO
T813 SQL/JSON: JSON_ARRAYAGG with ORDER BY NO
diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index 42646171e5..cde870b463 100644
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -124,7 +124,7 @@ static void addlit(char *ytext, int yleng, core_yyscan_t
yyscanner);
static void addlitchar(unsigned char ychar, core_yyscan_t yyscanner);
static char *litbufdup(core_yyscan_t yyscanner);
static unsigned char unescape_single_char(unsigned char c, core_yyscan_t
yyscanner);
-static int process_integer_literal(const char *token, YYSTYPE *lval);
+static int process_integer_literal(const char *token, YYSTYPE *lval, int
base);
static void addunicode(pg_wchar c, yyscan_t yyscanner);
#define yyerror(msg) scanner_yyerror(msg, yyscanner)
@@ -385,25 +385,40 @@ operator {op_chars}+
* Unary minus is not part of a number here. Instead we pass it separately to
* parser, and there it gets coerced via doNegate().
*
- * {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
+ * {numericfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
*
* {realfail1} and {realfail2} are added to prevent the need for scanner
* backup when the {real} rule fails to match completely.
*/
-digit [0-9]
-
-integer {digit}+
-decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
-decimalfail {digit}+\.\.
-real ({integer}|{decimal})[Ee][-+]?{digit}+
-realfail1 ({integer}|{decimal})[Ee]
-realfail2 ({integer}|{decimal})[Ee][-+]
-
-integer_junk {integer}{ident_start}
-decimal_junk {decimal}{ident_start}
+decdigit [0-9]
+hexdigit [0-9A-Fa-f]
+octdigit [0-7]
+bindigit [0-1]
+
+decinteger {decdigit}+
+hexinteger 0[xX]{hexdigit}+
+octinteger 0[oO]{octdigit}+
+bininteger 0[bB]{bindigit}+
+
+hexfail 0[xX]
+octfail 0[oO]
+binfail 0[bB]
+
+numeric (({decinteger}\.{decinteger}?)|(\.{decinteger}))
+numericfail {decdigit}+\.\.
+
+real ({decinteger}|{numeric})[Ee][-+]?{decdigit}+
+realfail1 ({decinteger}|{numeric})[Ee]
+realfail2 ({decinteger}|{numeric})[Ee][-+]
+
+decinteger_junk {decinteger}{ident_start}
+hexinteger_junk {hexinteger}{ident_start}
+octinteger_junk {octinteger}{ident_start}
+bininteger_junk {bininteger}{ident_start}
+numeric_junk {numeric}{ident_start}
real_junk {real}{ident_start}
-param \${integer}
+param \${decinteger}
other .
@@ -979,20 +994,41 @@ other .
return PARAM;
}
-{integer} {
+{decinteger} {
+ SET_YYLLOC();
+ return process_integer_literal(yytext,
yylval, 10);
+ }
+{hexinteger} {
+ SET_YYLLOC();
+ return process_integer_literal(yytext +
2, yylval, 16);
+ }
+{octinteger} {
+ SET_YYLLOC();
+ return process_integer_literal(yytext +
2, yylval, 8);
+ }
+{bininteger} {
SET_YYLLOC();
- return process_integer_literal(yytext,
yylval);
+ return process_integer_literal(yytext +
2, yylval, 2);
+ }
+{hexfail} {
+ yyerror("invalid hexadecimal integer");
+ }
+{octfail} {
+ yyerror("invalid octal integer");
}
-{decimal} {
+{binfail} {
+ yyerror("invalid binary integer");
+ }
+{numeric} {
SET_YYLLOC();
yylval->str = pstrdup(yytext);
return FCONST;
}
-{decimalfail} {
+{numericfail} {
/* throw back the .., and treat as
integer */
yyless(yyleng - 2);
SET_YYLLOC();
- return process_integer_literal(yytext,
yylval);
+ return process_integer_literal(yytext,
yylval, 10);
}
{real} {
SET_YYLLOC();
@@ -1007,11 +1043,23 @@ other .
SET_YYLLOC();
yyerror("trailing junk after numeric
literal");
}
-{integer_junk} {
+{decinteger_junk} {
+ SET_YYLLOC();
+ yyerror("trailing junk after numeric
literal");
+ }
+{hexinteger_junk} {
+ SET_YYLLOC();
+ yyerror("trailing junk after numeric
literal");
+ }
+{octinteger_junk} {
+ SET_YYLLOC();
+ yyerror("trailing junk after numeric
literal");
+ }
+{bininteger_junk} {
SET_YYLLOC();
yyerror("trailing junk after numeric
literal");
}
-{decimal_junk} {
+{numeric_junk} {
SET_YYLLOC();
yyerror("trailing junk after numeric
literal");
}
@@ -1307,17 +1355,17 @@ litbufdup(core_yyscan_t yyscanner)
}
/*
- * Process {integer}. Note this will also do the right thing with {decimal},
+ * Process {*integer}. Note this will also do the right thing with {numeric},
* ie digits and a decimal point.
*/
static int
-process_integer_literal(const char *token, YYSTYPE *lval)
+process_integer_literal(const char *token, YYSTYPE *lval, int base)
{
int val;
char *endptr;
errno = 0;
- val = strtoint(token, &endptr, 10);
+ val = strtoint(token, &endptr, base);
if (*endptr != '\0' || errno == ERANGE)
{
/* integer too large (or contains decimal pt), treat it as a
float */
diff --git a/src/backend/utils/adt/numutils.c b/src/backend/utils/adt/numutils.c
index ebc2d222a3..776612874f 100644
--- a/src/backend/utils/adt/numutils.c
+++ b/src/backend/utils/adt/numutils.c
@@ -173,6 +173,17 @@ pg_atoi(const char *s, int size, int c)
return (int32) l;
}
+static const int8 hexlookup[128] = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
+ -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+};
+
/*
* Convert input string to a signed 16 bit integer.
*
@@ -208,6 +219,48 @@ pg_strtoint16(const char *s)
goto invalid_syntax;
/* process digits */
+ if (ptr[0] == '0' && (ptr[1] == 'x' || ptr[1] == 'X'))
+ {
+ ptr += 2;
+ while (*ptr && isxdigit((unsigned char) *ptr))
+ {
+ int8 digit = hexlookup[(unsigned char) *ptr];
+
+ if (unlikely(pg_mul_s16_overflow(tmp, 16, &tmp)) ||
+ unlikely(pg_sub_s16_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+
+ ptr++;
+ }
+ }
+ else if (ptr[0] == '0' && (ptr[1] == 'o' || ptr[1] == 'O'))
+ {
+ ptr += 2;
+
+ while (*ptr && (*ptr >= '0' && *ptr <= '7'))
+ {
+ int8 digit = (*ptr++ - '0');
+
+ if (unlikely(pg_mul_s16_overflow(tmp, 8, &tmp)) ||
+ unlikely(pg_sub_s16_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+ }
+ }
+ else if (ptr[0] == '0' && (ptr[1] == 'b' || ptr[1] == 'B'))
+ {
+ ptr += 2;
+
+ while (*ptr && (*ptr >= '0' && *ptr <= '1'))
+ {
+ int8 digit = (*ptr++ - '0');
+
+ if (unlikely(pg_mul_s16_overflow(tmp, 2, &tmp)) ||
+ unlikely(pg_sub_s16_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+ }
+ }
+ else
+ {
while (*ptr && isdigit((unsigned char) *ptr))
{
int8 digit = (*ptr++ - '0');
@@ -216,6 +269,7 @@ pg_strtoint16(const char *s)
unlikely(pg_sub_s16_overflow(tmp, digit, &tmp)))
goto out_of_range;
}
+ }
/* allow trailing whitespace, but not other trailing chars */
while (*ptr != '\0' && isspace((unsigned char) *ptr))
@@ -284,6 +338,48 @@ pg_strtoint32(const char *s)
goto invalid_syntax;
/* process digits */
+ if (ptr[0] == '0' && (ptr[1] == 'x' || ptr[1] == 'X'))
+ {
+ ptr += 2;
+ while (*ptr && isxdigit((unsigned char) *ptr))
+ {
+ int8 digit = hexlookup[(unsigned char) *ptr];
+
+ if (unlikely(pg_mul_s32_overflow(tmp, 16, &tmp)) ||
+ unlikely(pg_sub_s32_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+
+ ptr++;
+ }
+ }
+ else if (ptr[0] == '0' && (ptr[1] == 'o' || ptr[1] == 'O'))
+ {
+ ptr += 2;
+
+ while (*ptr && (*ptr >= '0' && *ptr <= '7'))
+ {
+ int8 digit = (*ptr++ - '0');
+
+ if (unlikely(pg_mul_s32_overflow(tmp, 8, &tmp)) ||
+ unlikely(pg_sub_s32_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+ }
+ }
+ else if (ptr[0] == '0' && (ptr[1] == 'b' || ptr[1] == 'B'))
+ {
+ ptr += 2;
+
+ while (*ptr && (*ptr >= '0' && *ptr <= '1'))
+ {
+ int8 digit = (*ptr++ - '0');
+
+ if (unlikely(pg_mul_s32_overflow(tmp, 2, &tmp)) ||
+ unlikely(pg_sub_s32_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+ }
+ }
+ else
+ {
while (*ptr && isdigit((unsigned char) *ptr))
{
int8 digit = (*ptr++ - '0');
@@ -292,6 +388,7 @@ pg_strtoint32(const char *s)
unlikely(pg_sub_s32_overflow(tmp, digit, &tmp)))
goto out_of_range;
}
+ }
/* allow trailing whitespace, but not other trailing chars */
while (*ptr != '\0' && isspace((unsigned char) *ptr))
@@ -368,6 +465,48 @@ pg_strtoint64(const char *s)
goto invalid_syntax;
/* process digits */
+ if (ptr[0] == '0' && (ptr[1] == 'x' || ptr[1] == 'X'))
+ {
+ ptr += 2;
+ while (*ptr && isxdigit((unsigned char) *ptr))
+ {
+ int8 digit = hexlookup[(unsigned char) *ptr];
+
+ if (unlikely(pg_mul_s64_overflow(tmp, 16, &tmp)) ||
+ unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+
+ ptr++;
+ }
+ }
+ else if (ptr[0] == '0' && (ptr[1] == 'o' || ptr[1] == 'O'))
+ {
+ ptr += 2;
+
+ while (*ptr && (*ptr >= '0' && *ptr <= '7'))
+ {
+ int8 digit = (*ptr++ - '0');
+
+ if (unlikely(pg_mul_s64_overflow(tmp, 8, &tmp)) ||
+ unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+ }
+ }
+ else if (ptr[0] == '0' && (ptr[1] == 'b' || ptr[1] == 'B'))
+ {
+ ptr += 2;
+
+ while (*ptr && (*ptr >= '0' && *ptr <= '1'))
+ {
+ int8 digit = (*ptr++ - '0');
+
+ if (unlikely(pg_mul_s64_overflow(tmp, 2, &tmp)) ||
+ unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
+ goto out_of_range;
+ }
+ }
+ else
+ {
while (*ptr && isdigit((unsigned char) *ptr))
{
int8 digit = (*ptr++ - '0');
@@ -376,6 +515,7 @@ pg_strtoint64(const char *s)
unlikely(pg_sub_s64_overflow(tmp, digit, &tmp)))
goto out_of_range;
}
+ }
/* allow trailing whitespace, but not other trailing chars */
while (*ptr != '\0' && isspace((unsigned char) *ptr))
diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l
index 4cd5e69d00..4c95eb60e8 100644
--- a/src/fe_utils/psqlscan.l
+++ b/src/fe_utils/psqlscan.l
@@ -323,25 +323,40 @@ operator {op_chars}+
* Unary minus is not part of a number here. Instead we pass it separately to
* parser, and there it gets coerced via doNegate().
*
- * {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
+ * {numericfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
*
* {realfail1} and {realfail2} are added to prevent the need for scanner
* backup when the {real} rule fails to match completely.
*/
-digit [0-9]
-
-integer {digit}+
-decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
-decimalfail {digit}+\.\.
-real ({integer}|{decimal})[Ee][-+]?{digit}+
-realfail1 ({integer}|{decimal})[Ee]
-realfail2 ({integer}|{decimal})[Ee][-+]
-
-integer_junk {integer}{ident_start}
-decimal_junk {decimal}{ident_start}
+decdigit [0-9]
+hexdigit [0-9A-Fa-f]
+octdigit [0-7]
+bindigit [0-1]
+
+decinteger {decdigit}+
+hexinteger 0[xX]{hexdigit}+
+octinteger 0[oO]{octdigit}+
+bininteger 0[bB]{bindigit}+
+
+hexfail 0[xX]
+octfail 0[oO]
+binfail 0[bB]
+
+numeric (({decinteger}\.{decinteger}?)|(\.{decinteger}))
+numericfail {decdigit}+\.\.
+
+real ({decinteger}|{numeric})[Ee][-+]?{decdigit}+
+realfail1 ({decinteger}|{numeric})[Ee]
+realfail2 ({decinteger}|{numeric})[Ee][-+]
+
+decinteger_junk {decinteger}{ident_start}
+hexinteger_junk {hexinteger}{ident_start}
+octinteger_junk {octinteger}{ident_start}
+bininteger_junk {bininteger}{ident_start}
+numeric_junk {numeric}{ident_start}
real_junk {real}{ident_start}
-param \${integer}
+param \${decinteger}
/* psql-specific: characters allowed in variable names */
variable_char [A-Za-z\200-\377_0-9]
@@ -845,13 +860,31 @@ other .
ECHO;
}
-{integer} {
+{decinteger} {
+ ECHO;
+ }
+{hexinteger} {
+ ECHO;
+ }
+{octinteger} {
+ ECHO;
+ }
+{bininteger} {
+ ECHO;
+ }
+{hexfail} {
ECHO;
}
-{decimal} {
+{octfail} {
ECHO;
}
-{decimalfail} {
+{binfail} {
+ ECHO;
+ }
+{numeric} {
+ ECHO;
+ }
+{numericfail} {
/* throw back the .., and treat as
integer */
yyless(yyleng - 2);
ECHO;
@@ -865,10 +898,19 @@ other .
{realfail2} {
ECHO;
}
-{integer_junk} {
+{decinteger_junk} {
+ ECHO;
+ }
+{hexinteger_junk} {
+ ECHO;
+ }
+{octinteger_junk} {
+ ECHO;
+ }
+{bininteger_junk} {
ECHO;
}
-{decimal_junk} {
+{numeric_junk} {
ECHO;
}
{real_junk} {
diff --git a/src/interfaces/ecpg/preproc/pgc.l
b/src/interfaces/ecpg/preproc/pgc.l
index e641095496..fde211a33f 100644
--- a/src/interfaces/ecpg/preproc/pgc.l
+++ b/src/interfaces/ecpg/preproc/pgc.l
@@ -57,7 +57,7 @@ static bool include_next;
#define startlit() (literalbuf[0] = '\0', literallen = 0)
static void addlit(char *ytext, int yleng);
static void addlitchar(unsigned char);
-static int process_integer_literal(const char *token, YYSTYPE *lval);
+static int process_integer_literal(const char *token, YYSTYPE *lval, int
base);
static void parse_include(void);
static bool ecpg_isspace(char ch);
static bool isdefine(void);
@@ -351,25 +351,40 @@ operator {op_chars}+
* Unary minus is not part of a number here. Instead we pass it separately to
* parser, and there it gets coerced via doNegate().
*
- * {decimalfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
+ * {numericfail} is used because we would like "1..10" to lex as 1, dot_dot,
10.
*
* {realfail1} and {realfail2} are added to prevent the need for scanner
* backup when the {real} rule fails to match completely.
*/
-digit [0-9]
-
-integer {digit}+
-decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
-decimalfail {digit}+\.\.
-real ({integer}|{decimal})[Ee][-+]?{digit}+
-realfail1 ({integer}|{decimal})[Ee]
-realfail2 ({integer}|{decimal})[Ee][-+]
-
-integer_junk {integer}{ident_start}
-decimal_junk {decimal}{ident_start}
+decdigit [0-9]
+hexdigit [0-9A-Fa-f]
+octdigit [0-7]
+bindigit [0-1]
+
+decinteger {decdigit}+
+hexinteger 0[xX]{hexdigit}+
+octinteger 0[oO]{octdigit}+
+bininteger 0[bB]{bindigit}+
+
+hexfail 0[xX]
+octfail 0[oO]
+binfail 0[bB]
+
+numeric (({decinteger}\.{decinteger}?)|(\.{decinteger}))
+numericfail {decdigit}+\.\.
+
+real ({decinteger}|{numeric})[Ee][-+]?{decdigit}+
+realfail1 ({decinteger}|{numeric})[Ee]
+realfail2 ({decinteger}|{numeric})[Ee][-+]
+
+decinteger_junk {decinteger}{ident_start}
+hexinteger_junk {hexinteger}{ident_start}
+octinteger_junk {octinteger}{ident_start}
+bininteger_junk {bininteger}{ident_start}
+numeric_junk {numeric}{ident_start}
real_junk {real}{ident_start}
-param \${integer}
+param \${decinteger}
/* special characters for other dbms */
/* we have to react differently in compat mode */
@@ -399,9 +414,6 @@ include_next
[iI][nN][cC][lL][uU][dD][eE]_[nN][eE][xX][tT]
import [iI][mM][pP][oO][rR][tT]
undef [uU][nN][dD][eE][fF]
-/* C version of hex number */
-xch 0[xX][0-9A-Fa-f]*
-
ccomment "//".*\n
if [iI][fF]
@@ -414,7 +426,7 @@ endif [eE][nN][dD][iI][fF]
struct [sS][tT][rR][uU][cC][tT]
exec_sql {exec}{space}*{sql}{space}*
-ipdigit ({digit}|{digit}{digit}|{digit}{digit}{digit})
+ipdigit
({decdigit}|{decdigit}{decdigit}|{decdigit}{decdigit}{decdigit})
ip {ipdigit}\.{ipdigit}\.{ipdigit}\.{ipdigit}
/* we might want to parse all cpp include files */
@@ -929,17 +941,20 @@ cppline
{space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+
} /* <SQL> */
<C,SQL>{
-{integer} {
- return process_integer_literal(yytext,
&base_yylval);
+{decinteger} {
+ return process_integer_literal(yytext,
&base_yylval, 10);
+ }
+{hexinteger} {
+ return process_integer_literal(yytext +
2, &base_yylval, 16);
}
-{decimal} {
+{numeric} {
base_yylval.str = mm_strdup(yytext);
return FCONST;
}
-{decimalfail} {
+{numericfail} {
/* throw back the .., and treat as
integer */
yyless(yyleng - 2);
- return process_integer_literal(yytext,
&base_yylval);
+ return process_integer_literal(yytext,
&base_yylval, 10);
}
{real} {
base_yylval.str = mm_strdup(yytext);
@@ -948,18 +963,25 @@ cppline
{space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+
{realfail1} {
/*
* throw back the [Ee], and figure out
whether what
- * remains is an {integer} or {decimal}.
+ * remains is an {decinteger} or
{numeric}.
*/
yyless(yyleng - 1);
- return process_integer_literal(yytext,
&base_yylval);
+ return process_integer_literal(yytext,
&base_yylval, 10);
}
{realfail2} {
/* throw back the [Ee][+-], and proceed
as above */
yyless(yyleng - 2);
- return process_integer_literal(yytext,
&base_yylval);
+ return process_integer_literal(yytext,
&base_yylval, 10);
}
} /* <C,SQL> */
+<SQL>{octinteger} {
+ return process_integer_literal(yytext +
2, &base_yylval, 8);
+ }
+<SQL>{bininteger} {
+ return process_integer_literal(yytext +
2, &base_yylval, 2);
+ }
+
<SQL>{
:{identifier}((("->"|\.){identifier})|(\[{array}\]))* {
base_yylval.str = mm_strdup(yytext+1);
@@ -1015,19 +1037,6 @@ cppline
{space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+
return S_ANYTHING;
}
<C>{ccomment} { ECHO; }
-<C>{xch} {
- char* endptr;
-
- errno = 0;
- base_yylval.ival =
strtoul((char *)yytext,&endptr,16);
- if (*endptr != '\0' || errno ==
ERANGE)
- {
- errno = 0;
- base_yylval.str =
mm_strdup(yytext);
- return SCONST;
- }
- return ICONST;
- }
<C>{cppinclude} {
if (system_includes)
{
@@ -1552,17 +1561,17 @@ addlitchar(unsigned char ychar)
}
/*
- * Process {integer}. Note this will also do the right thing with {decimal},
+ * Process {*integer}. Note this will also do the right thing with {numeric},
* ie digits and a decimal point.
*/
static int
-process_integer_literal(const char *token, YYSTYPE *lval)
+process_integer_literal(const char *token, YYSTYPE *lval, int base)
{
int val;
char *endptr;
errno = 0;
- val = strtoint(token, &endptr, 10);
+ val = strtoint(token, &endptr, base);
if (*endptr != '\0' || errno == ERANGE)
{
/* integer too large (or contains decimal pt), treat it as a
float */
diff --git a/src/test/regress/expected/int2.out
b/src/test/regress/expected/int2.out
index 55ea7202cd..220e1493e8 100644
--- a/src/test/regress/expected/int2.out
+++ b/src/test/regress/expected/int2.out
@@ -306,3 +306,22 @@ FROM (VALUES (-2.5::numeric),
2.5 | 3
(7 rows)
+-- non-decimal literals
+SELECT int2 '0b100101';
+ int2
+------
+ 37
+(1 row)
+
+SELECT int2 '0o273';
+ int2
+------
+ 187
+(1 row)
+
+SELECT int2 '0x42F';
+ int2
+------
+ 1071
+(1 row)
+
diff --git a/src/test/regress/expected/int4.out
b/src/test/regress/expected/int4.out
index 9d20b3380f..6fdbd58b40 100644
--- a/src/test/regress/expected/int4.out
+++ b/src/test/regress/expected/int4.out
@@ -437,3 +437,22 @@ SELECT lcm((-2147483648)::int4, 1::int4); -- overflow
ERROR: integer out of range
SELECT lcm(2147483647::int4, 2147483646::int4); -- overflow
ERROR: integer out of range
+-- non-decimal literals
+SELECT int4 '0b100101';
+ int4
+------
+ 37
+(1 row)
+
+SELECT int4 '0o273';
+ int4
+------
+ 187
+(1 row)
+
+SELECT int4 '0x42F';
+ int4
+------
+ 1071
+(1 row)
+
diff --git a/src/test/regress/expected/int8.out
b/src/test/regress/expected/int8.out
index 36540ec456..edd15a4353 100644
--- a/src/test/regress/expected/int8.out
+++ b/src/test/regress/expected/int8.out
@@ -932,3 +932,22 @@ SELECT lcm((-9223372036854775808)::int8, 1::int8); --
overflow
ERROR: bigint out of range
SELECT lcm(9223372036854775807::int8, 9223372036854775806::int8); -- overflow
ERROR: bigint out of range
+-- non-decimal literals
+SELECT int8 '0b100101';
+ int8
+------
+ 37
+(1 row)
+
+SELECT int8 '0o273';
+ int8
+------
+ 187
+(1 row)
+
+SELECT int8 '0x42F';
+ int8
+------
+ 1071
+(1 row)
+
diff --git a/src/test/regress/expected/numerology.out
b/src/test/regress/expected/numerology.out
index 2f176ccb52..5313b47d3b 100644
--- a/src/test/regress/expected/numerology.out
+++ b/src/test/regress/expected/numerology.out
@@ -3,14 +3,33 @@
-- Test various combinations of numeric types and functions.
--
--
--- Trailing junk in numeric literals
+-- numeric literals
--
+SELECT 0b100101;
+ ?column?
+----------
+ 37
+(1 row)
+
+SELECT 0o273;
+ ?column?
+----------
+ 187
+(1 row)
+
+SELECT 0x42F;
+ ?column?
+----------
+ 1071
+(1 row)
+
+-- error cases
SELECT 123abc;
ERROR: trailing junk after numeric literal at or near "123a"
LINE 1: SELECT 123abc;
^
SELECT 0x0o;
-ERROR: trailing junk after numeric literal at or near "0x"
+ERROR: trailing junk after numeric literal at or near "0x0o"
LINE 1: SELECT 0x0o;
^
SELECT 1_2_3;
@@ -41,6 +60,42 @@ SELECT 0.0e+a;
ERROR: trailing junk after numeric literal at or near "0.0e+"
LINE 1: SELECT 0.0e+a;
^
+SELECT 0b;
+ERROR: invalid binary integer at or near "SELECT 0b"
+LINE 1: SELECT 0b;
+ ^
+SELECT 1b;
+ERROR: trailing junk after numeric literal at or near "1b"
+LINE 1: SELECT 1b;
+ ^
+SELECT 0b0x;
+ERROR: trailing junk after numeric literal at or near "0b0x"
+LINE 1: SELECT 0b0x;
+ ^
+SELECT 0o;
+ERROR: invalid octal integer at or near "SELECT 0o"
+LINE 1: SELECT 0o;
+ ^
+SELECT 1o;
+ERROR: trailing junk after numeric literal at or near "1o"
+LINE 1: SELECT 1o;
+ ^
+SELECT 0o0x;
+ERROR: trailing junk after numeric literal at or near "0o0x"
+LINE 1: SELECT 0o0x;
+ ^
+SELECT 0x;
+ERROR: invalid hexadecimal integer at or near "SELECT 0x"
+LINE 1: SELECT 0x;
+ ^
+SELECT 1x;
+ERROR: trailing junk after numeric literal at or near "1x"
+LINE 1: SELECT 1x;
+ ^
+SELECT 0x0y;
+ERROR: trailing junk after numeric literal at or near "0x0y"
+LINE 1: SELECT 0x0y;
+ ^
--
-- Test implicit type conversions
-- This fails for Postgres v6.1 (and earlier?)
diff --git a/src/test/regress/sql/int2.sql b/src/test/regress/sql/int2.sql
index 613b344704..0dee22fe6d 100644
--- a/src/test/regress/sql/int2.sql
+++ b/src/test/regress/sql/int2.sql
@@ -112,3 +112,10 @@ CREATE TABLE INT2_TBL(f1 int2);
(0.5::numeric),
(1.5::numeric),
(2.5::numeric)) t(x);
+
+
+-- non-decimal literals
+
+SELECT int2 '0b100101';
+SELECT int2 '0o273';
+SELECT int2 '0x42F';
diff --git a/src/test/regress/sql/int4.sql b/src/test/regress/sql/int4.sql
index 55ec07a147..2a69b1614e 100644
--- a/src/test/regress/sql/int4.sql
+++ b/src/test/regress/sql/int4.sql
@@ -176,3 +176,10 @@ CREATE TABLE INT4_TBL(f1 int4);
SELECT lcm((-2147483648)::int4, 1::int4); -- overflow
SELECT lcm(2147483647::int4, 2147483646::int4); -- overflow
+
+
+-- non-decimal literals
+
+SELECT int4 '0b100101';
+SELECT int4 '0o273';
+SELECT int4 '0x42F';
diff --git a/src/test/regress/sql/int8.sql b/src/test/regress/sql/int8.sql
index 32940b4daa..b7ad696dd8 100644
--- a/src/test/regress/sql/int8.sql
+++ b/src/test/regress/sql/int8.sql
@@ -250,3 +250,10 @@ CREATE TABLE INT8_TBL(q1 int8, q2 int8);
SELECT lcm((-9223372036854775808)::int8, 1::int8); -- overflow
SELECT lcm(9223372036854775807::int8, 9223372036854775806::int8); -- overflow
+
+
+-- non-decimal literals
+
+SELECT int8 '0b100101';
+SELECT int8 '0o273';
+SELECT int8 '0x42F';
diff --git a/src/test/regress/sql/numerology.sql
b/src/test/regress/sql/numerology.sql
index 70447a95fa..fd7e02e536 100644
--- a/src/test/regress/sql/numerology.sql
+++ b/src/test/regress/sql/numerology.sql
@@ -3,10 +3,16 @@
-- Test various combinations of numeric types and functions.
--
+
--
--- Trailing junk in numeric literals
+-- numeric literals
--
+SELECT 0b100101;
+SELECT 0o273;
+SELECT 0x42F;
+
+-- error cases
SELECT 123abc;
SELECT 0x0o;
SELECT 1_2_3;
@@ -17,6 +23,19 @@
SELECT 0.0e;
SELECT 0.0e+a;
+SELECT 0b;
+SELECT 1b;
+SELECT 0b0x;
+
+SELECT 0o;
+SELECT 1o;
+SELECT 0o0x;
+
+SELECT 0x;
+SELECT 1x;
+SELECT 0x0y;
+
+
--
-- Test implicit type conversions
-- This fails for Postgres v6.1 (and earlier?)
--
2.33.1
