Changeset: 58c81fe1e454 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=58c81fe1e454
Added Files:
sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.sql
sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.err
sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.out
Modified Files:
sql/server/sql_parser.y
sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
sql/test/BugTracker-2014/Tests/BOM-in-string.Bug-3641.stable.out
sql/test/BugTracker-2016/Tests/All
sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_JdbcClient.stable.out
sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_MapiClient.stable.out
Branch: Jul2015
Log Message:
When parsing a string in SQL, record the number of Unicode code points, not
bytes.
This fixes bug 3930.
Also added test.
diffs (truncated from 315 to 300 lines):
diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -57,6 +57,40 @@
#define MAX_HEX_DIGITS 16
#endif
+static inline int
+UTF8_strlen(const char *val)
+{
+ const unsigned char *s = (const unsigned char *) val;
+ int pos = 0;
+
+ while (*s) {
+ int c = *s++;
+
+ pos++;
+ if (c < 0xC0)
+ continue;
+ if (*s++ < 0x80)
+ return int_nil;
+ if (c < 0xE0)
+ continue;
+ if (*s++ < 0x80)
+ return int_nil;
+ if (c < 0xF0)
+ continue;
+ if (*s++ < 0x80)
+ return int_nil;
+ if (c < 0xF8)
+ continue;
+ if (*s++ < 0x80)
+ return int_nil;
+ if (c < 0xFC)
+ continue;
+ if (*s++ < 0x80)
+ return int_nil;
+ }
+ return pos;
+}
+
%}
/* KNOWN NOT DONE OF sql'99
*
@@ -728,7 +762,7 @@ set_statement:
| set sqlSESSION AUTHORIZATION ident
{ dlist *l = L();
sql_subtype t;
- sql_find_subtype(&t, "char", _strlen($4), 0 );
+ sql_find_subtype(&t, "char", UTF8_strlen($4), 0 );
append_string(l, sa_strdup(SA, "current_user"));
append_symbol(l,
_newAtomNode( _atom_string(&t, sql2str($4))) );
@@ -736,7 +770,7 @@ set_statement:
| set SCHEMA ident
{ dlist *l = L();
sql_subtype t;
- sql_find_subtype(&t, "char", _strlen($3), 0 );
+ sql_find_subtype(&t, "char", UTF8_strlen($3), 0 );
append_string(l, sa_strdup(SA, "current_schema"));
append_symbol(l,
_newAtomNode( _atom_string(&t, sql2str($3))) );
@@ -744,7 +778,7 @@ set_statement:
| set user '=' ident
{ dlist *l = L();
sql_subtype t;
- sql_find_subtype(&t, "char", _strlen($4), 0 );
+ sql_find_subtype(&t, "char", UTF8_strlen($4), 0 );
append_string(l, sa_strdup(SA, "current_user"));
append_symbol(l,
_newAtomNode( _atom_string(&t, sql2str($4))) );
@@ -752,7 +786,7 @@ set_statement:
| set ROLE ident
{ dlist *l = L();
sql_subtype t;
- sql_find_subtype(&t, "char", _strlen($3), 0);
+ sql_find_subtype(&t, "char", UTF8_strlen($3), 0);
append_string(l, sa_strdup(SA, "current_role"));
append_symbol(l,
_newAtomNode( _atom_string(&t, sql2str($3))) );
@@ -4143,7 +4177,7 @@ user:
literal:
string { char *s = sql2str($1);
- int len = _strlen(s);
+ int len = UTF8_strlen(s);
sql_subtype t;
sql_find_subtype(&t, "char", len, 0 );
$$ = _newAtomNode( _atom_string(&t, s)); }
diff --git a/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
b/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
--- a/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
+++ b/sql/test/BugTracker-2009/Tests/utf8_bug.SF-2822855.stable.out
@@ -39,7 +39,7 @@ Ready.
% .L # table_name
% Liever euro # name
% char # type
-% 13 # length
+% 11 # length
[ "Liever €uro" ]
# 20:06:13 >
diff --git a/sql/test/BugTracker-2014/Tests/BOM-in-string.Bug-3641.stable.out
b/sql/test/BugTracker-2014/Tests/BOM-in-string.Bug-3641.stable.out
--- a/sql/test/BugTracker-2014/Tests/BOM-in-string.Bug-3641.stable.out
+++ b/sql/test/BugTracker-2014/Tests/BOM-in-string.Bug-3641.stable.out
@@ -64,7 +64,7 @@ Ready.
% .L # table_name
% single_value # name
% char # type
-% 5 # length
+% 3 # length
[ " " ]
# 09:47:46 >
diff --git a/sql/test/BugTracker-2016/Tests/All
b/sql/test/BugTracker-2016/Tests/All
--- a/sql/test/BugTracker-2016/Tests/All
+++ b/sql/test/BugTracker-2016/Tests/All
@@ -2,3 +2,4 @@ LEFT-JOIN_with_OR_conditions_triggers_as
incorrect_column_name_in_OR_condition_of_LEFT-JOIN_crashes_mserver.Bug-3909
copy_into_with_header.Bug-3910
column_name_conflict.Bug-3912
+prepare-utf8.Bug-3930
diff --git a/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.sql
b/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.sql
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.sql
@@ -0,0 +1,8 @@
+start transaction;
+
+create table bug3930 (a varchar(4));
+
+prepare select * from bug3930 where a = ?;
+exec **('ä123');
+
+rollback;
diff --git a/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.err
b/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.err
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.err
@@ -0,0 +1,37 @@
+stderr of test 'prepare-utf8.Bug-3930` in directory 'sql/test/BugTracker-2016`
itself:
+
+
+# 15:40:49 >
+# 15:40:49 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=34042" "--set"
"mapi_usock=/var/tmp/mtest-27311/.s.monetdb.34042" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/ufs/sjoerd/Monet-stable/var/MonetDB/mTests_sql_test_BugTracker-2016"
"--set" "mal_listing=0" "--set" "embedded_r=yes"
+# 15:40:49 >
+
+# builtin opt gdk_dbpath = /ufs/sjoerd/Monet-stable/var/monetdb5/dbfarm/demo
+# builtin opt gdk_debug = 0
+# builtin opt gdk_vmtrim = no
+# builtin opt monet_prompt = >
+# builtin opt monet_daemon = no
+# builtin opt mapi_port = 50000
+# builtin opt mapi_open = false
+# builtin opt mapi_autosense = false
+# builtin opt sql_optimizer = default_pipe
+# builtin opt sql_debug = 0
+# cmdline opt gdk_nr_threads = 0
+# cmdline opt mapi_open = true
+# cmdline opt mapi_port = 34042
+# cmdline opt mapi_usock = /var/tmp/mtest-27311/.s.monetdb.34042
+# cmdline opt monet_prompt =
+# cmdline opt mal_listing = 2
+# cmdline opt gdk_dbpath =
/ufs/sjoerd/Monet-stable/var/MonetDB/mTests_sql_test_BugTracker-2016
+# cmdline opt mal_listing = 0
+# cmdline opt embedded_r = yes
+# cmdline opt gdk_debug = 536870922
+
+# 15:40:50 >
+# 15:40:50 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-27311" "--port=34042"
+# 15:40:50 >
+
+
+# 15:40:50 >
+# 15:40:50 > "Done."
+# 15:40:50 >
+
diff --git a/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.out
b/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.out
new file mode 100644
--- /dev/null
+++ b/sql/test/BugTracker-2016/Tests/prepare-utf8.Bug-3930.stable.out
@@ -0,0 +1,87 @@
+stdout of test 'prepare-utf8.Bug-3930` in directory 'sql/test/BugTracker-2016`
itself:
+
+
+# 15:40:49 >
+# 15:40:49 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set"
"mapi_open=true" "--set" "mapi_port=34042" "--set"
"mapi_usock=/var/tmp/mtest-27311/.s.monetdb.34042" "--set" "monet_prompt="
"--forcemito" "--set" "mal_listing=2"
"--dbpath=/ufs/sjoerd/Monet-stable/var/MonetDB/mTests_sql_test_BugTracker-2016"
"--set" "mal_listing=0" "--set" "embedded_r=yes"
+# 15:40:49 >
+
+# MonetDB 5 server v11.21.14 (hg id: e8c4edcbfabb+)
+# This is an unreleased version
+# Serving database 'mTests_sql_test_BugTracker-2016', using 8 threads
+# Compiled for x86_64-unknown-linux-gnu/64bit with 64bit OIDs and 128bit
integers dynamically linked
+# Found 15.589 GiB available main-memory.
+# Copyright (c) 1993-July 2008 CWI.
+# Copyright (c) August 2008-2015 MonetDB B.V., all rights reserved
+# Visit http://www.monetdb.org/ for further information
+# Listening for connection requests on mapi:monetdb://madrid.da.cwi.nl:34042/
+# Listening for UNIX domain connection requests on
mapi:monetdb:///var/tmp/mtest-27311/.s.monetdb.34042
+# MonetDB/GIS module loaded
+# Start processing logs sql/sql_logs version 52200
+# Finished processing logs sql/sql_logs
+# MonetDB/SQL module loaded
+# MonetDB/R module loaded
+
+Ready.
+# SQL catalog created, loading sql scripts once
+# loading sql script: 09_like.sql
+# loading sql script: 10_math.sql
+# loading sql script: 11_times.sql
+# loading sql script: 12_url.sql
+# loading sql script: 13_date.sql
+# loading sql script: 14_inet.sql
+# loading sql script: 15_querylog.sql
+# loading sql script: 16_tracelog.sql
+# loading sql script: 17_temporal.sql
+# loading sql script: 20_vacuum.sql
+# loading sql script: 21_dependency_functions.sql
+# loading sql script: 22_clients.sql
+# loading sql script: 23_skyserver.sql
+# loading sql script: 24_zorder.sql
+# loading sql script: 25_debug.sql
+# loading sql script: 26_sysmon.sql
+# loading sql script: 27_rejects.sql
+# loading sql script: 39_analytics.sql
+# loading sql script: 39_analytics_hge.sql
+# loading sql script: 40_geom.sql
+# loading sql script: 40_json.sql
+# loading sql script: 40_json_hge.sql
+# loading sql script: 41_md5sum.sql
+# loading sql script: 45_uuid.sql
+# loading sql script: 46_gsl.sql
+# loading sql script: 51_sys_schema_extension.sql
+# loading sql script: 72_fits.sql
+# loading sql script: 74_netcdf.sql
+# loading sql script: 75_storagemodel.sql
+# loading sql script: 80_statistics.sql
+# loading sql script: 80_udf.sql
+# loading sql script: 80_udf_hge.sql
+# loading sql script: 85_bam.sql
+# loading sql script: 90_generator.sql
+# loading sql script: 90_generator_hge.sql
+# loading sql script: 99_system.sql
+
+# 15:40:50 >
+# 15:40:50 > "mclient" "-lsql" "-ftest" "-Eutf-8" "-i" "-e"
"--host=/var/tmp/mtest-27311" "--port=34042"
+# 15:40:50 >
+
+#start transaction;
+#create table bug3930 (a varchar(4));
+#prepare select * from bug3930 where a = ?;
+#prepare select * from bug3930 where a = ?;
+% .prepare, .prepare, .prepare, .prepare, .prepare,
.prepare # table_name
+% type, digits, scale, schema, table, column # name
+% varchar, int, int, str, str, str # type
+% 7, 1, 1, 0, 7, 1 # length
+[ "varchar", 4, 0, "", "bug3930", "a" ]
+[ "varchar", 4, 0, NULL, NULL, NULL ]
+#exec 2('ä123');
+% sys.bug3930 # table_name
+% a # name
+% varchar # type
+% 0 # length
+#rollback;
+
+# 15:40:50 >
+# 15:40:50 > "Done."
+# 15:40:50 >
+
diff --git
a/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_JdbcClient.stable.out
b/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_JdbcClient.stable.out
--- a/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_JdbcClient.stable.out
+++ b/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_JdbcClient.stable.out
@@ -30,18 +30,18 @@ Ready.
# 12:55:12 > Mtimeout -timeout 60 java -jar
/net/pegasus.ins.cwi.nl/export/scratch1/fabian/monetdb/current/build-pegasus.ins.cwi.nl/sql/src/jdbc/jdbcclient-1.2.jar
-h localhost -p 45579 -f
../../../../../../../../../net/pegasus.ins.cwi.nl/export/scratch1/fabian/monetdb/current/sql/src/test/bugs/Tests/../bug-sf-1041324_weird-char.sql
# 12:55:12 >
-+----------------------------------------------------------------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------------------------------------------+
| single_value
|
-+==================================================================================================================================+
++================================================================================================================================+
|
http://shopping.b92.net/index.php?IDS=11&ID_Group=39&ID_prod=754&PROD_NAME=Belgrade+Coffee+Shop+4&GROUP_NAME=Muzi�a+produkcija
|
-+----------------------------------------------------------------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------------------------------------------+
1 row
-+----------------------------------------------------------------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------------------------------------------+
| single_value
|
-+==================================================================================================================================+
++================================================================================================================================+
| SAVE 25¢ ON ICE CREAM
|
-+----------------------------------------------------------------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------------------------------------------+
1 row
diff --git
a/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_MapiClient.stable.out
b/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_MapiClient.stable.out
--- a/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_MapiClient.stable.out
+++ b/sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324_MapiClient.stable.out
@@ -29,13 +29,13 @@ Ready.
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
