Changeset: ad077f405804 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=ad077f405804 Added Files: buildtools/autogen/scripts/sql2c.py sql/scripts/09_like.sql.c sql/scripts/09_like.sql.h sql/scripts/10_math.sql.c sql/scripts/10_math.sql.h sql/scripts/11_times.sql.c sql/scripts/11_times.sql.h sql/scripts/12_url.sql.c sql/scripts/12_url.sql.h sql/scripts/13_date.sql.c sql/scripts/13_date.sql.h sql/scripts/14_inet.sql.c sql/scripts/14_inet.sql.h sql/scripts/15_querylog.sql.c sql/scripts/15_querylog.sql.h sql/scripts/16_tracelog.sql.c sql/scripts/16_tracelog.sql.h sql/scripts/17_temporal.sql.c sql/scripts/17_temporal.sql.h sql/scripts/18_index.sql.c sql/scripts/18_index.sql.h sql/scripts/20_vacuum.sql.c sql/scripts/20_vacuum.sql.h sql/scripts/21_dependency_views.sql.c sql/scripts/21_dependency_views.sql.h sql/scripts/22_clients.sql.c sql/scripts/22_clients.sql.h sql/scripts/23_skyserver.sql.c sql/scripts/23_skyserver.sql.h sql/scripts/25_debug.sql.c sql/scripts/25_debug.sql.h sql/scripts/26_sysmon.sql.c sql/scripts/26_sysmon.sql.h sql/scripts/27_rejects.sql.c sql/scripts/27_rejects.sql.h sql/scripts/39_analytics.sql.c sql/scripts/39_analytics.sql.h sql/scripts/39_analytics_hge.sql.c sql/scripts/39_analytics_hge.sql.h sql/scripts/40_json.sql.c sql/scripts/40_json.sql.h sql/scripts/40_json_hge.sql.c sql/scripts/40_json_hge.sql.h sql/scripts/41_md5sum.sql.c sql/scripts/41_md5sum.sql.h sql/scripts/45_uuid.sql.c sql/scripts/45_uuid.sql.h sql/scripts/46_profiler.sql.c sql/scripts/46_profiler.sql.h sql/scripts/51_sys_schema_extension.sql.c sql/scripts/51_sys_schema_extension.sql.h sql/scripts/60_wlcr.sql.c sql/scripts/60_wlcr.sql.h sql/scripts/75_storagemodel.sql.c sql/scripts/75_storagemodel.sql.h sql/scripts/80_statistics.sql.c sql/scripts/80_statistics.sql.h sql/scripts/99_system.sql.c sql/scripts/99_system.sql.h sql/scripts/sql_scripts.c sql/scripts/sql_scripts.h sql/scripts/sql_scripts_hge.c sql/scripts/sql_scripts_hge.h Modified Files: sql/backends/monet5/Makefile.ag sql/backends/monet5/sql_scenario.c sql/scripts/Makefile.ag Branch: in-memory Log Message:
Progress on generating C files from SQL scripts. diffs (truncated from 2704 to 300 lines): diff --git a/buildtools/autogen/scripts/sql2c.py b/buildtools/autogen/scripts/sql2c.py new file mode 100644 --- /dev/null +++ b/buildtools/autogen/scripts/sql2c.py @@ -0,0 +1,216 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V. + +from __future__ import print_function + +import os +import sys + + +if len(sys.argv) < 2: + raise Exception("One argument must be given to sql2c.py") + +sql_content_file = open(sys.argv[1], 'r') +sql_content = sql_content_file.read() +sql_content_file.close() + +# get the file name and extension +base = os.path.basename(sys.argv[1]) +split = os.path.splitext(base) +filebasename = split[0] + +sql_c_output_file = open(filebasename + ".sql.c", 'w') +sql_h_output_file = open(filebasename + ".sql.h", 'w') + +# write the common header +insert1 = ( + '/*\n' + '* This Source Code Form is subject to the terms of the Mozilla Public\n' + '* License, v. 2.0. If a copy of the MPL was not distributed with this\n' + '* file, You can obtain one at http://mozilla.org/MPL/2.0/.\n' + '*\n' + '* Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.\n' + '*/\n' + '\n' + '// This file was generated automatically through bootstrap.py.\n// Do not edit this file directly.\n' + '\n' + '#include "monetdb_config.h"\n' + '\n' + '#include "{0}.sql.h"\n' + '#include "gdk.h"\n' + '#include "mal_client.h"\n' + '#include "sql_catalog.h"\n' + '\n' + '#include <assert.h>\n' + '\n' + 'extern str SQLstatementIntern(Client c, str *expr, str nme, bit execute, bit output, res_table **result);\n' + '\n' + 'str\n' + 'sql_install_{0}(Client c)\n' + '{{\n' + '\tsize_t bufsize = 16384, pos = 0;\n' + '\tchar *buf = GDKmalloc(bufsize), *err = NULL;\n' + '\n' + '\tif (buf == NULL)\n' + '\t\tthrow(SQL, "sql.install_{1}", SQLSTATE(HY001) MAL_MALLOC_FAIL);\n' + '\tpos += snprintf(buf + pos, bufsize - pos, "').format(filebasename, filebasename) + +sql_c_output_file.write(insert1) + +# Let's remove comments from the sql script with a Markov chain :) Bugs might still be there +# STATES 0 - OK, 1 in # comment, 2 in -- comment, 3 in /* comment, 4 inside ' string, 5 inside " string, +# 6 inside whitespaces +CACHE_SIZE = 4096 + +buffer = ['\0'] * CACHE_SIZE +cur_state = 0 +current_pointer = 0 +i = 0 +endloop = len(sql_content) - 1 + + +def write_to_buffer(input_c): + global current_pointer, sql_c_output_file, buffer + if current_pointer == CACHE_SIZE: + sql_c_output_file.write("".join(buffer)) + current_pointer = 0 + buffer[current_pointer] = input_c + current_pointer += 1 + + +while i < endloop: + c = sql_content[i] + + if cur_state == 1: + if c == '\n': + cur_state = 0 + i += 1 + continue + elif cur_state == 2: + if c == '\n': + cur_state = 0 + i += 1 + continue + elif cur_state == 3: + if c == '*' and sql_content[i + 1] == '/': + cur_state = 0 + i += 1 + i += 1 + continue + elif cur_state == 4: + if c == '\\': + write_to_buffer('\\') + write_to_buffer('\\') + write_to_buffer('\\') + write_to_buffer(sql_content[i + 1]) + i += 2 + else: + if c == '\'': + cur_state = 0 + write_to_buffer(c) + i += 1 + continue + elif cur_state == 5: + if c == '\\': + write_to_buffer('\\') + write_to_buffer('\\') + write_to_buffer('\\') + write_to_buffer(sql_content[i + 1]) + i += 2 + else: + if c == '"': + cur_state = 0 + write_to_buffer('\\') + write_to_buffer(c) + i += 1 + continue + elif cur_state == 6: + if c not in (' ', '\t', '\n'): + cur_state = 0 + continue + i += 1 + continue + + if c == '#': + cur_state = 1 + i += 1 + continue + elif c == '-' and sql_content[i + 1] == '-': + cur_state = 2 + i += 2 + continue + elif c == '/' and sql_content[i + 1] == '*': + cur_state = 3 + i += 2 + continue + elif c == '\'': + write_to_buffer(c) + cur_state = 4 + i += 1 + continue + elif c == '"': + write_to_buffer('\\') + write_to_buffer(c) + cur_state = 5 + i += 1 + continue + elif c in (' ', '\t', '\n'): + if c == '\n': + write_to_buffer(' ') + else: + write_to_buffer(c) + cur_state = 6 + i += 1 + continue + + write_to_buffer(c) + i += 1 + +if current_pointer > 0: + sql_c_output_file.write("".join(buffer[:current_pointer])) + +# insert more common stuff +insert2 = ( +'");\n' +'\n' +'\tpos += snprintf(buf + pos, bufsize - pos, "commit;\\n");\n' +'\n' +'\tassert(pos < bufsize);\n' +'\tprintf("#Loading: {0}{1}\\n");\n' +'\terr = SQLstatementIntern(c, &buf, "install", 1, 0, NULL);\n' +'\tGDKfree(buf);\n' +'\treturn err;\n' +'}}\n').format(filebasename, split[1]) + +sql_c_output_file.write(insert2) +sql_c_output_file.close() + +# write header file +insert3 = ( + '/*\n' + '* This Source Code Form is subject to the terms of the Mozilla Public\n' + '* License, v. 2.0. If a copy of the MPL was not distributed with this\n' + '* file, You can obtain one at http://mozilla.org/MPL/2.0/.\n' + '*\n' + '* Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.\n' + '*/\n' + '\n' + '// This file was generated automatically through bootstrap.py.\n// Do not edit this file directly.\n' + '\n' + '#ifndef SQL_{0}_H\n' + '#define SQL_{0}_H\n' + '\n' + '#include "monetdb_config.h"\n' + '\n' + '#include "gdk.h"\n' + '#include "mal_client.h"\n' + '\n' + 'extern str sql_install_{1}(Client c);\n' + '\n' + '#endif //SQL_{0}_H\n').format(filebasename.upper(), filebasename) + +sql_h_output_file.write(insert3) +sql_h_output_file.close() diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag --- a/sql/backends/monet5/Makefile.ag +++ b/sql/backends/monet5/Makefile.ag @@ -6,7 +6,7 @@ SUBDIRS = NOT_WIN32?vaults UDF generator -INCLUDES = ../../include ../../common ../../storage ../../server \ +INCLUDES = ../../include ../../common ../../storage ../../server ../../scripts \ ../../../monetdb5/modules/atoms \ ../../../monetdb5/modules/kernel \ ../../../monetdb5/mal \ diff --git a/sql/backends/monet5/sql_scenario.c b/sql/backends/monet5/sql_scenario.c --- a/sql/backends/monet5/sql_scenario.c +++ b/sql/backends/monet5/sql_scenario.c @@ -45,6 +45,10 @@ #include "opt_mitosis.h" #include <unistd.h> #include "sql_upgrades.h" +#include "sql_scripts.h" +#ifdef HAVE_HGE +#include "sql_scripts_hge.h" +#endif static int SQLinitialized = 0; static int SQLnewcatalog = 0; @@ -469,98 +473,17 @@ SQLinit(Client c) SQLnewcatalog = 0; maybeupgrade = 0; -#ifdef HAVE_EMBEDDED - size_t createdb_len = strlen(createdb_inline); - buffer* createdb_buf; - stream* createdb_stream; - bstream* createdb_bstream; - if ((createdb_buf = GDKmalloc(sizeof(buffer))) == NULL) { - MT_lock_unset(&sql_contextLock); - throw(MAL, "createdb", SQLSTATE(HY001) MAL_MALLOC_FAIL); - } - buffer_init(createdb_buf, createdb_inline, createdb_len); - if ((createdb_stream = buffer_rastream(createdb_buf, "createdb.sql")) == NULL) { - MT_lock_unset(&sql_contextLock); - GDKfree(createdb_buf); - throw(MAL, "createdb", SQLSTATE(HY001) MAL_MALLOC_FAIL); - } - if ((createdb_bstream = bstream_create(createdb_stream, createdb_len)) == NULL) { - MT_lock_unset(&sql_contextLock); - close_stream(createdb_stream); - GDKfree(createdb_buf); - throw(MAL, "createdb", SQLSTATE(HY001) MAL_MALLOC_FAIL); - } - if (bstream_next(createdb_bstream) >= 0) - msg = SQLstatementIntern(c, &createdb_bstream->buf, "sql.init", TRUE, FALSE, NULL); - else - msg = createException(MAL, "createdb", SQLSTATE(42000) "Could not load inlined createdb script"); + if ((msg = install_sql_scripts(c)) != MAL_SUCCEED) + return msg; +#ifdef HAVE_HGE + if ((msg = install_sql_scripts_hge(c)) != MAL_SUCCEED) + return msg; +#endif - bstream_destroy(createdb_bstream); - GDKfree(createdb_buf); if (m->sa) sa_destroy(m->sa); m->sa = NULL; m->sqs = NULL; - -#else - char path[FILENAME_MAX]; - str fullname; - - snprintf(path, FILENAME_MAX, "createdb"); - slash_2_dir_sep(path); - fullname = MSP_locate_sqlscript(path, 1); - if (fullname) { - str filename = fullname; - str p, n, newmsg= MAL_SUCCEED; _______________________________________________ checkin-list mailing list [email protected] https://www.monetdb.org/mailman/listinfo/checkin-list
