Changeset: 392da9cd5277 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=392da9cd5277
Added Files:
buildtools/autogen/scripts/mal2h.py
monetdb5/modules/atoms/00_json_hge.mal.h
monetdb5/modules/atoms/batxml.mal.h
monetdb5/modules/atoms/blob.mal.h
monetdb5/modules/atoms/color.mal.h
monetdb5/modules/atoms/identifier.mal.h
monetdb5/modules/atoms/inet.mal.h
monetdb5/modules/atoms/json.mal.h
monetdb5/modules/atoms/mtime.mal.h
monetdb5/modules/atoms/str.mal.h
monetdb5/modules/atoms/streams.mal.h
monetdb5/modules/atoms/url.mal.h
monetdb5/modules/atoms/uuid.mal.h
monetdb5/modules/atoms/xml.mal.h
monetdb5/modules/kernel/00_aggr_hge.mal.h
monetdb5/modules/kernel/aggr.mal.h
monetdb5/modules/kernel/alarm.mal.h
monetdb5/modules/kernel/algebra.mal.h
monetdb5/modules/kernel/bat5.mal.h
monetdb5/modules/kernel/batcolor.mal.h
monetdb5/modules/kernel/batmmath.mal.h
monetdb5/modules/kernel/batstr.mal.h
monetdb5/modules/kernel/group.mal.h
monetdb5/modules/kernel/logger.mal.h
monetdb5/modules/kernel/microbenchmark.mal.h
monetdb5/modules/kernel/mmath.mal.h
monetdb5/modules/kernel/status.mal.h
monetdb5/modules/mal/00_batExtensions_hge.mal.h
monetdb5/modules/mal/00_batcalc_hge.mal.h
monetdb5/modules/mal/00_calc_hge.mal.h
monetdb5/modules/mal/00_iterator_hge.mal.h
monetdb5/modules/mal/00_language_hge.mal.h
monetdb5/modules/mal/00_mal_mapi_hge.mal.h
monetdb5/modules/mal/00_mkey_hge.mal.h
monetdb5/modules/mal/01_batcalc.mal.h
monetdb5/modules/mal/01_calc.mal.h
monetdb5/modules/mal/batExtensions.mal.h
monetdb5/modules/mal/batmtime.mal.h
monetdb5/modules/mal/bbp.mal.h
monetdb5/modules/mal/clients.mal.h
monetdb5/modules/mal/factories.mal.h
monetdb5/modules/mal/groupby.mal.h
monetdb5/modules/mal/inspect.mal.h
monetdb5/modules/mal/iterator.mal.h
monetdb5/modules/mal/json_util.mal.h
monetdb5/modules/mal/language.mal.h
monetdb5/modules/mal/mal_io.mal.h
monetdb5/modules/mal/mal_mapi.mal.h
monetdb5/modules/mal/manifold.mal.h
monetdb5/modules/mal/manual.mal.h
monetdb5/modules/mal/mat.mal.h
monetdb5/modules/mal/mdb.mal.h
monetdb5/modules/mal/mkey.mal.h
monetdb5/modules/mal/oltp.mal.h
monetdb5/modules/mal/orderidx.mal.h
monetdb5/modules/mal/pcre.mal.h
monetdb5/modules/mal/profiler.mal.h
monetdb5/modules/mal/querylog.mal.h
monetdb5/modules/mal/remote.mal.h
monetdb5/modules/mal/sabaoth.mal.h
monetdb5/modules/mal/sample.mal.h
monetdb5/modules/mal/sysmon.mal.h
monetdb5/modules/mal/tablet.mal.h
monetdb5/modules/mal/tokenizer.mal.h
monetdb5/modules/mal/transaction.mal.h
monetdb5/modules/mal/txtsim.mal.h
monetdb5/modules/mal/wlc.mal.h
monetdb5/optimizer/optimizer.mal.h
monetdb5/scheduler/run_adder.mal.h
monetdb5/scheduler/run_isolate.mal.h
monetdb5/scheduler/run_memo.mal.h
sql/backends/monet5/40_sql.mal.h
sql/backends/monet5/41_sql_hge.mal.h
sql/backends/monet5/generator/90_generator.mal.h
sql/backends/monet5/generator/90_generator_hge.mal.h
sql/backends/monet5/generator/generator.mal.h
sql/backends/monet5/generator/generator_hge.mal.h
sql/backends/monet5/sql.mal.h
sql/backends/monet5/sql_aggr_bte.mal.h
sql/backends/monet5/sql_aggr_dbl.mal.h
sql/backends/monet5/sql_aggr_flt.mal.h
sql/backends/monet5/sql_aggr_hge.mal.h
sql/backends/monet5/sql_aggr_int.mal.h
sql/backends/monet5/sql_aggr_lng.mal.h
sql/backends/monet5/sql_aggr_sht.mal.h
sql/backends/monet5/sql_decimal.mal.h
sql/backends/monet5/sql_decimal_hge.mal.h
sql/backends/monet5/sql_hge.mal.h
sql/backends/monet5/sql_inspect.mal.h
sql/backends/monet5/sql_rank.mal.h
sql/backends/monet5/sql_rank_hge.mal.h
sql/backends/monet5/sql_transaction.mal.h
sql/backends/monet5/sqlcatalog.mal.h
sql/backends/monet5/wlr.mal.h
Removed Files:
monetdb5/modules/atoms/00_json_hge.include
monetdb5/modules/atoms/batxml.include
monetdb5/modules/atoms/blob.include
monetdb5/modules/atoms/color.include
monetdb5/modules/atoms/identifier.include
monetdb5/modules/atoms/inet.include
monetdb5/modules/atoms/json.include
monetdb5/modules/atoms/mtime.include
monetdb5/modules/atoms/str.include
monetdb5/modules/atoms/streams.include
monetdb5/modules/atoms/url.include
monetdb5/modules/atoms/uuid.include
monetdb5/modules/atoms/xml.include
monetdb5/modules/kernel/00_aggr_hge.include
monetdb5/modules/kernel/aggr.include
monetdb5/modules/kernel/alarm.include
monetdb5/modules/kernel/algebra.include
monetdb5/modules/kernel/bat5.include
monetdb5/modules/kernel/batcolor.include
monetdb5/modules/kernel/batmmath.include
monetdb5/modules/kernel/batstr.include
monetdb5/modules/kernel/group.include
monetdb5/modules/kernel/logger.include
monetdb5/modules/kernel/microbenchmark.include
monetdb5/modules/kernel/mmath.include
monetdb5/modules/kernel/status.include
monetdb5/modules/mal/00_batExtensions_hge.include
monetdb5/modules/mal/00_batcalc_hge.include
monetdb5/modules/mal/00_calc_hge.include
monetdb5/modules/mal/00_iterator_hge.include
monetdb5/modules/mal/00_language_hge.include
monetdb5/modules/mal/00_mal_mapi_hge.include
monetdb5/modules/mal/00_mkey_hge.include
monetdb5/modules/mal/01_batcalc.include
monetdb5/modules/mal/01_calc.include
monetdb5/modules/mal/batExtensions.include
monetdb5/modules/mal/batmtime.include
monetdb5/modules/mal/bbp.include
monetdb5/modules/mal/clients.include
monetdb5/modules/mal/factories.include
monetdb5/modules/mal/groupby.include
monetdb5/modules/mal/inspect.include
monetdb5/modules/mal/iterator.include
monetdb5/modules/mal/json_util.include
monetdb5/modules/mal/language.include
monetdb5/modules/mal/mal_init.mal
monetdb5/modules/mal/mal_io.include
monetdb5/modules/mal/mal_mapi.include
monetdb5/modules/mal/manifold.include
monetdb5/modules/mal/manual.include
monetdb5/modules/mal/mat.include
monetdb5/modules/mal/mdb.include
monetdb5/modules/mal/mkey.include
monetdb5/modules/mal/oltp.include
monetdb5/modules/mal/orderidx.include
monetdb5/modules/mal/pcre.include
monetdb5/modules/mal/profiler.include
monetdb5/modules/mal/querylog.include
monetdb5/modules/mal/remote.include
monetdb5/modules/mal/sabaoth.include
monetdb5/modules/mal/sample.include
monetdb5/modules/mal/sysmon.include
monetdb5/modules/mal/tablet.include
monetdb5/modules/mal/tokenizer.include
monetdb5/modules/mal/transaction.include
monetdb5/modules/mal/txtsim.include
monetdb5/modules/mal/wlc.include
monetdb5/modules/prepare.sh
monetdb5/optimizer/optimizer.include
monetdb5/scheduler/run_adder.include
monetdb5/scheduler/run_isolate.include
monetdb5/scheduler/run_memo.include
sql/backends/monet5/40_sql.include
sql/backends/monet5/41_sql_hge.include
sql/backends/monet5/generator/90_generator.include
sql/backends/monet5/generator/90_generator_hge.include
sql/backends/monet5/generator/generator.include
sql/backends/monet5/generator/generator_hge.include
sql/backends/monet5/sql.include
sql/backends/monet5/sql_aggr_bte.include
sql/backends/monet5/sql_aggr_dbl.include
sql/backends/monet5/sql_aggr_flt.include
sql/backends/monet5/sql_aggr_hge.include
sql/backends/monet5/sql_aggr_int.include
sql/backends/monet5/sql_aggr_lng.include
sql/backends/monet5/sql_aggr_sht.include
sql/backends/monet5/sql_decimal.include
sql/backends/monet5/sql_decimal_hge.include
sql/backends/monet5/sql_hge.include
sql/backends/monet5/sql_inspect.include
sql/backends/monet5/sql_rank.include
sql/backends/monet5/sql_rank_hge.include
sql/backends/monet5/sql_transaction.include
sql/backends/monet5/sqlcatalog.include
sql/backends/monet5/wlr.include
Modified Files:
buildtools/autogen/scripts/sql2h.py
monetdb5/mal/Makefile.ag
monetdb5/mal/mal_embedded.c
monetdb5/mal/mal_session.c
monetdb5/modules/mal/remote.mal
monetdb5/optimizer/optimizer.mal
sql/backends/monet5/Makefile.ag
sql/backends/monet5/sql_scenario.c
tools/mserver/mserver5.c
Branch: translate-scripts
Log Message:
MAL scripts cleanup. Remove comments while translating MAL scripts into C
header files.
diffs (truncated from 52084 to 300 lines):
diff --git a/buildtools/autogen/scripts/sql2h.py
b/buildtools/autogen/scripts/mal2h.py
copy from buildtools/autogen/scripts/sql2h.py
copy to buildtools/autogen/scripts/mal2h.py
--- a/buildtools/autogen/scripts/sql2h.py
+++ b/buildtools/autogen/scripts/mal2h.py
@@ -10,8 +10,8 @@ import os
import argparse
-parser = argparse.ArgumentParser(description='Convert MonetDB SQL scripts to C
header files to be inlined')
-parser.add_argument('file', metavar='F', type=str, help='The SQL file to
convert')
+parser = argparse.ArgumentParser(description='Convert MonetDB MAL scripts to C
header files to be inlined')
+parser.add_argument('file', metavar='F', type=str, help='The MAL file to
convert')
parser.add_argument('-t', '--trim', dest='trim', action='store_true',
help='trim whitespaces')
# check arguments and veracity of the input file first
@@ -29,16 +29,16 @@ if file_stat.st_size > (1 << 29):
# get the file name and extension
base = os.path.basename(args.file)
split = os.path.splitext(base)
-if len(split) < 2 or split[1] != '.sql':
- raise Exception("Only .sql files are supported")
+if len(split) < 2 or split[1] != '.mal':
+ raise Exception("Only .mal files are supported")
filebasename = split[0]
# output file will be written on the same directory
-sql_h_output_file = open(os.path.join(os.path.dirname(args.file),
filebasename) + ".sql.h", 'w')
+mal_h_output_file = open(os.path.join(os.path.dirname(args.file),
filebasename) + split[1] + ".h", 'w')
-sql_content_file = open(args.file, 'r')
-sql_content = sql_content_file.read()
-sql_content_file.close()
+mal_content_file = open(args.file, 'r')
+mal_content = mal_content_file.read()
+mal_content_file.close()
# write the common header, plus the C array entry
insert1 = (
@@ -50,34 +50,34 @@ insert1 = (
'* Copyright 1997 - July 2008 CWI, August 2008 - 2019 MonetDB B.V.\n'
'*/\n'
'\n'
-'// This file was generated automatically with sql2h.py. Do not edit this file
directly.\n'
+'// This file was generated automatically with mal2h.py. Do not edit this file
directly.\n'
'{{ "{0}", "').format(filebasename)
-sql_h_output_file.write(insert1)
+mal_h_output_file.write(insert1)
-# Let's remove comments from the sql script with a Markov chain :) Bugs might
still be there
-# STATES 0 - OK, 1 in # comment, 2 in -- comment, 3 in /* comment, 4 inside '
string, 5 inside " string,
-# 6 inside whitespaces
+# Let's remove comments from the mal script with a Markov chain :) Bugs might
still be there
+# STATES 0 - OK, 1 in # comment, 2 between comment keyword and comment block,
3 inside address comment block,
+# 4 inside ' string, 5 inside " string, 6 inside whitespaces
CACHE_SIZE = file_stat.st_blksize # we will set the cache size to the
filesystem blocksize
buffer = ['\0'] * CACHE_SIZE
cur_state = 0
current_pointer = 0
i = 0
-endloop = len(sql_content) - 1
+endloop = len(mal_content) - 1
def write_to_buffer(input_c):
- global current_pointer, sql_h_output_file, buffer
+ global current_pointer, mal_h_output_file, buffer
if current_pointer == CACHE_SIZE:
- sql_h_output_file.write("".join(buffer))
+ mal_h_output_file.write("".join(buffer))
current_pointer = 0
buffer[current_pointer] = input_c
current_pointer += 1
while i < endloop:
- c = sql_content[i]
+ c = mal_content[i]
if cur_state == 1:
if c == '\n':
@@ -85,36 +85,47 @@ while i < endloop:
i += 1
continue
elif cur_state == 2:
- if c == '\n':
- cur_state = 0
+ if c == '"':
+ cur_state = 3
i += 1
continue
elif cur_state == 3:
- if c == '*' and i + 1 < endloop and sql_content[i + 1] == '/':
+ if c == '\\':
+ i += 2
+ elif c == '"':
cur_state = 0
i += 1
- i += 1
+ else:
+ i += 1
continue
elif cur_state == 4:
if c == '\\' and i + 1 < endloop:
write_to_buffer('\\')
write_to_buffer('\\')
write_to_buffer('\\')
- write_to_buffer(sql_content[i + 1])
+ write_to_buffer(mal_content[i + 1])
i += 2
+ elif c == '\n':
+ write_to_buffer('\\')
+ write_to_buffer('n')
+ i += 1
else:
if c == '\'':
cur_state = 0
write_to_buffer(c)
i += 1
continue
- elif cur_state == 5 and i + 1 < endloop:
- if c == '\\':
+ elif cur_state == 5:
+ if c == '\\' and i + 1 < endloop:
write_to_buffer('\\')
write_to_buffer('\\')
write_to_buffer('\\')
- write_to_buffer(sql_content[i + 1])
+ write_to_buffer(mal_content[i + 1])
i += 2
+ elif c == '\n':
+ write_to_buffer('\\')
+ write_to_buffer('n')
+ i += 1
else:
if c == '"':
cur_state = 0
@@ -133,19 +144,19 @@ while i < endloop:
cur_state = 1
i += 1
continue
- elif c == '-' and i + 1 < endloop and sql_content[i + 1] == '-':
- cur_state = 2
- i += 2
- continue
- elif c == '/' and i + 1 < endloop and sql_content[i + 1] == '*':
- cur_state = 3
- i += 2
- continue
elif c == '\'':
write_to_buffer(c)
cur_state = 4
i += 1
continue
+ elif c == 'c':
+ if i + 7 < endloop and mal_content[i:i+8] == 'comment ':
+ cur_state = 2
+ i += 7
+ else:
+ write_to_buffer('c')
+ i += 1
+ continue
elif c == '"':
write_to_buffer('\\')
write_to_buffer(c)
@@ -172,8 +183,8 @@ while i < endloop:
i += 1
if current_pointer > 0:
- sql_h_output_file.write("".join(buffer[:current_pointer]))
+ mal_h_output_file.write("".join(buffer[:current_pointer]))
# finish C array entry
-sql_h_output_file.write('" }}, \n'.format())
-sql_h_output_file.close()
+mal_h_output_file.write('" }}, \n'.format())
+mal_h_output_file.close()
diff --git a/buildtools/autogen/scripts/sql2h.py
b/buildtools/autogen/scripts/sql2h.py
--- a/buildtools/autogen/scripts/sql2h.py
+++ b/buildtools/autogen/scripts/sql2h.py
@@ -34,7 +34,7 @@ if len(split) < 2 or split[1] != '.sql':
filebasename = split[0]
# output file will be written on the same directory
-sql_h_output_file = open(os.path.join(os.path.dirname(args.file),
filebasename) + ".sql.h", 'w')
+sql_h_output_file = open(os.path.join(os.path.dirname(args.file),
filebasename) + split[1] + ".h", 'w')
sql_content_file = open(args.file, 'r')
sql_content = sql_content_file.read()
@@ -102,19 +102,27 @@ while i < endloop:
write_to_buffer('\\')
write_to_buffer(sql_content[i + 1])
i += 2
+ elif c == '\n':
+ write_to_buffer('\\')
+ write_to_buffer('n')
+ i += 1
else:
if c == '\'':
cur_state = 0
write_to_buffer(c)
i += 1
continue
- elif cur_state == 5 and i + 1 < endloop:
- if c == '\\':
+ elif cur_state == 5:
+ if c == '\\' and i + 1 < endloop:
write_to_buffer('\\')
write_to_buffer('\\')
write_to_buffer('\\')
write_to_buffer(sql_content[i + 1])
i += 2
+ elif c == '\n':
+ write_to_buffer('\\')
+ write_to_buffer('n')
+ i += 1
else:
if c == '"':
cur_state = 0
diff --git a/monetdb5/mal/Makefile.ag b/monetdb5/mal/Makefile.ag
--- a/monetdb5/mal/Makefile.ag
+++ b/monetdb5/mal/Makefile.ag
@@ -11,6 +11,9 @@ INCLUDES = ../../common/options \
../../gdk \
../optimizer \
../modules/mal \
+ ../modules/kernel \
+ ../modules/atoms \
+ ../scheduler \
$(openssl_CFLAGS)
MTSAFE
diff --git a/monetdb5/mal/mal_embedded.c b/monetdb5/mal/mal_embedded.c
--- a/monetdb5/mal/mal_embedded.c
+++ b/monetdb5/mal/mal_embedded.c
@@ -25,106 +25,89 @@ static struct{
{
// Include the MAL definitions files in the proper order.
-#include "../modules/mal/mdb.include"
-#include "../modules/kernel/alarm.include"
-#include "../modules/kernel/mmath.include"
-#include "../modules/atoms/streams.include"
+#include "mdb.mal.h"
+#include "alarm.mal.h"
+#include "mmath.mal.h"
+#include "streams.mal.h"
-#include "../modules/kernel/bat5.include"
-#include "../modules/mal/batExtensions.include"
-#include "../modules/kernel/algebra.include"
-#include "../modules/mal/orderidx.include"
-#include "../modules/kernel/status.include"
-#include "../modules/mal/groupby.include"
-#include "../modules/kernel/group.include"
-#include "../modules/kernel/aggr.include"
-#include "../modules/mal/mkey.include"
+#include "bat5.mal.h"
+#include "batExtensions.mal.h"
+#include "algebra.mal.h"
+#include "orderidx.mal.h"
+#include "status.mal.h"
+#include "groupby.mal.h"
+#include "group.mal.h"
+#include "aggr.mal.h"
+#include "mkey.mal.h"
-#include "../modules/atoms/blob.include"
-#include "../modules/atoms/color.include"
-#include "../modules/atoms/str.include"
-#include "../modules/atoms/url.include"
-#include "../modules/atoms/uuid.include"
-#include "../modules/atoms/json.include"
-#include "../modules/mal/json_util.include"
-#include "../modules/atoms/mtime.include"
-#include "../modules/atoms/inet.include"
-#include "../modules/atoms/identifier.include"
-#include "../modules/atoms/xml.include"
-#include "../modules/atoms/batxml.include"
+#include "blob.mal.h"
+#include "color.mal.h"
+#include "str.mal.h"
+#include "url.mal.h"
+#include "uuid.mal.h"
+#include "json.mal.h"
+#include "json_util.mal.h"
+#include "mtime.mal.h"
+#include "inet.mal.h"
+#include "identifier.mal.h"
+#include "xml.mal.h"
+#include "batxml.mal.h"
-#include "../modules/kernel/batmmath.include"
-#include "../modules/mal/batmtime.include"
-#include "../modules/kernel/batstr.include"
-#include "../modules/kernel/batcolor.include"
+#include "batmmath.mal.h"
+#include "batmtime.mal.h"
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
