MonetDB: analytics - Distinguish unbounded limit from others, du...
Changeset: 255c9d707cac for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=255c9d707cac
Modified Files:
gdk/gdk_analytic.c
Branch: analytics
Log Message:
Distinguish unbounded limit from others, due to possible overflow in
calculation on range frames.
diffs (94 lines):
diff --git a/gdk/gdk_analytic.c b/gdk/gdk_analytic.c
--- a/gdk/gdk_analytic.c
+++ b/gdk/gdk_analytic.c
@@ -123,6 +123,19 @@ GDKanalyticaldiff(BAT *r, BAT *b, BAT *p
*rb = --curval; \
} while(0);
+#define ANALYTICAL_WINDOW_BOUNDS_FIXED_UNBOUNDED_START(TPE) \
+ do {\
+ TPE *bl = pbp; \
+ for(; pbphttps://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: analytics - Range frame with bounded preceding/followin...
Changeset: 33b3903d8346 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=33b3903d8346
Modified Files:
sql/server/rel_select.c
sql/server/sql_parser.h
sql/server/sql_parser.y
Branch: analytics
Log Message:
Range frame with bounded preceding/following and no ordering is not reliable to
implement.
diffs (89 lines):
diff --git a/sql/server/rel_select.c b/sql/server/rel_select.c
--- a/sql/server/rel_select.c
+++ b/sql/server/rel_select.c
@@ -4707,8 +4707,11 @@ rel_rankop(mvc *sql, sql_rel **rel, symb
return sql_error(sql, 02, SQLSTATE(42000) "OVER: frame
extend only possible with aggregation");
if(!obe && frame_type == FRAME_GROUPS)
return sql_error(sql, 02, SQLSTATE(42000) "GROUPS frame
requires an order by expression");
- if(!obe && frame_type == FRAME_RANGE)
+ if(!obe && frame_type == FRAME_RANGE) {
+ if(d->data.sym->token == SQL_FRAME_PRECEDING ||
d->next->data.sym->token == SQL_FRAME_FOLLOWING)
+ return sql_error(sql, 02, SQLSTATE(42000)
"RANGE frame with PRECEDING or FOLLOWING offsets requires an order by
expression");
frame_type = FRAME_ALL; //special case, iterate the
entire partition
+ }
if(calculate_window_bounds(sql, , , s, gbe ? pe :
NULL, obe ? obe->t->data : in, fstart, fend, frame_type, excl) == NULL)
return NULL;
diff --git a/sql/server/sql_parser.h b/sql/server/sql_parser.h
--- a/sql/server/sql_parser.h
+++ b/sql/server/sql_parser.h
@@ -145,7 +145,11 @@ typedef enum tokens {
SQL_FUNC,
SQL_AGGR,
SQL_RANK,
- SQL_FRAME,
+ SQL_FRAME_DEFINITION,
+ SQL_FRAME_CURRENT,
+ SQL_FRAME_UNBOUNDED,
+ SQL_FRAME_FOLLOWING,
+ SQL_FRAME_PRECEDING,
SQL_COMPARE,
SQL_FILTER,
SQL_ROUTINE,
diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y
--- a/sql/server/sql_parser.y
+++ b/sql/server/sql_parser.y
@@ -4175,7 +4175,7 @@ window_order_clause:
window_frame_clause:
/* empty */ { $$ = NULL; }
|window_frame_units window_frame_extent window_frame_exclusion
- { $$ = _symbol_create_list( SQL_FRAME, append_int(append_int($2, $1),
$3)); }
+ { $$ = _symbol_create_list( SQL_FRAME_DEFINITION,
append_int(append_int($2, $1), $3)); }
;
window_frame_units:
@@ -4185,14 +4185,14 @@ window_frame_units:
;
window_frame_extent:
- window_frame_start { $$ = append_symbol(append_symbol(L(), $1),
_symbol_create_int(SQL_FRAME, 0)); }
+ window_frame_start { $$ = append_symbol(append_symbol(L(), $1),
_symbol_create_int(SQL_FRAME_CURRENT, 0)); }
|window_frame_between { $$ = $1; }
;
window_frame_start:
- UNBOUNDED PRECEDING { $$ = _symbol_create_int(SQL_FRAME,
GDK_int_max); }
- |posint PRECEDING{ $$ = _symbol_create_int(SQL_FRAME, $1); }
- |CURRENT ROW { $$ = _symbol_create_int(SQL_FRAME,
0); }
+ UNBOUNDED PRECEDING { $$ = _symbol_create_int(SQL_FRAME_UNBOUNDED,
GDK_int_max); }
+ |posint PRECEDING{ $$ = _symbol_create_int(SQL_FRAME_PRECEDING,
$1); }
+ |CURRENT ROW { $$ =
_symbol_create_int(SQL_FRAME_CURRENT, 0); }
;
window_frame_between:
@@ -4200,9 +4200,9 @@ window_frame_between:
;
window_frame_end:
- UNBOUNDED FOLLOWING { $$ = _symbol_create_int(SQL_FRAME,
GDK_int_max); }
- |posint FOLLOWING{ $$ = _symbol_create_int(SQL_FRAME, $1); }
- |CURRENT ROW { $$ = _symbol_create_int(SQL_FRAME,
0); }
+ UNBOUNDED FOLLOWING { $$ = _symbol_create_int(SQL_FRAME_UNBOUNDED,
GDK_int_max); }
+ |posint FOLLOWING{ $$ = _symbol_create_int(SQL_FRAME_FOLLOWING,
$1); }
+ |CURRENT ROW { $$ =
_symbol_create_int(SQL_FRAME_CURRENT, 0); }
;
window_frame_exclusion:
@@ -6411,7 +6411,11 @@ char *token2string(int token)
SQL(FUNC);
SQL(AGGR);
SQL(RANK);
- SQL(FRAME);
+ SQL(FRAME_DEFINITION);
+ SQL(FRAME_CURRENT);
+ SQL(FRAME_UNBOUNDED);
+ SQL(FRAME_FOLLOWING);
+ SQL(FRAME_PRECEDING);
SQL(COMPARE);
SQL(FILTER);
SQL(ROUTINE);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Added missing sql parser tokens for string co...
Changeset: da30ec21c9a7 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=da30ec21c9a7 Modified Files: sql/server/sql_parser.y Branch: default Log Message: Added missing sql parser tokens for string conversion. diffs (23 lines): diff --git a/sql/server/sql_parser.y b/sql/server/sql_parser.y --- a/sql/server/sql_parser.y +++ b/sql/server/sql_parser.y @@ -6372,6 +6372,7 @@ char *token2string(int token) TR(MODE); SQL(INSERT); SQL(DELETE); + SQL(TRUNCATE); SQL(UPDATE); SQL(CROSS); SQL(JOIN); @@ -6443,6 +6444,11 @@ char *token2string(int token) SQL(XMLTEXT); SQL(XMLVALIDATE); SQL(XMLNAMESPACES); + SQL(MERGE_PARTITION); + SQL(PARTITION_LIST); + SQL(PARTITION_RANGE); + SQL(PARTITION_COLUMN); + SQL(PARTITION_EXPRESSION); } return "unknown"; /* just needed for broken compilers ! */ } ___ checkin-list mailing list checkin-list@monetdb.org https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: analytics - Merge with default
Changeset: c323fb1e3d7f for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=c323fb1e3d7f Added Files: sql/test/BugTracker-2018/Tests/timestamp-as-boolean.Bug-6642.sql sql/test/BugTracker-2018/Tests/timestamp-as-boolean.Bug-6642.stable.err sql/test/BugTracker-2018/Tests/timestamp-as-boolean.Bug-6642.stable.out sql/test/BugTracker-2018/Tests/timestamp-roundtrip.Bug-6640.sql sql/test/BugTracker-2018/Tests/timestamp-roundtrip.Bug-6640.stable.err sql/test/BugTracker-2018/Tests/timestamp-roundtrip.Bug-6640.stable.out Modified Files: clients/Tests/exports.stable.out clients/examples/C/sample0.c clients/examples/C/sample1.c clients/examples/C/sample4.c clients/examples/C/smack00.c clients/examples/C/smack01.c clients/mapiclient/mclient.c clients/mapiclient/mhelp.c clients/mapiclient/msqldump.c clients/mapilib/ChangeLog clients/mapilib/mapi.c clients/mapilib/mapi.h clients/odbc/driver/SQLExecute.c clients/odbc/driver/SQLGetTypeInfo.c common/stream/stream.c common/stream/stream.h gdk/gdk_align.c gdk/gdk_bat.c gdk/gdk_batop.c gdk/gdk_bbp.c gdk/gdk_cross.c gdk/gdk_heap.c gdk/gdk_join.c gdk/gdk_logger.c gdk/gdk_orderidx.c gdk/gdk_posix.c gdk/gdk_project.c gdk/gdk_select.c gdk/gdk_storage.c gdk/gdk_tm.c gdk/gdk_unique.c gdk/gdk_utils.c monetdb5/mal/mal_client.c monetdb5/mal/mal_client.h monetdb5/mal/mal_import.c monetdb5/mal/mal_readline.c monetdb5/mal/mal_session.c monetdb5/mal/mal_session.h monetdb5/modules/atoms/mtime.c monetdb5/modules/mal/mal_mapi.c monetdb5/modules/mal/projectionpath.c monetdb5/modules/mal/remote.c monetdb5/modules/mal/tablet.c monetdb5/optimizer/opt_dataflow.c sql/backends/monet5/mal_backend.c sql/backends/monet5/mal_backend.h sql/backends/monet5/rel_bin.c sql/backends/monet5/sql_scenario.c sql/backends/monet5/sql_statement.c sql/include/sql_hash.h sql/server/rel_select.c sql/server/sql_scan.c sql/storage/bat/bat_logger.c sql/test/BugTracker-2018/Tests/All tools/merovingian/daemon/client.c tools/merovingian/daemon/config/monetdbd.in tools/merovingian/daemon/multiplex-funnel.c Branch: analytics Log Message: Merge with default diffs (truncated from 6464 to 300 lines): diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out --- a/clients/Tests/exports.stable.out +++ b/clients/Tests/exports.stable.out @@ -479,7 +479,6 @@ MapiMsg mapi_bind_numeric(MapiHdl hdl, i MapiMsg mapi_bind_var(MapiHdl hdl, int fnr, int type, void *ptr); MapiMsg mapi_cache_freeup(MapiHdl hdl, int percentage); MapiMsg mapi_cache_limit(Mapi mid, int limit); -MapiMsg mapi_cache_shuffle(MapiHdl hdl, int percentage); MapiMsg mapi_clear_bindings(MapiHdl hdl); MapiMsg mapi_clear_params(MapiHdl hdl); MapiMsg mapi_close_handle(MapiHdl hdl); @@ -500,19 +499,19 @@ MapiMsg mapi_fetch_reset(MapiHdl hdl); int mapi_fetch_row(MapiHdl hdl); MapiMsg mapi_finish(MapiHdl hdl); MapiHdl mapi_get_active(Mapi mid); -int mapi_get_autocommit(Mapi mid); -char *mapi_get_dbname(Mapi mid); +bool mapi_get_autocommit(Mapi mid); +const char *mapi_get_dbname(Mapi mid); int mapi_get_digits(MapiHdl hdl, int fnr); int mapi_get_field_count(MapiHdl hdl); stream *mapi_get_from(Mapi mid); -char *mapi_get_host(Mapi mid); -char *mapi_get_lang(Mapi mid); +const char *mapi_get_host(Mapi mid); +const char *mapi_get_lang(Mapi mid); int64_t mapi_get_last_id(MapiHdl hdl); int mapi_get_len(MapiHdl hdl, int fnr); int64_t mapi_get_maloptimizertime(MapiHdl hdl); -char *mapi_get_mapi_version(Mapi mid); -char *mapi_get_monet_version(Mapi mid); -char *mapi_get_motd(Mapi mid); +const char *mapi_get_mapi_version(Mapi mid); +const char *mapi_get_monet_version(Mapi mid); +const char *mapi_get_motd(Mapi mid); char *mapi_get_name(MapiHdl hdl, int fnr); char *mapi_get_query(MapiHdl hdl); int64_t mapi_get_querytime(MapiHdl hdl); @@ -523,19 +522,19 @@ int64_t mapi_get_sqloptimizertime(MapiHd char *mapi_get_table(MapiHdl hdl, int fnr); int mapi_get_tableid(MapiHdl hdl); stream *mapi_get_to(Mapi mid); -int mapi_get_trace(Mapi mid); +bool mapi_get_trace(Mapi mid); char *mapi_get_type(MapiHdl hdl, int fnr); -char *mapi_get_uri(Mapi mid); -char *mapi_get_user(Mapi mid); -int mapi_is_connected(Mapi mid); +const char *mapi_get_uri(Mapi mid); +const char *mapi_get_user(Mapi mid); +bool mapi_is_connected(Mapi mid); MapiMsg mapi_log(Mapi mid, const char *nme); Mapi mapi_mapi(const char *host, int port, const char *username, const char *password, const char *lang, const char *dbname); Mapi mapi_mapiuri(const char *url, const char
MonetDB: analytics - Major cleanup in gdk_analytic. My implement...
Changeset: 4aab4a55d975 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4aab4a55d975
Modified Files:
gdk/gdk_analytic.c
gdk/gdk_analytic.h
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql_rank.c
sql/backends/monet5/sql_rank.h
sql/backends/monet5/sql_rank.mal
sql/backends/monet5/sql_rank.mal.sh
sql/backends/monet5/sql_rank_hge.mal
sql/backends/monet5/sql_rank_hge.mal.sh
sql/common/sql_types.c
sql/include/sql_catalog.h
sql/server/rel_select.c
Branch: analytics
Log Message:
Major cleanup in gdk_analytic. My implementation of range frame was indeed the
groups frame. With this cleanup, rows, range and groups frames are properly
implemented.
Meanwhile as the code was growing significantly for each frame implementation,
it was becoming difficult to maintain. The MAL generation was changed for
aggregations on a window where each row overlap is a calculated with
sql.window_start_bound and sql.window_end_bound calls. This approach simplifies
the implementation for each aggregate and upcoming steps such as varying row
bounds and the EXCLUDE clause.
diffs (truncated from 4970 to 300 lines):
diff --git a/gdk/gdk_analytic.c b/gdk/gdk_analytic.c
--- a/gdk/gdk_analytic.c
+++ b/gdk/gdk_analytic.c
@@ -15,9 +15,9 @@
do { \
TPE *bp = (TPE*)Tloc(b, 0); \
TPE prev = *bp, *end = bp + cnt; \
- if(rp) { \
- for(; bpbl; bs--, curval++) { \
+ if (ABSOLUTE(v - *bs) > blimit) \
+ break; \
+ } \
+ *rb = curval; \
+ } \
+ } while(0);
+
+#define ANALYTICAL_WINDOW_BOUNDS_FIXED_RANGE_END(TPE) \
+ do { \
+ TPE *bs, v, blimit = (TPE) limit; \
+ int curval; \
+ for(; pbp blimit) \
+ break;\
+ } \
+ *rb = curval; \
+ } \
+ } while(0);
+
+#define ANALYTICAL_WINDOW_BOUNDS_FIXED_GROUPS_START(TPE) \
+ do { \
+ TPE *bl = pbp-1, *bs, v; \
+ int curval; \
+ BUN rlimit; \
+ for(; pbpbl; bs--, curval++) { \
+ if(v != *bs) { \
+ if(rlimit == 0) \
+ break;
\
+ rlimit--;\
+ v = *bs; \
+ }\
+ }\
+ *rb = curval;\
+ }\
+ } while(0);
+
+#define ANALYTICAL_WINDOW_BOUNDS_FIXED_GROUPS_END(TPE) \
+ do { \
+ TPE *bs, v;\
+ int curval;\
+ BUN rlimit;\
+ for(; pbpj; l--, curval++) { \
+ if (ABSOLUTE(atomcmp(v, BUNtail(bpi, l))) >
llimit) \
+ break;\
+ } \
+ *rb = curval; \
+ } \
+ } while(0);
+
+#define ANALYTICAL_WINDOW_BOUNDS_VARSIZED_RANGE_END \
+ do {\
+ void *v;\
+ int curval, llimit = (int)limit;\
+ for(; k
llimit) \
+ break; \
+ } \
+ *rb = curval; \
+ } \
+ } while(0);
+
MonetDB: default - Modernize: use __func__.
Changeset: a6ffd8d79acc for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=a6ffd8d79acc
Modified Files:
clients/mapilib/mapi.c
Branch: default
Log Message:
Modernize: use __func__.
diffs (truncated from 536 to 300 lines):
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -891,44 +891,48 @@ struct MapiStatement {
* routine. It assures a working connection and proper reset of
* the error status of the Mapi structure.
*/
-#define mapi_check(X,C)
\
+#define mapi_check(X) \
do {\
- debugprint("entering %s\n", (C)); \
+ debugprint("entering %s\n", __func__); \
assert(X); \
if (!(X)->connected) { \
- mapi_setError((X), "Connection lost", (C), MERROR); \
+ mapi_setError((X), "Connection lost", \
+ __func__, MERROR);\
return (X)->error; \
} \
mapi_clrError(X); \
} while (0)
-#define mapi_check0(X,C) \
+#define mapi_check0(X) \
do {\
- debugprint("entering %s\n", (C)); \
+ debugprint("entering %s\n", __func__); \
assert(X); \
if (!(X)->connected) { \
- mapi_setError((X), "Connection lost", (C), MERROR); \
+ mapi_setError((X), "Connection lost", \
+ __func__, MERROR);\
return 0; \
} \
mapi_clrError(X); \
} while (0)
-#define mapi_hdl_check(X,C)\
+#define mapi_hdl_check(X) \
do {\
- debugprint("entering %s\n", (C)); \
+ debugprint("entering %s\n", __func__); \
assert(X); \
assert((X)->mid); \
if (!(X)->mid->connected) { \
- mapi_setError((X)->mid, "Connection lost", (C),
MERROR); \
+ mapi_setError((X)->mid, "Connection lost", \
+ __func__, MERROR);\
return (X)->mid->error; \
} \
mapi_clrError((X)->mid);\
} while (0)
-#define mapi_hdl_check0(X,C) \
+#define mapi_hdl_check0(X) \
do {\
- debugprint("entering %s\n", (C)); \
+ debugprint("entering %s\n", __func__); \
assert(X); \
assert((X)->mid); \
if (!(X)->mid->connected) { \
- mapi_setError((X)->mid, "Connection lost", (C),
MERROR); \
+ mapi_setError((X)->mid, "Connection lost", \
+ __func__, MERROR);\
return 0; \
} \
mapi_clrError((X)->mid);\
@@ -1279,28 +1283,28 @@ mapi_explain_result(MapiHdl hdl, FILE *f
stream *
mapi_get_to(Mapi mid)
{
- mapi_check0(mid, "mapi_get_to");
+ mapi_check0(mid);
return mid->to;
}
stream *
mapi_get_from(Mapi mid)
{
- mapi_check0(mid, "mapi_get_from");
+ mapi_check0(mid);
return mid->from;
}
bool
mapi_get_trace(Mapi mid)
{
-
MonetDB: default - Removed unused function mapi_cache_shuffle.
Changeset: bb5607cf823c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=bb5607cf823c
Modified Files:
clients/Tests/exports.stable.out
clients/mapilib/ChangeLog
clients/mapilib/mapi.c
clients/mapilib/mapi.h
Branch: default
Log Message:
Removed unused function mapi_cache_shuffle.
diffs (96 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -479,7 +479,6 @@ MapiMsg mapi_bind_numeric(MapiHdl hdl, i
MapiMsg mapi_bind_var(MapiHdl hdl, int fnr, int type, void *ptr);
MapiMsg mapi_cache_freeup(MapiHdl hdl, int percentage);
MapiMsg mapi_cache_limit(Mapi mid, int limit);
-MapiMsg mapi_cache_shuffle(MapiHdl hdl, int percentage);
MapiMsg mapi_clear_bindings(MapiHdl hdl);
MapiMsg mapi_clear_params(MapiHdl hdl);
MapiMsg mapi_close_handle(MapiHdl hdl);
diff --git a/clients/mapilib/ChangeLog b/clients/mapilib/ChangeLog
--- a/clients/mapilib/ChangeLog
+++ b/clients/mapilib/ChangeLog
@@ -2,5 +2,6 @@
# This file is updated with Maddlog
* Wed Sep 12 2018 Sjoerd Mullender
+- Removed function mapi_cache_shuffle.
- Removed function mapi_stream_query.
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -258,7 +258,6 @@
* @item mapi_bind_var() @tabBind typed C-variable to a field
* @item mapi_cache_freeup() @tab Forcefully shuffle fraction for cache
refreshment
* @item mapi_cache_limit()@tab Set the tuple cache limit
- * @item mapi_cache_shuffle() @tab Set shuffle fraction for cache refreshment
* @item mapi_clear_bindings() @tab Clear all field bindings
* @item mapi_clear_params() @tab Clear all parameter bindings
* @item mapi_close_handle() @tabClose query handle and free resources
@@ -632,16 +631,6 @@
* non-read elements. Filling the cache quicker than reading leads to an
* error.
*
- * @item MapiMsg mapi_cache_shuffle(MapiHdl hdl, int percentage)
- *
- * Make room in the cache by shuffling percentage tuples out of the
- * cache. It is sometimes handy to do so, for example, when your
- * application is stream-based and you process each tuple as it arrives
- * and still need a limited look-back. This percentage can be set
- * between 0 to 100. Making shuffle= 100% (default) leads to paging
- * behavior, while shuffle==1 leads to a sliding window over a tuple
- * stream with 1% refreshing.
- *
* @item MapiMsg mapi_cache_freeup(MapiHdl hdl, int percentage)
*
* Forcefully shuffle the cache making room for new rows. It ignores the
@@ -783,7 +772,6 @@ struct MapiParam {
*/
struct MapiRowBuf {
int rowlimit; /* maximum number of rows to cache */
- int shuffle;/* percentage of rows to shuffle upon overflow
*/
int limit; /* current storage space limit */
int writer;
int reader;
@@ -1442,7 +1430,6 @@ new_result(MapiHdl hdl)
result->fields = NULL;
result->cache.rowlimit = hdl->mid->cachelimit;
- result->cache.shuffle = 100;
result->cache.limit = 0;
result->cache.writer = 0;
result->cache.reader = -1;
@@ -4253,19 +4240,6 @@ mapi_cache_limit(Mapi mid, int limit)
}
MapiMsg
-mapi_cache_shuffle(MapiHdl hdl, int percentage)
-{
- /* clean out superflous space TODO */
- mapi_hdl_check(hdl, "mapi_cache_shuffle");
- if (percentage < 0 || percentage > 100) {
- return mapi_setError(hdl->mid, "Illegal percentage",
"mapi_cache_shuffle", MERROR);
- }
- if (hdl->result)
- hdl->result->cache.shuffle = percentage;
- return MOK;
-}
-
-MapiMsg
mapi_fetch_reset(MapiHdl hdl)
{
mapi_hdl_check(hdl, "mapi_fetch_reset");
diff --git a/clients/mapilib/mapi.h b/clients/mapilib/mapi.h
--- a/clients/mapilib/mapi.h
+++ b/clients/mapilib/mapi.h
@@ -171,7 +171,6 @@ mapi_export MapiMsg mapi_query_done(Mapi
mapi_export MapiHdl mapi_send(Mapi mid, const char *cmd);
mapi_export MapiMsg mapi_read_response(MapiHdl hdl);
mapi_export MapiMsg mapi_cache_limit(Mapi mid, int limit);
-mapi_export MapiMsg mapi_cache_shuffle(MapiHdl hdl, int percentage);
mapi_export MapiMsg mapi_cache_freeup(MapiHdl hdl, int percentage);
mapi_export MapiMsg mapi_seek_row(MapiHdl hdl, int64_t rowne, int whence);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Removed unused function mapi_stream_query.
Changeset: 0357d712bc13 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=0357d712bc13
Modified Files:
clients/Tests/exports.stable.out
clients/mapilib/ChangeLog
clients/mapilib/mapi.c
clients/mapilib/mapi.h
Branch: default
Log Message:
Removed unused function mapi_stream_query.
diffs (74 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -563,7 +563,6 @@ MapiMsg mapi_set_size_header(Mapi mid, i
int mapi_split_line(MapiHdl hdl);
MapiMsg mapi_start_talking(Mapi mid);
MapiMsg mapi_store_field(MapiHdl hdl, int fnr, int outtype, void *outparam);
-MapiHdl mapi_stream_query(Mapi mid, const char *cmd, int windowsize);
MapiMsg mapi_timeout(Mapi mid, unsigned int time);
void mapi_trace(Mapi mid, bool flag);
char *mapi_unquote(char *msg);
diff --git a/clients/mapilib/ChangeLog b/clients/mapilib/ChangeLog
--- a/clients/mapilib/ChangeLog
+++ b/clients/mapilib/ChangeLog
@@ -1,3 +1,6 @@
# ChangeLog file for mapilib
# This file is updated with Maddlog
+* Wed Sep 12 2018 Sjoerd Mullender
+- Removed function mapi_stream_query.
+
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -302,7 +302,6 @@
* @item mapi_rows_affected() @tab Obtain number of rows changed
* @item mapi_seek_row() @tabMove row reader to specific location in
cache
* @item mapi_setAutocommit() @tabSet auto-commit flag
- * @item mapi_stream_query() @tab Send query and prepare for reading tuple
stream
* @item mapi_table() @tabGet current table name
* @item mapi_timeout()@tabSet timeout for long-running
queries[TODO]
* @item mapi_trace() @tabSet trace flag
@@ -4211,30 +4210,6 @@ mapi_query_done(MapiHdl hdl)
return ret == MOK && hdl->needmore ? MMORE : ret;
}
-/*
- * Stream queries are requests to the database engine that produce a stream
- * of answers of indefinite length. Elements are eaten away using the normal
way.
- * The stream ends upon encountering of the prompt.
- * A stream query can not rely on upfront caching.
- * The stream query also ensures that the cache contains a window
- * over the stream by shuffling tuples once it is filled.
- */
-MapiHdl
-mapi_stream_query(Mapi mid, const char *cmd, int windowsize)
-{
- MapiHdl hdl;
- int cachelimit = mid->cachelimit;
-
- mapi_check0(mid, "mapi_stream_query");
-
- mid->cachelimit = windowsize;
- hdl = mapi_query(mid, cmd);
- mid->cachelimit = cachelimit;
- if (hdl != NULL)
- mapi_cache_shuffle(hdl, 100);
- return hdl;
-}
-
MapiMsg
mapi_cache_limit(Mapi mid, int limit)
{
diff --git a/clients/mapilib/mapi.h b/clients/mapilib/mapi.h
--- a/clients/mapilib/mapi.h
+++ b/clients/mapilib/mapi.h
@@ -170,7 +170,6 @@ mapi_export MapiMsg mapi_query_part(Mapi
mapi_export MapiMsg mapi_query_done(MapiHdl hdl);
mapi_export MapiHdl mapi_send(Mapi mid, const char *cmd);
mapi_export MapiMsg mapi_read_response(MapiHdl hdl);
-mapi_export MapiHdl mapi_stream_query(Mapi mid, const char *cmd, int
windowsize);
mapi_export MapiMsg mapi_cache_limit(Mapi mid, int limit);
mapi_export MapiMsg mapi_cache_shuffle(MapiHdl hdl, int percentage);
mapi_export MapiMsg mapi_cache_freeup(MapiHdl hdl, int percentage);
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Don't split huge query in mapi_query into chu...
Changeset: 35ae7f84c124 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=35ae7f84c124
Modified Files:
clients/mapilib/mapi.c
Branch: default
Log Message:
Don't split huge query in mapi_query into chunks.
The server reads all input before starting to process, so the premise
of the split is incorrect. (And the implementation was also incorrect
-- it was never tested apparently.)
diffs (103 lines):
diff --git a/clients/mapilib/mapi.c b/clients/mapilib/mapi.c
--- a/clients/mapilib/mapi.c
+++ b/clients/mapilib/mapi.c
@@ -4001,9 +4001,6 @@ read_into_cache(MapiHdl hdl, int lookahe
}
}
-#define MAXQUERYSIZE (100*1024)
-#define QUERYBLOCK (16*1024)
-
static MapiMsg
mapi_execute_internal(MapiHdl hdl)
{
@@ -4026,83 +4023,12 @@ mapi_execute_internal(MapiHdl hdl)
printf("mapi_query:%zu:%s\n", size, cmd);
}
if (mid->languageId == LANG_SQL) {
- if (size > MAXQUERYSIZE) {
- /* If the query is large and we don't do
- anything about it, deadlock may occur: the
- client (we) blocks writing the query while
- the server blocks writing the initial
- results. Therefore we split up large
- queries into smaller batches. The split is
- done simplistically, but we are prepared to
- receive the secondary prompt. We cache the
- results of all but the last batch.
- There is a problem if auto-commit is on:
- the server commits on batch boundaries, so
- if one of our batches ends at the end of a
- (sub)query, the server commits prematurely.
- Another problem is when the query we
- received is incomplete. We should tell the
- server that there is no more, but we
- don't. */
- size_t i = 0;
-
- while (i < size) {
- mid->active = hdl;
- mnstr_write(mid->to, "S", 1, 1);
- if (mid->tracelog) {
- mapi_log_header(mid, "W");
- mnstr_printf(mid->tracelog, "S");
- }
- check_stream(mid, mid->to, "write error on
stream", "mapi_execute", mid->error);
- do {
- size_t n;
-
- hdl->needmore = false;
- if ((n = QUERYBLOCK) > size - i)
- n = size - i;
- mnstr_write(mid->to, cmd + i, 1, n);
- if (mid->tracelog) {
- mnstr_write(mid->tracelog, cmd
+ i, 1, n);
- mnstr_flush(mid->tracelog);
- }
- check_stream(mid, mid->to, "write error
on stream", "mapi_execute", mid->error);
- i += n;
- if (i == size) {
- if (mid->languageId ==
LANG_SQL) {
- mnstr_write(mid->to,
";", 1, 1);
- check_stream(mid,
mid->to, "write error on stream", "mapi_execute", mid->error);
- if (mid->tracelog) {
-
mnstr_write(mid->tracelog, ";", 1, 1);
-
mnstr_flush(mid->tracelog);
- }
- }
- mnstr_write(mid->to, "\n", 1,
1);
- if (mid->tracelog) {
-
mnstr_write(mid->tracelog, "\n", 1, 1);
-
mnstr_flush(mid->tracelog);
- }
- check_stream(mid, mid->to,
"write error on stream", "mapi_execute", mid->error);
- }
- mnstr_flush(mid->to);
- check_stream(mid, mid->to, "write error
on stream", "mapi_execute", mid->error);
- if
MonetDB: default - Use bool.
Changeset: 698c3e10fed0 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=698c3e10fed0
Modified Files:
common/stream/stream.c
monetdb5/mal/mal_client.c
monetdb5/mal/mal_readline.c
monetdb5/mal/mal_session.c
monetdb5/modules/mal/mal_mapi.c
monetdb5/modules/mal/remote.c
monetdb5/modules/mal/tablet.c
sql/backends/monet5/sql_scenario.c
sql/server/sql_scan.c
Branch: default
Log Message:
Use bool.
diffs (truncated from 309 to 300 lines):
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -3216,7 +3216,7 @@ struct icstream {
stream *s;
char buffer[BUFSIZ];
size_t buflen;
- int eof;
+ bool eof;
};
static ssize_t
@@ -3330,7 +3330,7 @@ ic_read(stream *restrict s, void *restri
break;
case 0:
/* end of file */
- ic->eof = 1;
+ ic->eof = true;
if (ic->buflen > 0) {
/* incomplete input */
s->errnr = MNSTR_READ_ERROR;
@@ -3382,7 +3382,7 @@ ic_read(stream *restrict s, void *restri
* next call (i.e. keep ic->eof set), otherwise we
* must clear it so that the next call will cause the
* underlying stream to be read again */
- ic->eof = 0;
+ ic->eof = false;
}
return (ssize_t) ((elmsize * cnt - outbytesleft) / elmsize);
}
@@ -3493,7 +3493,7 @@ ic_open(iconv_t cd, stream *restrict ss,
ic->cd = cd;
ic->s = ss;
ic->buflen = 0;
- ic->eof = 0;
+ ic->eof = false;
return s;
}
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -566,7 +566,7 @@ MCreadClient(Client c)
if (!isa_block_stream(c->fdout) && c->promptlength > 0)
mnstr_write(c->fdout, c->prompt,
c->promptlength, 1);
mnstr_flush(c->fdout);
- in->eof = 0;
+ in->eof = false;
}
while ((rd = bstream_next(in)) > 0 && !in->eof) {
sum += rd;
diff --git a/monetdb5/mal/mal_readline.c b/monetdb5/mal/mal_readline.c
--- a/monetdb5/mal/mal_readline.c
+++ b/monetdb5/mal/mal_readline.c
@@ -132,7 +132,7 @@ readConsole(Client cntxt)
return 1;
}
bailout:
- cntxt->fdin->eof = 1;
+ cntxt->fdin->eof = true;
return -1;
}
#endif
diff --git a/monetdb5/mal/mal_session.c b/monetdb5/mal/mal_session.c
--- a/monetdb5/mal/mal_session.c
+++ b/monetdb5/mal/mal_session.c
@@ -598,7 +598,7 @@ MALreader(Client c)
int r = 1;
if (c == mal_clients) {
r = readConsole(c);
- if (r < 0 && c->fdin->eof == 0)
+ if (r < 0 && !c->fdin->eof)
r = MCreadClient(c);
if (r > 0)
return MAL_SUCCEED;
diff --git a/monetdb5/modules/mal/mal_mapi.c b/monetdb5/modules/mal/mal_mapi.c
--- a/monetdb5/modules/mal/mal_mapi.c
+++ b/monetdb5/modules/mal/mal_mapi.c
@@ -266,7 +266,7 @@ doChallenge(void *data)
GDKsyserror("SERVERlisten:"MAL_MALLOC_FAIL);
return;
}
- bs->eof = 1;
+ bs->eof = true;
MSscheduleClient(buf, challenge, bs, fdout, protocol, buflen);
}
diff --git a/monetdb5/modules/mal/remote.c b/monetdb5/modules/mal/remote.c
--- a/monetdb5/modules/mal/remote.c
+++ b/monetdb5/modules/mal/remote.c
@@ -1091,7 +1091,7 @@ str RMTbatload(Client cntxt, MalBlkPtr m
throw(MAL, "remote.load", SQLSTATE(HY001) MAL_MALLOC_FAIL);
/* grab the input stream and start reading */
- fdin->eof = 0;
+ fdin->eof = false;
len = fdin->pos;
while (len < fdin->len || bstream_next(fdin) > 0) {
/* newline hunting (how spartan) */
@@ -1388,7 +1388,7 @@ str RMTbincopyfrom(Client cntxt, MalBlkP
* rest is binary data directly on the stream. We get the first
* line from the buffered stream we have here, and pass it on
* together with the raw stream we have. */
- cntxt->fdin->eof = 0; /* in case it was before */
+ cntxt->fdin->eof = false; /* in case it was before */
if (bstream_next(cntxt->fdin) <= 0)
throw(MAL, "remote.bincopyfrom", "expected JSON header");
diff --git a/monetdb5/modules/mal/tablet.c b/monetdb5/modules/mal/tablet.c
--- a/monetdb5/modules/mal/tablet.c
+++ b/monetdb5/modules/mal/tablet.c
@@ -388,7 +388,8 @@ output_line_lookup(char **buf, size_t *l
return 0;
}
-static int
+/* returns TRUE if there is/might be more */
+static bool
tablet_read_more(bstream *in, stream *out, size_t n)
{
MonetDB: default - Better error handling in malLoadScript.
Changeset: 141504537aa9 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=141504537aa9
Modified Files:
monetdb5/mal/mal_import.c
Branch: default
Log Message:
Better error handling in malLoadScript.
diffs (51 lines):
diff --git a/monetdb5/mal/mal_import.c b/monetdb5/mal/mal_import.c
--- a/monetdb5/mal/mal_import.c
+++ b/monetdb5/mal/mal_import.c
@@ -79,28 +79,31 @@ malOpenSource(str file)
* to find out how long the input is.
*/
static str
-malLoadScript(Client c, str name, bstream **fdin)
+malLoadScript(str name, bstream **fdin)
{
stream *fd;
size_t sz;
fd = malOpenSource(name);
- if (fd == 0 || mnstr_errnr(fd) == MNSTR_OPEN_ERROR) {
- mnstr_destroy(fd);
+ if (fd == NULL || mnstr_errnr(fd) == MNSTR_OPEN_ERROR) {
+ close_stream(fd);
throw(MAL, "malInclude", "could not open file: %s", name);
}
sz = getFileSize(fd);
if (sz > (size_t) 1 << 29) {
- mnstr_destroy(fd);
+ close_stream(fd);
throw(MAL, "malInclude", "file %s too large to process", name);
}
*fdin = bstream_create(fd, sz == 0 ? (size_t) (2 * 128 * BLOCK) : sz);
if(*fdin == NULL) {
- mnstr_destroy(fd);
+ close_stream(fd);
throw(MAL, "malInclude", MAL_MALLOC_FAIL);
}
- if (bstream_next(*fdin) < 0)
- mnstr_printf(c->fdout, "!WARNING: could not read %s\n", name);
+ if (bstream_next(*fdin) < 0) {
+ bstream_destroy(*fdin);
+ *fdin = NULL;
+ throw(MAL, "malInclude", "could not read %s", name);
+ }
return MAL_SUCCEED;
}
#endif
@@ -210,7 +213,7 @@ malInclude(Client c, str name, int listi
c->srcFile = filename;
c->yycur = 0;
c->bak = NULL;
- if ((msg = malLoadScript(c, filename, >fdin)) ==
MAL_SUCCEED) {
+ if ((msg = malLoadScript(filename, >fdin)) ==
MAL_SUCCEED) {
parseMAL(c, c->curprg, 1, INT_MAX);
bstream_destroy(c->fdin);
} else {
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
MonetDB: default - Before reallocating more space for a bstream,...
Changeset: 398e7bbb5056 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=398e7bbb5056
Modified Files:
common/stream/stream.c
Branch: default
Log Message:
Before reallocating more space for a bstream, check whether it's needed.
diffs (67 lines):
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -5207,7 +5207,7 @@ bstream_create(stream *s, size_t size)
ssize_t
bstream_read(bstream *s, size_t size)
{
- ssize_t rd;
+ ssize_t rd, rd1 = 0;
if (s == NULL)
return -1;
@@ -5215,6 +5215,8 @@ bstream_read(bstream *s, size_t size)
if (s->eof)
return 0;
+ assert(s->buf != NULL);
+
if (s->pos > 0) {
if (s->pos < s->len) {
/* move all data and end of string marker */
@@ -5225,15 +5227,29 @@ bstream_read(bstream *s, size_t size)
s->pos = 0;
}
- assert(s->buf != NULL);
if (s->len == s->size) {
+ size_t sz = size > 8192 ? 8192 : size;
+ char tmpbuf[8192];
+
+ /* before we realloc more space, see if there is a need */
+ if ((rd1 = s->s->read(s->s, tmpbuf, 1, sz)) == 0) {
+ s->eof = true;
+ return 0;
+ }
+ if (rd1 < 0)
+ return rd1;
char *p;
- size_t ns = s->size + size + 8192;
+ size_t ns = s->size + size;
if ((p = realloc(s->buf, ns + 1)) == NULL) {
return -1;
}
s->size = ns;
s->buf = p;
+ memcpy(s->buf + s->len, tmpbuf, rd1);
+ s->len += rd1;
+ size -= rd1;
+ if (size == 0)
+ return rd1;
}
if (s->len + size > s->size)
@@ -5246,11 +5262,11 @@ bstream_read(bstream *s, size_t size)
if (rd == 0) {
s->eof = true;
- return 0;
+ return rd1;
}
s->len += (size_t) rd;
s->buf[s->len] = 0; /* fill in the spare with EOS */
- return rd;
+ return rd + rd1;
}
#ifdef _POSIX2_LINE_MAX
___
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list
