MonetDB: analytics - Implemented lag window function in MAL and ...
Changeset: 85eadf47444d for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=85eadf47444d Modified Files: gdk/gdk_analytic.c gdk/gdk_analytic.h sql/backends/monet5/sql_rank.c sql/backends/monet5/sql_rank.h sql/backends/monet5/sql_rank.mal sql/backends/monet5/sql_rank.mal.sh sql/common/sql_types.c Branch: analytics Log Message: Implemented lag window function in MAL and GDK layers. Now I have to check how I'm going to extend the SQL compiler to support aggregates with 3 parameters. diffs (truncated from 1134 to 300 lines): diff --git a/gdk/gdk_analytic.c b/gdk/gdk_analytic.c --- a/gdk/gdk_analytic.c +++ b/gdk/gdk_analytic.c @@ -162,13 +162,15 @@ GDKanalyticaldiff(BAT *r, BAT *b, BAT *c } while(0); gdk_return -GDKanalyticalntile(BAT *r, BAT *b, BAT *p, BAT *o, int tpe, ptr ntile) +GDKanalyticalntile(BAT *r, BAT *b, BAT *p, BAT *o, int tpe, const void* restrict ntile) { BUN cnt = BATcount(b); bit *np, *pnp; bool has_nils = false; gdk_return gdk_res = GDK_SUCCEED; + assert(ntile); + switch (tpe) { case TYPE_bte: ANALYTICAL_NTILE_IMP(bte) @@ -496,6 +498,7 @@ GDKanalyticalnthvalue(BAT *r, BAT *b, BA gdk_return gdk_res = GDK_SUCCEED; bool has_nils = false; + assert(is_lng_nil(nth) || nth >= 0); (void) o; switch (tpe) { case TYPE_bte: @@ -584,6 +587,142 @@ finish: #undef ANALYTICAL_NTHVALUE_IMP +#define ANALYTICAL_LAG_IMP(TPE) \ + do { \ + TPE *rp, *rb, *bp, *end, def = *((TPE *) default_value); \ + bp = (TPE*)Tloc(b, 0); \ + rb = rp = (TPE*)Tloc(r, 0); \ + if(is_lng_nil(lag)) {\ + has_nils = true; \ + end = rb + cnt; \ + for(; rb 0) { \ + for(; i 0) {\ + for(; i 0) {\ + for(; i= 0); + + (void) o; + switch (tpe) { + case TYPE_bte: + ANALYTICAL_LAG_IMP(bte) + break; + case TYPE_sht: + ANALYTICAL_LAG_IMP(sht) + break; + case TYPE_int: + ANALYTICAL_LAG_IMP(int) + break; + case TYPE_lng: + ANALYTICAL_LAG_IMP(lng) + break; +#ifdef HAVE_HGE + case TYPE_hge: + ANALYTICAL_LAG_IMP(hge) + break; +#endif + case TYPE_flt: + ANALYTICAL_LAG_IMP(flt) + break; + case TYPE_dbl: + ANALYTICAL_LAG_IMP(dbl) + break; + default: { + } + } +finish: + BATsetcount(r, cnt); + r->tnonil = !has_nils; + r->tnil = has_nils; + return gdk_res; +} + +#undef ANALYTICAL_LAG_IMP + +gdk_return +GDKanalyticallead(BAT *r, BAT *b, BAT *p, BAT *o, lng lead, const void* restrict default_value, int tpe) +{ + //int (*atomcmp)(const void *, const void *); + //const void *nil; + BUN /*i, j,*/ cnt = BATcount(b); + //bit *np; + gdk_return gdk_res = GDK_SUCCEED; + bool has_nils = false; + + assert(default_value); + assert(is_lng_nil(lead) || lead <= 0); + + (void) o; + (void) p; + (void) tpe; +//finish: + BATsetcount(r, cnt); + r->tnonil = !has_nils; + r->tnil = has_nils; + return gdk_res; +} + #define ANALYTICAL_LIMIT_IMP(TPE, OP)\ do { \ TPE *rp, *rb, *restrict bp, *end, curval;\ @@ -807,6 +946,7 @@ GDKanalyticalcount(BAT *r, BAT *b, BAT * BUN i, cnt = BATcount(b); gdk_return gdk_res = GDK_SUCCEED; + assert(ignore_nils); (void) o; if(!*ignore_nils || b->T.nonil) { bit *np, *pnp; diff --git a/gdk/gdk_analytic.h b/gdk/gdk_analytic.h --- a/gdk/gdk_analytic.h +++ b/gdk/gdk_analytic.h @@ -17,13 +17,15 @@ #include "gdk.h" gdk_export gdk_return GDKanalyticaldiff(BAT *r, BAT *b, BAT *c, int tpe); -gdk_export gdk_return GDKanalyticalntile(BAT *r, BAT *b, BAT *p, BAT *o, int tpe, ptr ntile); +gdk_export gdk_return GDKanalyticalntile(BAT *r, BAT *b, BAT *p, BAT *o, int tpe, const void* restrict ntile);
MonetDB: Aug2018 - A table can be declared outside of a sequenti...
Changeset: b8367e69fcdc for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=b8367e69fcdc Added Files: sql/test/miscellaneous/Tests/declared_tables.sql sql/test/miscellaneous/Tests/declared_tables.stable.err sql/test/miscellaneous/Tests/declared_tables.stable.out Modified Files: sql/server/rel_schema.c sql/server/rel_semantic.c sql/test/miscellaneous/Tests/All Branch: Aug2018 Log Message: A table can be declared outside of a sequential block as well. diffs (168 lines): diff --git a/sql/server/rel_schema.c b/sql/server/rel_schema.c --- a/sql/server/rel_schema.c +++ b/sql/server/rel_schema.c @@ -2336,6 +2336,7 @@ rel_schemas(mvc *sql, symbol *s) l->h->next->data.i_val, /* drop_action */ l->h->next->next->data.i_val); /* if exists */ } break; + case SQL_DECLARE_TABLE: case SQL_CREATE_TABLE: { dlist *l = s->data.lval; diff --git a/sql/server/rel_semantic.c b/sql/server/rel_semantic.c --- a/sql/server/rel_semantic.c +++ b/sql/server/rel_semantic.c @@ -129,6 +129,7 @@ rel_semantic(mvc *sql, symbol *s) case SQL_CREATE_SCHEMA: case SQL_DROP_SCHEMA: + case SQL_DECLARE_TABLE: case SQL_CREATE_TABLE: case SQL_CREATE_VIEW: case SQL_DROP_TABLE: diff --git a/sql/test/miscellaneous/Tests/All b/sql/test/miscellaneous/Tests/All --- a/sql/test/miscellaneous/Tests/All +++ b/sql/test/miscellaneous/Tests/All @@ -1,2 +1,3 @@ +declared_tables #now trace_test diff --git a/sql/test/miscellaneous/Tests/declared_tables.sql b/sql/test/miscellaneous/Tests/declared_tables.sql new file mode 100644 --- /dev/null +++ b/sql/test/miscellaneous/Tests/declared_tables.sql @@ -0,0 +1,6 @@ +start transaction; +declare table iamdeclared (a int, b varchar(16)); +insert into iamdeclared values (1, 'one'); +select a, b from iamdeclared; +drop table iamdeclared; +rollback; diff --git a/sql/test/miscellaneous/Tests/declared_tables.stable.err b/sql/test/miscellaneous/Tests/declared_tables.stable.err new file mode 100644 --- /dev/null +++ b/sql/test/miscellaneous/Tests/declared_tables.stable.err @@ -0,0 +1,35 @@ +stderr of test 'declared_tables` in directory 'sql/test/miscellaneous` itself: + + +# 14:29:32 > +# 14:29:32 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=39353" "--set" "mapi_usock=/var/tmp/mtest-20678/.s.monetdb.39353" "--set" "monet_prompt=" "--forcemito" "--dbpath=/home/ferreira/repositories/MonetDB-Aug2018/BUILD/var/MonetDB/mTests_sql_test_miscellaneous" "--set" "embedded_c=true" +# 14:29:32 > + +# builtin opt gdk_dbpath = /home/ferreira/repositories/MonetDB-Aug2018/BUILD/var/monetdb5/dbfarm/demo +# builtin opt gdk_debug = 0 +# builtin opt gdk_vmtrim = no +# builtin opt monet_prompt = > +# builtin opt monet_daemon = no +# builtin opt mapi_port = 5 +# builtin opt mapi_open = false +# builtin opt mapi_autosense = false +# builtin opt sql_optimizer = default_pipe +# builtin opt sql_debug = 0 +# cmdline opt gdk_nr_threads = 0 +# cmdline opt mapi_open = true +# cmdline opt mapi_port = 39353 +# cmdline opt mapi_usock = /var/tmp/mtest-20678/.s.monetdb.39353 +# cmdline opt monet_prompt = +# cmdline opt gdk_dbpath = /home/ferreira/repositories/MonetDB-Aug2018/BUILD/var/MonetDB/mTests_sql_test_miscellaneous +# cmdline opt embedded_c = true +# cmdline opt gdk_debug = 553648138 + +# 14:29:32 > +# 14:29:32 > "mclient" "-lsql" "-ftest" "-tnone" "-Eutf-8" "-i" "-e" "--host=/var/tmp/mtest-20678" "--port=39353" +# 14:29:32 > + + +# 14:29:33 > +# 14:29:33 > "Done." +# 14:29:33 > + diff --git a/sql/test/miscellaneous/Tests/declared_tables.stable.out b/sql/test/miscellaneous/Tests/declared_tables.stable.out new file mode 100644 --- /dev/null +++ b/sql/test/miscellaneous/Tests/declared_tables.stable.out @@ -0,0 +1,83 @@ +stdout of test 'declared_tables` in directory 'sql/test/miscellaneous` itself: + + +# 14:29:32 > +# 14:29:32 > "mserver5" "--debug=10" "--set" "gdk_nr_threads=0" "--set" "mapi_open=true" "--set" "mapi_port=39353" "--set" "mapi_usock=/var/tmp/mtest-20678/.s.monetdb.39353" "--set" "monet_prompt=" "--forcemito" "--dbpath=/home/ferreira/repositories/MonetDB-Aug2018/BUILD/var/MonetDB/mTests_sql_test_miscellaneous" "--set" "embedded_c=true" +# 14:29:32 > + +# MonetDB 5 server v11.31.4 +# This is an unreleased version +# Serving database 'mTests_sql_test_miscellaneous', using 8 threads +# Compiled for x86_64-pc-linux-gnu/64bit with 128bit integers +# Found 15.492 GiB available main-memory. +# Copyright (c) 1993 - July 2008 CWI. +# Copyright (c) August 2008 - 2018 MonetDB B.V., all rights reserved +# Visit https://www.monetdb.org/ for further information +# Listening for connection requests on mapi:monetdb://wired-142.cwi.nl:39353/ +# Listening for UNIX domain connection requests on
MonetDB: analytics - Ntile and nth_value single atom case handling.
Changeset: dbe149c2877e for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=dbe149c2877e Modified Files: sql/backends/monet5/sql_rank.c Branch: analytics Log Message: Ntile and nth_value single atom case handling. diffs (173 lines): diff --git a/sql/backends/monet5/sql_rank.c b/sql/backends/monet5/sql_rank.c --- a/sql/backends/monet5/sql_rank.c +++ b/sql/backends/monet5/sql_rank.c @@ -476,28 +476,37 @@ SQLcume_dist(Client cntxt, MalBlkPtr mb, gdk_code = GDKanalyticalntile(r, b, p, o, TYPE_##TPE, ntile); \ } while(0); +#define NTILE_VALUE_SINGLE_IMP(TPE) \ + do { \ + TPE val = *(TPE*) ntile, *rres = (TPE*) res; \ + if(!is_##TPE##_nil(val) && val < 1) \ + throw(SQL, "sql.ntile", SQLSTATE(42000) "ntile must be greater than zero"); \ + *rres = (is_##TPE##_nil(val) || val > 1) ? TPE##_nil : *(TPE*) in; \ + } while(0); + str SQLntile(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) { + int tp1, tp2; + (void)cntxt; if (pci->argc != 5 || (getArgType(mb, pci, 3) != TYPE_bit && getBatType(getArgType(mb, pci, 3)) != TYPE_bit) || (getArgType(mb, pci, 4) != TYPE_bit && getBatType(getArgType(mb, pci, 4)) != TYPE_bit)) { throw(SQL, "sql.ntile", SQLSTATE(42000) "ntile(:any_1,:number,:bit,:bit)"); } - if (isaBatType(getArgType(mb, pci, 1))) { + tp1 = getArgType(mb, pci, 1), tp2 = getArgType(mb, pci, 2); + if (isaBatType(tp2)) + throw(SQL, "sql.ntile", SQLSTATE(42000) "ntile first argument must a single atom"); + + if (isaBatType(tp1)) { BUN cnt; bat *res = getArgReference_bat(stk, pci, 0); - int tp2 = getArgType(mb, pci, 2); BAT *b = BATdescriptor(*getArgReference_bat(stk, pci, 1)), *p = NULL, *o = NULL, *r; if (!b) throw(SQL, "sql.ntile", SQLSTATE(HY005) "Cannot access column descriptor"); cnt = BATcount(b); gdk_return gdk_code; - if (isaBatType(tp2)) { - BBPunfix(b->batCacheid); - throw(SQL, "sql.ntile", SQLSTATE(42000) "ntile second argument must a single atom"); - } switch (tp2) { case TYPE_bte: NTILE_IMP(bte) @@ -529,14 +538,37 @@ SQLntile(Client cntxt, MalBlkPtr mb, Mal else throw(SQL, "sql.ntile", SQLSTATE(HY001) "Unknown GDK error"); } else { - int *res = getArgReference_int(stk, pci, 0); + ptr res = getArgReference_ptr(stk, pci, 0); + ptr in = getArgReference_ptr(stk, pci, 1); + ptr ntile = getArgReference_ptr(stk, pci, 2); - *res = 1; + switch (tp2) { + case TYPE_bte: + NTILE_VALUE_SINGLE_IMP(bte) + break; + case TYPE_sht: + NTILE_VALUE_SINGLE_IMP(sht) + break; + case TYPE_int: + NTILE_VALUE_SINGLE_IMP(int) + break; + case TYPE_lng: + NTILE_VALUE_SINGLE_IMP(lng) + break; +#ifdef HAVE_HGE + case TYPE_hge: + NTILE_VALUE_SINGLE_IMP(hge) + break; +#endif + default: + throw(SQL, "sql.ntile", SQLSTATE(42000) "ntile not available for %s", ATOMname(tp2)); + } } return MAL_SUCCEED; } #undef NTILE_IMP +#undef NTILE_VALUE_SINGLE_IMP static str SQLanalytics_args(BAT **r, BAT **b, BAT **p, BAT **o, Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, @@ -664,17 +696,30 @@ SQLlast_value(Client cntxt, MalBlkPtr mb gdk_code = GDKanalyticalnthvalue(r, b, p, o, cast_value, tp1); \ } while(0); +#define NTH_VALUE_SINGLE_IMP(TPE) \ + do { \ + TPE val = *(TPE*) nth, *rres = (TPE*) res; \ + if(!is_##TPE##_nil(val) && val < 1) \ + throw(SQL, "sql.nth_value", SQLSTATE(42000) "nth_value
MonetDB: analytics - Merge with default
Changeset: 5a23b1f40497 for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=5a23b1f40497 Added Files: sql/test/BugTracker-2018/Tests/comment-on-table-is-null.Bug-6639.sql sql/test/BugTracker-2018/Tests/comment-on-table-is-null.Bug-6639.stable.err sql/test/BugTracker-2018/Tests/comment-on-table-is-null.Bug-6639.stable.out sql/test/BugTracker-2018/Tests/crash-after-call-non-existing-loader.sql sql/test/BugTracker-2018/Tests/crash-after-call-non-existing-loader.stable.err sql/test/BugTracker-2018/Tests/crash-after-call-non-existing-loader.stable.out Modified Files: MonetDB.spec NT/rules.msc clients/examples/C/Makefile.ag clients/mapiclient/Makefile.ag common/stream/Makefile.ag common/stream/monetdb-stream.pc.in configure.ag geom/lib/Makefile.ag geom/monetdb5/Makefile.ag monetdb5/tools/monetdb5.pc.in sql/test/BugTracker-2018/Tests/All tools/merovingian/client/Makefile.ag tools/merovingian/daemon/Makefile.ag tools/mserver/Makefile.ag Branch: analytics Log Message: Merge with default diffs (truncated from 1465 to 300 lines): diff --git a/MonetDB.spec b/MonetDB.spec --- a/MonetDB.spec +++ b/MonetDB.spec @@ -2,29 +2,29 @@ %define version 11.32.0 %{!?buildno: %global buildno %(date +%Y%m%d)} -# groups of related archs -%define all_x86 i386 i586 i686 - -%ifarch %{all_x86} -%define bits 32 -%else -%define bits 64 -%define with_int128 1 +# Use %bcond_with to add a --with option; i.e., "without" is default. +# Use %bcond_without to add a --without option; i.e., "with" is default. +# The --with OPTION and --without OPTION arguments can be passed on +# the commandline of both rpmbuild and mock. + +# On 64 bit architectures we build "hugeint" packages. +%if "%{?_lib}" == "lib64" +%bcond_without hugeint %endif %define release %{buildno}%{?dist} # On RedHat Enterprise Linux and derivatives, if the Extra Packages -# for Enterprise Linux (EPEL) repository is available, you can define -# the _with_epel macro. When using mock to build the RPMs, this can -# be done using the --with=epel option to mock. +# for Enterprise Linux (EPEL) repository is available, you can enable +# its use by providing rpmbuild or mock with the "--with epel" option. # If the EPEL repository is availabe, or if building for Fedora, all # optional sub packages can be built. We indicate that here by # setting the macro fedpkgs to 1. If the EPEL repository is not # available and we are not building for Fedora, we set fedpkgs to 0. %if %{?rhel:1}%{!?rhel:0} # RedHat Enterprise Linux (or CentOS or Scientific Linux) -%if %{?_with_epel:1}%{!?_with_epel:0} +%bcond_with epel +%if %{with epel} # EPEL is enabled through the command line %define fedpkgs 1 %else @@ -45,7 +45,7 @@ # up-to-date version of RHEL. %if %{fedpkgs} %if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7 -%define with_geos 1 +%bcond_without geos %endif %endif @@ -55,73 +55,50 @@ # available if EPEL is enabled, and then only on version 7. %if %{fedpkgs} %if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7 -# If the _without_lidar macro is not set, the MonetDB-lidar RPM will -# be created. The macro can be set when using mock by passing it the -# flag --without=lidar. -%if %{?_without_lidar:0}%{!?_without_lidar:1} -%define with_lidar 1 -%endif +# By default create the MonetDB-lidar package on Fedora and RHEL 7 +%bcond_without lidar %endif %endif %if %{?rhel:0}%{!?rhel:1} -# If the _without_samtools macro is not set, the MonetDB-bam-MonetDB5 -# RPM will be created. The macro can be set when using mock by -# passing it the flag --without=samtools. +# By default create the MonetDB-bam-MonetDB5 package. # Note that the samtools-devel RPM is not available on RedHat # Enterprise Linux and derivatives, even with EPEL availabe. # (Actually, at the moment of writing, samtools-devel is available in # EPEL for RHEL 6, but not for RHEL 7. We don't make the distinction # here and just not build the MonetDB-bam-MonetDB5 RPM.) -%if %{?_without_samtools:0}%{!?_without_samtools:1} -%define with_samtools 1 -%endif +%bcond_without samtools %endif -# If the _without_pcre macro is not set, the PCRE library is used for -# the implementation of the SQL LIKE and ILIKE operators. Otherwise -# the POSIX regex functions are used. The macro can be set when using -# mock by passing it the flag --without=pcre. -%if %{?_without_pcre:0}%{!?_without_pcre:1} -%define with_pcre 1 +# By default use PCRE for the implementation of the SQL LIKE and ILIKE +# operators. Otherwise the POSIX regex functions are used. +%bcond_without pcre + +%if %{fedpkgs} +# By default, create teh MonetDB-R package. +%bcond_without rintegration +%endif + +# On Fedora and RHEL 7, create the MonetDB-python2 package. +# On RHEL 6, numpy is too old. +%if %{?rhel:0}%{!?rhel:1} || 0%{?rhel} >= 7 +%bcond_without pyintegration %endif %if
MonetDB: analytics - Implemented nth_value on a window, plus nti...
Changeset: d263423703ba for MonetDB URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=d263423703ba Modified Files: gdk/gdk_analytic.c gdk/gdk_analytic.h sql/backends/monet5/sql_rank.c sql/backends/monet5/sql_rank.h sql/backends/monet5/sql_rank.mal sql/backends/monet5/sql_rank.mal.sh sql/backends/monet5/sql_rank_hge.mal sql/backends/monet5/sql_rank_hge.mal.sh sql/common/sql_types.c sql/server/rel_select.c sql/server/sql_scan.c sql/test/analytics/Tests/analytics01.sql sql/test/analytics/Tests/analytics01.stable.out Branch: analytics Log Message: Implemented nth_value on a window, plus ntile cleanup. diffs (truncated from 1009 to 300 lines): diff --git a/gdk/gdk_analytic.c b/gdk/gdk_analytic.c --- a/gdk/gdk_analytic.c +++ b/gdk/gdk_analytic.c @@ -103,7 +103,6 @@ GDKanalyticaldiff(BAT *r, BAT *b, BAT *c #define NTILE_CALC(TPE) \ do { \ - TPE val = *(TPE *)ntile; \ if((BUN)val >= cnt) { \ i = 1;\ for(; rbtnonil = true; - r->tnil = false; + r->tnonil = !has_nils; + r->tnil = has_nils; if(o) { r->tsorted = o->tsorted; r->trevsorted = o->trevsorted; @@ -430,6 +436,154 @@ finish: #undef ANALYTICAL_LAST_IMP +#define ANALYTICAL_NTHVALUE_IMP(TPE) \ + do { \ + TPE *rp, *rb, *pbp, *bp, *end, curval;\ + pbp = bp = (TPE*)Tloc(b, 0); \ + rb = rp = (TPE*)Tloc(r, 0); \ + end = rp + cnt; \ + if(is_lng_nil(nth)) { \ + has_nils = true; \ + for(; rp (TPE) (bp - pbp)) { \ + curval = TPE##_nil; \ + } else { \ + curval = *(pbp + nth);\ + } \ + if(is_##TPE##_nil(curval))\ + has_nils = true; \ + for(; rb (TPE) (bp - pbp)) { \ + curval = TPE##_nil; \ + } else { \ + curval = *(pbp + nth);\ + } \ + if(is_##TPE##_nil(curval))\ + has_nils = true; \ + for(; rb (TPE) cnt) { \ + curval = TPE##_nil; \ + } else { \ + curval = *(bp + nth); \ + } \ + if(is_##TPE##_nil(curval))\ + has_nils = true; \ + for(; rp (lng)(i - j)) { + curval = nil; + } else { + curval = BUNtail(bpi, nth); + } + if((*atomcmp)(curval, nil) == 0) + has_nils = true; + for (;j < i; j++) { + if ((gdk_res = BUNappend(r, curval, false)) != GDK_SUCCEED) + goto finish; + } + } + } + if(nth > (lng)(i - j)) { + curval = nil; + } else { + curval = BUNtail(bpi, nth); + } + if((*atomcmp)(curval, nil) == 0) + has_nils = true; + for (;j < i; j++) { + if ((gdk_res = BUNappend(r, curval, false)) != GDK_SUCCEED) + goto finish; + } + } else { /* single value, ie no ordering */ + if(nth > (lng)cnt) { +