Changeset: 18d0f88695a4 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=18d0f88695a4 Modified Files: MonetDB/src/gdk/gdk.mx MonetDB/src/gdk/gdk_utils.mx MonetDB4/tests/Availability/Tests/00_Mserver.stable.out MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.STATIC MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.Windows MonetDB4/tests/BugTracker/Tests/project_on_BAT_with_batFirst.SF-2020806.stable.out MonetDB4/tests/suite_08/Tests/test_01.stable.out MonetDB5/conf/monetdb5.conf.5 MonetDB5/src/mal/mal.mx MonetDB5/src/modules/mal/mdb.mx MonetDB5/src/optimizer/opt_mitosis.mx MonetDB5/src/tools/Tests/mserver5--help.stable.err MonetDB5/src/tools/mserver5.mx sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.err sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.out sql/src/test/BugTracker/Tests/cardinality_violation.SF-1240701.stable.err testing/src/Mtest.py.in Branch: Oct2010 Log Message:
replace config variable "gdk_testing" by debug bit 29 (536870912) nick-named "forcemito" Like with the config variable "gdk_testing" as introduced by Sjoerd in changset fa142513c49c http://dev.monetdb.org/hg/MonetDB/rev/fa142513c49c the purpose of the "forcemito" debug/testing switch is to forcefully activate mitosis even on small tables, i.e., split small tables in as many (tiny) pieces as there are cores (threads) available. This allows us to test mitosis functionality without requiring large data sets --- at the expense of a potentially significant interpretation overhead for unnecessary (then) large plans. Hence, the default behavior of MonetDB is the one that is considered most suitable for "production" environments, i.e., mitosis splits tables into pieces only when the size of the tables suggests that the performance benefits of parallel execution outweighs the extra interpretation overhead for larger plans. To enforce mitosis also on small tables (for testing the mitosis functionality), one needs to explicitly call mserver5 with option "--forcemito" or set debug bit 29 (e.g., via "--debug=536870912"). Mtest.py does automatically use "--forcemito" for testing convenience. diffs (258 lines): diff -r f04657d95045 -r 18d0f88695a4 MonetDB/src/gdk/gdk.mx --- a/MonetDB/src/gdk/gdk.mx Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB/src/gdk/gdk.mx Thu Oct 07 18:08:40 2010 +0200 @@ -2709,6 +2709,9 @@ #define EXTENDMASK (1<<28) #define EXTENDDEBUG if (GDKdebug & EXTENDMASK) +#define FORCEMITOMASK (1<<29) +#define FORCEMITODEBUG if (GDKdebug & FORCEMITOMASK) + gdk_export int GDKnrofthreads; gdk_export int GDKembedded; gdk_export int GDKprotected; diff -r f04657d95045 -r 18d0f88695a4 MonetDB/src/gdk/gdk_utils.mx --- a/MonetDB/src/gdk/gdk_utils.mx Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB/src/gdk/gdk_utils.mx Thu Oct 07 18:08:40 2010 +0200 @@ -1667,7 +1667,6 @@ GDKnr_threads = GDKgetenv_int("gdk_nr_threads", 0); if (GDKnr_threads == 0) GDKnr_threads = MT_check_nr_cores(); - GDK_testing = GDKgetenv_istrue("gdk_testing"); #ifdef NATIVE_WIN32 if (GDKnr_threads) GDK_mmap_minsize /= GDKnr_threads; @@ -1693,16 +1692,10 @@ @h gdk_export int GDKstopped; gdk_export int GDKnr_threads; -gdk_export int GDK_testing; /* used to switch on/off code for testing purposes */ @c int GDKstopped = 1; int GDKnr_threads = 0; -/* GDK_testing is used to switch code on/off that is meant to be used - during testing only (an example is splitting small tables in the - mitosis optimizer to test mitosis/mergetable). The value is set on - the command line using --set gdk_testing=true by Mtest.py. */ -int GDK_testing = 0; @h gdk_export void GDKexit(int status); diff -r f04657d95045 -r 18d0f88695a4 MonetDB4/tests/Availability/Tests/00_Mserver.stable.out --- a/MonetDB4/tests/Availability/Tests/00_Mserver.stable.out Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB4/tests/Availability/Tests/00_Mserver.stable.out Thu Oct 07 18:08:40 2010 +0200 @@ -81,7 +81,7 @@ [ "MODULES", "str", "str", 48, 0, "clean", "load", "pers", 0, 2 ] [ "adm_atomtbl", "str", "int", 15, 0, "dirty", "load", "tran", 1, 1 ] [ "adm_fcntbl", "str", "oid", 1036, 0, "dirty", "load", "tran", 1, 1 ] -[ "monet_environment", "str", "str", 43, 0, "dirty", "load", "tran", 1, 1 ] +[ "monet_environment", "str", "str", 42, 0, "dirty", "load", "tran", 1, 1 ] [ "monet_fcn_aut", "oid", "str", 965, 0, "dirty", "load", "tran", 1, 2 ] [ "monet_fcn_cnt", "oid", "int", 1036, 0, "dirty", "load", "tran", 1, 2 ] [ "monet_fcn_dat", "oid", "str", 965, 0, "dirty", "load", "tran", 1, 2 ] diff -r f04657d95045 -r 18d0f88695a4 MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.STATIC --- a/MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.STATIC Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.STATIC Thu Oct 07 18:08:40 2010 +0200 @@ -75,7 +75,7 @@ [ "MODULES", "str", "str", 46, 0, "clean", "load", "pers", 0, 2 ] [ "adm_atomtbl", "str", "int", 15, 0, "dirty", "load", "tran", 1, 1 ] [ "adm_fcntbl", "str", "oid", 1036, 0, "dirty", "load", "tran", 1, 1 ] -[ "monet_environment", "str", "str", 43, 0, "dirty", "load", "tran", 1, 1 ] +[ "monet_environment", "str", "str", 42, 0, "dirty", "load", "tran", 1, 1 ] [ "monet_fcn_aut", "oid", "str", 965, 0, "dirty", "load", "tran", 1, 2 ] [ "monet_fcn_cnt", "oid", "int", 1036, 0, "dirty", "load", "tran", 1, 2 ] [ "monet_fcn_dat", "oid", "str", 965, 0, "dirty", "load", "tran", 1, 2 ] diff -r f04657d95045 -r 18d0f88695a4 MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.Windows --- a/MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.Windows Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB4/tests/Availability/Tests/00_Mserver.stable.out.Windows Thu Oct 07 18:08:40 2010 +0200 @@ -76,7 +76,7 @@ [ "MODULES", "str", "str", 47, 0, "clean", "load", "pers", 0, 2 ] [ "adm_atomtbl", "str", "int", 15, 0, "dirty", "load", "tran", 1, 1 ] [ "adm_fcntbl", "str", "oid", 1036, 0, "dirty", "load", "tran", 1, 1 ] -[ "monet_environment", "str", "str", 43, 0, "dirty", "load", "tran", 1, 1 ] +[ "monet_environment", "str", "str", 42, 0, "dirty", "load", "tran", 1, 1 ] [ "monet_fcn_aut", "oid", "str", 965, 0, "dirty", "load", "tran", 1, 2 ] [ "monet_fcn_cnt", "oid", "int", 1036, 0, "dirty", "load", "tran", 1, 2 ] [ "monet_fcn_dat", "oid", "str", 965, 0, "dirty", "load", "tran", 1, 2 ] diff -r f04657d95045 -r 18d0f88695a4 MonetDB4/tests/BugTracker/Tests/project_on_BAT_with_batFirst.SF-2020806.stable.out --- a/MonetDB4/tests/BugTracker/Tests/project_on_BAT_with_batFirst.SF-2020806.stable.out Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB4/tests/BugTracker/Tests/project_on_BAT_with_batFirst.SF-2020806.stable.out Thu Oct 07 18:08:40 2010 +0200 @@ -41,7 +41,6 @@ [ "gdk_mem_maxsize ", 1 ] [ "gdk_mem_pagebits", 1 ] [ "gdk_nr_threads", 1 ] -[ "gdk_testing", 1 ] [ "gdk_version", 1 ] [ "gdk_vmtrim", 1 ] [ "host", 1 ] diff -r f04657d95045 -r 18d0f88695a4 MonetDB4/tests/suite_08/Tests/test_01.stable.out --- a/MonetDB4/tests/suite_08/Tests/test_01.stable.out Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB4/tests/suite_08/Tests/test_01.stable.out Thu Oct 07 18:08:40 2010 +0200 @@ -284,7 +284,6 @@ [ "xrpc_trusted", "http://monetdb.cwi.nl;http://127.0.0.1;http://localhost;/ufs/manegold/_/scratch0/Monet/Testing/Stable/prefix.--enable-strict_--disable-debug_--enable-optimize_--enable-assert/share/MonetDB/xrpc/export"; ] [ "monet_cwd", "/net/rig.ins.cwi.nl/export/scratch0/manegold/Monet/Testing/Stable/build.--enable-strict_--disable-debug_--enable-optimize_--enable-assert/MonetDB4/mTests/tests/suite_08" ] [ "monet_version", "4.28.0" ] -[ "gdk_testing", "true" ] #~EndVariableOutput~# # Pseudo bat to map thread to name diff -r f04657d95045 -r 18d0f88695a4 MonetDB5/conf/monetdb5.conf.5 --- a/MonetDB5/conf/monetdb5.conf.5 Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB5/conf/monetdb5.conf.5 Thu Oct 07 18:08:40 2010 +0200 @@ -90,6 +90,17 @@ for debugging (GDK developers, only) 134217728 = OPTMASK = trace the actions, decisions and effects of MAL optimizers + 536870912 = FORCEMITOMASK = forcefully activate mitosis even on + small tables, i.e., split small tables + in as many (tiny) pieces as there are + cores (threads) available; + this allows us to test mitosis + functionality without requiring large + data sets (--- at the expense of a + potentially significant interpretation + overhead for unnecessary large plans); + used only for development & testing; + set automatically by Mtest.py .EE Note that mserver5 recognizes a series of command line options as well: .EX @@ -103,6 +114,7 @@ --xproperties (XPROPMASK ) --performance (JOINPROPMASK | DEADBEEFMASK) --optimizers (OPTMASK) + --forcemito (FORCEMITOMASK) .EE Default: 0 diff -r f04657d95045 -r 18d0f88695a4 MonetDB5/src/mal/mal.mx --- a/MonetDB5/src/mal/mal.mx Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB5/src/mal/mal.mx Thu Oct 07 18:08:40 2010 +0200 @@ -221,6 +221,7 @@ #define GRPxproperties (XPROPMASK ) #define GRPperformance (JOINPROPMASK | DEADBEEFMASK) #define GRPoptimizers (1<<27) /* == OPTMASK; cf., MonetDB/src/gdk/gdk.mx */ +#define GRPforcemito (FORCEMITOMASK) @c #include <mal_config.h> #include <mal.h> diff -r f04657d95045 -r 18d0f88695a4 MonetDB5/src/modules/mal/mdb.mx --- a/MonetDB5/src/modules/mal/mdb.mx Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB5/src/modules/mal/mdb.mx Thu Oct 07 18:08:40 2010 +0200 @@ -433,6 +433,8 @@ GDKdebug |= GRPperformance; if( strcmp("xproperties",*flg)==0) GDKdebug |= GRPxproperties; + if( strcmp("forcemito",*flg)==0) + GDKdebug |= GRPforcemito; return MAL_SUCCEED; } diff -r f04657d95045 -r 18d0f88695a4 MonetDB5/src/optimizer/opt_mitosis.mx --- a/MonetDB5/src/optimizer/opt_mitosis.mx Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB5/src/optimizer/opt_mitosis.mx Thu Oct 07 18:08:40 2010 +0200 @@ -146,8 +146,9 @@ r= (BUN) (monet_memory /typewidth/GDKnr_threads); /* how much fits */ if (rowcnt > r ) pieces = (int) (rowcnt /r+1); - if (GDK_testing && pieces < GDKnr_threads ) - pieces = GDKnr_threads; /* split up anyway when testing */ + FORCEMITODEBUG + if (pieces < GDKnr_threads ) + pieces = GDKnr_threads; /* split up anyway when testing */ if (pieces > MAXSLICES) pieces = MAXSLICES; /* cut off potential plan explosion */ diff -r f04657d95045 -r 18d0f88695a4 MonetDB5/src/tools/Tests/mserver5--help.stable.err --- a/MonetDB5/src/tools/Tests/mserver5--help.stable.err Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB5/src/tools/Tests/mserver5--help.stable.err Thu Oct 07 18:08:40 2010 +0200 @@ -32,6 +32,7 @@ --xproperties --performance --optimizers + --forcemito --debug=<bitmask> # 10:03:11 > diff -r f04657d95045 -r 18d0f88695a4 MonetDB5/src/tools/mserver5.mx --- a/MonetDB5/src/tools/mserver5.mx Thu Oct 07 16:46:35 2010 +0200 +++ b/MonetDB5/src/tools/mserver5.mx Thu Oct 07 18:08:40 2010 +0200 @@ -154,6 +154,7 @@ fprintf(stderr," --xproperties\n"); fprintf(stderr," --performance\n"); fprintf(stderr," --optimizers\n"); + fprintf(stderr," --forcemito\n"); fprintf(stderr," --debug=<bitmask>\n"); exit(0); @@ -309,6 +310,7 @@ {"optimizers",0,0,0}, {"performance",0,0,0}, {"xproperties",0,0,0}, + {"forcemito",0,0,0}, {0, 0, 0, 0} }; @@ -423,6 +425,10 @@ grpdebug |= GRPxproperties; break; } + if (strcmp(long_options[option_index].name, "forcemito") == 0) { + grpdebug |= GRPforcemito; + break; + } if (strcmp(long_options[option_index].name, "performance") == 0) { grpdebug |= GRPperformance; break; diff -r f04657d95045 -r 18d0f88695a4 sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.err --- a/sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.err Thu Oct 07 16:46:35 2010 +0200 +++ b/sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.err Thu Oct 07 18:08:40 2010 +0200 @@ -70,7 +70,7 @@ MAPI = mone...@ottar:39527 QUERY = select * from env() as env where name = ( select 'prefix' from env() as env ); -ERROR = !SQLException:zero_or_one:cardinality violation (52>1) +ERROR = !SQLException:zero_or_one:cardinality violation (51>1) # 13:22:15 > diff -r f04657d95045 -r 18d0f88695a4 sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.out --- a/sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.out Thu Oct 07 16:46:35 2010 +0200 +++ b/sql/src/test/BugDay_2005-12-19_2.9.3/Tests/select_from_env.SF-1240701.1242164.stable.out Thu Oct 07 18:08:40 2010 +0200 @@ -54,7 +54,6 @@ [ "standoff_end" ] [ "config" ] [ "gdk_nr_threads" ] -[ "gdk_testing" ] [ "xrpc_open" ] [ "xrpc_port" ] [ "mal_listing" ] diff -r f04657d95045 -r 18d0f88695a4 sql/src/test/BugTracker/Tests/cardinality_violation.SF-1240701.stable.err --- a/sql/src/test/BugTracker/Tests/cardinality_violation.SF-1240701.stable.err Thu Oct 07 16:46:35 2010 +0200 +++ b/sql/src/test/BugTracker/Tests/cardinality_violation.SF-1240701.stable.err Thu Oct 07 18:08:40 2010 +0200 @@ -82,7 +82,7 @@ MAPI = mone...@ottar:39527 QUERY = select * from env() as env where name = ( select 'prefix' from env() as env ); -ERROR = !SQLException:zero_or_one:cardinality violation (52>1) +ERROR = !SQLException:zero_or_one:cardinality violation (51>1) MAPI = mone...@ottar:39527 QUERY = select * from columns where name = (select columns.name from _tables, columns where _tables.id = columns.table_id); ERROR = !SQLException:zero_or_one:cardinality violation (288>1) diff -r f04657d95045 -r 18d0f88695a4 testing/src/Mtest.py.in --- a/testing/src/Mtest.py.in Thu Oct 07 16:46:35 2010 +0200 +++ b/testing/src/Mtest.py.in Thu Oct 07 18:08:40 2010 +0200 @@ -3481,10 +3481,10 @@ exe = {} exe['Mtimeout'] = CheckExec('Mtimeout') , 'Mtimeout -timeout %d ' % par['TIMEOUT'] if par['M5']: - exe['Mserver'] = CheckExec('mserver5') , '%s mserver5 "--config=%s" --debug=%s --set gdk_nr_threads=%s --set gdk_testing=true %s %s %s --set mapi_open=true --set xrpc_open=true --set mapi_port=%s --set xrpc_port=%s --set monet_prompt= --set mal_listing=2 --trace %s' % \ + exe['Mserver'] = CheckExec('mserver5') , '%s mserver5 "--config=%s" --debug=%s --set gdk_nr_threads=%s %s %s %s --set mapi_open=true --set xrpc_open=true --set mapi_port=%s --set xrpc_port=%s --set monet_prompt= --trace --forcemito --set mal_listing=2 %s' % \ (env['setDBG'], config, env['GDK_DEBUG'], env['GDK_NR_THREADS'], env['setMONETDB_MOD_PATH'], env['setGDK_DBFARM'], env['setXRPC_DOCROOT'], env['MAPIPORT'], env['XRPCPORT'], env['MSERVER_SET']) if par['M4']: - exe['Mserver'] = CheckExec('Mserver') , '%s Mserver "--config=%s" --debug=%s --set gdk_nr_threads=%s --set gdk_testing=true %s %s %s --set mapi_open=true --set xrpc_open=true --set mapi_port=%s --set xrpc_port=%s --set monet_prompt= --trace %s' % \ + exe['Mserver'] = CheckExec('Mserver') , '%s Mserver "--config=%s" --debug=%s --set gdk_nr_threads=%s %s %s %s --set mapi_open=true --set xrpc_open=true --set mapi_port=%s --set xrpc_port=%s --set monet_prompt= --trace %s' % \ (env['setDBG'], config, env['GDK_DEBUG'], env['GDK_NR_THREADS'], env['setMONETDB_MOD_PATH'], env['setGDK_DBFARM'], env['setXRPC_DOCROOT'], env['MAPIPORT'], env['XRPCPORT'], env['MSERVER_SET']) exe['Mdiff'] = CheckExec('Mdiff') , 'Mdiff' exe['python'] = CheckExec(sys.executable) , sys.executable _______________________________________________ Checkin-list mailing list [email protected] http://mail.monetdb.org/mailman/listinfo/checkin-list
