Changeset: 44be5313feb1 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=44be5313feb1
Modified Files:
monetdb5/mal/mal_client.c
monetdb5/mal/mal_resource.c
monetdb5/mal/mal_resource.h
monetdb5/mal/mal_runtime.c
Branch: default
Log Message:
Tighten the scope of locking the mal_running global and improve thread delays
for long runnong queries.
Ideally, the worker thread should not be stopped, but the query should be
delayed and the worker redirected
to another one. This calls for major changes in the dataflow queue.
diffs (149 lines):
diff --git a/monetdb5/mal/mal_client.c b/monetdb5/mal/mal_client.c
--- a/monetdb5/mal/mal_client.c
+++ b/monetdb5/mal/mal_client.c
@@ -475,16 +475,14 @@ MCstopClients(Client cntxt)
int
MCactiveClients(void)
{
- int freeclient=0, finishing=0, running=0, blocked = 0;
+ int finishing=0, running = 0;
Client cntxt = mal_clients;
for(cntxt = mal_clients; cntxt<mal_clients+MAL_MAXCLIENTS; cntxt++){
- freeclient += (cntxt->mode == FREECLIENT);
finishing += (cntxt->mode == FINISHCLIENT);
running += (cntxt->mode == RUNCLIENT);
- blocked += (cntxt->mode == BLOCKCLIENT);
}
- return finishing+running;
+ return finishing + running;
}
void
diff --git a/monetdb5/mal/mal_resource.c b/monetdb5/mal/mal_resource.c
--- a/monetdb5/mal/mal_resource.c
+++ b/monetdb5/mal/mal_resource.c
@@ -158,16 +158,15 @@ MALadmission(lng argclaim, lng hotclaim)
}
#endif
-/* Delay threads if too much competition arises and memory becomes a scarce
resource.
+/* Delay a thread if too much competition arises and memory becomes a scarce
resource.
* If in the mean time memory becomes free, or too many sleeping re-enable
worker.
* It may happen that all threads enter the wait state. So, keep one running
at all time
- * By keeping the query start time in the client record we can delay
- * them when resource stress occurs.
+ * By keeping the query start time in the client record we can delay them when
resource stress occurs.
*/
ATOMIC_TYPE mal_running = ATOMIC_VAR_INIT(0);
void
-MALresourceFairness(lng usec)
+MALresourceFairness(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci,
lng usec)
{
#ifdef FAIRNESS_THRESHOLD
size_t rss;
@@ -175,15 +174,24 @@ MALresourceFairness(lng usec)
int delayed= 0;
int users = 2;
+ (void) cntxt;
+ (void) mb;
+ (void) stk;
+ (void) pci;
+
+ /* don't punish queries whose last instruction was fast to executed in
the first place */
if ( usec <= TIMESLICE)
return;
+
/* use GDKmem_cursize as MT_getrss() is too expensive */
rss = GDKmem_cursize();
/* ample of memory available*/
if ( rss <= MEMORY_THRESHOLD )
return;
+ (void) ATOMIC_INC(&mal_running);
+
/* worker reporting time spent in usec! */
clk = usec / 1000;
@@ -193,7 +201,7 @@ MALresourceFairness(lng usec)
#endif
/* always keep one running to avoid all waiting */
- while (clk > DELAYUNIT && users > 1 && (int) ATOMIC_GET(&mal_running) >
GDKnr_threads && rss > MEMORY_THRESHOLD) {
+ while (clk > DELAYUNIT && users > 1 && rss > MEMORY_THRESHOLD && (int)
ATOMIC_GET(&mal_running) > GDKnr_threads ) {
if ( delayed++ == 0){
PARDEBUG fprintf(stderr, "#delay initial
["LLFMT"] memory %zu[%f]\n", clk, rss, MEMORY_THRESHOLD );
}
@@ -207,6 +215,7 @@ MALresourceFairness(lng usec)
rss = GDKmem_cursize();
clk -= DELAYUNIT;
}
+ (void) ATOMIC_DEC(&mal_running);
#else
(void) usec;
#endif
diff --git a/monetdb5/mal/mal_resource.h b/monetdb5/mal/mal_resource.h
--- a/monetdb5/mal/mal_resource.h
+++ b/monetdb5/mal/mal_resource.h
@@ -12,7 +12,8 @@
#include "mal_interpreter.h"
#include "matomic.h"
-#define TIMESLICE (3 * 60 * 1000 * 1000) /* usec , 3 minute high priority */
+#define LONGRUNNING (60 * 1000 * 1000) /* usec , 60 seconds high priority */
+#define TIMESLICE (3 * 1000 * 1000) /* usec , 3 seconds high priority */
#define DELAYUNIT 2 /* ms delay in parallel processing decisions */
#define MAX_DELAYS 1000 /* never wait more then 2000 ms */
@@ -28,7 +29,7 @@ mal_export int MALadmission(lng argclaim
#define FAIRNESS_THRESHOLD (MAX_DELAYS * DELAYUNIT)
mal_export lng getMemoryClaim(MalBlkPtr mb, MalStkPtr stk, InstrPtr pci, int
i, int flag);
-mal_export void MALresourceFairness(lng usec);
+mal_export void MALresourceFairness(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci, lng usec);
mal_export size_t MALrunningThreads(void);
#endif /* _MAL_RESOURCE_H*/
diff --git a/monetdb5/mal/mal_runtime.c b/monetdb5/mal/mal_runtime.c
--- a/monetdb5/mal/mal_runtime.c
+++ b/monetdb5/mal/mal_runtime.c
@@ -180,10 +180,6 @@ runtimeProfileBegin(Client cntxt, MalBlk
/* always collect the MAL instruction execution time */
pci->clock = prof->ticks = GDKusec();
- /* keep track of actual running instructions over BATs */
- if( isaBatType(getArgType(mb, pci, 0)) )
- (void) ATOMIC_INC(&mal_running);
-
/* emit the instruction upon start as well */
if(malProfileMode > 0 )
profilerEvent(mb, stk, pci, TRUE, cntxt->username);
@@ -202,10 +198,6 @@ runtimeProfileExit(Client cntxt, MalBlkP
cntxt->inprogress[tid].pci = 0;
}
- assert(pci);
- if( isaBatType(getArgType(mb, pci, 0)) )
- (void) ATOMIC_DEC(&mal_running);
-
assert(prof);
/* always collect the MAL instruction execution time */
pci->ticks = ticks - prof->ticks;
@@ -220,10 +212,10 @@ runtimeProfileExit(Client cntxt, MalBlkP
malProfileMode = 1;
}
cntxt->active = FALSE;
- /* Reduce worker threads of non-admin long running transaction if
needed */
- /* The super user can always proceed */
- if ( cntxt->user != MAL_ADMIN )
- MALresourceFairness(ticks - mb->starttime);
+ /* Reduce worker threads of non-admin long running transaction if
needed.
+ * the punishment is equal to the duration of the last instruction */
+ if ( cntxt->user != MAL_ADMIN && ticks - mb->starttime > LONGRUNNING )
+ MALresourceFairness(cntxt, mb, stk, pci, pci->ticks);
}
/*
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
