Update of /cvsroot/monetdb/MonetDB5/src/modules/mal
In directory sc8-pr-cvs16.sourceforge.net:/tmp/cvs-serv27837
Modified Files:
tablet.mx
Log Message:
Loading files should avoid BATextends as much as possible.
One heuristic is to estimate the number of records to be expected
by reading a small portion first.
Index: tablet.mx
===================================================================
RCS file: /cvsroot/monetdb/MonetDB5/src/modules/mal/tablet.mx,v
retrieving revision 1.89
retrieving revision 1.90
diff -u -d -r1.89 -r1.90
--- tablet.mx 25 May 2007 06:22:13 -0000 1.89
+++ tablet.mx 4 Jun 2007 07:09:05 -0000 1.90
@@ -1652,6 +1652,30 @@
}
#define SIZE 1*1024*1024
[EMAIL PROTECTED]
+Estimate the size of a BAT to avoid multiple extends.
[EMAIL PROTECTED]
+static size_t
+estimator(char *datafile){
+ long size;
+ char buf[BUFSIZ+1],*s=buf;
+ size_t nr=0;
+ FILE *f;
+ f= fopen(datafile,"R");
+ if(f == NULL) return 0;
+
+ fseek(f,0L,SEEK_END);
+ size= ftell(f);
+ buf[BUFSIZ]=0;
+ if( fread(buf, 0, BUFSIZ,f) >0){
+ for(; *s; s++)
+ if( *s=='\n') nr++;
+ }
+ fclose(f);
+ if( nr == 0)
+ return size/BUFSIZ;
+ return size/BUFSIZ/nr;
+}
BAT *
TABLETload(BAT *names, BAT *seps, BAT *types, char *datafile, size_t nr)
@@ -1667,6 +1691,8 @@
}
return NULL;
}
+ if( nr== 0)
+ nr= estimator(datafile);
res = TABLETinput(names, seps, types, b = bstream_create(s, SIZE),
NULL, nr);
bstream_destroy(b);
stream_close(s);
-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
_______________________________________________
Monetdb-checkins mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/monetdb-checkins
