Changeset: 7ed723670162 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7ed723670162
Modified Files:
monetdb5/mal/mal_import.c
monetdb5/modules/mal/tokenizer.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_scenario.c
Branch: Oct2014
Log Message:
When creating bstreams for files, create them using the file size.
This way, a large file will be read completely.
diffs (143 lines):
diff --git a/monetdb5/mal/mal_import.c b/monetdb5/mal/mal_import.c
--- a/monetdb5/mal/mal_import.c
+++ b/monetdb5/mal/mal_import.c
@@ -92,13 +92,15 @@ static str
malLoadScript(Client c, str name, bstream **fdin)
{
stream *fd;
+ FILE *f;
+ struct stat st;
fd = malOpenSource(name);
if (mnstr_errnr(fd) == MNSTR_OPEN_ERROR) {
mnstr_destroy(fd);
throw(MAL, "malInclude", "could not open file: %s", name);
}
- *fdin = bstream_create(fd, 128 * BLOCK);
+ *fdin = bstream_create(fd, (f = getFile(fd)) != NULL &&
fstat(fileno(f), &st) == 0 ? (size_t) st.st_size : (size_t) (128 * BLOCK));
if (bstream_next(*fdin) < 0)
mnstr_printf(c->fdout, "!WARNING: could not read %s\n", name);
return MAL_SUCCEED;
@@ -267,11 +269,13 @@ evalFile(Client c, str fname, int listin
mnstr_printf(c->fdout, "#WARNING: could not open file:
%s\n",
filename);
} else {
+ FILE *f;
+ struct stat st;
c->srcFile = filename;
c->yycur = 0;
c->bak = NULL;
MSinitClientPrg(c, "user", "main"); /*
re-initialize context */
- MCpushClientInput(c, bstream_create(fd, 128 * BLOCK),
c->listing, "");
+ MCpushClientInput(c, bstream_create(fd, (f =
getFile(fd)) != NULL && fstat(fileno(f), &st) == 0 ? (size_t) st.st_size :
(size_t) (128 * BLOCK)), c->listing, "");
msg = runScenario(c);
}
filename = p + 1;
@@ -281,11 +285,13 @@ evalFile(Client c, str fname, int listin
if( fd == 0) mnstr_destroy(fd);
msg = createException(MAL,"mal.eval", "WARNING: could not open
file: %s\n", filename);
} else {
+ FILE *f;
+ struct stat st;
c->srcFile = filename;
c->yycur = 0;
c->bak = NULL;
MSinitClientPrg(c, "user", "main"); /* re-initialize
context */
- MCpushClientInput(c, bstream_create(fd, 128 * BLOCK),
c->listing, "");
+ MCpushClientInput(c, bstream_create(fd, (f = getFile(fd)) !=
NULL && fstat(fileno(f), &st) == 0 ? (size_t) st.st_size : (size_t) (128 *
BLOCK)), c->listing, "");
msg = runScenario(c);
}
GDKfree(fname);
diff --git a/monetdb5/modules/mal/tokenizer.c b/monetdb5/modules/mal/tokenizer.c
--- a/monetdb5/modules/mal/tokenizer.c
+++ b/monetdb5/modules/mal/tokenizer.c
@@ -388,7 +388,7 @@ TKNZRappend(oid *pos, str *s)
return MAL_SUCCEED;
}
-#define SIZE 1 * 1024 * 1024
+#define SIZE (1 * 1024 * 1024)
str
TKNZRdepositFile(int *r, str *fnme)
{
@@ -399,6 +399,8 @@ TKNZRdepositFile(int *r, str *fnme)
char buf[PATHLENGTH];
oid pos;
str msg= MAL_SUCCEED;
+ FILE *f;
+ struct stat st;
if (TRANS == NULL)
throw(MAL, "tokenizer", "no tokenizer store open");
@@ -416,7 +418,7 @@ TKNZRdepositFile(int *r, str *fnme)
close_stream(fs);
throw(MAL, "tokenizer.depositFile", RUNTIME_FILE_NOT_FOUND
"%s", buf);
}
- bs = bstream_create(fs, SIZE);
+ bs = bstream_create(fs, (f = getFile(fs)) != NULL && fstat(fileno(f),
&st) == 0 ? (size_t) st.st_size : (size_t) SIZE);
if (bs == NULL)
throw(MAL, "tokenizer.depositFile", MAL_MALLOC_FAIL);
while (bstream_read(bs, bs->size - (bs->len - bs->pos)) != 0 &&
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -2739,6 +2739,8 @@ mvc_import_table_wrap(Client cntxt, MalB
bstream *s;
stream *ss;
str utf8 = "UTF-8";
+ FILE *f;
+ struct stat st;
(void) mb; /* NOT USED */
if ((msg = checkSQLContext(cntxt)) != NULL)
@@ -2769,11 +2771,15 @@ mvc_import_table_wrap(Client cntxt, MalB
mnstr_destroy(ss);
throw(IO, "streams.open", "could not open file '%s': %s",
filename, strerror(errnr));
}
+ if ((f = getFile(ss)) != NULL && fstat(fileno(f), &st) == 0)
+ s = bstream_create(ss, (size_t) st.st_size);
+ else {
#if SIZEOF_VOID_P == 4
- s = bstream_create(ss, 0x20000);
+ s = bstream_create(ss, 0x20000);
#else
- s = bstream_create(ss, 0x2000000);
+ s = bstream_create(ss, 0x2000000);
#endif
+ }
#ifdef WIN32
fix_windows_newline(tsep);
fix_windows_newline(rsep);
diff --git a/sql/backends/monet5/sql_scenario.c
b/sql/backends/monet5/sql_scenario.c
--- a/sql/backends/monet5/sql_scenario.c
+++ b/sql/backends/monet5/sql_scenario.c
@@ -1184,9 +1184,8 @@ SQLinitClient(Client c)
if (fd) {
struct stat st;
- if (fstat(fileno(getFile(fd)), &st) < 0)
- st.st_size = 128 * BLOCK;
- bfd = bstream_create(fd, (size_t)
st.st_size);
+ FILE *f;
+ bfd = bstream_create(fd, (f =
getFile(fd)) != NULL && fstat(fileno(f), &st) == 0 ? (size_t) st.st_size :
(size_t) (128 * BLOCK));
if (bfd && bstream_next(bfd) >= 0)
msg = SQLstatementIntern(c,
&bfd->buf, "sql.init", TRUE, FALSE);
bstream_destroy(bfd);
@@ -1581,6 +1580,8 @@ SQLinclude(Client cntxt, MalBlkPtr mb, M
str msg = MAL_SUCCEED, fullname;
str *expr;
mvc *m;
+ FILE *f;
+ struct stat st;
fullname = MSP_locate_sqlscript(*name, 0);
if (fullname == NULL)
@@ -1590,7 +1591,7 @@ SQLinclude(Client cntxt, MalBlkPtr mb, M
mnstr_destroy(fd);
throw(MAL, "sql.include", "could not open file: %s\n", *name);
}
- bfd = bstream_create(fd, 128 * BLOCK);
+ bfd = bstream_create(fd, (f = getFile(fd)) != NULL && fstat(fileno(f),
&st) == 0 ? (size_t) st.st_size : (size_t) (128 * BLOCK));
if (bstream_next(bfd) < 0) {
bstream_destroy(bfd);
throw(MAL, "sql.include", "could not read %s\n", *name);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
