Changeset: fa5b726b204c for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=fa5b726b204c
Modified Files:
clients/Tests/exports.stable.out
common/stream/stream.c
common/stream/stream.h
monetdb5/mal/mal_import.c
monetdb5/modules/mal/tokenizer.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_scenario.c
Branch: Oct2014
Log Message:
Fix use of bstream_create.
Only use a buffer the size of the input file if we need it.
If the file is too large, complain.
diffs (226 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -2813,6 +2813,7 @@ stream *file_rstream(FILE *fp, const cha
stream *file_wastream(FILE *fp, const char *name);
stream *file_wstream(FILE *fp, const char *name);
FILE *getFile(stream *s);
+size_t getFileSize(stream *s);
stream *iconv_rstream(stream *ss, const char *charset, const char *name);
stream *iconv_wstream(stream *ss, const char *charset, const char *name);
int isa_block_stream(stream *s);
diff --git a/common/stream/stream.c b/common/stream/stream.c
--- a/common/stream/stream.c
+++ b/common/stream/stream.c
@@ -71,8 +71,16 @@
#include <stdarg.h> /* va_alist.. */
#include <assert.h>
+#ifdef HAVE_SYS_TYPES_H
+# include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+# include <sys/stat.h>
+#endif
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif
#ifdef HAVE_NETDB_H
-# include <sys/types.h>
# include <netinet/in_systm.h>
# include <netinet/in.h>
# include <netinet/ip.h>
@@ -718,6 +726,18 @@ getFile(stream *s)
return (FILE *) s->stream_data.p;
}
+size_t
+getFileSize(stream *s)
+{
+ if (s->read == file_read) {
+ struct stat stb;
+
+ fstat(fileno((FILE *) s->stream_data.p), &stb);
+ return (size_t) stb.st_size;
+ }
+ return 0; /* unknown */
+}
+
static stream *
open_stream(const char *filename, const char *flags)
{
diff --git a/common/stream/stream.h b/common/stream/stream.h
--- a/common/stream/stream.h
+++ b/common/stream/stream.h
@@ -172,6 +172,7 @@ stream_export stream *file_rastream(FILE
stream_export stream *file_wastream(FILE *fp, const char *name);
stream_export FILE *getFile(stream *s);
+stream_export size_t getFileSize(stream *s);
stream_export stream *iconv_rstream(stream *ss, const char *charset, const
char *name);
stream_export stream *iconv_wstream(stream *ss, const char *charset, const
char *name);
diff --git a/monetdb5/mal/mal_import.c b/monetdb5/mal/mal_import.c
--- a/monetdb5/mal/mal_import.c
+++ b/monetdb5/mal/mal_import.c
@@ -86,21 +86,24 @@ malOpenSource(str file)
* where an intermediate file is used to pass commands around.
* Since the parser needs access to the complete block, we first have
* to find out how long the input is.
- * For the time being, we assume at most 1Mb.
*/
static str
malLoadScript(Client c, str name, bstream **fdin)
{
stream *fd;
- FILE *f;
- struct stat st;
+ size_t sz;
fd = malOpenSource(name);
if (mnstr_errnr(fd) == MNSTR_OPEN_ERROR) {
mnstr_destroy(fd);
throw(MAL, "malInclude", "could not open file: %s", name);
}
- *fdin = bstream_create(fd, (f = getFile(fd)) != NULL &&
fstat(fileno(f), &st) == 0 ? (size_t) st.st_size : (size_t) (128 * BLOCK));
+ sz = getFileSize(fd);
+ if (sz > (size_t) 1 << 29) {
+ mnstr_destroy(fd);
+ throw(MAL, "malInclude", "file %s too large to process", name);
+ }
+ *fdin = bstream_create(fd, sz == 0 ? (size_t) (128 * BLOCK) : sz);
if (bstream_next(*fdin) < 0)
mnstr_printf(c->fdout, "!WARNING: could not read %s\n", name);
return MAL_SUCCEED;
@@ -269,13 +272,11 @@ evalFile(Client c, str fname, int listin
mnstr_printf(c->fdout, "#WARNING: could not open file:
%s\n",
filename);
} else {
- FILE *f;
- struct stat st;
c->srcFile = filename;
c->yycur = 0;
c->bak = NULL;
MSinitClientPrg(c, "user", "main"); /*
re-initialize context */
- MCpushClientInput(c, bstream_create(fd, (f =
getFile(fd)) != NULL && fstat(fileno(f), &st) == 0 ? (size_t) st.st_size :
(size_t) (128 * BLOCK)), c->listing, "");
+ MCpushClientInput(c, bstream_create(fd, 128 * BLOCK),
c->listing, "");
msg = runScenario(c);
}
filename = p + 1;
@@ -285,13 +286,11 @@ evalFile(Client c, str fname, int listin
if( fd == 0) mnstr_destroy(fd);
msg = createException(MAL,"mal.eval", "WARNING: could not open
file: %s\n", filename);
} else {
- FILE *f;
- struct stat st;
c->srcFile = filename;
c->yycur = 0;
c->bak = NULL;
MSinitClientPrg(c, "user", "main"); /* re-initialize
context */
- MCpushClientInput(c, bstream_create(fd, (f = getFile(fd)) !=
NULL && fstat(fileno(f), &st) == 0 ? (size_t) st.st_size : (size_t) (128 *
BLOCK)), c->listing, "");
+ MCpushClientInput(c, bstream_create(fd, 128 * BLOCK),
c->listing, "");
msg = runScenario(c);
}
GDKfree(fname);
diff --git a/monetdb5/modules/mal/tokenizer.c b/monetdb5/modules/mal/tokenizer.c
--- a/monetdb5/modules/mal/tokenizer.c
+++ b/monetdb5/modules/mal/tokenizer.c
@@ -399,8 +399,6 @@ TKNZRdepositFile(int *r, str *fnme)
char buf[PATHLENGTH];
oid pos;
str msg= MAL_SUCCEED;
- FILE *f;
- struct stat st;
if (TRANS == NULL)
throw(MAL, "tokenizer", "no tokenizer store open");
@@ -418,7 +416,7 @@ TKNZRdepositFile(int *r, str *fnme)
close_stream(fs);
throw(MAL, "tokenizer.depositFile", RUNTIME_FILE_NOT_FOUND
"%s", buf);
}
- bs = bstream_create(fs, (f = getFile(fs)) != NULL && fstat(fileno(f),
&st) == 0 ? (size_t) st.st_size : (size_t) SIZE);
+ bs = bstream_create(fs, SIZE);
if (bs == NULL)
throw(MAL, "tokenizer.depositFile", MAL_MALLOC_FAIL);
while (bstream_read(bs, bs->size - (bs->len - bs->pos)) != 0 &&
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -2739,8 +2739,6 @@ mvc_import_table_wrap(Client cntxt, MalB
bstream *s;
stream *ss;
str utf8 = "UTF-8";
- FILE *f;
- struct stat st;
(void) mb; /* NOT USED */
if ((msg = checkSQLContext(cntxt)) != NULL)
@@ -2771,15 +2769,11 @@ mvc_import_table_wrap(Client cntxt, MalB
mnstr_destroy(ss);
throw(IO, "streams.open", "could not open file '%s': %s",
filename, strerror(errnr));
}
- if ((f = getFile(ss)) != NULL && fstat(fileno(f), &st) == 0)
- s = bstream_create(ss, (size_t) st.st_size);
- else {
#if SIZEOF_VOID_P == 4
- s = bstream_create(ss, 0x20000);
+ s = bstream_create(ss, 0x20000);
#else
- s = bstream_create(ss, 0x2000000);
+ s = bstream_create(ss, 0x2000000);
#endif
- }
#ifdef WIN32
fix_windows_newline(tsep);
fix_windows_newline(rsep);
diff --git a/sql/backends/monet5/sql_scenario.c
b/sql/backends/monet5/sql_scenario.c
--- a/sql/backends/monet5/sql_scenario.c
+++ b/sql/backends/monet5/sql_scenario.c
@@ -1183,12 +1183,17 @@ SQLinitClient(Client c)
filename = p + 1;
if (fd) {
- struct stat st;
- FILE *f;
- bfd = bstream_create(fd, (f =
getFile(fd)) != NULL && fstat(fileno(f), &st) == 0 ? (size_t) st.st_size :
(size_t) (128 * BLOCK));
- if (bfd && bstream_next(bfd) >= 0)
- msg = SQLstatementIntern(c,
&bfd->buf, "sql.init", TRUE, FALSE);
- bstream_destroy(bfd);
+ size_t sz;
+ sz = getFileSize(fd);
+ if (sz > (size_t) 1 << 29) {
+ mnstr_destroy(fd);
+ msg = createException(MAL,
"createdb", "file %s too large to process", filename);
+ } else {
+ bfd = bstream_create(fd, sz ==
0 ? (size_t) (128 * BLOCK) : sz);
+ if (bfd && bstream_next(bfd) >=
0)
+ msg =
SQLstatementIntern(c, &bfd->buf, "sql.init", TRUE, FALSE);
+ bstream_destroy(bfd);
+ }
if (m->sa)
sa_destroy(m->sa);
m->sa = NULL;
@@ -1580,8 +1585,7 @@ SQLinclude(Client cntxt, MalBlkPtr mb, M
str msg = MAL_SUCCEED, fullname;
str *expr;
mvc *m;
- FILE *f;
- struct stat st;
+ size_t sz;
fullname = MSP_locate_sqlscript(*name, 0);
if (fullname == NULL)
@@ -1591,7 +1595,12 @@ SQLinclude(Client cntxt, MalBlkPtr mb, M
mnstr_destroy(fd);
throw(MAL, "sql.include", "could not open file: %s\n", *name);
}
- bfd = bstream_create(fd, (f = getFile(fd)) != NULL && fstat(fileno(f),
&st) == 0 ? (size_t) st.st_size : (size_t) (128 * BLOCK));
+ sz = getFileSize(fd);
+ if (sz > (size_t) 1 << 29) {
+ mnstr_destroy(fd);
+ throw(MAL, "sql.include", "file %s too large to process",
fullname);
+ }
+ bfd = bstream_create(fd, sz == 0 ? (size_t) (128 * BLOCK) : sz);
if (bstream_next(bfd) < 0) {
bstream_destroy(bfd);
throw(MAL, "sql.include", "could not read %s\n", *name);
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
