Changeset: 711e95eec056 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=711e95eec056
Modified Files:
sql/src/backends/monet5/vaults/fits.mx
Branch: default
Log Message:
Added loading of string columns;
More efficient load using BATappend and a temporary bat;
Fixed duplicate table names and non unique column identifiers.
diffs (216 lines):
diff -r 9c7e5f98aaa9 -r 711e95eec056 sql/src/backends/monet5/vaults/fits.mx
--- a/sql/src/backends/monet5/vaults/fits.mx Wed Dec 15 14:59:54 2010 +0100
+++ b/sql/src/backends/monet5/vaults/fits.mx Wed Dec 15 16:31:10 2010 +0100
@@ -73,8 +73,8 @@
@c
#include "fits.h"
-#define FITS_INS_COL "INSERT INTO fits_columns(id, name, type, units,table_id)
\
- VALUES(%d, '%s','%s','%s',%d);"
+#define FITS_INS_COL "INSERT INTO fits_columns(id, name, type, units, number,
table_id) \
+ VALUES(%d,'%s','%s','%s',%d,%d);"
str
FITStest(int *res, str *fname)
@@ -127,6 +127,7 @@
mvc_create_column_(m, fits_col, "name","varchar",80);
mvc_create_column_(m, fits_col, "type","varchar",80);
mvc_create_column_(m, fits_col, "units","varchar",10);
+ mvc_create_column_(m, fits_col, "number","int",32);
mvc_create_column_(m, fits_col, "table_id","int",32);
}
@@ -135,8 +136,7 @@
static int
fits2mtype( int t )
{
- printf("%d\n", t );
- switch(t) {
+ switch(t) {
case TBIT:
case TLOGICAL:
return TYPE_bit;
@@ -170,8 +170,7 @@
static int
fits2subtype(sql_subtype *tpe, int t, long rep, long wid)
{
-
- printf("Tpcode %d, rep %ld, wid %ld\n", t,rep,wid);
+ (void) rep;
switch(t) {
case TBIT:
case TLOGICAL:
@@ -182,7 +181,7 @@
sql_find_subtype(tpe, "char", 1, 0 );
break;
case TSTRING:
- sql_find_subtype(tpe, "varchar", (unsigned int)rep, 0);
+ sql_find_subtype(tpe, "varchar", (unsigned int)wid, 0);
break;
case TUSHORT:
case TSHORT:
@@ -221,8 +220,8 @@
str fname = *(str*) getArgReference(stk, pci, 1);
fitsfile *fptr; /* pointer to the FITS file */
int status = 0, i, j, hdutype, hdunum = 1, cnum = 0;
- oid fid, tid, rid = oid_nil;
- char tname[BUFSIZ], *s, bname[BUFSIZ]; /*, stmt[BUFSIZ];*/
+ oid fid, tid, cid, rid = oid_nil;
+ char tname[BUFSIZ], *tname_low, *s, bname[BUFSIZ], stmt[BUFSIZ];
long tbcol;
char cname[BUFSIZ], tform[BUFSIZ], tunit[BUFSIZ], tnull[BUFSIZ],
tdisp[BUFSIZ];
double tscal, tzero;
@@ -287,12 +286,15 @@
status = 0;
}
else { /* check table name for existence in the fits catalog */
+ tname_low = toLower(tname);
col = mvc_bind_column(m, fits_tbl, "name");
- rid = table_funcs.column_find_row(m->session->tr, col,
tname, NULL);
+ rid = table_funcs.column_find_row(m->session->tr, col,
tname_low, NULL);
/* or as regular SQL table */
- tbl = mvc_bind_table(m, sch, tname);
- if ( rid != oid_nil || tbl )
+ tbl = mvc_bind_table(m, sch, tname_low);
+ if ( rid != oid_nil || tbl ){
snprintf(tname,BUFSIZ,"%s_%d",bname,i);
+ tname_low = toLower(tname);
+ }
}
fits_get_num_cols(fptr, &cnum, &status);
@@ -300,7 +302,7 @@
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "id"), &tid, TYPE_int);
store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, fits_tbl, "name"), toLower(tname),
TYPE_str);
+ mvc_bind_column(m, fits_tbl, "name"), tname_low,
TYPE_str);
store_funcs.append_col(m->session->tr,
mvc_bind_column(m, fits_tbl, "columns"), &cnum,
TYPE_str);
store_funcs.append_col(m->session->tr,
@@ -309,26 +311,13 @@
mvc_bind_column(m, fits_tbl, "hdu"), &i, TYPE_int);
/* read columns description */
- /* s = stmt;*/
-
- for ( j = 1; j <= cnum; j++){
+ s = stmt;
+ col = mvc_bind_column(m, fits_col, "id");
+ cid = store_funcs.count_col(col) + 1;
+ for ( j = 1; j <= cnum; j++, cid++){
fits_get_acolparms(fptr, j, cname, &tbcol, tunit, tform,
&tscal, &tzero, tnull, tdisp, &status);
- /* fits_get_coltype(fptr, j,
&tpcode, &rep, &wid, &status);*/
-
- /* snprintf(stmt, BUFSIZ,
FITS_INS_COL, j, cname, tform, tunit, (int)tid);
- printf("%s\n",stmt);
+ snprintf(stmt, BUFSIZ, FITS_INS_COL, (int)cid, cname,
tform, tunit, j, (int)tid);
msg = SQLstatementIntern(cntxt,
&s,"fits.attach",TRUE,FALSE);
- */
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, fits_col, "id"), &j,
TYPE_int);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, fits_col, "name"),
toLower(cname), TYPE_str);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, fits_col, "type"), tform,
TYPE_str);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, fits_col, "table_id"), &tid,
TYPE_int);
- store_funcs.append_col(m->session->tr,
- mvc_bind_column(m, fits_col, "units"), tunit,
TYPE_str);
}
tid++;
}
@@ -349,10 +338,11 @@
str fname;
str msg = MAL_SUCCEED;
oid rid = oid_nil, frid = oid_nil;
- int status = 0, cnum = 0, fid, hdu, hdutype, *tpcode = NULL, j,
anynull, mtype, i;
- long rep, wid, rows;
- char keywrd[80], **cname, *v, nm[FLEN_VALUE];
- /* BAT *b;*/
+ int status = 0, cnum = 0, fid, hdu, hdutype, i, j, anynull, mtype;
+ int *tpcode = NULL;
+ long *rep=NULL, *wid=NULL, rows;
+ char keywrd[80], **cname, **v, nm[FLEN_VALUE];
+ BAT *b, *tmp;
ptr nilptr;
@@ -409,6 +399,8 @@
tbl = mvc_create_table(m, sch, tname, tt_table, 0, SQL_PERSIST, 0,
cnum);
tpcode = (int *)GDKzalloc(sizeof(int) * cnum);
+ rep = (long *)GDKzalloc(sizeof(long) * cnum);
+ wid = (long *)GDKzalloc(sizeof(long) * cnum);
cname = (char **)GDKzalloc(sizeof(char *) * cnum);
for ( j = 1; j <= cnum; j++ ){
@@ -420,8 +412,8 @@
status = 0;
}
cname[j-1] = GDKstrdup(toLower(nm));
- fits_get_coltype(fptr, j, &tpcode[j-1], &rep, &wid, &status);
- fits2subtype(&tpe, tpcode[j-1], rep, wid);
+ fits_get_coltype(fptr, j, &tpcode[j-1], &rep[j-1], &wid[j-1],
&status);
+ fits2subtype(&tpe, tpcode[j-1], rep[j-1], wid[j-1]);
mvc_create_column(m, tbl, cname[j-1], &tpe);
}
@@ -430,25 +422,48 @@
for ( j = 1; j <= cnum; j++ ){
mtype = fits2mtype(tpcode[j-1]);
nilptr = ATOMnil(mtype);
- col = mvc_bind_column(m, tbl, cname[j-1]);
- v = GDKzalloc(rows * ATOMsize(mtype));
- /* b = store_funcs.bind_col(m->session->tr, col,
RD_INS);
- fits_read_col(fptr, tpcode[j-1], j, 1, 1, rows,
nilptr, BUNtloc(bat_iterator(b),BUNfirst(b)), &anynull, &status ); */
- fits_read_col(fptr, tpcode[j-1], j, 1, 1, rows, nilptr, (void
*)v, &anynull, &status );
+
+ if (mtype != TYPE_str ){
+ tmp = BATnew(TYPE_void, mtype, rows);
+ BATseqbase(tmp, 0);
+ fits_read_col(fptr, tpcode[j-1], j, 1, 1, rows, nilptr,
(void *) BUNtloc(bat_iterator(tmp),BUNfirst(tmp)), &anynull, &status );
+ }
+ else {
+ v = (char **) GDKzalloc(sizeof(char *) * rows);
+ for(i = 0; i < rows; i++)
+ v[i] = GDKzalloc(wid[j-1]);
+ fits_read_col(fptr, tpcode[j-1], j, 1, 1, rows, nilptr,
(void *)v, &anynull, &status );
+ }
+
if ( status ){
char buf[FLEN_ERRMSG+1];
fits_read_errmsg(buf);
msg = createException(MAL, "fits.loadtable", "Cannot
load column %s of %s table: %s.\n", cname[j-1], tname, buf);
- GDKfree(v);
+ /* GDKfree(v); */
goto finish;
}
- for (i = 0; i<rows; i++)
- store_funcs.append_col(m->session->tr, col, v + i *
ATOMsize(mtype), mtype);
- GDKfree(v);
+
+ col = mvc_bind_column(m, tbl, cname[j-1]);
+ if (mtype != TYPE_str ){
+ BATsetcount(tmp, rows);
+ b = store_funcs.bind_col(m->session->tr, col, RDONLY);
+ BATappend(b,tmp,TRUE);
+ BBPunfix(b->batCacheid);
+ BBPreclaim(tmp);
+ }
+ else {
+ for (i = 0; i<rows; i++)
+ store_funcs.append_col(m->session->tr, col,
v[i], mtype);
+ for(i = 0; i < rows; i++)
+ GDKfree(v[i]);
+ GDKfree(v);
+ }
}
finish:
GDKfree(tpcode);
+ GDKfree(rep);
+ GDKfree(wid);
GDKfree(cname);
fits_close_file(fptr, &status);
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
