Changeset: f84d11a7df55 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=f84d11a7df55
Modified Files:
sql/src/backends/monet5/vaults/mseed.mx
Branch: default
Log Message:
Revamping mseed
Mseed now has separate functions to update the mseed catalog
and (SQL) functions to materialize an individual mseed file.
The latter is the needed for delayed loaded at query time.
diffs (truncated from 1000 to 300 lines):
diff -r 5346d08a7014 -r f84d11a7df55 MonetDB5/src/modules/mal/tablet.mx
--- a/MonetDB5/src/modules/mal/tablet.mx Mon Dec 27 20:11:33 2010 +0100
+++ b/MonetDB5/src/modules/mal/tablet.mx Wed Dec 29 09:15:46 2010 +0100
@@ -1352,11 +1352,16 @@
if (bats == NULL)
return NULL;
for (i = 0; i < as->nr_attrs; i++) {
+ int GDKdebug_bak = GDKdebug;
BAT *b = fmt[i].c[0];
- BAT *bv = BATslice(b, offset, BATcount(b));
+ BAT *bv = NULL;
+ BATsetaccess(b, BAT_READ);
+ bv = BATslice(b, offset, BATcount(b));
BUNins(bats, (ptr) fmt[i].name, (ptr) &bv->batCacheid, FALSE);
- BATsetaccess(b, BAT_READ);
+ /* we "mis"use BATpropcheck to set rather than verify
properties on
+ * the newly loaded slice; hence, we locally disable property
errors */
+ GDKdebug &= ~PROPMASK;
BATaccessBegin(bv, USE_ALL, MMAP_WILLNEED);
BATpropcheck(bv, BATPROPS_ALL);
/* drop the hashes, we don't need them now and they consume
space */
@@ -1366,6 +1371,7 @@
/* drop the hashes, we don't need them now and they consume
space */
HASHremove(BATmirror(b));
BATaccessEnd(bv, USE_ALL, MMAP_WILLNEED);
+ GDKdebug = GDKdebug_bak;
b->hkey &= bv->hkey;
b->tkey &= bv->tkey;
@@ -1377,6 +1383,7 @@
b->hsorted = 0;
if (b->tsorted != bv->tsorted)
b->tsorted = 0;
+ b->batDirty = TRUE;
if (cnt != BATcount(b)) {
if ( as->error == 0) /* a new error */
diff -r 5346d08a7014 -r f84d11a7df55 sql/src/backends/monet5/vaults/mseed.mx
--- a/sql/src/backends/monet5/vaults/mseed.mx Mon Dec 27 20:11:33 2010 +0100
+++ b/sql/src/backends/monet5/vaults/mseed.mx Wed Dec 29 09:15:46 2010 +0100
@@ -59,7 +59,7 @@
EXTERNAL NAME mseed.import;
CREATE FUNCTION mseedLoad(entry string)
-RETURNS int
+RETURNS TABLE(time timestamp, data int)
EXTERNAL NAME mseed.load;
@end verbatim
@@ -96,6 +96,10 @@
pattern load{unsafe}(entry:str)(:bat[:oid,:timestamp],:bat[:oid,:int])
address MseedLoad
comment "Load the content of a mseed file into BATs";
+
+pattern load{unsafe}(entry:str):bat[:str,:bat]
+address MseedLoadSQL
+comment "Load the content of a mseed file into SQL TABLE";
@{
@-
Performance experimentation on Eir on Dec 17, 2010.
@@ -150,6 +154,7 @@
#include "mal_client.h"
#include "mal_interpreter.h"
#include "mal_function.h"
+#include "mtime.h"
#include "libmseed.h"
#ifdef WIN32
@@ -163,7 +168,7 @@
#endif
vault_export str MseedImport(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
-vault_export str MseedLoad(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
+vault_export str MseedLoadSQL(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
#define _MSEED_DEBUG_
@@ -171,6 +176,7 @@
@c
#include "mseed.h"
#include "vault.h"
+#include "mtime.h"
str SQLstatementIntern(Client c, str *expr, str nme, int execute, bit output);
@@ -205,6 +211,8 @@
*ret = int_nil;
snprintf(file,BUFSIZ,"%s%c%s",vaultpath,DIR_SEP,*targetfile);
+ if ( access(file,R_OK) )
+ throw(MAL, "mseed.load", "Cannot access %s\n", file);
while ( (retcode = ms_readmsr (&msr, file, reclen, NULL, NULL, 1,
dataflag, verbose)) == MS_NOERROR )
{
t= MS_HPTIME2EPOCH(msr->starttime);
@@ -243,21 +251,121 @@
return msg;
}
+static str
+MseedLoadIntern(BAT **bbtime, BAT **bbdata, str targetfile)
+{
+ str msg = MAL_SUCCEED;
+ MSRecord *msr = 0;
+ BAT *btime, *bdata;
+
+ int verbose = 1;
+ //int ppackets = 2;
+ int reclen = -1;
+ int dataflag = 1;
+ int retcode;
+ int j;
+ time_t t;
+ struct tm *tm;
+ timestamp ts;
+ char file[BUFSIZ];
+ date d;
+ daytime dt;
+ tzone tz;
+ int ms,stepsize,minutes=0;
+
+ snprintf(file,BUFSIZ,"%s%c%s",vaultpath,DIR_SEP,targetfile);
+ if ( access(file,R_OK) )
+ throw(MAL, "mseed.load", "Cannot access %s\n", file);
+
+ btime = BATnew(TYPE_void,TYPE_timestamp,0);
+ BATseqbase(btime,0);
+ bdata = BATnew(TYPE_void,TYPE_int,0);
+ BATseqbase(bdata,0);
+ if ( btime == NULL || bdata == NULL ){
+ if ( btime) BBPreleaseref(btime->batCacheid);
+ if ( bdata) BBPreleaseref(bdata->batCacheid);
+ throw(MAL, "mseed.load", MAL_MALLOC_FAIL);
+ }
+ *bbtime = btime;
+ *bbdata = bdata;
+ MTIMEtzone_create(&tz, &minutes);
+ while ( (retcode = ms_readmsr (&msr, file, reclen, NULL, NULL, 1,
dataflag, verbose)) == MS_NOERROR )
+ {
+ stepsize = 1000000/ msr->samprate;
+ /* collect the statistics */
+ switch(msr->sampletype){
+ case 'i':
+ if (msr->datasamples)
+ for ( j=0;j< msr->samplecnt; j++){
+ t= MS_HPTIME2EPOCH(msr->starttime);
+ tm = gmtime(&t);
+ tm->tm_year += (tm->tm_year >
80?1900:2000);
+ ms = (msr->starttime % HPTMODULUS)/1000;
+ tm->tm_mon++;
+ MTIMEdate_create(&d,&tm->tm_year,
&tm->tm_mon, &tm->tm_mday);
+ MTIMEdaytime_create(&dt, &tm->tm_hour,
&tm->tm_min, &tm->tm_sec, &ms);
+ MTIMEtimestamp_create(&ts, &d, &dt,
&tz);
+ BUNappend(btime, (ptr) &ts , FALSE);
+ BUNappend(bdata, (ptr)(((int*)
msr->datasamples)+j) , FALSE);
+ msr->starttime += stepsize;
+ }
+ break;
+ case 'a': case 'f': case 'd':
+ default:
+ msg = createException(MAL,"mseed.load","data type not
yet implemented");
+ goto wrapup;
+ }
+ }
+wrapup:
+ /* Make sure everything is cleaned up */
+ ms_readmsr (&msr, NULL, 0, NULL, NULL, 0, 0, 0);
+ if ( retcode != MS_ENDOFFILE )
+ throw(MAL, "mseed.load", "Cannot read %s: %s\n", *targetfile,
ms_errorstr(retcode));
+ return msg;
+}
+
+str
+MseedLoadSQL(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ int *ret = (int*) getArgReference(stk,pci,0);
+ str *targetfile = (str*) getArgReference(stk,pci,1);
+ str msg = MAL_SUCCEED;
+ BAT *btime, *bdata, *table;
+
+ (void) cntxt;
+ (void) mb;
+
+ table = BATnew(TYPE_str,TYPE_bat,0);
+ if ( table == NULL)
+ throw(MAL, "mseed.load", MAL_MALLOC_FAIL);
+ msg = MseedLoadIntern(&btime, &bdata, *targetfile);
+ if ( msg == MAL_SUCCEED){
+ BUNins(table, (ptr)"time", (ptr)&btime->batCacheid, FALSE);
+ BUNins(table, (ptr)"data", (ptr)&bdata->batCacheid, FALSE);
+ BBPreleaseref(btime->batCacheid);
+ BBPreleaseref(bdata->batCacheid);
+ BBPkeepref(*ret= table->batCacheid);
+ }
+ return msg;
+}
+
str
MseedLoad(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
- int *ret = (int*) getArgReference(stk,pci,0);
-/* int *vid = (int*) getArgReference(stk,pci,1);
- str *targetfile = (str*) getArgReference(stk,pci,2); */
- str msg= MAL_SUCCEED;
+ int *ret0 = (int*) getArgReference(stk,pci,0);
+ int *ret1 = (int*) getArgReference(stk,pci,1);
+ str *targetfile = (str*) getArgReference(stk,pci,2);
+ str msg;
+ BAT *btime, *bdata;
+ (void) cntxt;
(void) mb;
- (void) cntxt;
- if ( msg == MAL_SUCCEED)
- *ret = *(int*) getArgReference(stk,pci,1);
- else
- *ret = int_nil;
+
+ msg = MseedLoadIntern(&btime, &bdata, *targetfile);
+ if ( msg == MAL_SUCCEED){
+ BBPkeepref(*ret0 = btime->batCacheid);
+ BBPkeepref(*ret1 = bdata->batCacheid);
+ }
return msg;
}
-
@}
diff -r 5346d08a7014 -r f84d11a7df55
sql/src/benchmarks/tpch/LOCKED/Tests/check1.sql
--- a/sql/src/benchmarks/tpch/LOCKED/Tests/check1.sql Mon Dec 27 20:11:33
2010 +0100
+++ b/sql/src/benchmarks/tpch/LOCKED/Tests/check1.sql Wed Dec 29 09:15:46
2010 +0100
@@ -6,3 +6,12 @@
select count(*) from region;
select count(*) from supplier;
select count(*) from lineitem;
+
+select * from region limit 9;
+select * from nation limit 9;
+select * from supplier limit 9;
+select * from customer limit 9;
+select * from part limit 9;
+select * from partsupp limit 9;
+select * from orders limit 9;
+select * from lineitem limit 9;
diff -r 5346d08a7014 -r f84d11a7df55
sql/src/benchmarks/tpch/LOCKED/Tests/check1.stable.out
--- a/sql/src/benchmarks/tpch/LOCKED/Tests/check1.stable.out Mon Dec 27
20:11:33 2010 +0100
+++ b/sql/src/benchmarks/tpch/LOCKED/Tests/check1.stable.out Wed Dec 29
09:15:46 2010 +0100
@@ -64,8 +64,116 @@
% wrd # type
% 5 # length
[ 60175 ]
+#select * from region limit 9;
+% sys.region, sys.region, sys.region # table_name
+% r_regionkey, r_name, r_comment # name
+% int, varchar, varchar # type
+% 1, 11, 111 # length
+[ 0, "AFRICA", "special Tiresias about the furiously even dolphins are
furi" ]
+[ 1, "AMERICA", "even, ironic theodolites according to the bold
platelets wa" ]
+[ 2, "ASIA", "silent, bold requests sleep slyly across the quickly sly
dependencies. furiously silent instructions alongside " ]
+[ 3, "EUROPE", "special, bold deposits haggle foxes. platelet" ]
+[ 4, "MIDDLE EAST", "furiously unusual packages use carefully above the
unusual, exp" ]
+#select * from nation limit 9;
+% sys.nation, sys.nation, sys.nation, sys.nation # table_name
+% n_nationkey, n_name, n_regionkey, n_comment # name
+% int, varchar, int, varchar # type
+% 1, 9, 1, 112 # length
+[ 0, "ALGERIA", 0, "final accounts wake quickly. special reques"
]
+[ 1, "ARGENTINA", 1, "idly final instructions cajole stealthily.
regular instructions wake carefully blithely express accounts. fluffi" ]
+[ 2, "BRAZIL", 1, "always pending pinto beans sleep sil" ]
+[ 3, "CANADA", 1, "foxes among the bold requests" ]
+[ 4, "EGYPT", 4, "pending accounts haggle furiously. furiously
bold accounts detect. platelets at the packages haggle caref" ]
+[ 5, "ETHIOPIA", 0, "fluffily ruthless requests integrate fluffily.
pending ideas wake blithely acco" ]
+[ 6, "FRANCE", 3, "even requests detect near the pendin" ]
+[ 7, "GERMANY", 3, "blithely ironic foxes grow. quickly pending
accounts are b" ]
+[ 8, "INDIA", 2, "ironic packages should have to are slyly
around the special, ironic accounts. iron" ]
+#select * from supplier limit 9;
+% sys.supplier, sys.supplier, sys.supplier, sys.supplier,
sys.supplier, sys.supplier, sys.supplier # table_name
+% s_suppkey, s_name, s_address, s_nationkey, s_phone,
s_acctbal, s_comment # name
+% int, varchar, varchar, int, varchar, decimal,
varchar # type
+% 1, 18, 35, 2, 15, 17, 93 # length
+[ 1, "Supplier#000000001", " N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ", 17,
"27-918-335-1736", 5755.94, "requests haggle carefully. accounts
sublate finally. carefully ironic pa" ]
+[ 2, "Supplier#000000002", "89eJ5ksX3ImxJQBvxObC,", 5,
"15-679-861-2259", 4032.68, "furiously stealthy frays thrash
alongside of the slyly express deposits. blithely regular req" ]
+[ 3, "Supplier#000000003", "q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3", 1,
"11-383-516-1199", 4192.40, "furiously regular instructions impress
slyly! carefu" ]
+[ 4, "Supplier#000000004", "Bk7ah4CK8SYQTepEmvMkkgMwg", 15,
"25-843-787-7479", 4641.08, "final ideas cajole. furiously close
dep" ]
+[ 5, "Supplier#000000005", "Gcdm2rJRzl5qlTVzc", 11,
"21-151-690-3663", -283.84, "carefully silent instructions are
slyly according t" ]
+[ 6, "Supplier#000000006", "tQxuVm7s7CnK", 14, "24-696-997-4969",
1365.79, "even requests wake carefully! fluffily final pinto beans run
slyly among t" ]
+[ 7, "Supplier#000000007", "s,4TicNGB4uO6PaSqNBUq", 23,
"33-990-965-2201", 6820.35, "carefully express packages believe
furiously after the fur" ]
+[ 8, "Supplier#000000008", "9Sq4bBH2FQEmaFOocY45sRTxo6yuoG", 17,
"27-498-742-3860", 7627.85, "carefully express escapades are slyly
" ]
+[ 9, "Supplier#000000009", "1KhUgZegwM3ua7dsYmekYBsK", 10,
"20-403-398-8662", 5302.37, "slyly regular decoys mold slyly ironic
dugouts. requests are carefully-- carefully" ]
+#select * from customer limit 9;
+% sys.customer, sys.customer, sys.customer, sys.customer,
sys.customer, sys.customer, sys.customer, sys.customer # table_name
+% c_custkey, c_name, c_address, c_nationkey, c_phone,
c_acctbal, c_mktsegment, c_comment # name
+% int, varchar, varchar, int, varchar, decimal,
varchar, varchar # type
+% 1, 18, 37, 2, 15, 17, 10, 114 # length
+[ 1, "Customer#000000001", "IVhzIApeRb ot,c,E", 15,
"25-989-741-2988", 711.56, "BUILDING", "regular, regular platelets are
fluffily according to the even attainments. blithely iron" ]
+[ 2, "Customer#000000002", "XSTf4,NCwDVaWNe6tEgvwfmRchLXak", 13,
"23-768-687-3665", 121.65, "AUTOMOBILE", "furiously special deposits
solve slyly. furiously even foxes wake alongside of the furiously ironic ideas.
pending" ]
+[ 3, "Customer#000000003", "MG9kdTD2WBHm", 1, "11-719-748-3364",
7498.12, "AUTOMOBILE", "special packages wake. slyly reg" ]
+[ 4, "Customer#000000004", "XxVSJsLAGtn", 4, "14-128-190-5944",
2866.83, "MACHINERY", "slyly final accounts sublate carefully. slyly
ironic asymptotes nod across the quickly regular pack" ]
+[ 5, "Customer#000000005", "KvpyuHCplrB84WgAiGV6sYpZq7Tj", 3,
"13-750-942-6364", 794.47, "HOUSEHOLD", "blithely final instructions
haggle; stealthy sauternes nod; carefully regu" ]
+[ 6, "Customer#000000006", "sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn", 20,
"30-114-968-4951", 7638.57, "AUTOMOBILE", "special deposits wake
along the ironic foxes. slyly regular deposits are furiously about the blith"
]
+[ 7, "Customer#000000007", "TcGe5gaZNgVePxU5kRrvXBfkasDTea", 18,
"28-190-982-9759", 9561.95, "AUTOMOBILE", "theodolites kindle
carefully carefully regular deposits. regular depe" ]
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
