Changeset: 4c0805a1e381 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=4c0805a1e381
Modified Files:
Branch: default
Log Message:
merge
diffs (truncated from 334 to 300 lines):
diff -r c0584f7eabe8 -r 4c0805a1e381 MonetDB5/src/modules/mal/tablet.mx
--- a/MonetDB5/src/modules/mal/tablet.mx Sat Dec 04 00:27:02 2010 +0100
+++ b/MonetDB5/src/modules/mal/tablet.mx Sat Dec 04 00:29:07 2010 +0100
@@ -442,7 +442,6 @@
@h
typedef struct Table_t {
- int rawmode; /* perform bypass load preparation */
char *sep; /* default separator */
str ttopbrk, tbotbrk; /* table brackets */
str rlbrk, rrbrk; /* row brackets */
@@ -1299,8 +1298,6 @@
len= strlen(nme);
assert(len <BUFSIZ-50);
-/* as->rawmode= TRUE; TRIAL CODE */
- as->rawmode = 0;
for (i = 0; i < as->nr_attrs; i++) {
fmt[i].c[0] = void_bat_create(fmt[i].adt, est);
fmt[i].ci[0] = bat_iterator(fmt[i].c[0]);
@@ -1308,13 +1305,6 @@
GDKerror("TABLETcreate_bats: Failed to create bat of
size " BUNFMT "\n", as->nr);
return -1;
}
- if (as->rawmode){
- fmt[i].batfile=
GDKstrdup(BBP_physical(ABS(fmt[i].c[0]->batCacheid)));
-
snprintf(nme+len,BUFSIZ-len,"%cbat%c"BUNFMT".raw",DIR_SEP,DIR_SEP,i);
- fmt[i].rawfile= GDKstrdup(nme);
- fmt[i].raw= open_wastream(fmt[i].rawfile);
- mnstr_printf(GDKout,"#mv %s
%s\n",fmt[i].rawfile,fmt[i].batfile);
- }
}
return 0;
}
diff -r c0584f7eabe8 -r 4c0805a1e381 MonetDB5/src/modules/mal/tablet_sql.mx
--- a/MonetDB5/src/modules/mal/tablet_sql.mx Sat Dec 04 00:27:02 2010 +0100
+++ b/MonetDB5/src/modules/mal/tablet_sql.mx Sat Dec 04 00:29:07 2010 +0100
@@ -467,6 +467,7 @@
int threads= (!maxrow || maxrow > (1<<16)) ? (GDKnr_threads < 16?
GDKnr_threads: 16):1;
lng lio =0, tio, t1 = 0, total=0, iototal=0;
int vmtrim = GDK_vm_trim;
+ str msg= MAL_SUCCEED;
for (i = 0; i < 16; i++)
ptask[i].cols = 0;
@@ -579,6 +580,15 @@
#endif
while ( (task->b->pos < task->b->len || !task->b->eof) && cnt < (BUN)
maxrow && res == 0) {
+ if(task->errbuf && task->errbuf[0] ) {
+ msg = catchKernelException(cntxt, msg);
+ if ( msg ){
+ showException(MAL,"copy_from",msg);
+ GDKfree(msg);
+ goto bailout;
+ }
+ }
+
if ( task->basesize <= task->b->size){
/* end of record not found within 32M */
/* most likely wrong delimiter */
@@ -813,9 +823,10 @@
bailout:
if (task) {
- for( i=0; i< as->nr_attrs; i++)
+ for( i=0; i< as->nr_attrs; i++){
if (task->fields[i])
GDKfree(task->fields[i]);
+ }
if (task->fields)
GDKfree(task->fields);
if (task->time)
diff -r c0584f7eabe8 -r 4c0805a1e381
sql/src/backends/monet5/vaults/Tests/vault00.mal
--- a/sql/src/backends/monet5/vaults/Tests/vault00.mal Sat Dec 04 00:27:02
2010 +0100
+++ b/sql/src/backends/monet5/vaults/Tests/vault00.mal Sat Dec 04 00:29:07
2010 +0100
@@ -1,3 +1,3 @@
-s:= vault.getDirectory();
+s:= vault.getdirectory();
io.print(s);
diff -r c0584f7eabe8 -r 4c0805a1e381 sql/src/backends/monet5/vaults/mseed.mx
--- a/sql/src/backends/monet5/vaults/mseed.mx Sat Dec 04 00:27:02 2010 +0100
+++ b/sql/src/backends/monet5/vaults/mseed.mx Sat Dec 04 00:29:07 2010 +0100
@@ -21,14 +21,26 @@
@a Martin Kersten
@v 0.1
@+ Mseed
-These routines are meant to interpret mseed files.
-It is derived from the mseed library example.
+These routines are meant to interpret mseed files in the context of a vault.
+The simplifying situation is that mseed has a single model.
The code base assumes that libmseed has been installed on your system.
+
+Furthermore, the mseed catalog initialization script should have been run.
+
@mal
module mseed;
pattern import(vid:int, fnme:str):str
-address MSEmseedImport
-comment "Dump the record content of an mseed file from the vault";
+address MseedImport
+comment "Dump the record content of an mseed file from the vault into the
mseed catalog";
+
+...@-
+The intended use is to simply rely on SQL to select elements
+in the vault table for import and to apply the mseed import operation.
+It returns the id list of succesful imports.
+...@verbatim
+select import(vid) from vault where ...
+...@end verbatim
+...@{
@h
#ifndef _MSEED_H
#define _MSEED_H
@@ -51,7 +63,7 @@
#define vault_export extern
#endif
-vault_export str MSEmseedImport(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
+vault_export str MseedImport(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
#define _MSEED_DEBUG_
@@ -66,7 +78,7 @@
VALUES(%d, %d,'%c','%s', '%s','%s','%s','%s',%f,%d,%s);"
str
-MSEmseedImport(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+MseedImport(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
str *ret = (str*) getArgReference(stk,pci,0);
str msg = MAL_SUCCEED;
@@ -84,7 +96,7 @@
int j;
time_t t;
struct tm *tm;
- char *s,*kind;
+ char *s, *basename,*kind;
char file[BUFSIZ];
char buf[BUFSIZ];
char starttime[BUFSIZ];
@@ -98,12 +110,26 @@
*ret = GDKstrdup(msg);
return msg;
}
- snprintf(file,BUFSIZ,"%s%c%s",vaultpath, DIR_SEP,*sourcefile);
- s= buf;
+ basename = strrchr(*sourcefile,DIR_SEP);
+ basename = basename ? basename: *sourcefile;
+ snprintf(file,BUFSIZ,"%s%c%s",vaultpath, DIR_SEP,basename);
- printf("FTP Processing %s -> %s\n",*sourcefile,file);
+ /* only fetch the file if it is not already in the local vault */
+ if ( access(file, R_OK) ){
+ mnstr_printf(cntxt->fdout, "FTP fetch %s ->
%s\n",*sourcefile,file);
+ msg= VLTftpget(&j, sourcefile, &basename);
+ if ( msg)
+ return msg;
+ /* remember the location of the copy */
+ s= buf;
+ snprintf(buf,BUFSIZ,"UPDATE vault SET target='%s' WHERE vid =
%d;", file,*vid);
+ msg =SQLstatementIntern(cntxt,&s,"mseed.import
file",TRUE,FALSE);
+ if ( msg)
+ return msg;
+ }
/* Loop over the input file */
+ s= buf;
while ( (retcode = ms_readmsr (&msr, file, reclen, NULL, NULL, 1,
dataflag, verbose)) == MS_NOERROR )
{
totalrecs++;
@@ -174,3 +200,5 @@
else *ret= 0;
return msg;
}
+
+...@}
diff -r c0584f7eabe8 -r 4c0805a1e381 sql/src/backends/monet5/vaults/mseed.sql
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/sql/src/backends/monet5/vaults/mseed.sql Sat Dec 04 00:29:07 2010 +0100
@@ -0,0 +1,41 @@
+-- this schema is intended to experiment with accessing mseed files
+drop FUNCTION mseedImport();
+drop table mseedCatalog;
+drop table mseedRepository;
+
+-- all records in the mseed files correspond to a row in the catalog
+CREATE TABLE mseedCatalog (
+mseed int, -- Vault file id
+chunk varchar(255), -- SQL volumn storage container name
+seqno int, -- SEED record sequence number,
should be between 0 and 999999
+ PRIMARY KEY (mseed,seqno),
+dataquality char, -- Data record indicator, should be
'D=data unknown qual',
+ -- 'R=raw no
quality', 'Q= quality controlled' or 'M'
+network varchar(11), -- Network
+station varchar(11), -- Station
+location varchar(11), -- Location
+channel varchar(11), -- Channel
+starttime timestamp, -- Record start time, the time
of the first sample, as a high precision epoch time
+samplerate double, -- Nominal sample rate (Hz)
+samplecnt int, -- Number of samples in record
+sampletype string, -- storage type in mseed record
+minval float, -- statistics for search later
+maxval float
+);
+
+-- this function inserts the mseed record information into the catalog
+-- errors are returned for off-line analysis.
+CREATE FUNCTION mseedImport(vid int)
+ RETURNS string
+EXTERNAL NAME mseed.import;
+
+-- mseed data volumns may appear in different formats
+-- we try to postpone them, assuming the optimizer can guide JIT.
+--CREATE TABLE chunkname (
+--time timestamp,
+--mseed int,
+--adata varchar(20), dependent on type
+--idata int,
+--fdata float,
+--ddata double
+--);
diff -r c0584f7eabe8 -r 4c0805a1e381 sql/src/backends/monet5/vaults/vault.mx
--- a/sql/src/backends/monet5/vaults/vault.mx Sat Dec 04 00:27:02 2010 +0100
+++ b/sql/src/backends/monet5/vaults/vault.mx Sat Dec 04 00:29:07 2010 +0100
@@ -25,10 +25,12 @@
a repository with datafiles whose integrity and use is shared between
MonetDB and a client repository.
-Once a vault is created, the system administrator can manually add files to
the catalog.
-A refresh() operation will fetch the files from the remote site using a path
-to an (ftp) directory.
-Often, this involves semantic knowledge of the target repository and the vault
kind.
+Once a vault is created, the system administrator can manually add files to
the
+vault catalog. A target directory, when set, denotes the location where
+temporary copies are to be stored.
+
+A import() operation will perform a batch copy of the files marked as to be
copied
+from the remote site using a path to an (ftp) directory.
If source and target files reside on the same file system then a symbolic link
is sufficient.
Alternatively, the vault can be populated using normal updates to the vault
catalog.
@@ -46,10 +48,6 @@
cached timestamp, -- copy stored locally
target string -- file name of source file in
vault
);
-
-create function vaultDir()
-returns string
-external "vault.getDirectory";
@end verbatim
The module is developed solely for a Linux environment.
@@ -64,13 +62,9 @@
address VLTgetDirectory
comment "Return the location of the root vault directory";
-command refresh(id:str)
-address VLTrefresh
-comment "Refresh the objects in the vault identified";
-
-command refresh(id:int)
-address VLTrefreshObject
-comment "Refresh the content of the vault for a specific object";
+command import(vid:int)
+address VLTimport
+comment "Import the vault object identified";
command ftpget(src:str,trg:str)
address VLTftpget
@@ -118,8 +112,7 @@
vault_export str VLTprelude(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
vault_export str VLTftpget(int *ret, str *source, str *target);
vault_export str VLTgetDirectory(str *ret);
-vault_export str VLTrefresh(int *ret, str *nme);
-vault_export str VLTrefreshObject(int *ret, int *id);
+vault_export str VLTimport(int *ret, str *nme);
vault_export str VLTepilogue(int *ret);
vault_export char vaultpath[BUFSIZ];
@@ -127,6 +120,7 @@
@- Module initializaton
@c
#include "vault.h"
+#include "mseed.h"
char vaultpath[BUFSIZ];
@-
@@ -175,6 +169,7 @@
//curl_global_init(CURL_GLOBAL_DEFAULT);
snprintf(path,BUFSIZ,"%s%c%s", vaultpath, DIR_SEP, *target);
+ mnstr_printf(GDKout,"#vault.ftpget: %s\n",path);
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, *source);
@@ -238,14 +233,11 @@
return MAL_SUCCEED;
}
_______________________________________________
Checkin-list mailing list
[email protected]
http://mail.monetdb.org/mailman/listinfo/checkin-list
