Changeset: 70a801d2ab02 for MonetDB URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=70a801d2ab02 Added Files: sql/backends/monet5/vaults/gadget/82_gadget.mal sql/backends/monet5/vaults/gadget/82_gadget.sql sql/backends/monet5/vaults/gadget/Makefile.ag sql/backends/monet5/vaults/gadget/gadget.c sql/backends/monet5/vaults/gadget/gadget.h sql/backends/monet5/vaults/gadget/gadget.mal sql/backends/monet5/vaults/gadget/gadgetFile.c sql/backends/monet5/vaults/gadget/gadgetFile.h sql/backends/monet5/vaults/gadget/peano.c sql/backends/monet5/vaults/gadget/peano.h Branch: data-vaults Log Message:
Add GADGET data vault code diffs (truncated from 3569 to 300 lines): diff --git a/sql/backends/monet5/vaults/gadget/82_gadget.mal b/sql/backends/monet5/vaults/gadget/82_gadget.mal new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/vaults/gadget/82_gadget.mal @@ -0,0 +1,20 @@ +# The contents of this file are subject to the MonetDB Public License +# Version 1.1 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# http://www.monetdb.org/Legal/MonetDBLicense +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +# License for the specific language governing rights and limitations +# under the License. +# +# The Original Code is the MonetDB Database System. +# +# The Initial Developer of the Original Code is CWI. +# Portions created by CWI are Copyright (C) 1997-July 2008 CWI. +# Copyright August 2008-2015 MonetDB B.V. +# All Rights Reserved. + +# This loads the MonetDB/SQL module +library gadget; +include gadget; diff --git a/sql/backends/monet5/vaults/gadget/82_gadget.sql b/sql/backends/monet5/vaults/gadget/82_gadget.sql new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/vaults/gadget/82_gadget.sql @@ -0,0 +1,11 @@ +create procedure gadgetAttach(fname string) external name gadget.attach; +create procedure gadgetLoad(tname string) external name gadget.load; +create procedure gadgetLoadAll(tname string) external name gadget.loadall; +create procedure gadgetListDir(dname string) external name gadget.listdir; +create procedure gadgetListDirPat(dname string, pattern string) external name gadget.listdirpattern; +create procedure gadgetListDirAll(dname string) external name gadget.listdirall; +create procedure gadgetAttachAll(*) external name gadget.attachall; +create procedure gadgetCheckTable(tname string) external name gadget.checktable; +create function gadgetPHkeyConvert(x real, y real, z real, baseSize real, nbits int) returns bigint external name gadget.phkeyconvert; +create function gadgetPHkeyInvert(phkey bigint, baseSize real, nbits int) returns table (x real, y real, z real, cSize real) external name gadget.phkeyinvert; + diff --git a/sql/backends/monet5/vaults/gadget/Makefile.ag b/sql/backends/monet5/vaults/gadget/Makefile.ag new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/vaults/gadget/Makefile.ag @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V. + +INCLUDES = ../ \ + ../../ \ + ../../../../include \ + ../../../../common \ + ../../../../storage \ + ../../../../server \ + ../../../../../monetdb5/modules/atoms \ + ../../../../../monetdb5/modules/kernel \ + ../../../../../monetdb5/mal \ + ../../../../../monetdb5/modules/mal \ + ../../../../../monetdb5/optimizer \ + ../../../../../monetdb5/scheduler \ + ../../../../../clients/mapilib \ + ../../../../../common/options \ + ../../../../../common/stream \ + ../../../../../common/utils \ + ../../../../../gdk \ + $(READLINE_INCS) + +lib__gadget = { + MODULE + DIR = libdir/monetdb5 + SOURCES = gadgetFile.c peano.c gadget.c + LIBS = ../../../../../monetdb5/tools/libmonetdb5 +} + +headers_gadgetmal = { + HEADERS = mal + DIR = libdir/monetdb5 + SOURCES = gadget.mal +} + +headers_gadgetsql = { + HEADERS = sql + DIR = libdir/monetdb5/createdb + SOURCES = 82_gadget.sql +} + +headers_gadgetautoload = { + HEADERS = mal + DIR = libdir/monetdb5/autoload + SOURCES = 82_gadget.mal +} + +EXTRA_DIST = 82_gadget.mal gadget.mal 82_gadget.sql gadgetFile.h peano.h gadget.h diff --git a/sql/backends/monet5/vaults/gadget/gadget.c b/sql/backends/monet5/vaults/gadget/gadget.c new file mode 100644 --- /dev/null +++ b/sql/backends/monet5/vaults/gadget/gadget.c @@ -0,0 +1,2337 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * + * Copyright 2008-2016 MonetDB B.V. + */ + +/* + * Authors: R. Goncalves + * + * This module contains primitives for accessing data in GADGET file format. + */ + +#include "monetdb_config.h" +#include <glob.h> + +/* clash with GDK? */ +#undef htype +#undef ttype + +#include "gadget.h" +#include "mutils.h" +#include "sql_mvc.h" +#include "sql_scenario.h" +#include "sql.h" +#include "clients.h" +#include "mal_exception.h" + +static MT_Lock mt_gadget_lock; + +#define GADGET_LOCK MT_lock_set(&mt_gadget_lock) +#define GADGET_UNLOCK MT_lock_unset(&mt_gadget_lock) + +#define ATTACHDIR "call gadgetAttach('%s');" + +static +int replaceCharacter(char s[], const char oldC, const char newC) { + int j = 0; + while (s[j] != '\0'){ + if (s[j] == oldC) { + s[j] = newC; + } + j++; + } + return 0; +} +#ifndef NDEBUG + +static +int printHeader(FILE *fp, Header h, char *fname) +{ + int res = 0; + + fprintf(fp,"fileName: %s\nnpartTotal: %d\nnpart[6]{ %d, %d, %d, %d, %d. %d}\nmass[6]{ %lf, %lf, %lf, %lf, %lf. %lf}\ntime: %lf\nredshift: %lf\nflag_sfr: %d\nflag_feedback: %d\nnpart_total[6]{ %d, %d, %d, %d, %d. %d}\nflag_cooling: %d\nnum_files: %d\nBoxSize: %lf\nOmega0: %lf\nOmegaLambda: %lf\nHubbleParameter: %lf\nflag_stellarage: %d\nflag_metals: %d\nhashtabsize: %d\nheaderOk: %d\nfirst_cell: %d\nlast_cell: %d\n",fname, h.npartTotal,h.npart[0], h.npart[1], h.npart[2], h.npart[3], h.npart[4], h.npart[5],h.mass[0], h.mass[1], h.mass[2], h.mass[3], h.mass[4], h.mass[5],h.time, h.redshift,h.flag_sfr, h.flag_feedback,h.npart_total[0], h.npart_total[1], h.npart_total[2], h.npart_total[3], h.npart_total[4], h.npart_total[5],h.flag_cooling, h.num_files,h.BoxSize, h.Omega0, h.OmegaLambda, h.HubbleParameter,h.flag_stellarage, h.flag_metals, h.hashtabsize,h.headerOk, h.first_cell, h.last_cell); + + return res; +} +#endif + +static void +gadgetInitCatalog(mvc *m) +{ + sql_schema *sch; + sql_table *gadget_fl, *gadget_tbl, *gadget_col; + + MT_lock_init(&mt_gadget_lock, "gadget.lock"); + + sch = mvc_bind_schema(m, "sys"); + + gadget_fl = mvc_bind_table(m, sch, "gadget_files"); + if (gadget_fl == NULL) { + gadget_fl = mvc_create_table(m, sch, "gadget_files", tt_table, 0, SQL_PERSIST, 0, 2); + mvc_create_column_(m, gadget_fl, "id", "int", 32); + mvc_create_column_(m, gadget_fl, "name", "varchar", 80); + } + + gadget_tbl = mvc_bind_table(m, sch, "gadget_tables"); + if (gadget_tbl == NULL) { + gadget_tbl = mvc_create_table(m, sch, "gadget_tables", tt_table, 0, SQL_PERSIST, 0, 37); + mvc_create_column_(m, gadget_tbl, "id", "int", 32); + mvc_create_column_(m, gadget_tbl, "file_id", "int", 32); + mvc_create_column_(m, gadget_tbl, "name", "varchar", 80); + mvc_create_column_(m, gadget_tbl, "npart1", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart2", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart3", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart4", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart5", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart6", "int", 32); + mvc_create_column_(m, gadget_tbl, "mass1", "double", 64); + mvc_create_column_(m, gadget_tbl, "mass2", "double", 64); + mvc_create_column_(m, gadget_tbl, "mass3", "double", 64); + mvc_create_column_(m, gadget_tbl, "mass4", "double", 64); + mvc_create_column_(m, gadget_tbl, "mass5", "double", 64); + mvc_create_column_(m, gadget_tbl, "mass6", "double", 64); + mvc_create_column_(m, gadget_tbl, "time", "double", 64); + mvc_create_column_(m, gadget_tbl, "redshift", "double", 64); + mvc_create_column_(m, gadget_tbl, "flag_sfr", "int", 32); + mvc_create_column_(m, gadget_tbl, "flag_feedback", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart_total1", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart_total2", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart_total3", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart_total4", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart_total5", "int", 32); + mvc_create_column_(m, gadget_tbl, "npart_total6", "int", 32); + mvc_create_column_(m, gadget_tbl, "flag_cooling", "int", 32); + mvc_create_column_(m, gadget_tbl, "num_files", "int", 32); + mvc_create_column_(m, gadget_tbl, "BoxSize", "double", 64); + mvc_create_column_(m, gadget_tbl, "Omega0", "double", 64); + mvc_create_column_(m, gadget_tbl, "OmegaLambda", "double", 64); + mvc_create_column_(m, gadget_tbl, "HubbleParameter", "double", 64); + mvc_create_column_(m, gadget_tbl, "flag_stellarage", "int", 32); + mvc_create_column_(m, gadget_tbl, "flag_metals", "int", 32); + mvc_create_column_(m, gadget_tbl, "hashtabsize", "int", 32); + mvc_create_column_(m, gadget_tbl, "headerOk", "int", 32); + mvc_create_column_(m, gadget_tbl, "first_cell", "int", 32); + mvc_create_column_(m, gadget_tbl, "last_cell", "int", 32); + } + + gadget_col = mvc_bind_table(m, sch, "gadget_columns"); + if (gadget_col == NULL) { + gadget_col = mvc_create_table(m, sch, "gadget_columns", tt_table, 0, SQL_PERSIST, 0, 15); + mvc_create_column_(m, gadget_col, "id", "int", 32); + mvc_create_column_(m, gadget_col, "file_id", "int", 32); + mvc_create_column_(m, gadget_col, "table_id", "int", 32); + mvc_create_column_(m, gadget_col, "ScaleX", "double", 64); + mvc_create_column_(m, gadget_col, "ScaleY", "double", 64); + mvc_create_column_(m, gadget_col, "ScaleZ", "double", 64); + mvc_create_column_(m, gadget_col, "OffsetX", "double", 64); + mvc_create_column_(m, gadget_col, "OffsetY", "double", 64); + mvc_create_column_(m, gadget_col, "OffsetZ", "double", 64); + mvc_create_column_(m, gadget_col, "MinX", "double", 64); + mvc_create_column_(m, gadget_col, "MinY", "double", 64); + mvc_create_column_(m, gadget_col, "MinZ", "double", 64); + mvc_create_column_(m, gadget_col, "MaxX", "double", 64); + mvc_create_column_(m, gadget_col, "MaxY", "double", 64); + mvc_create_column_(m, gadget_col, "MaxZ", "double", 64); + } +} + +str gadgetListDir(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) +{ +#if 0 + (void) cntxt; + (void) mb; + (void) stk; + (void) pci; + return MAL_SUCCEED; +#endif + str msg = MAL_SUCCEED; + str dir = *getArgReference_str(stk, pci, 1); + DIR *dp; + struct dirent *ep; + FILE *stream = NULL; + char *s; + (void)mb; + + dp = opendir(dir); + if (dp != NULL) { + char stmt[BUFSIZ]; + char fname[BUFSIZ]; + + s = stmt; + + while ((ep = readdir(dp)) != NULL && !msg) { + if (ep->d_name[0] != '.') { + snprintf(fname, BUFSIZ, "%s%s", dir, ep->d_name); + + if ( !(stream = fopen(fname, "rb")) ) { + msg = createException(MAL, "gadgetDir", "Missing GADGET file %s: %s", fname, strerror(errno)); + return msg; + } + fclose(stream); + snprintf(stmt, BUFSIZ, ATTACHDIR, fname); + msg = SQLstatementIntern(cntxt, &s, "gadget.listofdir", TRUE, FALSE, NULL); + } + } + (void)closedir(dp); + } else + msg = createException(MAL, "gadgetDir", "Couldn't open the directory"); + + return msg; +#if 0 + return MAL_SUCCEED; +#endif +} + +static void +getPHkeyMinMax(PHBins bins, int numBins, int i, int *min_phkey, int *max_phkey) +{ + int j, min = bins.id[0], max = bins.id[0]; + for (j = 1; j < numBins; j++) { + if (bins.id[j] < min) + min = bins.id[j]; + if (bins.id[j] > max) + max = bins.id[j]; + } + _______________________________________________ checkin-list mailing list [email protected] https://www.monetdb.org/mailman/listinfo/checkin-list
