Changeset: e871ab80d0f5 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=e871ab80d0f5
Added Files:
sql/backends/monet5/Tests/fwf.malC
sql/backends/monet5/fwf.c
sql/backends/monet5/fwf.h
sql/backends/monet5/fwf.mal
Modified Files:
sql/backends/monet5/Makefile.ag
Branch: fixed-width-format
Log Message:
First part of fixed-width-format import
diffs (248 lines):
diff --git a/sql/backends/monet5/Makefile.ag b/sql/backends/monet5/Makefile.ag
--- a/sql/backends/monet5/Makefile.ag
+++ b/sql/backends/monet5/Makefile.ag
@@ -46,7 +46,8 @@ lib__sql = {
sql_round.c sql_round_impl.h sql_bat2time.c \
sql_fround.c sql_fround_impl.h \
sql_orderidx.c sql_orderidx.h \
- sql_rank.c sql_rank.h
+ sql_rank.c sql_rank.h \
+ fwf.c fwf.h
LIBS = ../../server/libsqlserver \
../../storage/libstore \
../../storage/bat/libbatstore \
@@ -64,7 +65,8 @@ headers_mal = {
DIR = libdir/monetdb5
SOURCES = sql_aggr_bte.mal sql_aggr_flt.mal sql_aggr_dbl.mal
sql_aggr_int.mal sql_aggr_lng.mal \
sql_aggr_sht.mal sql_decimal.mal sql_inspect.mal \
- sql_rank.mal sql.mal
+ sql_rank.mal sql.mal \
+ fwf.mal
}
headers_mal_hge = {
diff --git a/sql/backends/monet5/Tests/fwf.malC
b/sql/backends/monet5/Tests/fwf.malC
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/Tests/fwf.malC
@@ -0,0 +1,25 @@
+# write.fwf(mtcars, "/tmp/mtcars", header=F)
+
+# CREATE TABLE mtcars (mpg DOUBLE PRECISION, cyl DOUBLE PRECISION, disp DOUBLE
PRECISION, hp DOUBLE PRECISION, drat DOUBLE PRECISION, wt DOUBLE PRECISION,
qsec DOUBLE PRECISION, vs DOUBLE PRECISION, am DOUBLE PRECISION, gear DOUBLE
PRECISION, carb DOUBLE PRECISION)
+
+#4 2 6 4 5 6 6 2 2 2 2
+# 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4
+
+sql.start2();
+
+bint:= bat.new(:oid,:int);
+bat.append(bint,4:int);
+bat.append(bint,2:int);
+bat.append(bint,6:int);
+bat.append(bint,4:int);
+bat.append(bint,5:int);
+bat.append(bint,6:int);
+bat.append(bint,6:int);
+bat.append(bint,2:int);
+bat.append(bint,2:int);
+bat.append(bint,2:int);
+bat.append(bint,2:int);
+
+include fwf;
+
+fwf.load("sys", "mtcars", "/tmp/mtcars", bint, 32:bte);
diff --git a/sql/backends/monet5/fwf.c b/sql/backends/monet5/fwf.c
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/fwf.c
@@ -0,0 +1,143 @@
+/* Ghetto programming warning. */
+
+#include "monetdb_config.h"
+#include "mal.h"
+#include "mal_stack.h"
+#include "mal_linker.h"
+#include "gdk_utils.h"
+#include "gdk.h"
+#include "mmath.h"
+#include "sql_catalog.h"
+#include "sql_execute.h"
+#include "fwf.h"
+
+str fwf_load(mvc *m, char* schema, char* table, char* filename, BAT *widths,
char padding) {
+ FILE *f;
+ size_t fsize;
+ struct stat stb;
+ char* fptr, *fcur = NULL;
+ size_t i;
+ size_t width_sum = 0;
+ sql_schema *s;
+ sql_table *t;
+ node *n;
+ str msg = MAL_SUCCEED;
+ size_t ncol;
+ size_t approx_nrows;
+ size_t row = 0;
+ BAT** appends;
+ char* line = NULL;
+ int *widths_ptr = (int*) widths->T->heap.base;
+
+ s = mvc_bind_schema(m, schema);
+ if (s == NULL)
+ throw(SQL, "sql.append", "Schema missing");
+ t = mvc_bind_table(m, s, table);
+ if (t == NULL)
+ throw(SQL, "sql.append", "Table missing");
+
+ f = fopen(filename, "r");
+ if (fstat(fileno(f), &stb) != 0) {
+ msg = createException(MAL, "fwf.load", "Could not stat file");
+ goto cleanup;
+ }
+ fsize = (size_t) stb.st_size;
+ fptr = GDKmmap(filename, MMAP_READ, fsize);
+ if (!fptr) {
+ msg = createException(MAL, "fwf.load", "Could not map file");
+ goto cleanup;
+ }
+
+ ncol = t->columns.set->cnt;
+ if (BATcount(widths) != ncol) {
+ msg = createException(MAL, "fwf.load", "Incorrect number of
widths supplied");
+ goto cleanup;
+ }
+
+ appends = malloc(sizeof(BAT*)*ncol);
+ if (!appends) {
+ msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL);
+ goto cleanup;
+ }
+
+ for (i = 0; i < ncol; i++) {
+ width_sum += widths_ptr[i];
+ }
+ line = GDKmalloc(width_sum + ncol + 1);
+ if (!line) {
+ msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL);
+ goto cleanup;
+ }
+
+ approx_nrows = fsize/width_sum;
+ i = 0;
+ for (n = t->columns.set->h; n; n = n->next) {
+ sql_column *c = n->data;
+ appends[i] = BATnew(TYPE_void, c->type.type->localtype,
approx_nrows, TRANSIENT) ;
+ if (!appends[i]) {
+ msg = createException(MAL, "fwf.load", MAL_MALLOC_FAIL);
+ goto cleanup;
+ }
+ i++;
+ }
+
+ for (fcur = fptr; fcur < fptr + fsize - width_sum;) {
+ for (i=0; i < ncol; i++) {
+ char* val_start = line;
+ char* val_end = line + widths_ptr[i] - 1;
+ int type = appends[i]->T->type;
+ strncpy(val_start, fcur, widths_ptr[i]);
+ while (*val_start == padding) val_start++;
+ while (*val_end == padding) val_end--;
+ val_end[1] = '\0';
+
+ if (type == TYPE_str) {
+ BUNappend(appends[i], &val_start, 0);
+ } else {
+ char* dst = appends[i]->T->heap.base +
BATatoms[type].size * row;
+ if ((*BATatoms[type].atomFromStr) (val_start,
(int*) &BATatoms[type].size, (void**) &dst) < 0) {
+ msg = createException(MAL, "fwf.load",
"Conversion error");
+ goto cleanup;
+ }
+ }
+ fcur += widths_ptr[i];
+ }
+ row++;
+ while (*fcur != '\n' && fcur < fptr + fsize - width_sum) fcur++;
+ fcur++;
+ }
+
+ if (!m->session->active) mvc_trans(m);
+ i = 0;
+ for (n = t->columns.set->h; n; n = n->next) {
+ BATsetcount(appends[i], row);
+ store_funcs.append_col(m->session->tr, n->data,
&appends[i]->batCacheid, TYPE_bat);
+ i++;
+ }
+ sqlcleanup(m, 0);
+
+cleanup:
+ // TODO this leaks like an SR71
+
+ return msg;
+}
+
+str fwf_load_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci) {
+
+ mvc *m = NULL;
+ str msg = MAL_SUCCEED;
+ str sname = *getArgReference_str(stk, pci, 1);
+ str tname = *getArgReference_str(stk, pci, 2);
+ str fname = *getArgReference_str(stk, pci, 3);
+ bat batid = *getArgReference_bat(stk, pci, 4);
+ bte padding = *getArgReference_bte(stk, pci, 5);
+
+ if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != NULL)
+ return msg;
+ if ((msg = checkSQLContext(cntxt)) != NULL)
+ return msg;
+
+ msg = fwf_load(m, sname, tname, fname, BATdescriptor(batid), padding);
+ return msg;
+}
+
diff --git a/sql/backends/monet5/fwf.h b/sql/backends/monet5/fwf.h
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/fwf.h
@@ -0,0 +1,25 @@
+
+#ifndef _FWF_H_
+#define _FWF_H_
+
+/* #define _DEBUG_FWF_ */
+
+#include "mal.h"
+#include "mal_exception.h"
+#include "mal_interpreter.h"
+
+#ifdef WIN32
+#if !defined(LIBMAL) && !defined(LIBATOMS) && !defined(LIBKERNEL) &&
!defined(LIBMAL) && !defined(LIBOPTIMIZER) && !defined(LIBSCHEDULER) &&
!defined(LIBMONETDB5)
+#define fwf_export extern __declspec(dllimport)
+#else
+#define fwf_export extern __declspec(dllexport)
+#endif
+#else
+#define fwf_export extern
+#endif
+
+
+fwf_export str fwf_load(mvc *m, char* schema, char* table, char* filename, BAT
*widths, char padding);
+fwf_export str fwf_load_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
+
+#endif
diff --git a/sql/backends/monet5/fwf.mal b/sql/backends/monet5/fwf.mal
new file mode 100644
--- /dev/null
+++ b/sql/backends/monet5/fwf.mal
@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.
+
+module fwf;
+
+pattern
load(schema:str,table:str,filename:str,widths:bat[:int],padding:bte):str
+address fwf_load_wrap
+comment "import fixed width files into MonetDB";
+
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
