Changeset: 7b69da406a78 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7b69da406a78
Added Files:
ctest/tools/monetdbe/example_proxy.c
design.txt
Modified Files:
clients/Tests/exports.stable.out
ctest/tools/monetdbe/CMakeLists.txt
sql/backends/monet5/sql_result.c
tools/monetdbe/monetdbe.c
Branch: monetdbe-proxy
Log Message:
succesful poc: monetdbe as remote-proxy based on columnary binary protocol.
diffs (truncated from 611 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -602,6 +602,7 @@ int mapi_fetch_row(MapiHdl hdl);
MapiMsg mapi_finish(MapiHdl hdl);
MapiHdl mapi_get_active(Mapi mid);
bool mapi_get_autocommit(Mapi mid);
+bool mapi_get_columnar_protocol(Mapi mid);
const char *mapi_get_dbname(Mapi mid);
int mapi_get_digits(MapiHdl hdl, int fnr);
int mapi_get_field_count(MapiHdl hdl);
@@ -660,6 +661,7 @@ int64_t mapi_rows_affected(MapiHdl hdl);
MapiMsg mapi_seek_row(MapiHdl hdl, int64_t rowne, int whence);
MapiHdl mapi_send(Mapi mid, const char *cmd);
MapiMsg mapi_setAutocommit(Mapi mid, bool autocommit);
+MapiMsg mapi_set_columnar_protocol(Mapi mid, bool columnar_protocol);
MapiMsg mapi_set_size_header(Mapi mid, bool value);
void mapi_setfilecallback(Mapi mid, char *(*getfunc)(void *priv, const char
*filename, bool binary, uint64_t offset, size_t *size), char *(*putfunc)(void
*priv, const char *filename, const void *data, size_t size), void *priv);
int mapi_split_line(MapiHdl hdl);
diff --git a/ctest/tools/monetdbe/CMakeLists.txt
b/ctest/tools/monetdbe/CMakeLists.txt
--- a/ctest/tools/monetdbe/CMakeLists.txt
+++ b/ctest/tools/monetdbe/CMakeLists.txt
@@ -20,6 +20,12 @@ target_link_libraries(example2
monetdbe)
#add_test(run_example2 example2)
+add_executable(example_proxy example_proxy.c)
+target_link_libraries(example_proxy
+ PRIVATE
+ monetdb_config_header
+ monetdbe)
+
add_executable(example_temporal example_temporal.c)
target_link_libraries(example_temporal
PRIVATE
diff --git a/ctest/tools/monetdbe/example_proxy.c
b/ctest/tools/monetdbe/example_proxy.c
new file mode 100644
--- /dev/null
+++ b/ctest/tools/monetdbe/example_proxy.c
@@ -0,0 +1,82 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright 1997 - July 2008 CWI, August 2008 - 2020 MonetDB B.V.
+ */
+
+#include "monetdbe.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <inttypes.h>
+
+#define error(msg) {fprintf(stderr, "Failure: %s\n", msg); return -1;}
+
+int
+main(void)
+{
+ char* err = NULL;
+ monetdbe_database mdbe = NULL;
+ monetdbe_result* result = NULL;
+
+ monetdbe_remote remote = {
+ .host = "127.0.0.1",
+ .port = 50001,
+ .username="monetdb",
+ .password="monetdb",
+ .lang="sql"};
+
+ monetdbe_options opt = {.remote = &remote};
+
+ // second argument is a string for the db directory or NULL for
in-memory mode
+ if (monetdbe_open(&mdbe,
"mapi:monetdb://127.0.0.1:50001?database=devdb1", &opt))
+ error("Failed to open database")
+
+ // Assumes the existance of a table test (x INT, y STRING) on the
remote.
+ if ((err = monetdbe_query(mdbe, "SELECT x, y FROM test; ", &result,
NULL)) != NULL)
+ error(err)
+
+ fprintf(stdout, "Query result with %zu cols and %"PRId64" rows\n",
result->ncols, result->nrows);
+ for (int64_t r = 0; r < result->nrows; r++) {
+ for (size_t c = 0; c < result->ncols; c++) {
+ monetdbe_column* rcol;
+ if ((err = monetdbe_result_fetch(result, &rcol, c)) !=
NULL)
+ error(err)
+ switch (rcol->type) {
+ case monetdbe_int32_t: {
+ monetdbe_column_int32_t * col =
(monetdbe_column_int32_t *) rcol;
+ if (col->data[r] == col->null_value) {
+ printf("NULL");
+ } else {
+ printf("%d", col->data[r]);
+ }
+ break;
+ }
+ case monetdbe_str: {
+ monetdbe_column_str * col =
(monetdbe_column_str *) rcol;
+ if (col->is_null(col->data[r])) {
+ printf("NULL");
+ } else {
+ printf("%s", (char*)
col->data[r]);
+ }
+ break;
+ }
+ default: {
+ printf("UNKNOWN");
+ }
+ }
+
+ if (c + 1 < result->ncols) {
+ printf(", ");
+ }
+ }
+ printf("\n");
+ }
+
+ if ((err = monetdbe_cleanup_result(mdbe, result)) != NULL)
+ error(err)
+ if (monetdbe_close(mdbe))
+ error("Failed to close database")
+ return 0;
+}
diff --git a/design.txt b/design.txt
new file mode 100644
--- /dev/null
+++ b/design.txt
@@ -0,0 +1,59 @@
+server afhandeling van xcommand calls
+ sql_scenario.c:SQLparser
+
+client kant van xcommand
+ bijvoorbeeld mapi.c:mapi_setAutocommit
+
+bepaling van MAPI protocol server kant
+ mal_mapi.c:doChallenge voor parsen en mal_session.c:MSscheduleClient zet
het Client.protocol field gelijk aan protocol
+
+Opmerkingen:
+mapi.c/h kent geen protocol verschillen zoals PROTOCOL_9 en
PROTOCOL_10/"PROT10".
+
+Via een aanvankelijk mapi xcommand aangeven dat we PROTOCOL_COLUMNAR
gebruiken. PROTOCOL_COLUMNAR moet er voor zorgen dat sql_result sets geen
tablet results gaat maken maar alleen de bats vasthoud.
+De headers moeten
+
+gdb -ex r --args mserver5 --dbpath=devdb --set "mapi_port=50000"
+gdb -ex r --args mserver5 --dbpath=devdb1 --set "mapi_port=50001"
+
+mclient -p 50001
+CREATE TABLE t1 (i int);
+INSERT INTO t1 VALUES (20), (10), (NULL), (30);
+
+CREATE REMOTE TABLE rt (i int) ON
'mapi:monetdb://127.0.0.1:50001/devdb1/sys/t1';
+
+SELECT * FROM rt;
+
+this will create the following mal function user.%4 on the local side in
sql_gencode.c:_create_relational_remote.
+
+function user.%4():bat[:int];
+ X_2:bat[:int] := bat.new(nil:int);
+ X_4:str := remote.connect("sys.rt":str, "msql":str);
+ remote.put(X_4:str, 15:int);
+ X_11:str := remote.put(X_4:str, "user":str);
+ X_13:str := remote.put(X_4:str, "l4":str);
+ X_15:str := remote.put(X_4:str, "\nproject (\ntable(sys.t1) [ \"t1\".\"i\"
as \"rt\".\"i\" ] COUNT \n) [ \"rt\".\"i\" ] REMOTE sys.rt\n":str);
+ X_17:str := remote.put(X_4:str, "":str);
+ X_19:str := remote.put(X_4:str, "int":str);
+ remote.exec(X_4:str, "sql":str, "register":str, X_11:str, X_13:str,
X_15:str, X_17:str, X_19:str);
+ remote.put(X_4:str, 5:int);
+ X_25:str := remote.put(X_4:str,
"df7b54d9-da0f-4f38-a40b-10f675da1e7e":str);
+ X_27:str := remote.put(X_4:str,
"8499eb56-2c5b-4509-97a5-ac4b90d3b309":str);
+ remote.exec(X_4:str, "remote":str, "register_supervisor":str, X_25:str,
X_27:str);
+ remote.register_supervisor("df7b54d9-da0f-4f38-a40b-10f675da1e7e":str,
"8499eb56-2c5b-4509-97a5-ac4b90d3b309":str);
+ X_30:str := remote.put(X_4:str, X_2:bat[:int]);
+ X_30:str := remote.exec(X_4:str, "user":str, "l4":str);
+ X_2:bat[:int] := remote.get(X_4:str, X_30:str);
+ remote.disconnect(X_4:str);
+ return X_2:bat[:int] := X_2:bat[:int];
+
+
+DESIGN
+Xset_protocol PROTOCOL_COLUMNAR
+sql_result checkt taal
+stuurt header op welke ook de volgorde van opsturen van de BAT's bepaald en
stuurt vervolgens de BAT's ook op in die volgorde via de onderliggende mapi
stream a la remote.
+
+de client leest de header stream en pakt daarna de onderliggende read stream
en vangt daarmee de bats op.
+
+
+
diff --git a/sql/backends/monet5/sql_result.c b/sql/backends/monet5/sql_result.c
--- a/sql/backends/monet5/sql_result.c
+++ b/sql/backends/monet5/sql_result.c
@@ -1628,6 +1628,76 @@ cleanup:
return fres;
}
+// TODO copied from/based on remote.c:RMTbincopyto
+static int
+mvc_export_table_columnar(stream *s, res_table *t, BAT *order) {
+ int i;
+
+ (void) order;
+
+ if (!t)
+ return -1;
+ if (!s)
+ return 0;
+
+ for (i = 1; i <= t->nr_cols; i++) {
+ res_col *c = t->cols + (i - 1);
+
+ if (!c->b)
+ break;
+
+ BAT* b = BATdescriptor(c->b);
+ if (b == NULL) {
+ while (--i >= 1)
+ BBPunfix(b->batCacheid);
+ return -1;
+ }
+
+ BAT* bn = b;
+
+ bool sendtheap = b->ttype != TYPE_void && b->tvarsized;
+
+ mnstr_printf(s, /*JSON*/"{"
+ "\"version\":1,"
+ "\"ttype\":%d,"
+ "\"hseqbase\":" OIDFMT ","
+ "\"tseqbase\":" OIDFMT ","
+ "\"tsorted\":%d,"
+ "\"trevsorted\":%d,"
+ "\"tkey\":%d,"
+ "\"tnonil\":%d,"
+ "\"tdense\":%d,"
+ "\"size\":" BUNFMT ","
+ "\"tailsize\":%zu,"
+ "\"theapsize\":%zu"
+ "}\n",
+ bn->ttype,
+ bn->hseqbase, bn->tseqbase,
+ bn->tsorted, bn->trevsorted,
+ bn->tkey,
+ bn->tnonil,
+ BATtdense(bn),
+ bn->batCount,
+ (size_t)bn->batCount * Tsize(bn),
+ sendtheap && bn->batCount > 0 ?
bn->tvheap->free : 0
+ );
+
+ if (bn->batCount > 0) {
+ mnstr_write(s, /* tail */
+ Tloc(bn, 0), bn->batCount * Tsize(bn), 1);
+ if (sendtheap)
+ mnstr_write(s, /* theap */
+ Tbase(bn), bn->tvheap->free, 1);
+ }
+
+ mnstr_flush(s);
+
+ BBPunfix(b->batCacheid);
+ }
+
+ return 0;
+}
+
static int
mvc_export_table(backend *b, stream *s, res_table *t, BAT *order, BUN offset,
BUN nr, const char *btag, const char *sep, const char *rsep, const char *ssep,
const char *ns)
{
@@ -2347,11 +2417,6 @@ mvc_export_result(backend *b, stream *s,
if (!s || !t)
return 0;
- if (b->client->protocol == PROTOCOL_10) {
- // Don't do anything
- return 0;
- }
-
/* Proudly supporting SQLstatementIntern's output flag */
if (b->output_format == OFMT_NONE) {
return 0;
@@ -2376,6 +2441,11 @@ mvc_export_result(backend *b, stream *s,
if (!order)
return -1;
+ if (b->client->protocol == PROTOCOL_COLUMNAR) {
+ if (mnstr_flush(s) < 0) return -1;
+ return mvc_export_table_columnar(s, t, order);
+ }
+
count = m->reply_size;
if (m->reply_size != -2 && (count <= 0 || count >= t->nr_rows)) {
count = t->nr_rows;
diff --git a/tools/monetdbe/monetdbe.c b/tools/monetdbe/monetdbe.c
--- a/tools/monetdbe/monetdbe.c
+++ b/tools/monetdbe/monetdbe.c
@@ -28,6 +28,7 @@
#include "mapi.h"
#include "monetdbe_mapi.h"
#include "msqldump.h"
+#include "sql_result.h"
#define UNUSED(x) (void)(x)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
